diff --git a/common/sampling.cpp b/common/sampling.cpp index 0fe1c7405e14..713c6887fefa 100644 --- a/common/sampling.cpp +++ b/common/sampling.cpp @@ -125,6 +125,8 @@ struct common_sampler { // for rejection sampling; independent of the draft, or the target distribution is not preserved std::mt19937 rng; + bool pure_argmax_fast = false; + void reset() { prev.clear(); @@ -342,10 +344,56 @@ struct common_sampler * common_sampler_init( } } + bool pure_argmax_fast = false; + if (params.mirostat == 0) { bool use_adaptive_p = false; // see below + const bool greedy = params.n_probs == 0 && !params.samplers.empty() && + ((params.samplers.back() == COMMON_SAMPLER_TYPE_TEMPERATURE && params.temp <= 0.0f && params.dynatemp_range == 0.0f) || + (params.samplers.back() == COMMON_SAMPLER_TYPE_TOP_K && params.top_k == 1)); + + bool pure_argmax_chain = greedy && !grmr && !rbudget && !getenv("GGML_VK_DISABLE_GREEDY_FAST"); + if (pure_argmax_chain) { + for (const auto & cnstr : params.samplers) { + switch (cnstr) { + case COMMON_SAMPLER_TYPE_TOP_K: + break; + case COMMON_SAMPLER_TYPE_TOP_P: + case COMMON_SAMPLER_TYPE_MIN_P: + if (params.min_keep < 1) { pure_argmax_chain = false; } + break; + case COMMON_SAMPLER_TYPE_TYPICAL_P: + if (params.typ_p < 1.0f) { pure_argmax_chain = false; } + break; + case COMMON_SAMPLER_TYPE_TOP_N_SIGMA: + if (params.top_n_sigma > 0.0f) { pure_argmax_chain = false; } + break; + case COMMON_SAMPLER_TYPE_TEMPERATURE: + if (params.temp > 0.0f || params.dynatemp_range != 0.0f) { pure_argmax_chain = false; } + break; + case COMMON_SAMPLER_TYPE_PENALTIES: + if (params.penalty_last_n != 0 && + (params.penalty_repeat != 1.0f || params.penalty_freq != 0.0f || params.penalty_present != 0.0f)) { + pure_argmax_chain = false; + } + break; + case COMMON_SAMPLER_TYPE_DRY: + if (params.dry_multiplier > 0.0f) { pure_argmax_chain = false; } + break; + case COMMON_SAMPLER_TYPE_XTC: + if (params.xtc_probability > 0.0f) { pure_argmax_chain = false; } + break; + default: + pure_argmax_chain = false; + break; + } + } + } + pure_argmax_fast = pure_argmax_chain && samplers.empty(); + + if (!pure_argmax_chain) { for (const auto & cnstr : params.samplers) { switch (cnstr) { case COMMON_SAMPLER_TYPE_DRY: @@ -396,14 +444,12 @@ struct common_sampler * common_sampler_init( GGML_ASSERT(false && "unknown sampler type"); } } + } if (use_adaptive_p) { // only if user explicitly included adaptive-p sampler samplers.push_back(llama_sampler_init_adaptive_p(params.adaptive_target, params.adaptive_decay, params.seed)); } else { // Keep distribution sampling when callers request probabilities. - const bool greedy = params.n_probs == 0 && !params.samplers.empty() && - ((params.samplers.back() == COMMON_SAMPLER_TYPE_TEMPERATURE && params.temp == 0.0f && params.dynatemp_range == 0.0f) || - (params.samplers.back() == COMMON_SAMPLER_TYPE_TOP_K && params.top_k == 1)); samplers.push_back(greedy ? llama_sampler_init_greedy() : llama_sampler_init_dist(params.seed)); } } else if (params.mirostat == 1) { @@ -442,6 +488,7 @@ struct common_sampler * common_sampler_init( /* .cur_p = */ {}, // mix it, the chain and the draft are seeded from this one too /* .rng = */ std::mt19937(llama_sampler_get_seed(chain) ^ 0x9e3779b9u), + /* .pure_argmax_fast = */ pure_argmax_fast, }; return result; @@ -526,6 +573,7 @@ struct common_sampler * common_sampler_clone(common_sampler * gsmpl) { /* .cur = */ gsmpl->cur, /* .cur_p = */ gsmpl->cur_p, /* .rng = */ gsmpl->rng, + /* .pure_argmax_fast = */ gsmpl->pure_argmax_fast, }; } @@ -547,6 +595,7 @@ void common_sampler_copy(const common_sampler * src, common_sampler * dst) { dst->cur_p = src->cur_p; dst->cur_p.data = src->cur_p.data ? dst->cur.data() : nullptr; // re-point to dst's buffer dst->rng = src->rng; + dst->pure_argmax_fast = src->pure_argmax_fast; dst->t_total_us = src->t_total_us; } @@ -617,6 +666,28 @@ llama_token common_sampler_sample(struct common_sampler * gsmpl, struct llama_co auto & chain = gsmpl->chain; auto & cur_p = gsmpl->cur_p; // initialized by set_logits + if (gsmpl->pure_argmax_fast && !gsmpl->grmr && !gsmpl->rbudget && + llama_get_sampled_token_ith(ctx, idx) == LLAMA_TOKEN_NULL && + !llama_get_sampled_probs_ith(ctx, idx) && + !llama_get_sampled_logits_ith(ctx, idx)) { + const float * logits = llama_get_logits_ith(ctx, idx); + GGML_ASSERT(logits != nullptr); + const int n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(llama_get_model(ctx))); + llama_token best_id = 0; + float best_val = logits[0]; + for (llama_token token_id = 1; token_id < n_vocab; ++token_id) { + const float v = logits[token_id]; + if (v > best_val) { + best_val = v; + best_id = token_id; + } + } + gsmpl->cur.resize(1); + gsmpl->cur[0] = llama_token_data{ best_id, best_val, 1.0f }; + cur_p = { gsmpl->cur.data(), 1, 0, true }; + return best_id; + } + gsmpl->set_logits(ctx, idx); // Check if a backend sampler has already sampled a token in which case we @@ -671,6 +742,28 @@ llama_token common_sampler_sample(struct common_sampler * gsmpl, struct llama_co // resampling: // if the token is not valid, sample again, but first apply the grammar sampler and then the sampling chain + if (gsmpl->pure_argmax_fast && !gsmpl->grmr && !gsmpl->rbudget && + llama_get_sampled_token_ith(ctx, idx) == LLAMA_TOKEN_NULL && + !llama_get_sampled_probs_ith(ctx, idx) && + !llama_get_sampled_logits_ith(ctx, idx)) { + const float * logits = llama_get_logits_ith(ctx, idx); + GGML_ASSERT(logits != nullptr); + const int n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(llama_get_model(ctx))); + llama_token best_id = 0; + float best_val = logits[0]; + for (llama_token token_id = 1; token_id < n_vocab; ++token_id) { + const float v = logits[token_id]; + if (v > best_val) { + best_val = v; + best_id = token_id; + } + } + gsmpl->cur.resize(1); + gsmpl->cur[0] = llama_token_data{ best_id, best_val, 1.0f }; + cur_p = { gsmpl->cur.data(), 1, 0, true }; + return best_id; + } + gsmpl->set_logits(ctx, idx); llama_sampler_apply(rbudget, &cur_p); diff --git a/common/speculative.cpp b/common/speculative.cpp index b40eb5535f73..8e88e41bd909 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -64,7 +64,11 @@ static void spec_retune( sparams.temp = cur.temp; // must be explicit, the default reseeds at random; mixed so it differs from the target's sparams.seed = cur.seed == LLAMA_DEFAULT_SEED ? cur.seed : cur.seed ^ 0x85ebca6bu; - sparams.samplers = { COMMON_SAMPLER_TYPE_TOP_K, COMMON_SAMPLER_TYPE_TEMPERATURE }; + if (cur.temp <= 0.0f && !getenv("GGML_VK_DISABLE_GREEDY_FAST")) { + sparams.samplers = { COMMON_SAMPLER_TYPE_TEMPERATURE }; + } else { + sparams.samplers = { COMMON_SAMPLER_TYPE_TOP_K, COMMON_SAMPLER_TYPE_TEMPERATURE }; + } smpls[seq_id].reset(common_sampler_init(model, sparams)); } diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-common.h b/ggml/src/ggml-vulkan/ggml-vulkan-common.h index 69028c04251b..e991bfead6d8 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan-common.h +++ b/ggml/src/ggml-vulkan/ggml-vulkan-common.h @@ -111,7 +111,7 @@ void ggml_vk_dsv4_hc_pre(ggml_backend_vk_context * ctx, vk_context& subctx, cons void ggml_vk_dsv4_hc_post(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * x, const ggml_tensor * residual, const ggml_tensor * post, const ggml_tensor * comb, ggml_tensor * dst, const ggml_tensor * gate_scale_in = nullptr); void ggml_vk_top_k_mask(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst); void ggml_vk_mul_mat(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx); -bool ggml_vk_use_mul_mat_vec_id(const struct ggml_cgraph * cgraph, int node_idx); +bool ggml_vk_use_mul_mat_vec_id(const ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx); void ggml_vk_mul_mat_id(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx); // flash-attn @@ -303,6 +303,14 @@ inline void ggml_vk_dispatch_pipeline(ggml_backend_vk_context* ctx, vk_context& GGML_ASSERT(pipeline->parameter_count == descriptor_buffer_infos.size()); GGML_ASSERT(pipeline->push_constant_size == push_constant_size(push_constants)); + ctx->stat_dispatches++; + { + // gen: GGML_VK_GRAPH_STATS=2 lists every dispatch (pipeline and workgroups) + static const int gen_stats_level = getenv("GGML_VK_GRAPH_STATS") ? atoi(getenv("GGML_VK_GRAPH_STATS")) : 0; + if (gen_stats_level >= 2) { + fprintf(stderr, "ggml_vk_dispatch: %s %u %u %u\n", pipeline->name.c_str(), wg0, wg1, wg2); + } + } const uint32_t descriptor_set_idx = ctx->descriptor_set_idx++; vk::DescriptorSet& descriptor_set = ctx->descriptor_sets[descriptor_set_idx]; diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-push-constants.h b/ggml/src/ggml-vulkan/ggml-vulkan-push-constants.h index caa852e28bf7..f978ce821cf8 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan-push-constants.h +++ b/ggml/src/ggml-vulkan/ggml-vulkan-push-constants.h @@ -15,6 +15,20 @@ struct vk_mat_mat_push_constants { uint32_t padded_N; }; +struct vk_mat_mat_fused_push_constants { + uint32_t M; uint32_t N; uint32_t K; + uint32_t stride_a; uint32_t stride_b; uint32_t stride_d; + uint32_t batch_stride_a; uint32_t batch_stride_b; uint32_t batch_stride_d; + uint32_t base_work_group_z; uint32_t num_batches; + uint32_t k_split; + uint32_t ne02; uint32_t ne12; uint32_t broadcast2; uint32_t broadcast3; + uint32_t padded_N; + uint32_t fusion_flags; + uint32_t pair_m0; + uint32_t stride_d2; + uint32_t batch_stride_d2; +}; + struct vk_mat_vec_push_constants { uint32_t ncols; uint32_t stride_a; @@ -29,6 +43,36 @@ struct vk_mat_vec_push_constants { uint32_t ne12; uint32_t broadcast2; uint32_t broadcast3; + uint32_t pair_m0; + uint32_t stride_d2; + uint32_t batch_stride_d2; + uint32_t lora_r; + float lora_scale0; + float lora_scale1; +}; + +struct vk_mat_mv_mma_push_constants { + uint32_t M; + uint32_t N; + uint32_t K; + uint32_t stride_a; + uint32_t stride_b; + uint32_t stride_d; + uint32_t batch_stride_a; + uint32_t batch_stride_b; + uint32_t batch_stride_d; + uint32_t fusion_flags; + uint32_t base_work_group_z; + uint32_t ne02; + uint32_t ne12; + uint32_t broadcast2; + uint32_t broadcast3; + uint32_t pair_m0; + uint32_t stride_d2; + uint32_t batch_stride_d2; + uint32_t lora_r; + float lora_scale0; + float lora_scale1; }; struct vk_mat_vec_p021_push_constants { @@ -66,6 +110,16 @@ struct vk_mat_mat_id_push_constants { uint32_t row_ids_offset; }; +struct vk_mat_mat_id_fused_push_constants { + uint32_t M; uint32_t N; uint32_t K; + uint32_t stride_a; uint32_t stride_b; uint32_t stride_d; + uint32_t batch_stride_a; uint32_t batch_stride_b; uint32_t batch_stride_d; + uint32_t nei0; uint32_t nei1; uint32_t nbi1; uint32_t ne11; + uint32_t n_experts; + uint32_t row_ids_offset; + uint32_t fusion_flags; +}; + struct vk_mat_vec_id_push_constants { uint32_t ncols; uint32_t stride_a; @@ -454,15 +508,23 @@ struct vk_op_multi_add_push_constants { uint32_t ne20; uint32_t ne21; uint32_t ne22; uint32_t ne23; // strides for srcs+dst - uint32_t nb[MAX_PARAMETER_COUNT][4]; + uint32_t nb[12][4]; uint32_t rms_partials; }; -static_assert(MAX_PARAMETER_COUNT == 12); - static_assert(sizeof(vk_op_multi_add_push_constants) <= 256); +struct vk_op_multi_add_rms_push_constants { + uint32_t ne0; + uint32_t nrows; + float eps; + uint32_t write_add_dst; + uint32_t nb1[10]; +}; + +static_assert(sizeof(vk_op_multi_add_rms_push_constants) <= 128); + struct vk_op_topk_moe_push_constants { uint32_t n_rows; uint32_t n_experts_push; @@ -730,6 +792,7 @@ struct vk_op_gated_delta_net_push_constants { uint32_t neq1, rq3; float scale; uint32_t K; + uint32_t sout_nb2; }; struct vk_op_ssm_scan_push_constants { @@ -746,6 +809,9 @@ struct vk_op_ssm_conv_push_constants { uint32_t nb11; uint32_t dst_nb0, dst_nb1, dst_nb2; uint32_t nc, ncs, nr, n_t, n_s; + uint32_t st_nb1, st_nb2; + uint32_t qkv_nb0, qkv_nb1, qkv_nb2; + uint32_t st_k, st_slot_nb; }; struct vk_op_conv2d_push_constants { diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-types.h b/ggml/src/ggml-vulkan/ggml-vulkan-types.h index e19e2d029cf3..8f629bde263b 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan-types.h +++ b/ggml/src/ggml-vulkan/ggml-vulkan-types.h @@ -217,9 +217,9 @@ static bool is_pow2(uint32_t x) { return x > 1 && (x & (x-1)) == 0; } #define VK_LOG_DEBUG(msg) ((void) 0) #endif // GGML_VULKAN_DEBUG -#define MAX_PARAMETER_COUNT 12 +#define MAX_PARAMETER_COUNT 14 -#define MAX_FUSED_ADDS (MAX_PARAMETER_COUNT - 3) +#define MAX_FUSED_ADDS 9 struct vk_pipeline_struct; @@ -250,6 +250,12 @@ struct vk_pipeline_struct { // linked list of pipelines for multiple compilation variants. // currently only used to compile a 64-bit indexing variant. vk_pipeline next; + // gen: mat-vec / matmul twins compiled lazily with spec constant = 1 (single-op epilogue), 2 (PAIR_MM), 3 (PAIR_GLU), 4 (fast BIAS/SCALE) + vk_pipeline gen_glu; + vk_pipeline gen_pair_mm; + vk_pipeline gen_pair_glu; + vk_pipeline gen_fast_ep; + vk_pipeline gen_lora_twin; }; typedef std::weak_ptr vk_pipeline_ref; @@ -362,6 +368,7 @@ struct vk_queue { }; static constexpr uint32_t mul_mat_vec_max_cols = 8; +static constexpr uint32_t mul_mat_vec_max_cols_limit = 32; static constexpr uint32_t p021_max_gqa_ratio = 8; @@ -650,6 +657,8 @@ enum rms_norm_mode { RMS_NORM_MUL_ROPE, RMS_NORM_MUL_ROPE_VIEW_SET_ROWS, RMS_NORM_VIEW_SET_ROWS, + RMS_NORM_SCALE, // gen + RMS_NORM_MUL_SILU_GATE, // gen RMS_NORM_COUNT, }; @@ -713,6 +722,10 @@ struct vk_device_struct { vk::Device device; uint32_t vendor_id; vk::DriverId driver_id; + bool borealis_medium_tiles = false; + bool borealis_roll_bk = false; + uint32_t mmv_max_cols = mul_mat_vec_max_cols; + uint32_t mmv_id_max_tokens = 8; vk_device_architecture architecture; std::unique_ptr compute_queue; std::unique_ptr transfer_queue; @@ -763,6 +776,7 @@ struct vk_device_struct { bool coopmat_bf16_support {}; bool coopmat_support_16x16x16_f16acc {}; bool coopmat_support_16x16x16_f32acc {}; + bool coopmat_support_8x8x8_f32acc {}; bool coopmat1_fa_support {}; uint32_t coopmat_m; uint32_t coopmat_n; @@ -823,11 +837,14 @@ struct vk_device_struct { vk_pipeline pipeline_dequant[GGML_TYPE_COUNT]; vk_pipeline pipeline_dequant_transpose[GGML_TYPE_COUNT]; // fused dequant+transpose for FA quant-KV - vk_pipeline pipeline_dequant_mul_mat_vec_f32_f32[DMMV_WG_SIZE_COUNT][GGML_TYPE_COUNT][mul_mat_vec_max_cols]; - vk_pipeline pipeline_dequant_mul_mat_vec_f16_f32[DMMV_WG_SIZE_COUNT][GGML_TYPE_COUNT][mul_mat_vec_max_cols]; + vk_pipeline pipeline_dequant_mul_mat_vec_f32_f32[DMMV_WG_SIZE_COUNT][GGML_TYPE_COUNT][mul_mat_vec_max_cols_limit]; + vk_pipeline pipeline_dequant_mul_mat_vec_f16_f32[DMMV_WG_SIZE_COUNT][GGML_TYPE_COUNT][mul_mat_vec_max_cols_limit]; vk_pipeline pipeline_dequant_mul_mat_vec_id_f32[DMMV_WG_SIZE_COUNT][GGML_TYPE_COUNT]; + vk_pipeline pipeline_mul_mv_mma[GGML_TYPE_COUNT][5][8][2]; + vk_pipeline pipeline_mul_mv_mma_f32_f16[5][8][2]; + vk_pipeline pipeline_mul_mv_mma_id[GGML_TYPE_COUNT][7][5][2]; - vk_pipeline pipeline_dequant_mul_mat_vec_q8_1_f32[DMMV_WG_SIZE_COUNT][GGML_TYPE_COUNT][mul_mat_vec_max_cols]; + vk_pipeline pipeline_dequant_mul_mat_vec_q8_1_f32[DMMV_WG_SIZE_COUNT][GGML_TYPE_COUNT][mul_mat_vec_max_cols_limit]; vk_pipeline pipeline_dequant_mul_mat_vec_id_q8_1_f32[DMMV_WG_SIZE_COUNT][GGML_TYPE_COUNT]; vk_pipeline pipeline_mul_mat_vec_p021_f16_f32[p021_max_gqa_ratio]; @@ -853,6 +870,7 @@ struct vk_device_struct { // indexed by num_additional_fused_ops == num_adds - 1 vk_pipeline pipeline_multi_add[MAX_FUSED_ADDS]; vk_pipeline pipeline_multi_add_rms[MAX_FUSED_ADDS]; + vk_pipeline pipeline_multi_add_rms_mul[MAX_FUSED_ADDS][3][2]; vk_pipeline pipeline_add_id_f32; @@ -860,6 +878,7 @@ struct vk_device_struct { vk_pipeline pipeline_concat_transpose_i8, pipeline_concat_transpose_i16, pipeline_concat_transpose_i32; vk_pipeline pipeline_upscale_nearest_f32, pipeline_upscale_bilinear_f32, pipeline_upscale_bicubic_f32, pipeline_upscale_bilinear_antialias_f32; vk_pipeline pipeline_scale_f32; + vk_pipeline pipeline_scale_zero_f32; vk_pipeline pipeline_log[2]; vk_pipeline pipeline_tri[2]; vk_pipeline pipeline_diag[2]; @@ -880,10 +899,12 @@ struct vk_device_struct { vk_pipeline pipeline_set_rows_i64[2][GGML_TYPE_COUNT]; vk_pipeline pipeline_norm_f32; vk_pipeline pipeline_group_norm_f32; - std::array pipeline_rms_norm_f32; - std::array pipeline_rms_norm_mul_f32; - std::array pipeline_rms_norm_mul_add_f32; - std::array pipeline_rms_norm_mul_add_mul_f32; + std::array pipeline_rms_norm_f32; + std::array pipeline_rms_norm_mul_f32; + std::array pipeline_rms_norm_scale_f32; // gen + std::array pipeline_rms_norm_mul_add_f32; + std::array pipeline_rms_norm_mul_silu_f32; // gen + std::array pipeline_rms_norm_mul_add_mul_f32; vk_pipeline pipeline_rms_norm_mul_add_partials_f32; vk_pipeline pipeline_rms_norm_mul_add_mul_partials_f32; vk_pipeline pipeline_rms_norm_set_rows_f32_f32; @@ -1002,11 +1023,20 @@ struct vk_device_struct { vk_pipeline pipeline_lightning_indexer_f32[GGML_TYPE_COUNT]; // [size_idx][kda] where size_idx: 0=d16, 1=d32, 2=d64, 3=d128 vk_pipeline pipeline_gated_delta_net[4][2]; + vk_pipeline pipeline_gated_delta_net_sout[4][2]; // gen: final state to a separate binding vk_pipeline pipeline_ssm_scan_f32_d128; vk_pipeline pipeline_ssm_scan_f32_d256; vk_pipeline pipeline_ssm_conv_f32; vk_pipeline pipeline_ssm_conv_silu_f32; vk_pipeline pipeline_ssm_conv_bias_silu_f32; + vk_pipeline pipeline_ssm_conv_1t_f32; + vk_pipeline pipeline_ssm_conv_silu_1t_f32; + vk_pipeline pipeline_ssm_conv_sout_f32; + vk_pipeline pipeline_ssm_conv_silu_sout_f32; + vk_pipeline pipeline_ssm_conv_concat_cpy_f32; + vk_pipeline pipeline_ssm_conv_concat_cpy_silu_f32; + vk_pipeline pipeline_ssm_conv_concat_cpy_mt_f32; + vk_pipeline pipeline_ssm_conv_concat_cpy_mt_silu_f32; vk_pipeline pipeline_opt_step_adamw_f32; vk_pipeline pipeline_opt_step_sgd_f32; std::map pipeline_conv2d_f32[CONV_SHAPE_COUNT]; @@ -1142,6 +1172,23 @@ typedef std::vector vk_sequence; #define MAT_VEC_FUSION_FLAGS_SCALE0 0x4 #define MAT_VEC_FUSION_FLAGS_SCALE1 0x8 +#define MAT_VEC_FUSION_FLAGS_GLU_A 0x10 // gen +#define MAT_VEC_FUSION_FLAGS_GLU_B 0x20 // gen +#define MAT_VEC_FUSION_FLAGS_GEN_TOKY 0x40 // gen: MUL_MAT_ID, all tokens in one dispatch (y = slot + nei0 * token) +#define MAT_VEC_FUSION_FLAGS_SIGMOID 0x80 // gen +#define MAT_VEC_FUSION_FLAGS_SOFTPLUS_GATE 0x100 // gen +#define MAT_VEC_FUSION_FLAGS_ROW_SCALE0 0x200 // gen +#define MAT_VEC_FUSION_FLAGS_PAIR_GLU 0x400 // gen: gate+up+swiglu in one mat-vec/mat-vec-id dispatch +#define MAT_VEC_FUSION_FLAGS_PAIR_MM 0x800 // gen: two mat-vecs sharing src1 in one dispatch +#define MAT_VEC_FUSION_FLAGS_PAIR_MM_SIGMOID1 0x1000 // gen: sigmoid epilogue on second mat-vec output +#define MAT_VEC_FUSION_FLAGS_SCALE0_TOK 0x2000 // s10: MUL_MAT_ID SCALE0 indexed per (slot, token) +#define MAT_VEC_FUSION_FLAGS_SCALE1_TOK 0x4000 // s10: MUL_MAT_ID SCALE1 indexed per (slot, token) +#define MAT_VEC_FUSION_FLAGS_COL_SPLIT 0x8000 // s10: multi-column batch_n split across y workgroups +#define MAT_VEC_FUSION_FLAGS_B_F16 0x10000 // s14: FP16 B buffer bound to mul_mv_mma{_id} +#define MAT_VEC_FUSION_FLAGS_D_F16_AUX1 0x20000 // s14: mul_mv_mma_id _pairglu writes FP16 D to AUX1 (prealloc_y) +#define MAT_VEC_FUSION_FLAGS_SKIP_F32_D 0x40000 // s14: mul_mv_mma_id _pairglu skips dead FP32 D store +#define MAT_VEC_FUSION_FLAGS_LORA0 0x80000 // fused dynamic LoRA0 epilogue +#define MAT_VEC_FUSION_FLAGS_LORA1 0x100000 // fused dynamic LoRA1 epilogue (up in PAIR_GLU, second mm in PAIR_MM) struct vk_staging_memcpy { vk_staging_memcpy(void * _dst, const void * _src, size_t _n) : dst(_dst), src(_src), n(_n) {} @@ -1281,6 +1328,36 @@ class vk_perf_logger { uint32_t print_count {}; }; +static constexpr uint32_t VK_PREALLOC_RING = 8; +static constexpr size_t VK_REPLAY_CACHE_MAX = 8; + +struct vk_graph_replay_entry { + uint64_t hash1 {}; + uint64_t hash2 {}; + uint64_t destroy_count {}; + uint64_t pre_x_handle {}, pre_y_handle {}, pre_sk_handle {}, pre_arp_handle {}, pre_mo_handle {}; + size_t pre_x_size {}, pre_y_size {}, pre_sk_size {}, pre_arp_size {}, pre_mo_size {}; + uint32_t stable_runs {}; + uint64_t lru_tick {}; + bool ready {}; + + std::vector sig_words; + + vk_command_pool cmd_pool; + vk::CommandBuffer cmd_buf {}; + std::vector memsets; + + std::vector descriptor_pools; + std::vector descriptor_sets; + std::vector> descriptor_set_bindings; + uint64_t descriptor_destroy_count {}; + + uint64_t total_flops {}; + uint64_t stat_dispatches {}; + uint64_t stat_syncs {}; + double ema_gpu_us {}; +}; + struct ggml_backend_vk_context { std::string name; @@ -1292,10 +1369,11 @@ struct ggml_backend_vk_context { size_t semaphore_idx, binary_semaphore_idx, event_idx; ggml_vk_garbage_collector gc; - size_t prealloc_size_x, prealloc_size_y, prealloc_size_split_k, prealloc_size_add_rms_partials, prealloc_size_add_rms_partials_offset; - vk_buffer prealloc_x, prealloc_y, prealloc_split_k, prealloc_add_rms_partials, sync_staging; + size_t prealloc_size_x, prealloc_size_y, prealloc_size_split_k, prealloc_size_add_rms_partials, prealloc_size_add_rms_partials_offset, prealloc_size_mask_opt {}; + vk_buffer prealloc_x, prealloc_y, prealloc_split_k, prealloc_add_rms_partials, prealloc_mask_opt, sync_staging; vk::Fence fence, almost_ready_fence; bool submit_pending {}; + bool fence_submitted {}; bool almost_ready_fence_pending {}; // Set before op_add and unset after op_rms_norm to indicate that the add should // write partial sums to accumulate the square of the vector components @@ -1303,6 +1381,54 @@ struct ggml_backend_vk_context { bool do_add_rms_partials; uint64_t last_total_flops {UINT64_MAX}; + // gen instrumentation (GGML_VK_GRAPH_STATS): per-graph counters + uint64_t stat_dispatches {}; + // gen: GATED_DELTA_NET writes its final state into this CPY node's destination (and the CPY is skipped) + const ggml_tensor * gen_gdn_state_out {}; + bool gen_gdn_zero_state {}; + // gen: RMS_NORM_SCALE writes into this SCALE node (found a few nodes later, which is then skipped) + const ggml_tensor * gen_rms_scale_dst {}; + const ggml_tensor * gen_rms_scale_dst2 {}; + const ggml_tensor * gen_rms_scale_rsrc2 {}; + // gen: dense MUL_MAT forward-elision epilogue target and auxiliary inputs (GGML_GEN_FUSE=mmgate,shexp) + const ggml_tensor * gen_mm_dst {}; + const ggml_tensor * gen_mm_f0 {}; + const ggml_tensor * gen_mm_f1 {}; + uint32_t gen_mm_flags {}; + // gen: paired dense MUL_MAT sharing src1 (GGML_GEN_FUSE=pairmm) + const ggml_tensor * gen_pair_mm1 {}; + const ggml_tensor * gen_pair_dst1 {}; + int gen_pair_mul0_rel {}; + // Fused dynamic LoRA epilogue state + const ggml_tensor * gen_lora_b0 {}; + const ggml_tensor * gen_lora_h0 {}; + const ggml_tensor * gen_lora_b1 {}; + const ggml_tensor * gen_lora_h1 {}; + const ggml_tensor * gen_lora_gate_src0 {}; + const ggml_tensor * gen_lora_up_src0 {}; + uint32_t gen_lora_r {}; + float gen_lora_scale0 {}; + float gen_lora_scale1 {}; + // Series 10: SSM_CONV fusion (mode 1 = CONCAT_SLIDING, mode 2 = WRITE_STATE_OUT, mode 3 = CONCAT_SLIDING_ROLLBACK) + int s10_ssm_conv_mode {}; + const ggml_tensor * s10_ssm_state_out {}; + const ggml_tensor * s10_ssm_old_st {}; + const ggml_tensor * s10_ssm_qkv {}; + const ggml_tensor * s10_ssm_conv_node {}; + uint32_t s10_ssm_st_k {}; + uint32_t s10_ssm_st_slot_nb {}; + bool s10_ssm_zero_state {}; + // Series 10: MULTI_ADD + RMS_NORM + [MUL] fusion + const ggml_tensor * s10_add_rms_dst {}; + const ggml_tensor * s10_add_rms_weight {}; + const ggml_tensor * s10_add_rms_add_dst {}; + float s10_add_rms_eps {}; + uint32_t s10_add_rms_num_adds {}; + uint32_t s10_add_rms_tree_mode {}; + uint32_t s10_add_rms_write_add {}; + const ggml_tensor * s10_add_rms_srcs[10] {}; + uint32_t gen_n1_partials_offset {}; + uint64_t stat_syncs {}; // Cache most recent tensor that was converted into prealloc_y, and what pipeline it used to convert. vk_pipeline_struct * prealloc_y_last_pipeline_used {}; @@ -1310,6 +1436,58 @@ struct ggml_backend_vk_context { // True when the K dimension in prealloc_y is padded. bool prealloc_y_last_k_padded {}; + // Series 9: K=4 ring-sliced prealloc_y and prealloc_split_k to eliminate WAR drain barriers + uint32_t prealloc_y_ring_idx {}; + size_t prealloc_y_ring_stride {}; + uint32_t prealloc_y_ring_count {1}; + bool prealloc_y_slot_need_sync[VK_PREALLOC_RING] {}; + vk_pipeline_struct * prealloc_y_last_pipeline_ring[VK_PREALLOC_RING] {}; + const ggml_tensor * prealloc_y_last_tensor_ring[VK_PREALLOC_RING] {}; + bool prealloc_y_last_k_padded_ring[VK_PREALLOC_RING] {}; + int prealloc_y_keep_slot {-1}; + bool gen_pair_glu_sole_mmid_next {}; + bool gen_rms_silu_sole_mm_next {}; + + uint32_t prealloc_split_k_ring_idx {}; + size_t prealloc_split_k_ring_stride {}; + uint32_t prealloc_split_k_ring_count {1}; + bool prealloc_split_k_slot_need_sync[VK_PREALLOC_RING] {}; + + // Series 9: Reuse count_experts output across MoE projections sharing non-negative ARGSORT ids + const ggml_tensor * last_expert_ids_tensor {}; + uint32_t last_expert_bn {}; + uint32_t last_expert_row_ids_offset {}; + uint32_t last_expert_n_as {}; + uint32_t last_expert_slot {}; + vk_subbuffer last_expert_count_buf {}; + bool last_expert_count_valid {}; + + // Series 13: Reuse flash_attn_mask_opt across FA layers sharing the identical mask tensor + const ggml_tensor * last_mask_opt_tensor {}; + VkBuffer last_mask_opt_vk_buf {}; + size_t last_mask_opt_offset {}; + uint32_t last_mask_opt_nem0 {}, last_mask_opt_nem1 {}, last_mask_opt_nem2 {}, last_mask_opt_nem3 {}; + size_t last_mask_opt_nb1 {}, last_mask_opt_nb2 {}, last_mask_opt_nb3 {}; + uint32_t last_mask_opt_br {}, last_mask_opt_bc {}; + bool last_mask_opt_valid {}; + bool prealloc_mask_opt_need_sync {}; + + // Series 9: Single-Front reusable command buffer LRU cache + zero-submit UMA async readbacks + std::vector replay_cache; + uint64_t replay_lru_clock {}; + uint64_t replay_probation_h1[8] {}; + uint64_t replay_probation_h2[8] {}; + uint64_t replay_probation_destroy[8] {}; + uint32_t replay_probation_clock {}; + uint32_t replay_consec_misses {}; + int32_t replay_miss_n_nodes {}; + bool recording_replay {}; + bool replay_abort {}; + vk_command_pool * active_replay_cmd_pool {}; + vk_graph_replay_entry * active_replay_entry {}; + int64_t replay_submit_us {}; + std::vector uma_out_memcpys; + // Track which nodes have been used since the last sync, and whether they were written to std::vector unsynced_nodes_written; std::vector unsynced_nodes_read; diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index b5b4264fb767..d507bdb3162f 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -338,13 +338,131 @@ static VkDeviceSize ggml_vk_get_max_buffer_range(const ggml_backend_vk_context * return range; } +static inline void ggml_vk_invalidate_prealloc_y_ring(ggml_backend_vk_context * ctx) { + ctx->prealloc_y_last_pipeline_used = nullptr; + ctx->prealloc_y_last_tensor_used = nullptr; + ctx->prealloc_y_last_k_padded = false; + for (uint32_t s = 0; s < VK_PREALLOC_RING; ++s) { + ctx->prealloc_y_last_pipeline_ring[s] = nullptr; + ctx->prealloc_y_last_tensor_ring[s] = nullptr; + ctx->prealloc_y_last_k_padded_ring[s] = false; + } +} + void ggml_vk_wait_for_fence(ggml_backend_vk_context * ctx) { + auto finish_fence = [&]() { + ctx->device->device.resetFences({ ctx->fence }); + ctx->fence_submitted = false; + if (ctx->active_replay_entry != nullptr) { + const double gpu_us = (double) (ggml_time_us() - ctx->replay_submit_us); + if (gpu_us > 0.0) { + vk_graph_replay_entry * e = ctx->active_replay_entry; + e->ema_gpu_us = e->ema_gpu_us == 0.0 ? gpu_us : (0.85 * e->ema_gpu_us + 0.15 * gpu_us); + } + ctx->active_replay_entry = nullptr; + } + }; + // Use waitForFences while most of the graph executes. Hopefully the CPU can sleep // during this wait. + bool gen_had_almost_ready = false; if (ctx->almost_ready_fence_pending) { VK_CHECK(ctx->device->device.waitForFences({ ctx->almost_ready_fence }, true, UINT64_MAX), "almost_ready_fence", ctx->device); ctx->device->device.resetFences({ ctx->almost_ready_fence }); ctx->almost_ready_fence_pending = false; + gen_had_almost_ready = true; + } + const int64_t gen_t_ar = ggml_time_us(); + + // gen: GGML_VK_FENCE_WAIT selects how the last part of the graph is waited for: + // spin (default) getFenceStatus + 1000 YIELDs per poll + // block waitForFences (the thread sleeps) + // wfe poll, then WFE (aarch64): the core is clock-gated until an event or the arch timer's event + // stream (~100 us) wakes it; it stays "busy" for the scheduler, so its frequency stays up + // nap poll, then nanosleep 50 us + static const int gen_wait_mode = [] { + const char * m = getenv("GGML_VK_FENCE_WAIT"); + if (m == nullptr || strcmp(m, "spin") == 0) return 0; + if (strcmp(m, "block") == 0) return 1; + if (strcmp(m, "wfe") == 0) return 2; + if (strcmp(m, "nap") == 0) return 3; + if (strcmp(m, "hybrid") == 0) return 4; + return 0; + }(); + // Series 9: Single-Front replay adaptive sleep-then-spin using per-graph EMA duration + if (ctx->active_replay_entry != nullptr && gen_wait_mode == 0 && ctx->active_replay_entry->ema_gpu_us > 500.0) { + static const int64_t replay_margin_us = getenv("GGML_VK_REPLAY_MARGIN_US") ? atoll(getenv("GGML_VK_REPLAY_MARGIN_US")) : 350; + const int64_t elapsed_us = ggml_time_us() - ctx->replay_submit_us; + const int64_t sleep_us = (int64_t) ctx->active_replay_entry->ema_gpu_us - elapsed_us - replay_margin_us; + if (sleep_us > 80) { + const vk::Result r = ctx->device->device.waitForFences({ ctx->fence }, true, (uint64_t) sleep_us * 1000); + if (r != vk::Result::eSuccess && r != vk::Result::eTimeout) { + throw vk::SystemError(vk::make_error_code(r), "ggml_vulkan: waitForFences (replay)"); + } + } + while (ctx->device->device.getFenceStatus(ctx->fence) != vk::Result::eSuccess) { + for (uint32_t k = 0; k < 250; ++k) { + YIELD(); + } + } + finish_fence(); + return; + } + // hybrid: sleep in waitForFences until GGML_VK_FENCE_MARGIN_US (default 300) before the expected end of the graph + // (moving average of the time from the almost-ready fence to the final fence), then spin + if (gen_wait_mode == 4 && gen_had_almost_ready) { + static const int64_t margin_us = getenv("GGML_VK_FENCE_MARGIN_US") ? atoll(getenv("GGML_VK_FENCE_MARGIN_US")) : 300; + static thread_local double ema_tail_us = 0.0; + const int64_t sleep_us = (int64_t) ema_tail_us - margin_us; + if (sleep_us > 50) { + const vk::Result r = ctx->device->device.waitForFences({ ctx->fence }, true, (uint64_t) sleep_us * 1000); + if (r != vk::Result::eSuccess && r != vk::Result::eTimeout) { + throw vk::SystemError(vk::make_error_code(r), "ggml_vulkan: waitForFences (hybrid)"); + } + } + for (;;) { + const vk::Result r = ctx->device->device.getFenceStatus(ctx->fence); + if (r == vk::Result::eSuccess) { + break; + } + if (r != vk::Result::eNotReady) { + throw vk::SystemError(vk::make_error_code(r), "ggml_vulkan: getFenceStatus"); + } + YIELD(); + } + const double tail = (double) (ggml_time_us() - gen_t_ar); + ema_tail_us = ema_tail_us == 0.0 ? tail : 0.8 * ema_tail_us + 0.2 * tail; + finish_fence(); + return; + } + const bool gen_block_wait = gen_wait_mode == 1; + if (gen_wait_mode == 2 || gen_wait_mode == 3) { + for (;;) { + const vk::Result r = ctx->device->device.getFenceStatus(ctx->fence); + if (r == vk::Result::eSuccess) { + break; + } + if (r != vk::Result::eNotReady) { + throw vk::SystemError(vk::make_error_code(r), "ggml_vulkan: getFenceStatus"); + } + if (gen_wait_mode == 2) { +#if defined(__aarch64__) + __asm__ __volatile__("wfe" ::: "memory"); +#else + YIELD(); +#endif + } else { + struct timespec ts = { 0, 50000 }; + nanosleep(&ts, nullptr); + } + } + finish_fence(); + return; + } + if (gen_block_wait) { + VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "fence (blocking)", ctx->device); + finish_fence(); + return; } // Spin (w/pause) waiting for the graph to finish executing. @@ -377,7 +495,14 @@ void ggml_vk_wait_for_fence(ggml_backend_vk_context * ctx) { YIELD(); } } - ctx->device->device.resetFences({ ctx->fence }); + finish_fence(); +} + +static void ggml_vk_ensure_fence_free(ggml_backend_vk_context * ctx) { + if (ctx->fence_submitted) { + ggml_vk_wait_for_fence(ctx); + ctx->submit_pending = false; + } } static bool ggml_vk_strip_decode_vector(const uint32_t * code, size_t word_count, std::vector & out) { @@ -672,10 +797,11 @@ static void ggml_vk_create_pipeline_func(vk_device& device, vk_pipeline& pipelin #if VK_HEADER_VERSION >= 287 // Roll the mul_mm BK loop on Asahi Linux. Skip bf16 and the mul_mmq pipelines. - if (device->driver_id == vk::DriverId::eMesaHoneykrisp && + if ((device->driver_id == vk::DriverId::eMesaHoneykrisp || device->borealis_roll_bk) && pipeline->name.rfind("matmul", 0) == 0 && pipeline->name.find("bf16") == std::string::npos && - pipeline->name.find("q8_1") == std::string::npos) { + pipeline->name.find("q8_1") == std::string::npos && + (!device->coopmat_support || (getenv("GGML_VK_CM1_ROLL_BK") && atoi(getenv("GGML_VK_CM1_ROLL_BK")) != 0))) { const uint32_t * src = spirv.empty() ? reinterpret_cast(spv_data) : spirv.data(); size_t src_n = spirv.empty() ? spv_size / sizeof(uint32_t) : spirv.size(); std::vector rolled; @@ -1157,6 +1283,12 @@ void ggml_vk_sync_buffers(ggml_backend_vk_context* ctx, vk_context& subctx) { if (ctx) { ctx->prealloc_x_need_sync = ctx->prealloc_y_need_sync = ctx->prealloc_split_k_need_sync = false; + ctx->prealloc_mask_opt_need_sync = false; + for (uint32_t s = 0; s < VK_PREALLOC_RING; ++s) { + ctx->prealloc_y_slot_need_sync[s] = false; + ctx->prealloc_split_k_slot_need_sync[s] = false; + } + ctx->stat_syncs++; } subctx->s->buffer->buf.pipelineBarrier( @@ -1206,6 +1338,31 @@ void ggml_vk_wait_events(vk_context& ctx, std::vector&& events) { ); } +// gen: GGML_VK_BATCH_INVARIANT=1 makes small batches (speculative verification) compute each token exactly as n=1: +// the KV split of flash attention is sized as for one token, and the n=1-only ADD+RMS_NORM partials fusion is off. +static bool gen_batch_invariant() { + static const bool v = getenv("GGML_VK_BATCH_INVARIANT") != nullptr && atoi(getenv("GGML_VK_BATCH_INVARIANT")) != 0; + return v; +} + +// Series 13: Speculative verification batches are small (k <= 8); prompts with n > 8 tokens do not need +// single-token bit-identity and can use full coopmat8x8 FA, mul_mv_mma, mul_mv_mma_id, and cm1 fusions. +static bool gen_batch_invariant_for_n(int64_t n) { + return gen_batch_invariant() && n <= 8; +} + +static bool ggml_vk_fast_fp16_enabled() { + static const bool v = getenv("GGML_VK_FAST_FP16") != nullptr && atoi(getenv("GGML_VK_FAST_FP16")) != 0; + return v; +} + +static bool ggml_vk_exact_s8_enabled() { + static const bool v = getenv("GGML_VK_EXACT_S8") != nullptr && atoi(getenv("GGML_VK_EXACT_S8")) != 0; + return v; +} + +static bool ggml_vk_is_apple_agx_coopmat(const vk_device& device); + static vk_fa_tuning_params get_fa_tuning_params_scalar(const vk_device& device, uint32_t hsk, uint32_t hsv, uint32_t n_rows, uint32_t n_kv, ggml_type k_type, ggml_type v_type, bool f32acc) { vk_fa_tuning_params result{}; @@ -1245,6 +1402,8 @@ static vk_fa_tuning_params get_fa_tuning_params_scalar(const vk_device& device, // row_split 1 means higher register use per row, so block size has to be adjusted if (result.row_split == 1) { result.block_rows = n_rows == 2 ? 2 : ((n_rows <= 4 || reduce_block_rows) ? 4 : 8); + } else if (ggml_vk_fast_fp16_enabled() && ggml_vk_is_apple_agx_coopmat(device) && hsv >= 256 && n_rows > 8 && n_rows <= 64) { + result.block_rows = 4; } else { result.block_rows = n_rows <= 4 ? 4 : ((n_rows <= 8 || reduce_block_rows) ? 8 : 16); } @@ -1299,10 +1458,20 @@ static vk_fa_tuning_params get_fa_tuning_params_coopmat1(const vk_device& device const uint32_t D = hsk | hsv; - const uint32_t coopmat_block_rows = 16; - const uint32_t coopmat_block_cols = 16; + static const int env_fa_wg256 = getenv("GGML_VK_FA_WG256") ? atoi(getenv("GGML_VK_FA_WG256")) : 1; + const bool use_agx_wg256 = !device->coopmat_support_16x16x16_f32acc && + device->coopmat_support_8x8x8_f32acc && + !ggml_vk_exact_s8_enabled() && + !gen_batch_invariant_for_n(n_rows) && + k_type == GGML_TYPE_F16 && v_type == GGML_TYPE_F16 && + hsv >= 128 && + env_fa_wg256 != 0; + + const uint32_t coopmat_block_rows = 16u; + const uint32_t coopmat_block_cols = 16u; - const uint32_t num_subgroups = 4; + const uint32_t num_subgroups = 4u; + const uint32_t wg_subgroups = use_agx_wg256 ? 8u : 4u; // Halves K/V re-reads; only wins on UMA, regresses on discrete RDNA4 const bool double_block_rows = device->uma && @@ -1311,7 +1480,7 @@ static vk_fa_tuning_params get_fa_tuning_params_coopmat1(const vk_device& device result.block_cols = coopmat_block_cols * num_subgroups; result.row_split = num_subgroups; result.subgroup_size = device->subgroup_size; - result.workgroup_size = num_subgroups * result.subgroup_size; + result.workgroup_size = wg_subgroups * result.subgroup_size; const uint32_t D_lsb = D ^ (D & (D-1)); // extract lowest set bit result.d_split = std::min(std::min(result.subgroup_size, 8u), D_lsb / 4); @@ -1401,7 +1570,7 @@ vk_fa_tuning_params get_fa_tuning_params(const vk_device& device, uint32_t hsk, } if (path == FA_COOPMAT1) { - bool shape_ok = (f32acc && device->coopmat_support_16x16x16_f32acc) || + bool shape_ok = (f32acc && (device->coopmat_support_16x16x16_f32acc || device->coopmat_support_8x8x8_f32acc)) || (!f32acc && device->coopmat_support_16x16x16_f16acc); vk_fa_tuning_params params = get_fa_tuning_params_coopmat1(device, hsk, hsv, n_rows, n_kv, k_type, v_type, f32acc, qk_ratio, allow_flip); bool shmem_ok = ggml_vk_flash_attn_coopmat_shmem_support(device, params, hsk, hsv, f32acc, k_type, v_type); @@ -1413,6 +1582,16 @@ vk_fa_tuning_params get_fa_tuning_params(const vk_device& device, uint32_t hsk, if (!shape_ok || !shmem_ok) { path = FA_SCALAR; + } else if (!device->coopmat_support_16x16x16_f32acc && device->coopmat_support_8x8x8_f32acc) { + static const int fa_8x8_min_rows = getenv("GGML_VK_FA_8X8_MIN_ROWS") ? atoi(getenv("GGML_VK_FA_8X8_MIN_ROWS")) : -1; + const int eff_fa_min_rows = (fa_8x8_min_rows >= 0) ? fa_8x8_min_rows : (n_kv >= 64 ? 4 : 8); + const bool kv_fp16_bf16 = (k_type == GGML_TYPE_F16 || k_type == GGML_TYPE_BF16) && + (v_type == GGML_TYPE_F16 || v_type == GGML_TYPE_BF16); + if (ggml_vk_exact_s8_enabled() || gen_batch_invariant_for_n(n_rows) || + !kv_fp16_bf16 || (hsk % 16 != 0) || (hsv % 16 != 0) || + (int)n_rows < eff_fa_min_rows) { + path = FA_SCALAR; + } } } @@ -1438,14 +1617,24 @@ vk_fa_pipeline_state get_fa_pipeline_state(const vk_device& device, const vk_fa_ const bool old_amd_windows = device->vendor_id == VK_VENDOR_ID_AMD && device->driver_id == vk::DriverId::eAmdProprietary && (device->architecture == AMD_GCN || device->architecture == AMD_RDNA1 || device->architecture == AMD_RDNA2); - uint32_t flags = (use_mask_opt ? 1 : 0) | - (use_mask ? 2 : 0) | - (use_logit_softcap ? 4 : 0) | - (old_amd_windows ? 8 : 0) | - (use_sparse ? 16 : 0) | - (params.q_in_regs ? 32 : 0) | - (params.flip ? 64 : 0) | - (params.flip_direct ? 128 : 0); + static const int env_fa_swizzle = getenv("GGML_VK_FA_SWIZZLE") ? atoi(getenv("GGML_VK_FA_SWIZZLE")) : 1; + const bool use_lpt_gqa_swizzle = (env_fa_swizzle != 0) && + params.path == FA_COOPMAT1 && + !device->coopmat_support_16x16x16_f32acc && + device->coopmat_support_8x8x8_f32acc && + k_type == GGML_TYPE_F16 && v_type == GGML_TYPE_F16 && + !use_sparse && + !ggml_vk_exact_s8_enabled(); + + uint32_t flags = (use_mask_opt ? 1 : 0) | + (use_mask ? 2 : 0) | + (use_logit_softcap ? 4 : 0) | + (old_amd_windows ? 8 : 0) | + (use_sparse ? 16 : 0) | + (params.q_in_regs ? 32 : 0) | + (params.flip ? 64 : 0) | + (params.flip_direct ? 128 : 0) | + (use_lpt_gqa_swizzle ? 256 : 0); const uint32_t subgroup_size = params.disable_subgroups ? 0 : params.subgroup_size; @@ -1662,7 +1851,7 @@ static vk_xe_fa_tuning_params get_xe_fa_tuning_params(const vk_device_struct * d // CM2 uses narrow tiles for 5 <= N <= 32; CM1 uses them for 9 <= N <= 32. static uint32_t ggml_vk_mul_mat_vec_max_cols(const vk_device & device) { - return device->coopmat2 && !device->disable_narrow_mul_mat ? 4u : mul_mat_vec_max_cols; + return device->coopmat2 && !device->disable_narrow_mul_mat ? 4u : device->mmv_max_cols; } static bool ggml_vk_matmul_shmem_support(const vk_device& device, const std::vector& warptile, bool mul_mat_id, ggml_type src0_type) { @@ -1707,13 +1896,16 @@ static bool ggml_vk_matmul_shmem_support(const vk_device& device, const std::vec // Needs to stay aligned with ggml_vk_mul_mm_spec. const bool intel_shmem_stride_pad_zero = device->vendor_id == VK_VENDOR_ID_INTEL && device->coopmat_support && device->driver_id == vk::DriverId::eIntelProprietaryWindows; - const uint32_t bank_conflict_offset = intel_shmem_stride_pad_zero ? 0 : (device->coopmat_support ? 8 : 1); + const bool agx_reshaped_dense = ggml_vk_is_apple_agx_coopmat(device) && !ggml_vk_exact_s8_enabled() && !mul_mat_id && !getenv("GGML_VK_NO_SLM_RESHAPE"); + const uint32_t bank_conflict_offset = (intel_shmem_stride_pad_zero || agx_reshaped_dense) ? 0 : (device->coopmat_support ? 8 : 1); const uint32_t type_size = device->fp16 ? sizeof(ggml_fp16_t) : sizeof(float); const uint32_t warps = warptile[0] / warptile[10]; const uint32_t load_bufs = (warptile[1] + warptile[2]) * (warptile[3] + bank_conflict_offset) * type_size; const uint32_t mmid_row_ids = mul_mat_id ? (warptile[2] * 2 * sizeof(uint16_t)) : 0; - const uint32_t coopmat_stage = device->coopmat_support ? warptile[7] * warptile[8] * warps * sizeof(float) : 0; + const uint32_t coopmat_stage = device->coopmat_support + ? (ggml_vk_is_apple_agx_coopmat(device) ? (ggml_vk_exact_s8_enabled() ? (uint32_t)sizeof(float) : 0u) : warptile[7] * warptile[8] * warps * (uint32_t)sizeof(float)) + : 0; const uint32_t total_size = load_bufs + mmid_row_ids + coopmat_stage + lut_size; const bool supported = total_size <= device->properties.limits.maxComputeSharedMemorySize; @@ -1724,6 +1916,19 @@ static bool ggml_vk_matmul_shmem_support(const vk_device& device, const std::vec return supported; } +static bool ggml_vk_is_apple_agx_coopmat(const vk_device& device) { + if (!device->coopmat_support || device->coopmat_m != 8 || device->coopmat_n != 8 || device->coopmat_k != 8) { + return false; + } +#if VK_HEADER_VERSION >= 287 + if (device->driver_id == vk::DriverId::eMesaHoneykrisp) { + return true; + } +#endif + return device->borealis_medium_tiles || device->borealis_roll_bk || + (device->vendor_id == 0x10005 && device->properties.deviceID == 0x6041); +} + static bool ggml_vk_matmul_int_shmem_support(const vk_device& device, const std::vector& warptile, bool mul_mat_id, ggml_type src0_type) { // FLOAT_TYPE in the shader is float16_t with fp16 support, otherwise float. @@ -1975,7 +2180,7 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { l_mmq_wg_denoms, m_mmq_wg_denoms, s_mmq_wg_denoms, l_mmq_wg_denoms_k, m_mmq_wg_denoms_k, s_mmq_wg_denoms_k, l_mmq_cm1_wg_denoms_k, m_mmq_cm1_wg_denoms_k, s_mmq_cm1_wg_denoms_k, - l_mmqid_wg_denoms, m_mmqid_wg_denoms, s_mmqid_wg_denoms; + l_mmqid_wg_denoms, m_mmqid_wg_denoms, s_mmqid_wg_denoms, gen_s_mmqid_denoms; uint32_t l_align, m_align, s_align; @@ -2131,6 +2336,80 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { } } + if (ggml_vk_is_apple_agx_coopmat(device)) { + if (ggml_vk_exact_s8_enabled()) { + s_warptile = { 32, 32, 16, 32, 32, 16, 2, tm_s, tn_s, tk_s, 32 }; + m_warptile = { 128, 64, 64, 16, 32, 32, 2, tm_m, tn_m, tk_m, 32 }; + l_warptile = { 256, 128, 64, 16, 32, 32, 2, tm_l, tn_l, tk_l, 32 }; + s_warptile_mmq = { 32, 32, 16, 32, 32, 16, 2, tm_s, tn_s, tk_s, 32 }; + m_warptile_mmq = { 128, 64, 64, 32, 32, 32, 2, tm_m, tn_m, tk_m, 32 }; + l_warptile_mmq = { 256, 128, 64, 32, 32, 32, 2, tm_l, tn_l, tk_l, 32 }; + s_warptile_id = s_warptile; + m_warptile_id = m_warptile; + l_warptile_id = l_warptile; + s_warptile_mmqid = { 128, 128, 16, 32, 64, 8, 2, tm_s, tn_s, tk_s, 32 }; + m_warptile_mmqid = m_warptile_mmq; + l_warptile_mmqid = l_warptile_mmq; + l_mmq_wg_denoms = l_wg_denoms = { 128, 64, 1 }; + m_mmq_wg_denoms = m_wg_denoms = { 64, 64, 1 }; + s_mmq_wg_denoms = s_wg_denoms = { 32, 16, 1 }; + l_mmqid_wg_denoms = { 128, 64, 1 }; + m_mmqid_wg_denoms = { 64, 64, 1 }; + s_mmqid_wg_denoms = { 128, 16, 1 }; + } else { + // AGX G17P 8x8x8 simdgroup_matrix (f32matmad8x8): keep <=16 8x8 accumulators (64 half-regs) per subgroup + // with reshaped 8x8 buf_a + buf_b (zero bank-pad waste) and BK=32 alignment. + s_warptile = { 64, 32, 32, 32, 32, 16, 2, tm_s, tn_s, tk_s, 32 }; + m_warptile = { 128, 64, 64, 32, 32, 32, 2, tm_m, tn_m, tk_m, 32 }; + l_warptile = { 256, 128, 64, 32, 32, 32, 2, tm_l, tn_l, tk_l, 32 }; + s_warptile_mmq = { 32, 32, 16, 32, 32, 16, 2, tm_s, tn_s, tk_s, 32 }; + m_warptile_mmq = { 128, 64, 64, 32, 32, 32, 2, tm_m, tn_m, tk_m, 32 }; + l_warptile_mmq = { 256, 128, 64, 32, 32, 32, 2, tm_l, tn_l, tk_l, 32 }; + s_warptile_id = s_warptile; + m_warptile_id = m_warptile; + l_warptile_id = l_warptile; + s_warptile_mmqid = { 128, 64, 32, 32, 32, 16, 2, tm_s, tn_s, tk_s, 32 }; + m_warptile_mmqid = { 128, 64, 64, 32, 32, 32, 2, tm_m, tn_m, tk_m, 32 }; + l_warptile_mmqid = l_warptile_mmq; + l_mmq_wg_denoms = l_wg_denoms = { 128, 64, 1 }; + m_wg_denoms = { 64, 64, 1 }; + m_mmq_wg_denoms = { 64, 64, 1 }; + s_wg_denoms = { 32, 32, 1 }; + s_mmq_wg_denoms = { 32, 16, 1 }; + l_mmqid_wg_denoms = { 128, 64, 1 }; + m_mmqid_wg_denoms = { 64, 64, 1 }; + s_mmqid_wg_denoms = { 64, 32, 1 }; + l_align = 32; + m_align = 32; + s_align = 32; + } + } + auto gen_parse_warptile = [](const char * env_name, std::vector& wt, std::array& denoms) { + if (const char * gs = getenv(env_name)) { + std::vector v; + for (const char * c = gs; *c; ) { char * end = nullptr; v.push_back((uint32_t) strtoul(c, &end, 10)); c = end; if (*c == ',') c++; else if (*c) break; } + GGML_ASSERT(v.size() == 11 && "warptile override needs 11 values"); + wt = v; + denoms = { v[1], v[2], 1 }; + } + }; + gen_parse_warptile("GGML_VK_MMQ_S", s_warptile_mmq, s_mmq_wg_denoms); + gen_parse_warptile("GGML_VK_MMQ_M", m_warptile_mmq, m_mmq_wg_denoms); + gen_parse_warptile("GGML_VK_MMQ_L", l_warptile_mmq, l_mmq_wg_denoms); + + // gen: GGML_VK_MMQID_S="BLOCK,BM,BN,BK,WM,WN,WMITER,TM,TN,TK,WARP" replaces the small quantized MUL_MAT_ID tile + // (and enables it on Borealis, which otherwise runs only the medium 64x64 tile): an expert with ~8 rows in a + // 64-column tile computes 56 empty columns. Each output's sum over K is unchanged. + if (const char * gs = getenv("GGML_VK_MMQID_S")) { + std::vector v; + for (const char * c = gs; *c; ) { char * end = nullptr; v.push_back((uint32_t) strtoul(c, &end, 10)); c = end; if (*c == ',') c++; else if (*c) break; } + GGML_ASSERT(v.size() == 11 && "GGML_VK_MMQID_S needs 11 values"); + s_warptile_mmqid = v; + gen_s_mmqid_denoms = { v[1], v[2], 1 }; + } else { + gen_s_mmqid_denoms = { s_warptile_mmqid[1], s_warptile_mmqid[2], 1 }; + } + const bool use_cm1_int = device->coopmat_int_support && (device->architecture == AMD_RDNA3 || device->architecture == AMD_RDNA4); @@ -2273,6 +2552,172 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { has_claimed_task = true; } } + + // gen: mat-vec / mul_mv_mma twins (mode: 1 = single-op epilogue, 2 = PAIR_MM, 3 = PAIR_GLU), compiled lazily when requested + const bool is_mmv_twin = strncmp(name, "mul_mat_vec", 11) == 0 && strstr(name, "p021") == nullptr && strstr(name, "_nc_") == nullptr; + const bool is_mma_twin = strncmp(name, "mul_mv_mma", 10) == 0; + const bool is_mma_id_twin = strncmp(name, "mul_mv_mma_id", 13) == 0; + if (is_mmv_twin || is_mma_twin) { + const size_t mode_idx = is_mma_twin ? 4u : 3u; + for (vk_pipeline * b = &base_pipeline; *b; b = &(*b)->next) { + const struct { vk_pipeline * slot; const char * sfx; uint32_t mode; } twins[4] = { + { &(*b)->gen_glu, "_glu", 1u }, + { &(*b)->gen_pair_mm, "_pairmm", 2u }, + { &(*b)->gen_pair_glu, "_pairglu", 3u }, + { &(*b)->gen_fast_ep, "_fastep", 4u }, + }; + for (const auto & tw : twins) { + if ((is_mma_id_twin && tw.mode == 2u) || (!is_mma_twin && tw.mode == 4u)) { + continue; + } + static const int env_pglu_nt1 = getenv("GGML_VK_MMA_ID_PGLU_NT1") ? atoi(getenv("GGML_VK_MMA_ID_PGLU_NT1")) : 1; + static const int env_fep_nt1 = getenv("GGML_VK_MMA_ID_FEP_NT1") ? atoi(getenv("GGML_VK_MMA_ID_FEP_NT1")) : 0; + static const int env_fep_nt = getenv("GGML_VK_MMA_ID_FEP_NT") ? atoi(getenv("GGML_VK_MMA_ID_FEP_NT")) : 2; + static const int env_fep_nt4_min_rt = getenv("GGML_VK_MMA_ID_FEP_NT4_MIN_RT") ? atoi(getenv("GGML_VK_MMA_ID_FEP_NT4_MIN_RT")) : 1; + static const int env_fep_nt4_max_rt = getenv("GGML_VK_MMA_ID_FEP_NT4_MAX_RT") ? atoi(getenv("GGML_VK_MMA_ID_FEP_NT4_MAX_RT")) : 3; + const bool mma_id_twin_nt1 = is_mma_twin && + specialization_constants.size() >= 4 && + specialization_constants[1] == 2u && + specialization_constants[3] == 1u && + ((tw.mode == 3u && ((env_pglu_nt1 == 1 && specialization_constants[2] >= 4u) || + (env_pglu_nt1 >= 2 && specialization_constants[2] >= 2u))) || + (is_mma_id_twin && tw.mode == 4u && ((env_fep_nt1 == 1 && specialization_constants[2] >= 4u) || + (env_fep_nt1 >= 2 && specialization_constants[2] >= 2u)))); + const bool mma_id_twin_nt4 = is_mma_id_twin && !mma_id_twin_nt1 && + tw.mode == 4u && env_fep_nt == 4 && + specialization_constants.size() >= 3 && + specialization_constants[0] == 4u && + specialization_constants[1] == 2u && + (int)specialization_constants[2] >= env_fep_nt4_min_rt && + (int)specialization_constants[2] <= env_fep_nt4_max_rt; + const std::array twin_denoms = mma_id_twin_nt1 + ? std::array{ wg_denoms[0] / 2u, wg_denoms[1], wg_denoms[2] } + : (mma_id_twin_nt4 + ? std::array{ wg_denoms[0] * 2u, wg_denoms[1], wg_denoms[2] } + : wg_denoms); + vk_pipeline & twin = *tw.slot; + if (!twin) { + twin = std::make_shared(); + } + if (!twin->initialized) { + twin->name = std::string(name) + tw.sfx; + twin->parameter_count = parameter_count; + twin->push_constant_size = push_constant_size; + twin->wg_denoms = twin_denoms; + twin->align = align; + twin->initialized = true; +#if defined(VK_EXT_shader_64bit_indexing) + twin->is_64b_indexing = (*b)->is_64b_indexing; +#endif + } + const bool can_lora_twin = !is_mma_id_twin && strncmp(name, "mul_mat_vec_id", 14) != 0 && parameter_count == 7u; + if (can_lora_twin) { + vk_pipeline & ltwin = twin->gen_lora_twin; + if (!ltwin) { + ltwin = std::make_shared(); + } + if (!ltwin->initialized) { + ltwin->name = twin->name + "_lora"; + ltwin->parameter_count = 11u; + ltwin->push_constant_size = push_constant_size; + ltwin->wg_denoms = twin_denoms; + ltwin->align = align; + ltwin->initialized = true; +#if defined(VK_EXT_shader_64bit_indexing) + ltwin->is_64b_indexing = (*b)->is_64b_indexing; +#endif + } + } + auto make_twin_sc = [&](bool with_lora) { + std::vector sc = specialization_constants; + if (mma_id_twin_nt1) { + sc[1] = 1u; + } else if (mma_id_twin_nt4) { + sc[1] = 4u; + } + if (!is_mma_id_twin && is_mma_twin && sc.size() >= 7 && sc[5] != 0u && sc[2] >= 4u && tw.mode == 3u) { + static const uint32_t env_twin_shb = getenv("GGML_VK_MMA_TWIN_SHB") ? (uint32_t)atoi(getenv("GGML_VK_MMA_TWIN_SHB")) : 12u; + sc[5] = env_twin_shb; + } + if (is_mma_id_twin && sc.size() >= 7 && sc[5] != 0u) { + const uint32_t wg_sg_val = sc[0]; + const uint32_t id_rt_val = sc[2]; + if (tw.mode == 3u) { + const uint32_t def_shb = (id_rt_val >= 4u) ? 6u : ((id_rt_val == 3u) ? 7u : 8u); + const uint32_t def_kp = (id_rt_val == 3u || id_rt_val == 4u || (wg_sg_val >= 8u && id_rt_val >= 3u)) ? 2u : 1u; + sc[5] = getenv("GGML_VK_MMA_PGLU_SHB") ? (uint32_t)atoi(getenv("GGML_VK_MMA_PGLU_SHB")) : def_shb; + sc[6] = getenv("GGML_VK_MMA_PGLU_KP") ? (uint32_t)atoi(getenv("GGML_VK_MMA_PGLU_KP")) : def_kp; + } else if (tw.mode == 4u) { + const uint32_t def_shb = (id_rt_val >= 4u) ? 6u : 7u; + const uint32_t def_kp = (id_rt_val >= 5u) ? ((wg_sg_val >= 8u) ? 2u : 1u) : ((id_rt_val >= 3u) ? 2u : 4u); + sc[5] = getenv("GGML_VK_MMA_FASTEP_SHB") ? (uint32_t)atoi(getenv("GGML_VK_MMA_FASTEP_SHB")) : def_shb; + sc[6] = getenv("GGML_VK_MMA_FASTEP_KP") ? (uint32_t)atoi(getenv("GGML_VK_MMA_FASTEP_KP")) : def_kp; + } + } + while (sc.size() < mode_idx) { + sc.push_back(1); + } + if (sc.size() == mode_idx) { + sc.push_back(tw.mode); + } else { + sc[mode_idx] = tw.mode; + } + if (with_lora) { + static constexpr uint32_t k_mmv_defs[7] = { 32u, 1u, 1u, 0u, 1u, 0u, 0u }; + static constexpr uint32_t k_mma_defs[10] = { 1u, 2u, 1u, 1u, 0u, 1u, 1u, 1u, 0u, 0u }; + const size_t lora_idx = is_mma_twin ? 9u : 6u; + const uint32_t * defs = is_mma_twin ? k_mma_defs : k_mmv_defs; + while (sc.size() < lora_idx) { + sc.push_back(defs[sc.size()]); + } + if (sc.size() == lora_idx) { + sc.push_back(1u); + } else { + sc[lora_idx] = 1u; + } + } + return sc; + }; + if (can_lora_twin && twin->gen_lora_twin.get() == requested.get() && !twin->gen_lora_twin->compiled) { + vk_pipeline & ltwin = twin->gen_lora_twin; + wait_pipeline = ltwin; + if (!ltwin->compile_pending) { + ltwin->compile_pending = true; + claimed_task.pipeline = ltwin; + claimed_task.spv_size = spv_size; + claimed_task.spv_data = spv_data; + claimed_task.entrypoint = entrypoint; + claimed_task.parameter_count = 11u; + claimed_task.wg_denoms = twin_denoms; + claimed_task.specialization_constants = make_twin_sc(true); + claimed_task.disable_robustness = disable_robustness; + claimed_task.require_full_subgroups = require_full_subgroups; + claimed_task.required_subgroup_size = required_subgroup_size; + has_claimed_task = true; + } + continue; + } + if (twin.get() != requested.get() || twin->compiled) { + continue; + } + wait_pipeline = twin; + if (!twin->compile_pending) { + twin->compile_pending = true; + claimed_task.pipeline = twin; + claimed_task.spv_size = spv_size; + claimed_task.spv_data = spv_data; + claimed_task.entrypoint = entrypoint; + claimed_task.parameter_count = parameter_count; + claimed_task.wg_denoms = twin_denoms; + claimed_task.specialization_constants = make_twin_sc(false); + claimed_task.disable_robustness = disable_robustness; + claimed_task.require_full_subgroups = require_full_subgroups; + claimed_task.required_subgroup_size = required_subgroup_size; + has_claimed_task = true; + } + } + } + } }; auto const &ggml_vk_create_pipeline2 = [&](vk_device& device, vk_pipeline& pipeline, const std::string &name, size_t spv_size, const void* spv_data, const char *entrypoint, @@ -2353,6 +2798,8 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { const void * spv_data; size_t spv_size; const char *name; + const bool use_8x8_fa = !device->coopmat_support_16x16x16_f32acc && device->coopmat_support_8x8x8_f32acc; + if (use_8x8_fa && !f32acc) continue; if (bf16_kv) { #if defined(VK_KHR_shader_bfloat16) && defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT) if (!device->coopmat_bf16_support) continue; @@ -2361,8 +2808,8 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { spv_size = flash_attn_f32_f16_bf16_flip_cm1_len; name = aligned ? "flash_attn_f32_bf16_aligned_flip_cm1" : "flash_attn_f32_bf16_flip_cm1"; } else { - spv_data = flash_attn_f32_f16_bf16_cm1_data; - spv_size = flash_attn_f32_f16_bf16_cm1_len; + spv_data = use_8x8_fa ? flash_attn_f32_f16_bf16_8x8_cm1_data : flash_attn_f32_f16_bf16_cm1_data; + spv_size = use_8x8_fa ? flash_attn_f32_f16_bf16_8x8_cm1_len : flash_attn_f32_f16_bf16_cm1_len; name = aligned ? "flash_attn_f32_bf16_aligned_cm1" : "flash_attn_f32_bf16_cm1"; } #else @@ -2373,8 +2820,16 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { else { spv_data = flash_attn_f32_f16_flip_f16acc_cm1_data; spv_size = flash_attn_f32_f16_flip_f16acc_cm1_len; } name = aligned ? "flash_attn_f32_f16_aligned_flip_cm1" : "flash_attn_f32_f16_flip_cm1"; } else { - if (f32acc) { spv_data = flash_attn_f32_f16_cm1_data; spv_size = flash_attn_f32_f16_cm1_len; } - else { spv_data = flash_attn_f32_f16_f16acc_cm1_data; spv_size = flash_attn_f32_f16_f16acc_cm1_len; } + if (use_8x8_fa) { + spv_data = flash_attn_f32_f16_8x8_cm1_data; + spv_size = flash_attn_f32_f16_8x8_cm1_len; + } else if (f32acc) { + spv_data = flash_attn_f32_f16_cm1_data; + spv_size = flash_attn_f32_f16_cm1_len; + } else { + spv_data = flash_attn_f32_f16_f16acc_cm1_data; + spv_size = flash_attn_f32_f16_f16acc_cm1_len; + } name = aligned ? "flash_attn_f32_f16_aligned_cm1" : "flash_attn_f32_f16_cm1"; } ggml_vk_create_pipeline(device, fa.second, name, spv_size, spv_data, "main", 8, @@ -2421,23 +2876,39 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { } #endif - auto const &ggml_vk_mul_mm_spec = [&device](std::vector spec, bool aligned) { + static const uint32_t agx_slm_reshape = (getenv("GGML_VK_NO_SLM_RESHAPE") || ggml_vk_exact_s8_enabled()) ? 0u : 1u; + const uint32_t agx_shmem_pad = ggml_vk_exact_s8_enabled() ? 4u : 2u; + const uint32_t env_mm_kunroll = getenv("GGML_VK_MM_KUNROLL") ? (uint32_t)atoi(getenv("GGML_VK_MM_KUNROLL")) : 9u; + const uint32_t agx_direct_mode = ggml_vk_exact_s8_enabled() ? 1u + : (env_mm_kunroll == 9u) ? 5u + : (env_mm_kunroll == 8u) ? 4u + : (env_mm_kunroll == 1u) ? 3u + : (env_mm_kunroll == 2u) ? 2u : 1u; + auto const &ggml_vk_mul_mm_spec = [&device, agx_shmem_pad, agx_direct_mode](std::vector spec, bool aligned) { spec.push_back(aligned ? 1u : 0u); // constantID=11: ALIGNED if (device->vendor_id == VK_VENDOR_ID_INTEL && device->coopmat_support && device->driver_id == vk::DriverId::eIntelProprietaryWindows) { spec.push_back(0u); // constantID=12: SHMEM_STRIDE_PAD = 0 spec.push_back(1u); // constantID=13: APPLY_SLM_A_RESHAPE = true + } else if (ggml_vk_is_apple_agx_coopmat(device)) { + spec.push_back(agx_shmem_pad); // constantID=12: SHMEM_STRIDE_PAD + spec.push_back(agx_slm_reshape); // constantID=13: APPLY_SLM_A_RESHAPE + spec.push_back(agx_direct_mode); // constantID=14: AGX_DIRECT_STORE } return spec; }; - auto const &ggml_vk_mul_mm_spec_quant = [&device](std::vector spec, bool aligned, uint32_t type) { + auto const &ggml_vk_mul_mm_spec_quant = [&device, agx_shmem_pad, agx_direct_mode](std::vector spec, bool aligned, uint32_t type) { spec.push_back(aligned ? 1u : 0u); // constantID=11: ALIGNED spec.push_back(type); // constantID=12: MmTypeA if (device->vendor_id == VK_VENDOR_ID_INTEL && device->coopmat_support && device->driver_id == vk::DriverId::eIntelProprietaryWindows) { spec.push_back(0u); // constantID=13: SHMEM_STRIDE_PAD = 0 spec.push_back(1u); // constantID=14: APPLY_SLM_A_RESHAPE = true + } else if (ggml_vk_is_apple_agx_coopmat(device)) { + spec.push_back(agx_shmem_pad); // constantID=13: SHMEM_STRIDE_PAD + spec.push_back(agx_slm_reshape); // constantID=14: APPLY_SLM_A_RESHAPE + spec.push_back(agx_direct_mode); // constantID=15: AGX_DIRECT_STORE } return spec; }; @@ -2474,7 +2945,9 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { uint32_t push_constant_size, uint32_t param_count, const spec_fn_t& spec_fn, bool disable_robustness = false, bool require_full_subgroups = false, uint32_t required_subgroup_size = 0, - bool create_aligned = true, bool pin_subgroup_to_warp = false + bool create_aligned = true, bool pin_subgroup_to_warp = false, + size_t fused_spv_len = 0, const void* fused_spv_data = nullptr, + uint32_t fused_pc_size = 0, uint32_t fused_param_count = 0, uint32_t fused_mode_const_id = 15 ) { auto& vec = device->pipeline_matmul[key]; const bool first_call = vec.empty(); @@ -2514,6 +2987,74 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { tc.wg_denoms, spec_fn(tc.warptile, true), tc.align, disable_robustness, rfs, rsgs); } + + if (fused_spv_data != nullptr) { + const bool can_pair_glu = tc.warptile.size() >= 8 && + tc.warptile[1] >= 16 && (tc.warptile[1] % 2 == 0) && + (tc.warptile[4] % 2 == 0) && + ((tc.warptile[4] / tc.warptile[7]) % 2 == 0); + auto attach_mm_twins = [&](vk_pipeline & base_pl, bool is_aligned) { + for (vk_pipeline * b = &base_pl; *b; b = &(*b)->next) { + const struct { + vk_pipeline * slot; + const char * sfx; + uint32_t mode; + std::array denoms; + bool enabled; + } twins[4] = { + { &(*b)->gen_glu, "_mmep", 1u, tc.wg_denoms, true }, + { &(*b)->gen_pair_glu, "_mmpairglu", 2u, { tc.wg_denoms[0] / 2u, tc.wg_denoms[1], tc.wg_denoms[2] }, can_pair_glu }, + { &(*b)->gen_pair_mm, "_mmpairmm", 3u, tc.wg_denoms, !key.mul_mat_id }, + { &(*b)->gen_fast_ep, key.mul_mat_id ? "_mmfastep" : "_mmbias", 4u, tc.wg_denoms, true }, + }; + for (const auto & tw : twins) { + if (!tw.enabled) continue; + vk_pipeline & twin = *tw.slot; + if (!twin) { + twin = std::make_shared(); + } + if (!twin->initialized) { + twin->name = (*b)->name + tw.sfx; + twin->parameter_count = fused_param_count; + twin->push_constant_size = fused_pc_size; + twin->wg_denoms = tw.denoms; + twin->align = (*b)->align; + twin->initialized = true; +#if defined(VK_EXT_shader_64bit_indexing) + twin->is_64b_indexing = (*b)->is_64b_indexing; +#endif + } + if (twin.get() != requested.get() || twin->compiled) { + continue; + } + wait_pipeline = twin; + if (!twin->compile_pending) { + twin->compile_pending = true; + std::vector sc = spec_fn(tc.warptile, is_aligned); + if (sc.size() == fused_mode_const_id - 3) sc.push_back(4u); // SHMEM_STRIDE_PAD + if (sc.size() == fused_mode_const_id - 2) sc.push_back(0u); // APPLY_SLM_A_RESHAPE + if (sc.size() == fused_mode_const_id - 1) sc.push_back(0u); // AGX_DIRECT_STORE + sc.push_back(tw.mode); // MM_FUSE_MODE + claimed_task.pipeline = twin; + claimed_task.spv_size = fused_spv_len; + claimed_task.spv_data = fused_spv_data; + claimed_task.entrypoint = "main"; + claimed_task.parameter_count = fused_param_count; + claimed_task.wg_denoms = tw.denoms; + claimed_task.specialization_constants = std::move(sc); + claimed_task.disable_robustness = disable_robustness; + claimed_task.require_full_subgroups = rfs; + claimed_task.required_subgroup_size = rsgs; + has_claimed_task = true; + } + } + } + }; + attach_mm_twins(vec[i].unaligned, false); + if (vec[i].aligned) { + attach_mm_twins(vec[i].aligned, true); + } + } } }; @@ -2576,7 +3117,7 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { } }; auto add_narrow_n_cm1 = [&](std::vector& configs, ggml_type type, bool is_id) { - if (is_id || configs.empty()) { + if (is_id || configs.empty() || (ggml_vk_is_apple_agx_coopmat(device) && ggml_vk_exact_s8_enabled())) { return; } add_narrow_n_cm1_tile(configs, type, 16); @@ -2584,7 +3125,17 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { }; std::vector tc_mm = {{s_warptile, s_wg_denoms, s_align}, {m_warptile, m_wg_denoms, m_align}, {l_warptile, l_wg_denoms, l_align}}; - std::vector tc_mmq = {{s_warptile_mmq, s_mmq_wg_denoms, s_align}, {m_warptile_mmq, m_mmq_wg_denoms, m_align}, {l_warptile_mmq, l_mmq_wg_denoms, l_align}}; + // gen: GGML_VK_MMQ_M="BLOCK,BM,BN,BK,WM,WN,WMITER,TM,TN,TK,WARP" replaces the medium quantized matmul tile (the only + // one Borealis runs), for a register/shared-memory tile sweep on the M4 Max + std::array gen_m_mmq_denoms = m_mmq_wg_denoms; + if (const char * gs = getenv("GGML_VK_MMQ_M")) { + std::vector v; + for (const char * c = gs; *c; ) { char * end = nullptr; v.push_back((uint32_t) strtoul(c, &end, 10)); c = end; if (*c == ',') c++; else if (*c) break; } + GGML_ASSERT(v.size() == 11 && "GGML_VK_MMQ_M needs 11 values"); + m_warptile_mmq = v; + gen_m_mmq_denoms = { v[1], v[2], 1 }; + } + std::vector tc_mmq = {{s_warptile_mmq, s_mmq_wg_denoms, s_align}, {m_warptile_mmq, gen_m_mmq_denoms, m_align}, {l_warptile_mmq, l_mmq_wg_denoms, l_align}}; #if defined(VK_NV_cooperative_matrix2) && defined(GGML_VULKAN_COOPMAT2_GLSLC_SUPPORT) if (device->coopmat2) { @@ -2710,23 +3261,35 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { const bool cm1_pin = device->vendor_id == VK_VENDOR_ID_INTEL; // Intel coopmat1 uses a dedicated large-tile config for quant matmul_id. - std::vector tc_mmq_id = tc_mmq; + const uint32_t s_mmqid_align = std::max(s_align, s_warptile_mmqid[1]); + std::vector tc_id = {{s_warptile_id, s_wg_denoms, s_align}, {m_warptile_id, m_wg_denoms, m_align}, {l_warptile_id, l_wg_denoms, l_align}}; + std::vector tc_mmq_id = { + {s_warptile_mmqid, gen_s_mmqid_denoms, s_mmqid_align}, + {m_warptile_mmqid, m_mmqid_wg_denoms, m_align}, + {l_warptile_mmqid, l_mmqid_wg_denoms, l_align}, + }; if (cm1_pin) { tc_mmq_id[2] = { { 512, 128, 128, 32, 32, 32, 2, device->coopmat_m, device->coopmat_n, device->coopmat_k, 32 }, { 128, 128, 1 }, 32 }; } auto cm1_create = [&](vk_matmul_pipeline_key key, const std::vector& tc_base, - const std::string& name, size_t len, const void* data, uint32_t pc_size, uint32_t pc) { + const std::string& name, size_t len, const void* data, uint32_t pc_size, uint32_t pc, + size_t flen = 0, const void* fdata = nullptr) { auto tc = filter_tc(tc_base, key.type_a, key.mul_mat_id); add_narrow_n_cm1(tc, key.type_a, key.mul_mat_id); - if (!tc.empty()) create_mm_pipelines(key, tc, name, len, data, pc_size, pc, cm1_spec, false, true, 0, true, cm1_pin); + const uint32_t fpc_size = key.mul_mat_id ? sizeof(vk_mat_mat_id_fused_push_constants) : sizeof(vk_mat_mat_fused_push_constants); + const uint32_t fpc = key.mul_mat_id ? 8u : 6u; + if (!tc.empty()) create_mm_pipelines(key, tc, name, len, data, pc_size, pc, cm1_spec, true, true, 0, true, cm1_pin, flen, fdata, fpc_size, fpc, 15); }; auto cm1_create_quant = [&](vk_matmul_pipeline_key key, const std::vector& tc_base, - const std::string& name, size_t len, const void* data, uint32_t pc_size, uint32_t pc) { + const std::string& name, size_t len, const void* data, uint32_t pc_size, uint32_t pc, + size_t flen = 0, const void* fdata = nullptr) { spec_fn_t qs = [&, type_a=key.type_a](const std::vector& wt, bool a) { return ggml_vk_mul_mm_spec_quant(wt, a, (uint32_t)type_a); }; auto tc = filter_tc(tc_base, key.type_a, key.mul_mat_id); add_narrow_n_cm1(tc, key.type_a, key.mul_mat_id); - if (!tc.empty()) create_mm_pipelines(key, tc, name, len, data, pc_size, pc, qs, false, true, 0, true, cm1_pin); + const uint32_t fpc_size = key.mul_mat_id ? sizeof(vk_mat_mat_id_fused_push_constants) : sizeof(vk_mat_mat_fused_push_constants); + const uint32_t fpc = key.mul_mat_id ? 8u : 6u; + if (!tc.empty()) create_mm_pipelines(key, tc, name, len, data, pc_size, pc, qs, true, true, 0, true, cm1_pin, flen, fdata, fpc_size, fpc, 16); }; // int8 MMQ helper: per-type cm1 shader, warptile passed as-is (carries DEVICE_ARCH in // spec constant WARP_SIZE_IDX+1), subgroup size pinned to the warptile WARP element, no aligned variant. @@ -2752,15 +3315,15 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { const bool rdna3 = device->architecture == AMD_RDNA3; const bool rdna4 = device->architecture == AMD_RDNA4; - cm1_create({GGML_TYPE_F32, GGML_TYPE_F32, false, false}, tc_mm, "matmul_f32_f32", matmul_f32_f32_cm1_len, matmul_f32_f32_cm1_data, sizeof(vk_mat_mat_push_constants), 3); - cm1_create({GGML_TYPE_F32, GGML_TYPE_F16, false, false}, tc_mm, "matmul_f32_f16", matmul_f32_f16_cm1_len, matmul_f32_f16_cm1_data, sizeof(vk_mat_mat_push_constants), 3); + cm1_create({GGML_TYPE_F32, GGML_TYPE_F32, false, false}, tc_mm, "matmul_f32_f32", matmul_f32_f32_cm1_len, matmul_f32_f32_cm1_data, sizeof(vk_mat_mat_push_constants), 3, matmul_f32_f32_fused_cm1_len, matmul_f32_f32_fused_cm1_data); + cm1_create({GGML_TYPE_F32, GGML_TYPE_F16, false, false}, tc_mm, "matmul_f32_f16", matmul_f32_f16_cm1_len, matmul_f32_f16_cm1_data, sizeof(vk_mat_mat_push_constants), 3, matmul_f32_f16_fused_cm1_len, matmul_f32_f16_fused_cm1_data); if (device->coopmat_acc_f16_support) { cm1_create({GGML_TYPE_F16, GGML_TYPE_F16, false, true}, tc_mm, "matmul_f16_f16acc", matmul_f16_f16acc_cm1_len, matmul_f16_f16acc_cm1_data, sizeof(vk_mat_mat_push_constants), 3); cm1_create({GGML_TYPE_F16, GGML_TYPE_F32, false, true}, tc_mm, "matmul_f16_f32_f16acc", matmul_f16_f32_f16acc_cm1_len, matmul_f16_f32_f16acc_cm1_data, sizeof(vk_mat_mat_push_constants), 3); } if (device->coopmat_acc_f32_support) { - cm1_create({GGML_TYPE_F16, GGML_TYPE_F16, false, false}, tc_mm, "matmul_f16", matmul_f16_cm1_len, matmul_f16_cm1_data, sizeof(vk_mat_mat_push_constants), 3); - cm1_create({GGML_TYPE_F16, GGML_TYPE_F32, false, false}, tc_mm, "matmul_f16_f32", matmul_f16_f32_cm1_len, matmul_f16_f32_cm1_data, sizeof(vk_mat_mat_push_constants), 3); + cm1_create({GGML_TYPE_F16, GGML_TYPE_F16, false, false}, tc_mm, "matmul_f16", matmul_f16_cm1_len, matmul_f16_cm1_data, sizeof(vk_mat_mat_push_constants), 3, matmul_f16_fused_cm1_len, matmul_f16_fused_cm1_data); + cm1_create({GGML_TYPE_F16, GGML_TYPE_F32, false, false}, tc_mm, "matmul_f16_f32", matmul_f16_f32_cm1_len, matmul_f16_f32_cm1_data, sizeof(vk_mat_mat_push_constants), 3, matmul_f16_f32_fused_cm1_len, matmul_f16_f32_fused_cm1_data); } #if defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT) if (device->coopmat_bf16_support) { @@ -2773,8 +3336,8 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { cm1_create_quant({type, GGML_TYPE_F16, false, true}, tc_mmq, "matmul_quant_f16_f16acc", matmul_quant_f16_f16acc_cm1_len, matmul_quant_f16_f16acc_cm1_data, sizeof(vk_mat_mat_push_constants), 3); } if (device->coopmat_acc_f32_support) { - cm1_create_quant({type, GGML_TYPE_F32, false, false}, tc_mmq, "matmul_quant_f32", matmul_quant_f32_cm1_len, matmul_quant_f32_cm1_data, sizeof(vk_mat_mat_push_constants), 3); - cm1_create_quant({type, GGML_TYPE_F16, false, false}, tc_mmq, "matmul_quant_f16", matmul_quant_f16_cm1_len, matmul_quant_f16_cm1_data, sizeof(vk_mat_mat_push_constants), 3); + cm1_create_quant({type, GGML_TYPE_F32, false, false}, tc_mmq, "matmul_quant_f32", matmul_quant_f32_cm1_len, matmul_quant_f32_cm1_data, sizeof(vk_mat_mat_push_constants), 3, matmul_quant_f32_fused_cm1_len, matmul_quant_f32_fused_cm1_data); + cm1_create_quant({type, GGML_TYPE_F16, false, false}, tc_mmq, "matmul_quant_f16", matmul_quant_f16_cm1_len, matmul_quant_f16_cm1_data, sizeof(vk_mat_mat_push_constants), 3, matmul_quant_f16_fused_cm1_len, matmul_quant_f16_fused_cm1_data); } } // The _f16 variants provide the f16 B-type pipeline used when y_non_contig converts f32->f16. @@ -2783,11 +3346,17 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { cm1_create({TYPE, GGML_TYPE_F32, false, true}, tc_mmq, "matmul_" #tstr "_f32_f16acc", matmul_##tstr##_f32_f16acc_cm1_len, matmul_##tstr##_f32_f16acc_cm1_data, sizeof(vk_mat_mat_push_constants), 3); \ cm1_create({TYPE, GGML_TYPE_F16, false, true}, tc_mmq, "matmul_" #tstr "_f16_f16acc", matmul_##tstr##_f16_f16acc_cm1_len, matmul_##tstr##_f16_f16acc_cm1_data, sizeof(vk_mat_mat_push_constants), 3); \ } \ - if (device->coopmat_acc_f32_support) { \ + if (device->coopmat_acc_f32_support && (TYPE) != GGML_TYPE_IQ4_XS && (TYPE) != GGML_TYPE_IQ4_NL) { \ cm1_create({TYPE, GGML_TYPE_F32, false, false}, tc_mmq, "matmul_" #tstr "_f32", matmul_##tstr##_f32_cm1_len, matmul_##tstr##_f32_cm1_data, sizeof(vk_mat_mat_push_constants), 3); \ cm1_create({TYPE, GGML_TYPE_F16, false, false}, tc_mmq, "matmul_" #tstr "_f16", matmul_##tstr##_f16_cm1_len, matmul_##tstr##_f16_cm1_data, sizeof(vk_mat_mat_push_constants), 3); \ } FOR_EACH_LUT_TYPE_NONFP4(X_CM1) + if (device->coopmat_acc_f32_support) { + cm1_create({GGML_TYPE_IQ4_XS, GGML_TYPE_F32, false, false}, tc_mmq, "matmul_iq4_xs_f32", matmul_iq4_xs_f32_cm1_len, matmul_iq4_xs_f32_cm1_data, sizeof(vk_mat_mat_push_constants), 3, matmul_iq4_xs_f32_fused_cm1_len, matmul_iq4_xs_f32_fused_cm1_data); + cm1_create({GGML_TYPE_IQ4_XS, GGML_TYPE_F16, false, false}, tc_mmq, "matmul_iq4_xs_f16", matmul_iq4_xs_f16_cm1_len, matmul_iq4_xs_f16_cm1_data, sizeof(vk_mat_mat_push_constants), 3, matmul_iq4_xs_f16_fused_cm1_len, matmul_iq4_xs_f16_fused_cm1_data); + cm1_create({GGML_TYPE_IQ4_NL, GGML_TYPE_F32, false, false}, tc_mmq, "matmul_iq4_nl_f32", matmul_iq4_nl_f32_cm1_len, matmul_iq4_nl_f32_cm1_data, sizeof(vk_mat_mat_push_constants), 3, matmul_iq4_nl_f32_fused_cm1_len, matmul_iq4_nl_f32_fused_cm1_data); + cm1_create({GGML_TYPE_IQ4_NL, GGML_TYPE_F16, false, false}, tc_mmq, "matmul_iq4_nl_f16", matmul_iq4_nl_f16_cm1_len, matmul_iq4_nl_f16_cm1_data, sizeof(vk_mat_mat_push_constants), 3, matmul_iq4_nl_f16_fused_cm1_len, matmul_iq4_nl_f16_fused_cm1_data); + } #if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT) if (device->ocp_fp4) { #define X_CM1_OCP(TYPE, tstr) \ @@ -2826,18 +3395,18 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { GGML_ASSERT(device->subgroup_ballot); - cm1_create({GGML_TYPE_F32, GGML_TYPE_F32, true, false}, tc_mm, "matmul_id_subgroup_f32_f32", matmul_id_subgroup_f32_f32_cm1_len, matmul_id_subgroup_f32_f32_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + cm1_create({GGML_TYPE_F32, GGML_TYPE_F32, true, false}, tc_id, "matmul_id_subgroup_f32_f32", matmul_id_subgroup_f32_f32_cm1_len, matmul_id_subgroup_f32_f32_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, matmul_id_subgroup_f32_f32_fused_cm1_len, matmul_id_subgroup_f32_f32_fused_cm1_data); if (device->coopmat_acc_f16_support) { - cm1_create({GGML_TYPE_F16, GGML_TYPE_F16, true, true}, tc_mm, "matmul_id_subgroup_f16_f16acc", matmul_id_subgroup_f16_f16acc_cm1_len, matmul_id_subgroup_f16_f16acc_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); - cm1_create({GGML_TYPE_F16, GGML_TYPE_F32, true, true}, tc_mm, "matmul_id_subgroup_f16_f32_f16acc", matmul_id_subgroup_f16_f32_f16acc_cm1_len, matmul_id_subgroup_f16_f32_f16acc_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + cm1_create({GGML_TYPE_F16, GGML_TYPE_F16, true, true}, tc_id, "matmul_id_subgroup_f16_f16acc", matmul_id_subgroup_f16_f16acc_cm1_len, matmul_id_subgroup_f16_f16acc_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + cm1_create({GGML_TYPE_F16, GGML_TYPE_F32, true, true}, tc_id, "matmul_id_subgroup_f16_f32_f16acc", matmul_id_subgroup_f16_f32_f16acc_cm1_len, matmul_id_subgroup_f16_f32_f16acc_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); } if (device->coopmat_acc_f32_support) { - cm1_create({GGML_TYPE_F16, GGML_TYPE_F16, true, false}, tc_mm, "matmul_id_subgroup_f16", matmul_id_subgroup_f16_cm1_len, matmul_id_subgroup_f16_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); - cm1_create({GGML_TYPE_F16, GGML_TYPE_F32, true, false}, tc_mm, "matmul_id_subgroup_f16_f32", matmul_id_subgroup_f16_f32_cm1_len, matmul_id_subgroup_f16_f32_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + cm1_create({GGML_TYPE_F16, GGML_TYPE_F16, true, false}, tc_id, "matmul_id_subgroup_f16", matmul_id_subgroup_f16_cm1_len, matmul_id_subgroup_f16_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, matmul_id_subgroup_f16_fused_cm1_len, matmul_id_subgroup_f16_fused_cm1_data); + cm1_create({GGML_TYPE_F16, GGML_TYPE_F32, true, false}, tc_id, "matmul_id_subgroup_f16_f32", matmul_id_subgroup_f16_f32_cm1_len, matmul_id_subgroup_f16_f32_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, matmul_id_subgroup_f16_f32_fused_cm1_len, matmul_id_subgroup_f16_f32_fused_cm1_data); } #if defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT) if (device->coopmat_bf16_support) { - cm1_create({GGML_TYPE_BF16, GGML_TYPE_BF16, true, false}, tc_mm, "matmul_id_subgroup_bf16", matmul_id_subgroup_bf16_cm1_len, matmul_id_subgroup_bf16_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + cm1_create({GGML_TYPE_BF16, GGML_TYPE_BF16, true, false}, tc_id, "matmul_id_subgroup_bf16", matmul_id_subgroup_bf16_cm1_len, matmul_id_subgroup_bf16_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); } #endif for (const auto type : non_lut_quant_types) { @@ -2846,8 +3415,8 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { cm1_create_quant({type, GGML_TYPE_F16, true, true}, tc_mmq_id, "matmul_id_subgroup_quant_f16_f16acc", matmul_id_subgroup_quant_f16_f16acc_cm1_len, matmul_id_subgroup_quant_f16_f16acc_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); } if (device->coopmat_acc_f32_support) { - cm1_create_quant({type, GGML_TYPE_F32, true, false}, tc_mmq_id, "matmul_id_subgroup_quant_f32", matmul_id_subgroup_quant_f32_cm1_len, matmul_id_subgroup_quant_f32_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); - cm1_create_quant({type, GGML_TYPE_F16, true, false}, tc_mmq_id, "matmul_id_subgroup_quant_f16", matmul_id_subgroup_quant_f16_cm1_len, matmul_id_subgroup_quant_f16_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); + cm1_create_quant({type, GGML_TYPE_F32, true, false}, tc_mmq_id, "matmul_id_subgroup_quant_f32", matmul_id_subgroup_quant_f32_cm1_len, matmul_id_subgroup_quant_f32_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, matmul_id_subgroup_quant_f32_fused_cm1_len, matmul_id_subgroup_quant_f32_fused_cm1_data); + cm1_create_quant({type, GGML_TYPE_F16, true, false}, tc_mmq_id, "matmul_id_subgroup_quant_f16", matmul_id_subgroup_quant_f16_cm1_len, matmul_id_subgroup_quant_f16_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, matmul_id_subgroup_quant_f16_fused_cm1_len, matmul_id_subgroup_quant_f16_fused_cm1_data); } } // The _f16 variants provide the f16 B-type pipeline used when y_non_contig converts f32->f16. @@ -2856,11 +3425,17 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { cm1_create({TYPE, GGML_TYPE_F32, true, true}, tc_mmq_id, "matmul_id_subgroup_" #tstr "_f32_f16acc", matmul_id_subgroup_##tstr##_f32_f16acc_cm1_len, matmul_id_subgroup_##tstr##_f32_f16acc_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); \ cm1_create({TYPE, GGML_TYPE_F16, true, true}, tc_mmq_id, "matmul_id_subgroup_" #tstr "_f16_f16acc", matmul_id_subgroup_##tstr##_f16_f16acc_cm1_len, matmul_id_subgroup_##tstr##_f16_f16acc_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); \ } \ - if (device->coopmat_acc_f32_support) { \ + if (device->coopmat_acc_f32_support && (TYPE) != GGML_TYPE_IQ4_XS && (TYPE) != GGML_TYPE_IQ4_NL) { \ cm1_create({TYPE, GGML_TYPE_F32, true, false}, tc_mmq_id, "matmul_id_subgroup_" #tstr "_f32", matmul_id_subgroup_##tstr##_f32_cm1_len, matmul_id_subgroup_##tstr##_f32_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); \ cm1_create({TYPE, GGML_TYPE_F16, true, false}, tc_mmq_id, "matmul_id_subgroup_" #tstr "_f16", matmul_id_subgroup_##tstr##_f16_cm1_len, matmul_id_subgroup_##tstr##_f16_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); \ } FOR_EACH_LUT_TYPE_NONFP4(X_CM1_ID) + if (device->coopmat_acc_f32_support) { + cm1_create({GGML_TYPE_IQ4_XS, GGML_TYPE_F32, true, false}, tc_mmq_id, "matmul_id_subgroup_iq4_xs_f32", matmul_id_subgroup_iq4_xs_f32_cm1_len, matmul_id_subgroup_iq4_xs_f32_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, matmul_id_subgroup_iq4_xs_f32_fused_cm1_len, matmul_id_subgroup_iq4_xs_f32_fused_cm1_data); + cm1_create({GGML_TYPE_IQ4_XS, GGML_TYPE_F16, true, false}, tc_mmq_id, "matmul_id_subgroup_iq4_xs_f16", matmul_id_subgroup_iq4_xs_f16_cm1_len, matmul_id_subgroup_iq4_xs_f16_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, matmul_id_subgroup_iq4_xs_f16_fused_cm1_len, matmul_id_subgroup_iq4_xs_f16_fused_cm1_data); + cm1_create({GGML_TYPE_IQ4_NL, GGML_TYPE_F32, true, false}, tc_mmq_id, "matmul_id_subgroup_iq4_nl_f32", matmul_id_subgroup_iq4_nl_f32_cm1_len, matmul_id_subgroup_iq4_nl_f32_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, matmul_id_subgroup_iq4_nl_f32_fused_cm1_len, matmul_id_subgroup_iq4_nl_f32_fused_cm1_data); + cm1_create({GGML_TYPE_IQ4_NL, GGML_TYPE_F16, true, false}, tc_mmq_id, "matmul_id_subgroup_iq4_nl_f16", matmul_id_subgroup_iq4_nl_f16_cm1_len, matmul_id_subgroup_iq4_nl_f16_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, matmul_id_subgroup_iq4_nl_f16_fused_cm1_len, matmul_id_subgroup_iq4_nl_f16_fused_cm1_data); + } #if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT) if (device->ocp_fp4) { #define X_CM1_ID_OCP(TYPE, tstr) \ @@ -2896,6 +3471,111 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { cm1_create_mmq({GGML_TYPE_Q6_K, GGML_TYPE_Q8_1, true, false}, tc_mmq_cm1_int_k, "matmul_id_subgroup_q6_k_q8_1", matmul_id_subgroup_q6_k_q8_1_cm1_len, matmul_id_subgroup_q6_k_q8_1_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); if (!rdna4) { cm1_create_mmq({GGML_TYPE_NVFP4, GGML_TYPE_Q8_1, true, false}, tc_mmq_cm1_int_k, "matmul_id_subgroup_nvfp4_q8_1", matmul_id_subgroup_nvfp4_q8_1_cm1_len, matmul_id_subgroup_nvfp4_q8_1_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); } } + + if (ggml_vk_is_apple_agx_coopmat(device) && !getenv("GGML_VK_DISABLE_MMA")) { + struct mma_spv_entry { + ggml_type type; + const char * tstr; + size_t dense_len; const void * dense_data; + size_t id_len; const void * id_data; + }; + const mma_spv_entry mma_spvs[] = { + { GGML_TYPE_F32, "f32", mul_mv_mma_f32_cm1_len, mul_mv_mma_f32_cm1_data, mul_mv_mma_id_f32_cm1_len, mul_mv_mma_id_f32_cm1_data }, + { GGML_TYPE_F16, "f16", mul_mv_mma_f16_cm1_len, mul_mv_mma_f16_cm1_data, mul_mv_mma_id_f16_cm1_len, mul_mv_mma_id_f16_cm1_data }, + { GGML_TYPE_Q4_0, "q4_0", mul_mv_mma_q4_0_cm1_len, mul_mv_mma_q4_0_cm1_data, mul_mv_mma_id_q4_0_cm1_len, mul_mv_mma_id_q4_0_cm1_data }, + { GGML_TYPE_Q4_1, "q4_1", mul_mv_mma_q4_1_cm1_len, mul_mv_mma_q4_1_cm1_data, mul_mv_mma_id_q4_1_cm1_len, mul_mv_mma_id_q4_1_cm1_data }, + { GGML_TYPE_Q5_0, "q5_0", mul_mv_mma_q5_0_cm1_len, mul_mv_mma_q5_0_cm1_data, mul_mv_mma_id_q5_0_cm1_len, mul_mv_mma_id_q5_0_cm1_data }, + { GGML_TYPE_Q5_1, "q5_1", mul_mv_mma_q5_1_cm1_len, mul_mv_mma_q5_1_cm1_data, mul_mv_mma_id_q5_1_cm1_len, mul_mv_mma_id_q5_1_cm1_data }, + { GGML_TYPE_Q8_0, "q8_0", mul_mv_mma_q8_0_cm1_len, mul_mv_mma_q8_0_cm1_data, mul_mv_mma_id_q8_0_cm1_len, mul_mv_mma_id_q8_0_cm1_data }, + { GGML_TYPE_Q4_K, "q4_k", mul_mv_mma_q4_k_cm1_len, mul_mv_mma_q4_k_cm1_data, mul_mv_mma_id_q4_k_cm1_len, mul_mv_mma_id_q4_k_cm1_data }, + { GGML_TYPE_Q5_K, "q5_k", mul_mv_mma_q5_k_cm1_len, mul_mv_mma_q5_k_cm1_data, mul_mv_mma_id_q5_k_cm1_len, mul_mv_mma_id_q5_k_cm1_data }, + { GGML_TYPE_Q6_K, "q6_k", mul_mv_mma_q6_k_cm1_len, mul_mv_mma_q6_k_cm1_data, mul_mv_mma_id_q6_k_cm1_len, mul_mv_mma_id_q6_k_cm1_data }, + { GGML_TYPE_IQ4_XS, "iq4_xs", mul_mv_mma_iq4_xs_cm1_len, mul_mv_mma_iq4_xs_cm1_data, mul_mv_mma_id_iq4_xs_cm1_len, mul_mv_mma_id_iq4_xs_cm1_data }, + { GGML_TYPE_IQ4_NL, "iq4_nl", mul_mv_mma_iq4_nl_cm1_len, mul_mv_mma_iq4_nl_cm1_data, mul_mv_mma_id_iq4_nl_cm1_len, mul_mv_mma_id_iq4_nl_cm1_data }, + }; + const uint32_t mma_use_shb = getenv("GGML_VK_MMA_SHB") ? (uint32_t)atoi(getenv("GGML_VK_MMA_SHB")) : 1u; + const uint32_t mma_id_kp = getenv("GGML_VK_MMA_ID_KP") ? (uint32_t)atoi(getenv("GGML_VK_MMA_ID_KP")) : 1u; + const uint32_t mma_id_wgsg0 = getenv("GGML_VK_MMA_ID_WGSG") ? (uint32_t)atoi(getenv("GGML_VK_MMA_ID_WGSG")) : 4u; + const uint32_t mma_id_nt0 = getenv("GGML_VK_MMA_ID_NT") ? (uint32_t)atoi(getenv("GGML_VK_MMA_ID_NT")) : 2u; + static constexpr struct { uint32_t wg_sg; uint32_t nt; uint32_t nsg; } mma_cfgs[8] = { + { 8u, 1u, 8u }, // ci=0: NSG=8, MSG=1, BM=8 + { 4u, 1u, 4u }, // ci=1: NSG=4, MSG=1, BM=8 + { 4u, 2u, 4u }, // ci=2: NSG=4, MSG=1, BM=16 + { 2u, 2u, 1u }, // ci=3: NSG=1, MSG=2, BM=32 (64 threads/WG, fills all 40 cores at m=2048!) + { 1u, 2u, 1u }, // ci=4: NSG=1, MSG=1, BM=16 (0 shmem, 0 barriers) + { 4u, 2u, 1u }, // ci=5: NSG=1, MSG=4, BM=64 (128 threads/WG, hoisted sh_b!) + { 8u, 1u, 1u }, // ci=6: NSG=1, MSG=8, BM=64 (256 threads/WG, 100% occupancy at RT=8!) + { 8u, 2u, 1u }, // ci=7: NSG=1, MSG=8, BM=128 (256 threads/WG, NT=1 BM=64 on _pairglu!) + }; + static constexpr uint32_t mma_rts[5] = { 1u, 2u, 4u, 6u, 8u }; + const uint32_t mma_f32_f16_shb = getenv("GGML_VK_MMA_F32_SHB") ? (uint32_t)atoi(getenv("GGML_VK_MMA_F32_SHB")) : 7u; + const uint32_t mma_bf16_shb = getenv("GGML_VK_MMA_BF16_SHB") ? (uint32_t)atoi(getenv("GGML_VK_MMA_BF16_SHB")) : 12u; + const uint32_t mma_bf16_kp = getenv("GGML_VK_MMA_BF16_KP") ? (uint32_t)atoi(getenv("GGML_VK_MMA_BF16_KP")) : 2u; + for (uint32_t rt_i = 0; rt_i < 5; ++rt_i) { + const uint32_t rt = mma_rts[rt_i]; + for (uint32_t ci = 0; ci < 8; ++ci) { + const uint32_t wg_sg = mma_cfgs[ci].wg_sg; + const uint32_t nt = mma_cfgs[ci].nt; + const uint32_t nsg = mma_cfgs[ci].nsg; + const uint32_t msg = wg_sg / nsg; + for (uint32_t bf16_i = 0; bf16_i < 2; ++bf16_i) { + const uint32_t shb = bf16_i ? mma_bf16_shb : ((rt >= 6u && !getenv("GGML_VK_MMA_F32_SHB")) ? 1u : mma_f32_f16_shb); + const uint32_t kp = bf16_i ? mma_bf16_kp : 1u; + const std::string pname = std::string("mul_mv_mma_f32_f16_rt") + std::to_string(rt) + "_c" + std::to_string(ci) + (bf16_i ? "_bf16" : ""); + ggml_vk_create_pipeline(device, device->pipeline_mul_mv_mma_f32_f16[rt_i][ci][bf16_i], + pname.c_str(), mul_mv_mma_f32_f16_cm1_len, mul_mv_mma_f32_f16_cm1_data, "main", + 7u, sizeof(vk_mat_mv_mma_push_constants), + { 8u * nt * msg, 8u * rt, 1u }, { wg_sg, nt, rt, nsg, 0u, shb, kp, 1u, bf16_i }, + 1u, true, true, 32u); + } + } + } + for (const auto & e : mma_spvs) { + for (uint32_t rt_i = 0; rt_i < 5; ++rt_i) { + const uint32_t rt = mma_rts[rt_i]; + if (rt >= 6u && e.type == GGML_TYPE_F32) { + continue; + } + for (uint32_t ci = 0; ci < 8; ++ci) { + const uint32_t wg_sg = mma_cfgs[ci].wg_sg; + const uint32_t nt = mma_cfgs[ci].nt; + const uint32_t nsg = mma_cfgs[ci].nsg; + const uint32_t msg = wg_sg / nsg; + for (uint32_t bf16_i = 0; bf16_i < 2; ++bf16_i) { + const uint32_t shb = bf16_i ? mma_bf16_shb : mma_use_shb; + const uint32_t kp = (bf16_i && e.type != GGML_TYPE_F32) ? mma_bf16_kp : 1u; + const std::string pname = std::string("mul_mv_mma_") + e.tstr + "_rt" + std::to_string(rt) + "_c" + std::to_string(ci) + (bf16_i ? "_bf16" : ""); + ggml_vk_create_pipeline(device, device->pipeline_mul_mv_mma[e.type][rt_i][ci][bf16_i], + pname.c_str(), e.dense_len, e.dense_data, "main", + 7u, sizeof(vk_mat_mv_mma_push_constants), + { 8u * nt * msg, 8u * rt, 1u }, { wg_sg, nt, rt, nsg, 0u, shb, kp, 1u, bf16_i }, + 1u, true, true, 32u); + } + } + } + static constexpr uint32_t mma_id_rts[7] = { 1u, 2u, 3u, 4u, 5u, 6u, 8u }; + for (uint32_t id_rt_i = 0; id_rt_i < 7; ++id_rt_i) { + const uint32_t id_rt = mma_id_rts[id_rt_i]; + if (id_rt >= 5u && e.type == GGML_TYPE_F32) { + continue; + } + for (uint32_t ici = 0; ici < 5; ++ici) { + const uint32_t wg_sg = (ici == 1u) ? 1u : ((ici == 2u) ? 2u : ((ici >= 3u) ? 8u : mma_id_wgsg0)); + const uint32_t id_nt = (ici == 4u) ? 1u : ((ici == 0u) ? mma_id_nt0 : 2u); + const uint32_t msg = wg_sg; + for (uint32_t bf16_i = 0; bf16_i < 2; ++bf16_i) { + const std::string id_pname = std::string("mul_mv_mma_id_") + e.tstr + "_rt" + std::to_string(id_rt) + + "_m" + std::to_string(msg) + "_nt" + std::to_string(id_nt) + (bf16_i ? "_bf16" : ""); + ggml_vk_create_pipeline(device, device->pipeline_mul_mv_mma_id[e.type][id_rt_i][ici][bf16_i], + id_pname.c_str(), e.id_len, e.id_data, "main", + 8u, sizeof(vk_mat_mat_id_fused_push_constants), + { 8u * id_nt * msg, 8u * id_rt, 1u }, { wg_sg, id_nt, id_rt, 1u, 0u, mma_use_shb, mma_id_kp, 1u, bf16_i }, + 1u, true, true, 32u); + } + } + } + } + } } else #endif // defined(VK_KHR_cooperative_matrix) && defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT) { @@ -2930,7 +3610,7 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { #endif std::vector tc_id = {{s_warptile_id, s_wg_denoms, s_align}, {m_warptile_id, m_wg_denoms, m_align}, {l_warptile_id, l_wg_denoms, l_align}}; - std::vector tc_mmqid = {{s_warptile_mmqid, s_mmq_wg_denoms, s_align}, {m_warptile_mmqid, m_mmq_wg_denoms, m_align}, {l_warptile_mmqid, l_mmq_wg_denoms, l_align}}; + std::vector tc_mmqid = {{s_warptile_mmqid, gen_s_mmqid_denoms, s_align}, {m_warptile_mmqid, m_mmq_wg_denoms, m_align}, {l_warptile_mmqid, l_mmq_wg_denoms, l_align}}; if (device->fp16) { // FP16 subgroup path - with dot2 runtime selection @@ -3183,6 +3863,34 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { if (n > crossover_medium) return 1; return 0; }; + } else if (ggml_vk_is_apple_agx_coopmat(device)) { + device->matmul_tile_selector = [](uint32_t m, uint32_t n, uint32_t /*k*/, uint32_t shader_core_count, + const std::vector& configs) -> uint32_t { + if (configs.size() <= 1) return 0; + const uint32_t cores = std::max(shader_core_count, 40u); + if (m <= 32 || n <= (ggml_vk_exact_s8_enabled() ? 24u : 16u)) return 0; + const uint32_t tiles_m = CEIL_DIV(m, configs[1].unaligned->wg_denoms[0]) * CEIL_DIV(n, configs[1].unaligned->wg_denoms[1]); + if (tiles_m < cores) return 0; + if (configs.size() == 2) return 1; + if (m < 128 || n < 64) return 1; + const uint32_t last = (uint32_t)configs.size() - 1; + const uint32_t tiles_l = CEIL_DIV(m, configs[last].unaligned->wg_denoms[0]) * CEIL_DIV(n, configs[last].unaligned->wg_denoms[1]); + static const uint32_t env_l_min = getenv("GGML_VK_CM1_L_MIN_TILES") ? (uint32_t)atoi(getenv("GGML_VK_CM1_L_MIN_TILES")) : 0u; + const uint32_t min_tiles_l = env_l_min > 0 ? env_l_min : (ggml_vk_exact_s8_enabled() ? (cores / 2u) : 140u); + if (tiles_l < min_tiles_l) return 1; + return last; + }; + device->matmul_id_tile_selector = [](uint32_t m, uint32_t n, uint32_t /*k*/, uint32_t /*shader_core_count*/, + const std::vector& configs) -> uint32_t { + if (configs.size() <= 1) return 0; + const uint32_t bn0 = configs[0].unaligned->wg_denoms[1]; + const uint32_t bn1 = configs[1].unaligned->wg_denoms[1]; + const uint32_t cross0 = std::min(3 * bn0, std::max(bn0, bn1 / 2)); + if (m <= 64 || n <= cross0) return 0; + if (configs.size() == 2) return 1; + if (m <= 128 || n <= bn1) return 1; + return (uint32_t)configs.size() - 1; + }; } else { device->matmul_tile_selector = [](uint32_t m, uint32_t n, uint32_t /*k*/, uint32_t /*shader_core_count*/, const std::vector& configs) -> uint32_t { @@ -3195,9 +3903,21 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { if (last == 0 || m <= 32 || n <= 32) return 0; if (last == 1) return 1; if (m <= 64 || n <= 64) return 1; + const uint32_t tiles_l = CEIL_DIV(m, configs[last].unaligned->wg_denoms[0]) * CEIL_DIV(n, configs[last].unaligned->wg_denoms[1]); + if (tiles_l < 64) return 1; return last; }; - device->matmul_id_tile_selector = device->matmul_tile_selector; + device->matmul_id_tile_selector = [](uint32_t m, uint32_t n, uint32_t /*k*/, uint32_t /*shader_core_count*/, + const std::vector& configs) -> uint32_t { + if (configs.size() <= 1) return 0; + const uint32_t bn0 = configs[0].unaligned->wg_denoms[1]; + const uint32_t bn1 = configs[1].unaligned->wg_denoms[1]; + const uint32_t cross0 = std::min(2 * bn0, std::max(bn0, bn1 / 2)); + if (m <= 32 || n <= cross0) return 0; + if (configs.size() == 2) return 1; + if (m <= 64 || n <= bn1) return 1; + return (uint32_t)configs.size() - 1; + }; } // mul mat vec @@ -3217,7 +3937,24 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { } else if (device->vendor_id == VK_VENDOR_ID_INTEL) { rm_stdq = 2; rm_stdq_int = 2; - } + } else if (ggml_vk_is_apple_agx_coopmat(device)) { + rm_stdq = 2; + rm_kq = 2; + } + // gen: GGML_VK_RM_STDQ / GGML_VK_RM_KQ override the rows each mat-vec workgroup computes (q8_0 etc. / K-quants); + // each row's arithmetic is unchanged, only the grouping (Metal uses 8 rows per threadgroup for q8_0, 4 for q4_K) + if (getenv("GGML_VK_RM_STDQ")) { rm_stdq = std::max(1, atoi(getenv("GGML_VK_RM_STDQ"))); } + if (getenv("GGML_VK_RM_KQ")) { rm_kq = std::max(1, atoi(getenv("GGML_VK_RM_KQ"))); } + uint32_t rm_stdq_id = rm_stdq; + uint32_t rm_kq_id = rm_kq; + if (getenv("GGML_VK_RM_STDQ_ID")) { rm_stdq_id = std::max(1, atoi(getenv("GGML_VK_RM_STDQ_ID"))); } + if (getenv("GGML_VK_RM_KQ_ID")) { rm_kq_id = std::max(1, atoi(getenv("GGML_VK_RM_KQ_ID"))); } + uint32_t dmmv_nsg = 1; + uint32_t dmmv_id_nsg = 1; + uint32_t fast_kq = (ggml_vk_is_apple_agx_coopmat(device) && !ggml_vk_exact_s8_enabled()) ? 2u : 0u; + if (getenv("GGML_VK_DMMV_NSG")) { dmmv_nsg = std::max(1, atoi(getenv("GGML_VK_DMMV_NSG"))); } + if (getenv("GGML_VK_DMMV_ID_NSG")) { dmmv_id_nsg = std::max(1, atoi(getenv("GGML_VK_DMMV_ID_NSG"))); } + if (getenv("GGML_VK_FAST_KQ")) { fast_kq = std::max(0, atoi(getenv("GGML_VK_FAST_KQ"))); } // RDNA3/4: above four columns, static 4 rows for all types bench faster than the default const bool is_rdna3_or_4 = device->vendor_id == VK_VENDOR_ID_AMD && (device->architecture == AMD_RDNA3 || device->architecture == AMD_RDNA4); auto const &rm_int_n = [&](uint32_t rows, uint32_t i) { return (is_rdna3_or_4 && i >= 4) ? 4u : rows; }; @@ -3229,7 +3966,7 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { } return is_rdna3_or_4 ? 4u : rows; }; - uint32_t rm_iq = 2 * rm_kq; + uint32_t rm_iq_id = 2 * rm_kq_id; const bool use_subgroups = device->subgroup_arithmetic; // The Imagination proprietary compiler rejects the subgroup-only dequant mul_mat_vec @@ -3244,8 +3981,10 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { const uint32_t force_subgroup_size = use_subgroups ? subgroup_size : 0; const uint32_t force_subgroup_size16 = use_subgroups16 ? subgroup_size16 : 0; - static constexpr uint32_t mul_mat_vec_num_bindings = 5; + const uint32_t dmmv_max_wg_inv = std::min(device->properties.limits.maxComputeWorkGroupInvocations, device->properties.limits.maxComputeWorkGroupSize[1]); + static constexpr uint32_t mul_mat_vec_num_bindings = 7; static constexpr uint32_t mul_mat_vec_id_num_bindings = 6; + const bool agx_no_robust = ggml_vk_is_apple_agx_coopmat(device); #if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT) #define OCP_DMMV_LEN(NAME, REDUC) (device->ocp_fp4 ? NAME ## _ocp_len[REDUC] : NAME ## _len[REDUC]) @@ -3267,64 +4006,76 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { (use_subgroups16 && w == DMMV_WG_SIZE_LARGE) ? SHADER_REDUCTION_MODE_HYBRID : SHADER_REDUCTION_MODE_SHMEM; - for (uint32_t i = 0; i < mul_mat_vec_max_cols; ++i) { - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_F32 ][i], "mul_mat_vec_f32_f32_f32", arr_dmmv_f32_f32_f32_len[reduc], arr_dmmv_f32_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {1, 1, 1}, {wg_size_subgroup, 1, i+1}, 1, false, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_F16 ][i], "mul_mat_vec_f16_f32_f32", arr_dmmv_f16_f32_f32_len[reduc], arr_dmmv_f16_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2, 1, 1}, {wg_size_subgroup, 2, i+1}, 1, false, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_BF16][i], "mul_mat_vec_bf16_f32_f32", arr_dmmv_bf16_f32_f32_len[reduc], arr_dmmv_bf16_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2, 1, 1}, {wg_size_subgroup, 2, i+1}, 1, false, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q1_0][i], "mul_mat_vec_q1_0_f32_f32", arr_dmmv_q1_0_f32_f32_len[reduc], arr_dmmv_q1_0_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q2_0][i], "mul_mat_vec_q2_0_f32_f32", arr_dmmv_q2_0_f32_f32_len[reduc], arr_dmmv_q2_0_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q4_0][i], "mul_mat_vec_q4_0_f32_f32", arr_dmmv_q4_0_f32_f32_len[reduc], arr_dmmv_q4_0_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q4_1][i], "mul_mat_vec_q4_1_f32_f32", arr_dmmv_q4_1_f32_f32_len[reduc], arr_dmmv_q4_1_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q5_0][i], "mul_mat_vec_q5_0_f32_f32", arr_dmmv_q5_0_f32_f32_len[reduc], arr_dmmv_q5_0_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q5_1][i], "mul_mat_vec_q5_1_f32_f32", arr_dmmv_q5_1_f32_f32_len[reduc], arr_dmmv_q5_1_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q8_0][i], "mul_mat_vec_q8_0_f32_f32", arr_dmmv_q8_0_f32_f32_len[reduc], arr_dmmv_q8_0_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {1*rm_stdq, 1, 1}, {wg_size_subgroup, 1*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q2_K][i], "mul_mat_vec_q2_k_f32_f32", arr_dmmv_q2_k_f32_f32_len[reduc16], arr_dmmv_q2_k_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q3_K][i], "mul_mat_vec_q3_k_f32_f32", arr_dmmv_q3_k_f32_f32_len[reduc16], arr_dmmv_q3_k_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q4_K][i], "mul_mat_vec_q4_k_f32_f32", arr_dmmv_q4_k_f32_f32_len[reduc16], arr_dmmv_q4_k_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q5_K][i], "mul_mat_vec_q5_k_f32_f32", arr_dmmv_q5_k_f32_f32_len[reduc16], arr_dmmv_q5_k_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q6_K][i], "mul_mat_vec_q6_k_f32_f32", arr_dmmv_q6_k_f32_f32_len[reduc16], arr_dmmv_q6_k_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_TQ1_0][i], "mul_mat_vec_tq1_0_f32_f32", arr_dmmv_tq1_0_f32_f32_len[reduc16], arr_dmmv_tq1_0_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_TQ2_0][i], "mul_mat_vec_tq2_0_f32_f32", arr_dmmv_tq2_0_f32_f32_len[reduc16], arr_dmmv_tq2_0_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ1_S][i], "mul_mat_vec_iq1_s_f32_f32", arr_dmmv_iq1_s_f32_f32_len[reduc16], arr_dmmv_iq1_s_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ1_M][i], "mul_mat_vec_iq1_m_f32_f32", arr_dmmv_iq1_m_f32_f32_len[reduc16], arr_dmmv_iq1_m_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ2_XXS][i], "mul_mat_vec_iq2_xxs_f32_f32", arr_dmmv_iq2_xxs_f32_f32_len[reduc16], arr_dmmv_iq2_xxs_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ2_XS][i], "mul_mat_vec_iq2_xs_f32_f32", arr_dmmv_iq2_xs_f32_f32_len[reduc16], arr_dmmv_iq2_xs_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ2_S][i], "mul_mat_vec_iq2_s_f32_f32", arr_dmmv_iq2_s_f32_f32_len[reduc16], arr_dmmv_iq2_s_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ3_XXS][i], "mul_mat_vec_iq3_xxs_f32_f32", arr_dmmv_iq3_xxs_f32_f32_len[reduc16], arr_dmmv_iq3_xxs_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ3_S][i], "mul_mat_vec_iq3_s_f32_f32", arr_dmmv_iq3_s_f32_f32_len[reduc16], arr_dmmv_iq3_s_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ4_XS][i], "mul_mat_vec_iq4_xs_f32_f32", arr_dmmv_iq4_xs_f32_f32_len[reduc16], arr_dmmv_iq4_xs_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ4_NL][i], "mul_mat_vec_iq4_nl_f32_f32", arr_dmmv_iq4_nl_f32_f32_len[reduc16], arr_dmmv_iq4_nl_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_MXFP4][i], "mul_mat_vec_mxfp4_f32_f32", OCP_DMMV_LEN(arr_dmmv_mxfp4_f32_f32, reduc16), OCP_DMMV_DATA(arr_dmmv_mxfp4_f32_f32, reduc16), "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_NVFP4][i], "mul_mat_vec_nvfp4_f32_f32", OCP_DMMV_LEN(arr_dmmv_nvfp4_f32_f32, reduc16), OCP_DMMV_DATA(arr_dmmv_nvfp4_f32_f32, reduc16), "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_F32 ][i], "mul_mat_vec_f32_f16_f32", arr_dmmv_f32_f16_f32_len[reduc], arr_dmmv_f32_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {1, 1, 1}, {wg_size_subgroup, 1, i+1}, 1, false, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_F16 ][i], "mul_mat_vec_f16_f16_f32", arr_dmmv_f16_f16_f32_len[reduc], arr_dmmv_f16_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2, 1, 1}, {wg_size_subgroup, 2, i+1}, 1, false, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_BF16][i], "mul_mat_vec_bf16_f16_f32", arr_dmmv_bf16_f16_f32_len[reduc], arr_dmmv_bf16_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2, 1, 1}, {wg_size_subgroup, 2, i+1}, 1, false, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q1_0][i], "mul_mat_vec_q1_0_f16_f32", arr_dmmv_q1_0_f16_f32_len[reduc], arr_dmmv_q1_0_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q2_0][i], "mul_mat_vec_q2_0_f16_f32", arr_dmmv_q2_0_f16_f32_len[reduc], arr_dmmv_q2_0_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q4_0][i], "mul_mat_vec_q4_0_f16_f32", arr_dmmv_q4_0_f16_f32_len[reduc], arr_dmmv_q4_0_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q4_1][i], "mul_mat_vec_q4_1_f16_f32", arr_dmmv_q4_1_f16_f32_len[reduc], arr_dmmv_q4_1_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q5_0][i], "mul_mat_vec_q5_0_f16_f32", arr_dmmv_q5_0_f16_f32_len[reduc], arr_dmmv_q5_0_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q5_1][i], "mul_mat_vec_q5_1_f16_f32", arr_dmmv_q5_1_f16_f32_len[reduc], arr_dmmv_q5_1_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q8_0][i], "mul_mat_vec_q8_0_f16_f32", arr_dmmv_q8_0_f16_f32_len[reduc], arr_dmmv_q8_0_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {1*rm_stdq, 1, 1}, {wg_size_subgroup, 1*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q2_K][i], "mul_mat_vec_q2_k_f16_f32", arr_dmmv_q2_k_f16_f32_len[reduc16], arr_dmmv_q2_k_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q3_K][i], "mul_mat_vec_q3_k_f16_f32", arr_dmmv_q3_k_f16_f32_len[reduc16], arr_dmmv_q3_k_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q4_K][i], "mul_mat_vec_q4_k_f16_f32", arr_dmmv_q4_k_f16_f32_len[reduc16], arr_dmmv_q4_k_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q5_K][i], "mul_mat_vec_q5_k_f16_f32", arr_dmmv_q5_k_f16_f32_len[reduc16], arr_dmmv_q5_k_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q6_K][i], "mul_mat_vec_q6_k_f16_f32", arr_dmmv_q6_k_f16_f32_len[reduc16], arr_dmmv_q6_k_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_TQ1_0][i], "mul_mat_vec_tq1_0_f16_f32", arr_dmmv_tq1_0_f16_f32_len[reduc16], arr_dmmv_tq1_0_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_TQ2_0][i], "mul_mat_vec_tq2_0_f16_f32", arr_dmmv_tq2_0_f16_f32_len[reduc16], arr_dmmv_tq2_0_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ1_S][i], "mul_mat_vec_iq1_s_f16_f32", arr_dmmv_iq1_s_f16_f32_len[reduc16], arr_dmmv_iq1_s_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ1_M][i], "mul_mat_vec_iq1_m_f16_f32", arr_dmmv_iq1_m_f16_f32_len[reduc16], arr_dmmv_iq1_m_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ2_XXS][i], "mul_mat_vec_iq2_xxs_f16_f32", arr_dmmv_iq2_xxs_f16_f32_len[reduc16], arr_dmmv_iq2_xxs_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ2_XS][i], "mul_mat_vec_iq2_xs_f16_f32", arr_dmmv_iq2_xs_f16_f32_len[reduc16], arr_dmmv_iq2_xs_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ2_S][i], "mul_mat_vec_iq2_s_f16_f32", arr_dmmv_iq2_s_f16_f32_len[reduc16], arr_dmmv_iq2_s_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ3_XXS][i], "mul_mat_vec_iq3_xxs_f16_f32", arr_dmmv_iq3_xxs_f16_f32_len[reduc16], arr_dmmv_iq3_xxs_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ3_S][i], "mul_mat_vec_iq3_s_f16_f32", arr_dmmv_iq3_s_f16_f32_len[reduc16], arr_dmmv_iq3_s_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ4_XS][i], "mul_mat_vec_iq4_xs_f16_f32", arr_dmmv_iq4_xs_f16_f32_len[reduc16], arr_dmmv_iq4_xs_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ4_NL][i], "mul_mat_vec_iq4_nl_f16_f32", arr_dmmv_iq4_nl_f16_f32_len[reduc16], arr_dmmv_iq4_nl_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_MXFP4][i], "mul_mat_vec_mxfp4_f16_f32", OCP_DMMV_LEN(arr_dmmv_mxfp4_f16_f32, reduc16), OCP_DMMV_DATA(arr_dmmv_mxfp4_f16_f32, reduc16), "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_NVFP4][i], "mul_mat_vec_nvfp4_f16_f32", OCP_DMMV_LEN(arr_dmmv_nvfp4_f16_f32, reduc16), OCP_DMMV_DATA(arr_dmmv_nvfp4_f16_f32, reduc16), "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + const uint32_t nsg_sub = (reduc == SHADER_REDUCTION_MODE_SUBGROUP) ? std::max(1u, std::min(dmmv_nsg, dmmv_max_wg_inv / wg_size_subgroup)) : 1u; + const uint32_t nsg_sub16 = (reduc16 == SHADER_REDUCTION_MODE_SUBGROUP) ? std::max(1u, std::min(dmmv_nsg, dmmv_max_wg_inv / wg_size_subgroup16)) : 1u; + const uint32_t nsg_id = (reduc == SHADER_REDUCTION_MODE_SUBGROUP) ? std::max(1u, std::min(dmmv_id_nsg, dmmv_max_wg_inv / wg_size_subgroup)) : 1u; + const uint32_t nsg_id16 = (reduc16 == SHADER_REDUCTION_MODE_SUBGROUP) ? std::max(1u, std::min(dmmv_id_nsg, dmmv_max_wg_inv / wg_size_subgroup16)) : 1u; + + const uint32_t mmv_pipe_cols = std::min(device->mmv_max_cols, mul_mat_vec_max_cols_limit); + for (uint32_t i = 0; i < mmv_pipe_cols; ++i) { + const uint32_t rm_stdq_c = (ggml_vk_is_apple_agx_coopmat(device) && i >= 4 && !getenv("GGML_VK_RM_STDQ")) ? 1u : rm_stdq; + const uint32_t rm_q8_c = (w == DMMV_WG_SIZE_LARGE && ggml_vk_is_apple_agx_coopmat(device) && !ggml_vk_exact_s8_enabled() && !getenv("GGML_VK_RM_STDQ")) ? 1u : rm_stdq_c; + const uint32_t rm_kq_c = (ggml_vk_is_apple_agx_coopmat(device) && i >= 4 && !getenv("GGML_VK_RM_KQ")) ? 1u : rm_kq; + const uint32_t rm_iq_c = 2u * rm_kq_c; + const uint32_t rm_f32_c = getenv("GGML_VK_RM_F32") ? (uint32_t)std::max(1, atoi(getenv("GGML_VK_RM_F32"))) : 1u; + const uint32_t fast_kq_c = (ggml_vk_is_apple_agx_coopmat(device) && !ggml_vk_exact_s8_enabled() && !getenv("GGML_VK_FAST_KQ") && i >= 1) ? 2u : fast_kq; + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_F32 ][i], "mul_mat_vec_f32_f32_f32", arr_dmmv_f32_f32_f32_len[reduc], arr_dmmv_f32_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_f32_c*nsg_sub, 1, 1}, {wg_size_subgroup, rm_f32_c, i+1, 0, nsg_sub}, 1, agx_no_robust, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_F16 ][i], "mul_mat_vec_f16_f32_f32", arr_dmmv_f16_f32_f32_len[reduc], arr_dmmv_f16_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*nsg_sub, 1, 1}, {wg_size_subgroup, 2, i+1, 0, nsg_sub}, 1, agx_no_robust, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_BF16][i], "mul_mat_vec_bf16_f32_f32", arr_dmmv_bf16_f32_f32_len[reduc], arr_dmmv_bf16_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*nsg_sub, 1, 1}, {wg_size_subgroup, 2, i+1, 0, nsg_sub}, 1, false, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q1_0][i], "mul_mat_vec_q1_0_f32_f32", arr_dmmv_q1_0_f32_f32_len[reduc], arr_dmmv_q1_0_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq_c*nsg_sub, 1, 1}, {wg_size_subgroup, 2*rm_stdq_c, i+1, 0, nsg_sub}, 1, true, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q2_0][i], "mul_mat_vec_q2_0_f32_f32", arr_dmmv_q2_0_f32_f32_len[reduc], arr_dmmv_q2_0_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq_c*nsg_sub, 1, 1}, {wg_size_subgroup, 2*rm_stdq_c, i+1, 0, nsg_sub}, 1, true, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q4_0][i], "mul_mat_vec_q4_0_f32_f32", arr_dmmv_q4_0_f32_f32_len[reduc], arr_dmmv_q4_0_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq_c*nsg_sub, 1, 1}, {wg_size_subgroup, 2*rm_stdq_c, i+1, 0, nsg_sub}, 1, true, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q4_1][i], "mul_mat_vec_q4_1_f32_f32", arr_dmmv_q4_1_f32_f32_len[reduc], arr_dmmv_q4_1_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq_c*nsg_sub, 1, 1}, {wg_size_subgroup, 2*rm_stdq_c, i+1, 0, nsg_sub}, 1, true, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q5_0][i], "mul_mat_vec_q5_0_f32_f32", arr_dmmv_q5_0_f32_f32_len[reduc], arr_dmmv_q5_0_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq_c*nsg_sub, 1, 1}, {wg_size_subgroup, 2*rm_stdq_c, i+1, 0, nsg_sub}, 1, true, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q5_1][i], "mul_mat_vec_q5_1_f32_f32", arr_dmmv_q5_1_f32_f32_len[reduc], arr_dmmv_q5_1_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq_c*nsg_sub, 1, 1}, {wg_size_subgroup, 2*rm_stdq_c, i+1, 0, nsg_sub}, 1, true, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q8_0][i], "mul_mat_vec_q8_0_f32_f32", arr_dmmv_q8_0_f32_f32_len[reduc], arr_dmmv_q8_0_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {1*rm_q8_c*nsg_sub, 1, 1}, {wg_size_subgroup, 1*rm_q8_c, i+1, 0, nsg_sub}, 1, true, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q2_K][i], "mul_mat_vec_q2_k_f32_f32", arr_dmmv_q2_k_f32_f32_len[reduc16], arr_dmmv_q2_k_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq_c, 1, 1}, {wg_size_subgroup16, rm_kq_c, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q3_K][i], "mul_mat_vec_q3_k_f32_f32", arr_dmmv_q3_k_f32_f32_len[reduc16], arr_dmmv_q3_k_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq_c, 1, 1}, {wg_size_subgroup16, rm_kq_c, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q4_K][i], "mul_mat_vec_q4_k_f32_f32", arr_dmmv_q4_k_f32_f32_len[reduc16], arr_dmmv_q4_k_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq_c*nsg_sub16, 1, 1}, {wg_size_subgroup16, rm_kq_c, i+1, 0, nsg_sub16, fast_kq_c}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q5_K][i], "mul_mat_vec_q5_k_f32_f32", arr_dmmv_q5_k_f32_f32_len[reduc16], arr_dmmv_q5_k_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq_c*nsg_sub16, 1, 1}, {wg_size_subgroup16, rm_kq_c, i+1, 0, nsg_sub16, fast_kq_c}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q6_K][i], "mul_mat_vec_q6_k_f32_f32", arr_dmmv_q6_k_f32_f32_len[reduc16], arr_dmmv_q6_k_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq_c*nsg_sub16, 1, 1}, {wg_size_subgroup16, rm_kq_c, i+1, 0, nsg_sub16, fast_kq_c}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_TQ1_0][i], "mul_mat_vec_tq1_0_f32_f32", arr_dmmv_tq1_0_f32_f32_len[reduc16], arr_dmmv_tq1_0_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq_c, 1, 1}, {wg_size_subgroup16, rm_kq_c, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_TQ2_0][i], "mul_mat_vec_tq2_0_f32_f32", arr_dmmv_tq2_0_f32_f32_len[reduc16], arr_dmmv_tq2_0_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq_c, 1, 1}, {wg_size_subgroup16, rm_kq_c, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ1_S][i], "mul_mat_vec_iq1_s_f32_f32", arr_dmmv_iq1_s_f32_f32_len[reduc16], arr_dmmv_iq1_s_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq_c, 1, 1}, {wg_size_subgroup16, rm_iq_c, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ1_M][i], "mul_mat_vec_iq1_m_f32_f32", arr_dmmv_iq1_m_f32_f32_len[reduc16], arr_dmmv_iq1_m_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq_c, 1, 1}, {wg_size_subgroup16, rm_iq_c, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ2_XXS][i], "mul_mat_vec_iq2_xxs_f32_f32", arr_dmmv_iq2_xxs_f32_f32_len[reduc16], arr_dmmv_iq2_xxs_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq_c, 1, 1}, {wg_size_subgroup16, rm_iq_c, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ2_XS][i], "mul_mat_vec_iq2_xs_f32_f32", arr_dmmv_iq2_xs_f32_f32_len[reduc16], arr_dmmv_iq2_xs_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq_c, 1, 1}, {wg_size_subgroup16, rm_iq_c, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ2_S][i], "mul_mat_vec_iq2_s_f32_f32", arr_dmmv_iq2_s_f32_f32_len[reduc16], arr_dmmv_iq2_s_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq_c, 1, 1}, {wg_size_subgroup16, rm_iq_c, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ3_XXS][i], "mul_mat_vec_iq3_xxs_f32_f32", arr_dmmv_iq3_xxs_f32_f32_len[reduc16], arr_dmmv_iq3_xxs_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq_c, 1, 1}, {wg_size_subgroup16, rm_iq_c, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ3_S][i], "mul_mat_vec_iq3_s_f32_f32", arr_dmmv_iq3_s_f32_f32_len[reduc16], arr_dmmv_iq3_s_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq_c, 1, 1}, {wg_size_subgroup16, rm_iq_c, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ4_XS][i], "mul_mat_vec_iq4_xs_f32_f32", arr_dmmv_iq4_xs_f32_f32_len[reduc16], arr_dmmv_iq4_xs_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq_c, 1, 1}, {wg_size_subgroup16, rm_iq_c, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_IQ4_NL][i], "mul_mat_vec_iq4_nl_f32_f32", arr_dmmv_iq4_nl_f32_f32_len[reduc16], arr_dmmv_iq4_nl_f32_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq_c, 1, 1}, {wg_size_subgroup16, rm_iq_c, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_MXFP4][i], "mul_mat_vec_mxfp4_f32_f32", OCP_DMMV_LEN(arr_dmmv_mxfp4_f32_f32, reduc16), OCP_DMMV_DATA(arr_dmmv_mxfp4_f32_f32, reduc16), "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq_c, 1, 1}, {wg_size_subgroup16, rm_iq_c, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_NVFP4][i], "mul_mat_vec_nvfp4_f32_f32", OCP_DMMV_LEN(arr_dmmv_nvfp4_f32_f32, reduc16), OCP_DMMV_DATA(arr_dmmv_nvfp4_f32_f32, reduc16), "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq_c, 1, 1}, {wg_size_subgroup16, rm_iq_c, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_F32 ][i], "mul_mat_vec_f32_f16_f32", arr_dmmv_f32_f16_f32_len[reduc], arr_dmmv_f32_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_f32_c*nsg_sub, 1, 1}, {wg_size_subgroup, rm_f32_c, i+1, 0, nsg_sub}, 1, false, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_F16 ][i], "mul_mat_vec_f16_f16_f32", arr_dmmv_f16_f16_f32_len[reduc], arr_dmmv_f16_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*nsg_sub, 1, 1}, {wg_size_subgroup, 2, i+1, 0, nsg_sub}, 1, false, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_BF16][i], "mul_mat_vec_bf16_f16_f32", arr_dmmv_bf16_f16_f32_len[reduc], arr_dmmv_bf16_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*nsg_sub, 1, 1}, {wg_size_subgroup, 2, i+1, 0, nsg_sub}, 1, false, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q1_0][i], "mul_mat_vec_q1_0_f16_f32", arr_dmmv_q1_0_f16_f32_len[reduc], arr_dmmv_q1_0_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq_c*nsg_sub, 1, 1}, {wg_size_subgroup, 2*rm_stdq_c, i+1, 0, nsg_sub}, 1, true, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q2_0][i], "mul_mat_vec_q2_0_f16_f32", arr_dmmv_q2_0_f16_f32_len[reduc], arr_dmmv_q2_0_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq_c*nsg_sub, 1, 1}, {wg_size_subgroup, 2*rm_stdq_c, i+1, 0, nsg_sub}, 1, true, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q4_0][i], "mul_mat_vec_q4_0_f16_f32", arr_dmmv_q4_0_f16_f32_len[reduc], arr_dmmv_q4_0_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq_c*nsg_sub, 1, 1}, {wg_size_subgroup, 2*rm_stdq_c, i+1, 0, nsg_sub}, 1, true, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q4_1][i], "mul_mat_vec_q4_1_f16_f32", arr_dmmv_q4_1_f16_f32_len[reduc], arr_dmmv_q4_1_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq_c*nsg_sub, 1, 1}, {wg_size_subgroup, 2*rm_stdq_c, i+1, 0, nsg_sub}, 1, true, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q5_0][i], "mul_mat_vec_q5_0_f16_f32", arr_dmmv_q5_0_f16_f32_len[reduc], arr_dmmv_q5_0_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq_c*nsg_sub, 1, 1}, {wg_size_subgroup, 2*rm_stdq_c, i+1, 0, nsg_sub}, 1, true, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q5_1][i], "mul_mat_vec_q5_1_f16_f32", arr_dmmv_q5_1_f16_f32_len[reduc], arr_dmmv_q5_1_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq_c*nsg_sub, 1, 1}, {wg_size_subgroup, 2*rm_stdq_c, i+1, 0, nsg_sub}, 1, true, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q8_0][i], "mul_mat_vec_q8_0_f16_f32", arr_dmmv_q8_0_f16_f32_len[reduc], arr_dmmv_q8_0_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {1*rm_q8_c*nsg_sub, 1, 1}, {wg_size_subgroup, 1*rm_q8_c, i+1, 0, nsg_sub}, 1, true, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q2_K][i], "mul_mat_vec_q2_k_f16_f32", arr_dmmv_q2_k_f16_f32_len[reduc16], arr_dmmv_q2_k_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq_c, 1, 1}, {wg_size_subgroup16, rm_kq_c, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q3_K][i], "mul_mat_vec_q3_k_f16_f32", arr_dmmv_q3_k_f16_f32_len[reduc16], arr_dmmv_q3_k_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq_c, 1, 1}, {wg_size_subgroup16, rm_kq_c, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q4_K][i], "mul_mat_vec_q4_k_f16_f32", arr_dmmv_q4_k_f16_f32_len[reduc16], arr_dmmv_q4_k_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq_c*nsg_sub16, 1, 1}, {wg_size_subgroup16, rm_kq_c, i+1, 0, nsg_sub16, fast_kq_c}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q5_K][i], "mul_mat_vec_q5_k_f16_f32", arr_dmmv_q5_k_f16_f32_len[reduc16], arr_dmmv_q5_k_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq_c*nsg_sub16, 1, 1}, {wg_size_subgroup16, rm_kq_c, i+1, 0, nsg_sub16, fast_kq_c}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q6_K][i], "mul_mat_vec_q6_k_f16_f32", arr_dmmv_q6_k_f16_f32_len[reduc16], arr_dmmv_q6_k_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq_c*nsg_sub16, 1, 1}, {wg_size_subgroup16, rm_kq_c, i+1, 0, nsg_sub16, fast_kq_c}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_TQ1_0][i], "mul_mat_vec_tq1_0_f16_f32", arr_dmmv_tq1_0_f16_f32_len[reduc16], arr_dmmv_tq1_0_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq_c, 1, 1}, {wg_size_subgroup16, rm_kq_c, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_TQ2_0][i], "mul_mat_vec_tq2_0_f16_f32", arr_dmmv_tq2_0_f16_f32_len[reduc16], arr_dmmv_tq2_0_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_kq_c, 1, 1}, {wg_size_subgroup16, rm_kq_c, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ1_S][i], "mul_mat_vec_iq1_s_f16_f32", arr_dmmv_iq1_s_f16_f32_len[reduc16], arr_dmmv_iq1_s_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq_c, 1, 1}, {wg_size_subgroup16, rm_iq_c, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ1_M][i], "mul_mat_vec_iq1_m_f16_f32", arr_dmmv_iq1_m_f16_f32_len[reduc16], arr_dmmv_iq1_m_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq_c, 1, 1}, {wg_size_subgroup16, rm_iq_c, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ2_XXS][i], "mul_mat_vec_iq2_xxs_f16_f32", arr_dmmv_iq2_xxs_f16_f32_len[reduc16], arr_dmmv_iq2_xxs_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq_c, 1, 1}, {wg_size_subgroup16, rm_iq_c, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ2_XS][i], "mul_mat_vec_iq2_xs_f16_f32", arr_dmmv_iq2_xs_f16_f32_len[reduc16], arr_dmmv_iq2_xs_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq_c, 1, 1}, {wg_size_subgroup16, rm_iq_c, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ2_S][i], "mul_mat_vec_iq2_s_f16_f32", arr_dmmv_iq2_s_f16_f32_len[reduc16], arr_dmmv_iq2_s_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq_c, 1, 1}, {wg_size_subgroup16, rm_iq_c, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ3_XXS][i], "mul_mat_vec_iq3_xxs_f16_f32", arr_dmmv_iq3_xxs_f16_f32_len[reduc16], arr_dmmv_iq3_xxs_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq_c, 1, 1}, {wg_size_subgroup16, rm_iq_c, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ3_S][i], "mul_mat_vec_iq3_s_f16_f32", arr_dmmv_iq3_s_f16_f32_len[reduc16], arr_dmmv_iq3_s_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq_c, 1, 1}, {wg_size_subgroup16, rm_iq_c, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ4_XS][i], "mul_mat_vec_iq4_xs_f16_f32", arr_dmmv_iq4_xs_f16_f32_len[reduc16], arr_dmmv_iq4_xs_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq_c, 1, 1}, {wg_size_subgroup16, rm_iq_c, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_IQ4_NL][i], "mul_mat_vec_iq4_nl_f16_f32", arr_dmmv_iq4_nl_f16_f32_len[reduc16], arr_dmmv_iq4_nl_f16_f32_data[reduc16], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq_c, 1, 1}, {wg_size_subgroup16, rm_iq_c, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_MXFP4][i], "mul_mat_vec_mxfp4_f16_f32", OCP_DMMV_LEN(arr_dmmv_mxfp4_f16_f32, reduc16), OCP_DMMV_DATA(arr_dmmv_mxfp4_f16_f32, reduc16), "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq_c, 1, 1}, {wg_size_subgroup16, rm_iq_c, i+1}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_NVFP4][i], "mul_mat_vec_nvfp4_f16_f32", OCP_DMMV_LEN(arr_dmmv_nvfp4_f16_f32, reduc16), OCP_DMMV_DATA(arr_dmmv_nvfp4_f16_f32, reduc16), "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {rm_iq_c, 1, 1}, {wg_size_subgroup16, rm_iq_c, i+1}, 1, true, use_subgroups16, force_subgroup_size16); #if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) if (device->integer_dot_product) { @@ -3354,34 +4105,35 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { #endif // GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT } - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_F32 ], "mul_mat_vec_id_f32_f32", arr_dmmv_id_f32_f32_f32_len[reduc], arr_dmmv_id_f32_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {1, 1, 1}, {wg_size_subgroup, 1}, 1, false, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_F16 ], "mul_mat_vec_id_f16_f32", arr_dmmv_id_f16_f32_f32_len[reduc], arr_dmmv_id_f16_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2, 1, 1}, {wg_size_subgroup, 2}, 1, false, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_BF16], "mul_mat_vec_id_bf16_f32", arr_dmmv_id_bf16_f32_f32_len[reduc], arr_dmmv_id_bf16_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2, 1, 1}, {wg_size_subgroup, 2}, 1, false, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q1_0], "mul_mat_vec_id_q1_0_f32", arr_dmmv_id_q1_0_f32_f32_len[reduc], arr_dmmv_id_q1_0_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq}, 1, true, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q2_0], "mul_mat_vec_id_q2_0_f32", arr_dmmv_id_q2_0_f32_f32_len[reduc], arr_dmmv_id_q2_0_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq}, 1, true, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q4_0], "mul_mat_vec_id_q4_0_f32", arr_dmmv_id_q4_0_f32_f32_len[reduc], arr_dmmv_id_q4_0_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq}, 1, true, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q4_1], "mul_mat_vec_id_q4_1_f32", arr_dmmv_id_q4_1_f32_f32_len[reduc], arr_dmmv_id_q4_1_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq}, 1, true, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q5_0], "mul_mat_vec_id_q5_0_f32", arr_dmmv_id_q5_0_f32_f32_len[reduc], arr_dmmv_id_q5_0_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq}, 1, true, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q5_1], "mul_mat_vec_id_q5_1_f32", arr_dmmv_id_q5_1_f32_f32_len[reduc], arr_dmmv_id_q5_1_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq}, 1, true, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q8_0], "mul_mat_vec_id_q8_0_f32", arr_dmmv_id_q8_0_f32_f32_len[reduc], arr_dmmv_id_q8_0_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {1*rm_stdq, 1, 1}, {wg_size_subgroup, 1*rm_stdq}, 1, true, use_subgroups, force_subgroup_size); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q2_K], "mul_mat_vec_id_q2_k_f32", arr_dmmv_id_q2_k_f32_f32_len[reduc16], arr_dmmv_id_q2_k_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q3_K], "mul_mat_vec_id_q3_k_f32", arr_dmmv_id_q3_k_f32_f32_len[reduc16], arr_dmmv_id_q3_k_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q4_K], "mul_mat_vec_id_q4_k_f32", arr_dmmv_id_q4_k_f32_f32_len[reduc16], arr_dmmv_id_q4_k_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q5_K], "mul_mat_vec_id_q5_k_f32", arr_dmmv_id_q5_k_f32_f32_len[reduc16], arr_dmmv_id_q5_k_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q6_K], "mul_mat_vec_id_q6_k_f32", arr_dmmv_id_q6_k_f32_f32_len[reduc16], arr_dmmv_id_q6_k_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_TQ1_0], "mul_mat_vec_id_tq1_0_f32", arr_dmmv_id_tq1_0_f32_f32_len[reduc16], arr_dmmv_id_tq1_0_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_TQ2_0], "mul_mat_vec_id_tq2_0_f32", arr_dmmv_id_tq2_0_f32_f32_len[reduc16], arr_dmmv_id_tq2_0_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_kq, 1, 1}, {wg_size_subgroup16, rm_kq}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ1_S], "mul_mat_vec_id_iq1_s_f32", arr_dmmv_id_iq1_s_f32_f32_len[reduc16], arr_dmmv_id_iq1_s_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ1_M], "mul_mat_vec_id_iq1_m_f32", arr_dmmv_id_iq1_m_f32_f32_len[reduc16], arr_dmmv_id_iq1_m_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ2_XXS], "mul_mat_vec_id_iq2_xxs_f32", arr_dmmv_id_iq2_xxs_f32_f32_len[reduc16], arr_dmmv_id_iq2_xxs_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ2_XS], "mul_mat_vec_id_iq2_xs_f32", arr_dmmv_id_iq2_xs_f32_f32_len[reduc16], arr_dmmv_id_iq2_xs_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ2_S], "mul_mat_vec_id_iq2_s_f32", arr_dmmv_id_iq2_s_f32_f32_len[reduc16], arr_dmmv_id_iq2_s_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ3_XXS], "mul_mat_vec_id_iq3_xxs_f32", arr_dmmv_id_iq3_xxs_f32_f32_len[reduc16], arr_dmmv_id_iq3_xxs_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ3_S], "mul_mat_vec_id_iq3_s_f32", arr_dmmv_id_iq3_s_f32_f32_len[reduc16], arr_dmmv_id_iq3_s_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ4_XS], "mul_mat_vec_id_iq4_xs_f32", arr_dmmv_id_iq4_xs_f32_f32_len[reduc16], arr_dmmv_id_iq4_xs_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ4_NL], "mul_mat_vec_id_iq4_nl_f32", arr_dmmv_id_iq4_nl_f32_f32_len[reduc16], arr_dmmv_id_iq4_nl_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_MXFP4], "mul_mat_vec_id_mxfp4_f32", OCP_DMMV_LEN(arr_dmmv_id_mxfp4_f32_f32, reduc16), OCP_DMMV_DATA(arr_dmmv_id_mxfp4_f32_f32, reduc16), "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}, 1, true, use_subgroups16, force_subgroup_size16); - ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_NVFP4], "mul_mat_vec_id_nvfp4_f32", OCP_DMMV_LEN(arr_dmmv_id_nvfp4_f32_f32, reduc16), OCP_DMMV_DATA(arr_dmmv_id_nvfp4_f32_f32, reduc16), "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq, 1, 1}, {wg_size_subgroup16, rm_iq}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_F32 ], "mul_mat_vec_id_f32_f32", arr_dmmv_id_f32_f32_f32_len[reduc], arr_dmmv_id_f32_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {1*nsg_id, 1, 1}, {wg_size_subgroup, 1, 1, 0, nsg_id}, 1, false, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_F16 ], "mul_mat_vec_id_f16_f32", arr_dmmv_id_f16_f32_f32_len[reduc], arr_dmmv_id_f16_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2*nsg_id, 1, 1}, {wg_size_subgroup, 2, 1, 0, nsg_id}, 1, false, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_BF16], "mul_mat_vec_id_bf16_f32", arr_dmmv_id_bf16_f32_f32_len[reduc], arr_dmmv_id_bf16_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2*nsg_id, 1, 1}, {wg_size_subgroup, 2, 1, 0, nsg_id}, 1, false, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q1_0], "mul_mat_vec_id_q1_0_f32", arr_dmmv_id_q1_0_f32_f32_len[reduc], arr_dmmv_id_q1_0_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2*rm_stdq_id*nsg_id, 1, 1}, {wg_size_subgroup, 2*rm_stdq_id, 1, 0, nsg_id}, 1, true, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q2_0], "mul_mat_vec_id_q2_0_f32", arr_dmmv_id_q2_0_f32_f32_len[reduc], arr_dmmv_id_q2_0_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2*rm_stdq_id*nsg_id, 1, 1}, {wg_size_subgroup, 2*rm_stdq_id, 1, 0, nsg_id}, 1, true, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q4_0], "mul_mat_vec_id_q4_0_f32", arr_dmmv_id_q4_0_f32_f32_len[reduc], arr_dmmv_id_q4_0_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2*rm_stdq_id*nsg_id, 1, 1}, {wg_size_subgroup, 2*rm_stdq_id, 1, 0, nsg_id}, 1, true, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q4_1], "mul_mat_vec_id_q4_1_f32", arr_dmmv_id_q4_1_f32_f32_len[reduc], arr_dmmv_id_q4_1_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2*rm_stdq_id*nsg_id, 1, 1}, {wg_size_subgroup, 2*rm_stdq_id, 1, 0, nsg_id}, 1, true, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q5_0], "mul_mat_vec_id_q5_0_f32", arr_dmmv_id_q5_0_f32_f32_len[reduc], arr_dmmv_id_q5_0_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2*rm_stdq_id*nsg_id, 1, 1}, {wg_size_subgroup, 2*rm_stdq_id, 1, 0, nsg_id}, 1, true, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q5_1], "mul_mat_vec_id_q5_1_f32", arr_dmmv_id_q5_1_f32_f32_len[reduc], arr_dmmv_id_q5_1_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2*rm_stdq_id*nsg_id, 1, 1}, {wg_size_subgroup, 2*rm_stdq_id, 1, 0, nsg_id}, 1, true, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q8_0], "mul_mat_vec_id_q8_0_f32", arr_dmmv_id_q8_0_f32_f32_len[reduc], arr_dmmv_id_q8_0_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {1*rm_stdq_id*nsg_id, 1, 1}, {wg_size_subgroup, 1*rm_stdq_id, 1, 0, nsg_id}, 1, true, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q2_K], "mul_mat_vec_id_q2_k_f32", arr_dmmv_id_q2_k_f32_f32_len[reduc16], arr_dmmv_id_q2_k_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_kq_id, 1, 1}, {wg_size_subgroup16, rm_kq_id}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q3_K], "mul_mat_vec_id_q3_k_f32", arr_dmmv_id_q3_k_f32_f32_len[reduc16], arr_dmmv_id_q3_k_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_kq_id, 1, 1}, {wg_size_subgroup16, rm_kq_id}, 1, true, use_subgroups16, force_subgroup_size16); + const uint32_t fast_kq_id_56 = (ggml_vk_is_apple_agx_coopmat(device) && !ggml_vk_exact_s8_enabled() && !getenv("GGML_VK_FAST_KQ")) ? 1u : fast_kq; + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q4_K], "mul_mat_vec_id_q4_k_f32", arr_dmmv_id_q4_k_f32_f32_len[reduc16], arr_dmmv_id_q4_k_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_kq_id*nsg_id16, 1, 1}, {wg_size_subgroup16, rm_kq_id, 1, 0, nsg_id16, fast_kq}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q5_K], "mul_mat_vec_id_q5_k_f32", arr_dmmv_id_q5_k_f32_f32_len[reduc16], arr_dmmv_id_q5_k_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_kq_id*nsg_id16, 1, 1}, {wg_size_subgroup16, rm_kq_id, 1, 0, nsg_id16, fast_kq_id_56}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q6_K], "mul_mat_vec_id_q6_k_f32", arr_dmmv_id_q6_k_f32_f32_len[reduc16], arr_dmmv_id_q6_k_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_kq_id*nsg_id16, 1, 1}, {wg_size_subgroup16, rm_kq_id, 1, 0, nsg_id16, fast_kq_id_56}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_TQ1_0], "mul_mat_vec_id_tq1_0_f32", arr_dmmv_id_tq1_0_f32_f32_len[reduc16], arr_dmmv_id_tq1_0_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_kq_id, 1, 1}, {wg_size_subgroup16, rm_kq_id}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_TQ2_0], "mul_mat_vec_id_tq2_0_f32", arr_dmmv_id_tq2_0_f32_f32_len[reduc16], arr_dmmv_id_tq2_0_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_kq_id, 1, 1}, {wg_size_subgroup16, rm_kq_id}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ1_S], "mul_mat_vec_id_iq1_s_f32", arr_dmmv_id_iq1_s_f32_f32_len[reduc16], arr_dmmv_id_iq1_s_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq_id, 1, 1}, {wg_size_subgroup16, rm_iq_id}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ1_M], "mul_mat_vec_id_iq1_m_f32", arr_dmmv_id_iq1_m_f32_f32_len[reduc16], arr_dmmv_id_iq1_m_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq_id, 1, 1}, {wg_size_subgroup16, rm_iq_id}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ2_XXS], "mul_mat_vec_id_iq2_xxs_f32", arr_dmmv_id_iq2_xxs_f32_f32_len[reduc16], arr_dmmv_id_iq2_xxs_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq_id, 1, 1}, {wg_size_subgroup16, rm_iq_id}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ2_XS], "mul_mat_vec_id_iq2_xs_f32", arr_dmmv_id_iq2_xs_f32_f32_len[reduc16], arr_dmmv_id_iq2_xs_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq_id, 1, 1}, {wg_size_subgroup16, rm_iq_id}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ2_S], "mul_mat_vec_id_iq2_s_f32", arr_dmmv_id_iq2_s_f32_f32_len[reduc16], arr_dmmv_id_iq2_s_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq_id, 1, 1}, {wg_size_subgroup16, rm_iq_id}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ3_XXS], "mul_mat_vec_id_iq3_xxs_f32", arr_dmmv_id_iq3_xxs_f32_f32_len[reduc16], arr_dmmv_id_iq3_xxs_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq_id, 1, 1}, {wg_size_subgroup16, rm_iq_id}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ3_S], "mul_mat_vec_id_iq3_s_f32", arr_dmmv_id_iq3_s_f32_f32_len[reduc16], arr_dmmv_id_iq3_s_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq_id, 1, 1}, {wg_size_subgroup16, rm_iq_id}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ4_XS], "mul_mat_vec_id_iq4_xs_f32", arr_dmmv_id_iq4_xs_f32_f32_len[reduc16], arr_dmmv_id_iq4_xs_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq_id, 1, 1}, {wg_size_subgroup16, rm_iq_id}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_IQ4_NL], "mul_mat_vec_id_iq4_nl_f32", arr_dmmv_id_iq4_nl_f32_f32_len[reduc16], arr_dmmv_id_iq4_nl_f32_f32_data[reduc16], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq_id, 1, 1}, {wg_size_subgroup16, rm_iq_id}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_MXFP4], "mul_mat_vec_id_mxfp4_f32", OCP_DMMV_LEN(arr_dmmv_id_mxfp4_f32_f32, reduc16), OCP_DMMV_DATA(arr_dmmv_id_mxfp4_f32_f32, reduc16), "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq_id, 1, 1}, {wg_size_subgroup16, rm_iq_id}, 1, true, use_subgroups16, force_subgroup_size16); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_NVFP4], "mul_mat_vec_id_nvfp4_f32", OCP_DMMV_LEN(arr_dmmv_id_nvfp4_f32_f32, reduc16), OCP_DMMV_DATA(arr_dmmv_id_nvfp4_f32_f32, reduc16), "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {rm_iq_id, 1, 1}, {wg_size_subgroup16, rm_iq_id}, 1, true, use_subgroups16, force_subgroup_size16); #if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) if (device->integer_dot_product) { @@ -3579,26 +4331,30 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { for (uint32_t i = 0; i < p021_max_gqa_ratio; ++i) { if (device->subgroup_arithmetic && device->subgroup_require_full_support) { - ggml_vk_create_pipeline2(device, device->pipeline_mul_mat_vec_p021_f16_f32[i], "mul_mat_vec_p021_f16_f32"+std::to_string(i+1), mul_mat_vec_p021_f16_f32_subgroup_add_len, mul_mat_vec_p021_f16_f32_subgroup_add_data, "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_p021_push_constants), {1, 1, 1}, {device->subgroup_size, i + 1}, 1, true, true); + ggml_vk_create_pipeline2(device, device->pipeline_mul_mat_vec_p021_f16_f32[i], "mul_mat_vec_p021_f16_f32"+std::to_string(i+1), mul_mat_vec_p021_f16_f32_subgroup_add_len, mul_mat_vec_p021_f16_f32_subgroup_add_data, "main", 5, sizeof(vk_mat_vec_p021_push_constants), {1, 1, 1}, {device->subgroup_size, i + 1}, 1, true, true); } else { - ggml_vk_create_pipeline2(device, device->pipeline_mul_mat_vec_p021_f16_f32[i], "mul_mat_vec_p021_f16_f32"+std::to_string(i+1), mul_mat_vec_p021_f16_f32_len, mul_mat_vec_p021_f16_f32_data, "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_p021_push_constants), {1, 1, 1}, {device->subgroup_size, i + 1}, 1, true); + ggml_vk_create_pipeline2(device, device->pipeline_mul_mat_vec_p021_f16_f32[i], "mul_mat_vec_p021_f16_f32"+std::to_string(i+1), mul_mat_vec_p021_f16_f32_len, mul_mat_vec_p021_f16_f32_data, "main", 5, sizeof(vk_mat_vec_p021_push_constants), {1, 1, 1}, {device->subgroup_size, i + 1}, 1, true); } } - ggml_vk_create_pipeline(device, device->pipeline_mul_mat_vec_nc_f16_f32, "mul_mat_vec_nc_f16_f32", mul_mat_vec_nc_f16_f32_len, mul_mat_vec_nc_f16_f32_data, "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_nc_push_constants), {1, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_mul_mat_vec_nc_f16_f32, "mul_mat_vec_nc_f16_f32", mul_mat_vec_nc_f16_f32_len, mul_mat_vec_nc_f16_f32_data, "main", 5, sizeof(vk_mat_vec_nc_push_constants), {1, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_norm_f32, "norm_f32", norm_f32_len, norm_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {1, 1, 1}, {}, 1, true); ggml_vk_create_pipeline(device, device->pipeline_group_norm_f32, "group_norm_f32", group_norm_f32_len, group_norm_f32_data, "main", 2, sizeof(vk_op_push_constants), {1, 1, 1}, {}, 1); - const bool rms_subgroups = (device->vendor_id == VK_VENDOR_ID_INTEL || device->vendor_id == VK_VENDOR_ID_NVIDIA || device->vendor_id == VK_VENDOR_ID_AMD) && device->subgroup_arithmetic && device->subgroup_require_full_support; + const bool rms_subgroups = (device->vendor_id == VK_VENDOR_ID_INTEL || device->vendor_id == VK_VENDOR_ID_NVIDIA || device->vendor_id == VK_VENDOR_ID_AMD || + (ggml_vk_is_apple_agx_coopmat(device) && !ggml_vk_exact_s8_enabled())) && + device->subgroup_arithmetic && device->subgroup_require_full_support; const uint32_t rms_set_rows_block_size = 128; #define RMS_SHADER(name) (rms_subgroups ? name##_subgroup_len : name##_len), (rms_subgroups ? name##_subgroup_data : name##_data) - // 128/256/512 block sizes, picked by row length - for (uint32_t i = 0; i < 3; ++i) { - const uint32_t block_size = 128u << i; + // 128/256/512 block sizes (+ 32 single-subgroup block size in slot 3 when subgroups are enabled), picked by row length + for (uint32_t i = 0; i < (rms_subgroups ? 4u : 3u); ++i) { + const uint32_t block_size = (i == 3u) ? 32u : (128u << i); const std::string suffix = "_" + std::to_string(block_size); ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_f32[i], "rms_norm_f32" + suffix, RMS_SHADER(rms_norm_f32), "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0, 0, block_size}, 1, true, rms_subgroups); ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_mul_f32[i], "rms_norm_mul_f32" + suffix, RMS_SHADER(rms_norm_f32), "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 0, block_size}, 1, true, rms_subgroups); + ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_scale_f32[i], "rms_norm_scale_f32" + suffix, RMS_SHADER(rms_norm_scale_f32), "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0, 0, block_size}, 1, true, rms_subgroups); ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_mul_add_f32[i], "rms_norm_mul_add_f32" + suffix, RMS_SHADER(rms_norm_mul_add_f32), "main", 5, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 0, block_size}, 1, true, rms_subgroups); + ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_mul_silu_f32[i], "rms_norm_mul_silu_f32" + suffix, RMS_SHADER(rms_norm_mul_silu_f32), "main", rms_subgroups ? 5u : 4u, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 0, block_size}, 1, true, rms_subgroups); ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_mul_add_mul_f32[i], "rms_norm_mul_add_mul_f32" + suffix, RMS_SHADER(rms_norm_mul_add_f32), "main", 5, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 1, block_size}, 1, true, rms_subgroups); } ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_add_partials_f32, "rms_norm_mul_add_partials_f32", rms_norm_mul_add_partials_f32_len, rms_norm_mul_add_partials_f32_data, "main", 6, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 0}, 1, true); @@ -3621,25 +4377,25 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_rms_norm_back_f32, "rms_norm_back_f32", rms_norm_back_f32_len, rms_norm_back_f32_data, "main", 3, sizeof(vk_op_push_constants), {1, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_l2_norm_f32, "l2_norm_f32", l2_norm_f32_len, l2_norm_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {1, 1, 1}, {}, 1, true); - ggml_vk_create_pipeline(device, device->pipeline_cpy_32, "cpy_32", cpy_32_len, cpy_32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_cpy_f32_f16, "cpy_f32_f16", cpy_f32_f16_len, cpy_f32_f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_cpy_16, "cpy_16", cpy_16_len, cpy_16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_cpy_f16_f32, "cpy_f16_f32", cpy_f16_f32_len, cpy_f16_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_cpy_f32_bf16,"cpy_f32_bf16",cpy_f32_bf16_len,cpy_f32_bf16_data,"main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_cpy_bf16_f32,"cpy_bf16_f32",cpy_bf16_f32_len,cpy_bf16_f32_data,"main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_cpy_bf16_f16,"cpy_bf16_f16",cpy_bf16_f16_len,cpy_bf16_f16_data,"main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_cpy_i32_f32, "cpy_i32_f32", cpy_i32_f32_len, cpy_i32_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_cpy_f32_i32, "cpy_f32_i32", cpy_f32_i32_len, cpy_f32_i32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - - ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_32, "contig_cpy_32", contig_cpy_32_len, contig_cpy_32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_f32_f16, "contig_cpy_f32_f16", contig_cpy_f32_f16_len, contig_cpy_f32_f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_16, "contig_cpy_16", contig_cpy_16_len, contig_cpy_16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_f16_f32, "contig_cpy_f16_f32", contig_cpy_f16_f32_len, contig_cpy_f16_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_f32_bf16,"contig_cpy_f32_bf16",contig_cpy_f32_bf16_len,contig_cpy_f32_bf16_data,"main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_bf16_f32,"contig_cpy_bf16_f32",contig_cpy_bf16_f32_len,contig_cpy_bf16_f32_data,"main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_bf16_f16,"contig_cpy_bf16_f16",contig_cpy_bf16_f16_len,contig_cpy_bf16_f16_data,"main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_i32_f32, "contig_cpy_i32_f32", contig_cpy_i32_f32_len, contig_cpy_i32_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_f32_i32, "contig_cpy_f32_i32", contig_cpy_f32_i32_len, contig_cpy_f32_i32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_cpy_32, "cpy_32", cpy_32_len, cpy_32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1, agx_no_robust); + ggml_vk_create_pipeline(device, device->pipeline_cpy_f32_f16, "cpy_f32_f16", cpy_f32_f16_len, cpy_f32_f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1, agx_no_robust); + ggml_vk_create_pipeline(device, device->pipeline_cpy_16, "cpy_16", cpy_16_len, cpy_16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1, agx_no_robust); + ggml_vk_create_pipeline(device, device->pipeline_cpy_f16_f32, "cpy_f16_f32", cpy_f16_f32_len, cpy_f16_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1, agx_no_robust); + ggml_vk_create_pipeline(device, device->pipeline_cpy_f32_bf16,"cpy_f32_bf16",cpy_f32_bf16_len,cpy_f32_bf16_data,"main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1, agx_no_robust); + ggml_vk_create_pipeline(device, device->pipeline_cpy_bf16_f32,"cpy_bf16_f32",cpy_bf16_f32_len,cpy_bf16_f32_data,"main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1, agx_no_robust); + ggml_vk_create_pipeline(device, device->pipeline_cpy_bf16_f16,"cpy_bf16_f16",cpy_bf16_f16_len,cpy_bf16_f16_data,"main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1, agx_no_robust); + ggml_vk_create_pipeline(device, device->pipeline_cpy_i32_f32, "cpy_i32_f32", cpy_i32_f32_len, cpy_i32_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1, agx_no_robust); + ggml_vk_create_pipeline(device, device->pipeline_cpy_f32_i32, "cpy_f32_i32", cpy_f32_i32_len, cpy_f32_i32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1, agx_no_robust); + + ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_32, "contig_cpy_32", contig_cpy_32_len, contig_cpy_32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1, agx_no_robust); + ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_f32_f16, "contig_cpy_f32_f16", contig_cpy_f32_f16_len, contig_cpy_f32_f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1, agx_no_robust); + ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_16, "contig_cpy_16", contig_cpy_16_len, contig_cpy_16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1, agx_no_robust); + ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_f16_f32, "contig_cpy_f16_f32", contig_cpy_f16_f32_len, contig_cpy_f16_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1, agx_no_robust); + ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_f32_bf16,"contig_cpy_f32_bf16",contig_cpy_f32_bf16_len,contig_cpy_f32_bf16_data,"main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1, agx_no_robust); + ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_bf16_f32,"contig_cpy_bf16_f32",contig_cpy_bf16_f32_len,contig_cpy_bf16_f32_data,"main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1, agx_no_robust); + ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_bf16_f16,"contig_cpy_bf16_f16",contig_cpy_bf16_f16_len,contig_cpy_bf16_f16_data,"main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1, agx_no_robust); + ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_i32_f32, "contig_cpy_i32_f32", contig_cpy_i32_f32_len, contig_cpy_i32_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1, agx_no_robust); + ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_f32_i32, "contig_cpy_f32_i32", contig_cpy_f32_i32_len, contig_cpy_f32_i32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1, agx_no_robust); ggml_vk_create_pipeline(device, device->pipeline_cpy_transpose_32, "cpy_transpose_32", cpy_transpose_32_len, cpy_transpose_32_data, "main", 2, sizeof(vk_op_unary_push_constants), {1, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_cpy_transpose_16, "cpy_transpose_16", cpy_transpose_16_len, cpy_transpose_16_data, "main", 2, sizeof(vk_op_unary_push_constants), {1, 1, 1}, {}, 1); @@ -3712,8 +4468,19 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { if (device->multi_add) { for (uint32_t i = 0; i < MAX_FUSED_ADDS; ++i) { - ggml_vk_create_pipeline2(device, device->pipeline_multi_add[i], "multi_add_f32_" + std::to_string(i+1), multi_add_f32_len, multi_add_f32_data, "main", MAX_PARAMETER_COUNT, sizeof(vk_op_multi_add_push_constants), {512, 1, 1}, {i+2}, 1); - ggml_vk_create_pipeline2(device, device->pipeline_multi_add_rms[i], "multi_add_rms_f32_" + std::to_string(i+1), multi_add_rms_f32_len, multi_add_rms_f32_data, "main", MAX_PARAMETER_COUNT, sizeof(vk_op_multi_add_push_constants), {512, 1, 1}, {i+2}, 1); + ggml_vk_create_pipeline2(device, device->pipeline_multi_add[i], "multi_add_f32_" + std::to_string(i+1), multi_add_f32_len, multi_add_f32_data, "main", 12, sizeof(vk_op_multi_add_push_constants), {512, 1, 1}, {i+2}, 1, agx_no_robust); + ggml_vk_create_pipeline2(device, device->pipeline_multi_add_rms[i], "multi_add_rms_f32_" + std::to_string(i+1), multi_add_rms_f32_len, multi_add_rms_f32_data, "main", 12, sizeof(vk_op_multi_add_push_constants), {512, 1, 1}, {i+2}, 1, agx_no_robust); + if (device->subgroup_basic && device->subgroup_arithmetic && device->subgroup_shuffle && device->subgroup_require_full_support) { + for (uint32_t tm = 0; tm < 3; ++tm) { + const uint32_t block_size = (tm == 2u) ? 128u : 512u; + for (uint32_t dm = 0; dm < 2; ++dm) { + ggml_vk_create_pipeline2(device, device->pipeline_multi_add_rms_mul[i][tm][dm], + "multi_add_rms_mul_f32_" + std::to_string(i+1) + "_tm" + std::to_string(tm) + "_dm" + std::to_string(dm), + multi_add_rms_mul_f32_len, multi_add_rms_mul_f32_data, "main", + 14, sizeof(vk_op_multi_add_rms_push_constants), {1, 1, 1}, {i+2, tm, dm, block_size}, 1, true, true); + } + } + } } } @@ -3735,7 +4502,8 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_upscale_bicubic_f32, "upscale_f32", upscale_f32_len, upscale_f32_data, "main", 2, sizeof(vk_op_upscale_push_constants), {512, 1, 1}, {GGML_SCALE_MODE_BICUBIC}, 1); ggml_vk_create_pipeline(device, device->pipeline_upscale_bilinear_antialias_f32, "upscale_f32", upscale_f32_len, upscale_f32_data, "main", 2, sizeof(vk_op_upscale_push_constants), {512, 1, 1}, {GGML_SCALE_MODE_BILINEAR | GGML_SCALE_FLAG_ANTIALIAS}, 1); - ggml_vk_create_pipeline(device, device->pipeline_scale_f32, "scale_f32", scale_f32_len, scale_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_scale_f32, "scale_f32", scale_f32_len, scale_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {0}, 1); + ggml_vk_create_pipeline(device, device->pipeline_scale_zero_f32, "scale_zero_f32", scale_f32_len, scale_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {1}, 1); ggml_vk_create_pipeline(device, device->pipeline_log[0], "log_f32", log_f32_len, log_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_log[1], "log_f16", log_f16_len, log_f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); @@ -3801,7 +4569,7 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_unary_mul[(idx)][dt][nr][ob], \ (#name "_mul" + std::string(ob ? "_b" : "") + "_" + dts_ + (nr ? "_norepeat" : "")).c_str(), \ len_, data_, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, \ - { (uint32_t) nr, (uint32_t) ob }, 1); \ + { (uint32_t) nr, (uint32_t) ob }, 1, agx_no_robust); \ } CREATE_UNARY_MUL(gelu, 0) @@ -3902,7 +4670,7 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { } } - ggml_vk_create_pipeline(device, device->pipeline_argmax_f32, "argmax_f32", argmax_f32_len, argmax_f32_data, "main", 2, sizeof(vk_op_push_constants), {1, 1, 1}, { device->subgroup_size }, 1); + ggml_vk_create_pipeline(device, device->pipeline_argmax_f32, "argmax_f32", argmax_f32_len, argmax_f32_data, "main", 2, sizeof(vk_op_push_constants), {1, 1, 1}, { ggml_vk_is_apple_agx_coopmat(device) ? 512u : device->subgroup_size }, 1, agx_no_robust); ggml_vk_create_pipeline(device, device->pipeline_sum_rows_f32, "sum_rows_f32", sum_rows_f32_len, sum_rows_f32_data, "main", 2, sizeof(vk_op_sum_rows_push_constants), {1, 1, 1}, { device->subgroup_size }, 1, true); ggml_vk_create_pipeline(device, device->pipeline_cross_entropy_loss_f32, "cross_entropy_loss_f32", cross_entropy_loss_f32_len, cross_entropy_loss_f32_data, "main", 3, sizeof(vk_op_push_constants), {1, 1, 1}, { device->subgroup_size }, 1); @@ -4042,9 +4810,11 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { device->subgroup_min_size <= 16u && device->subgroup_max_size >= 16u) { gdn_subgroup_size = 16u; } + static const bool gen_exact_s8 = getenv("GGML_VK_EXACT_S8") != nullptr && atoi(getenv("GGML_VK_EXACT_S8")) != 0; uint32_t lanes_per_column; - if (device->vendor_id == VK_VENDOR_ID_INTEL) { - // Intel Xe: full-width reduction (min rows/lane) ~10x over the COLS_PER_WG=8 rule. + if (device->vendor_id == VK_VENDOR_ID_INTEL || + (ggml_vk_is_apple_agx_coopmat(device) && !gen_exact_s8)) { + // Intel Xe & Apple AGX: full-width reduction (ROWS_PER_LANE=4 on sg32) eliminates 128+ reg pressure and multiplies occupancy 8x. lanes_per_column = std::min(gdn_subgroup_size, S_V); } else if (S_V >= 128u && device->subgroup_clustered) { // COLS_PER_WG=8: measured optimum (Ampere sg32->4, Vega20 sg64->8). @@ -4076,26 +4846,58 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { const bool use_clustered_reduce = device->subgroup_arithmetic && device->subgroup_clustered && need_partial_subgroup_reduce; const bool use_subgroup_reduce = device->subgroup_arithmetic && !need_partial_subgroup_reduce; const bool use_subgroup_ops = use_clustered_reduce || use_subgroup_reduce; - size_t gdn_len; + size_t gdn_len, gdn_sout_len; const void * gdn_data; + const void * gdn_sout_data; if (use_clustered_reduce) { gdn_len = gated_delta_net_f32_len; gdn_data = (const void *)gated_delta_net_f32_data; + gdn_sout_len = gated_delta_net_f32_sout_len; + gdn_sout_data = (const void *)gated_delta_net_f32_sout_data; } else if (use_subgroup_reduce) { gdn_len = gated_delta_net_f32_nocluster_len; gdn_data = (const void *)gated_delta_net_f32_nocluster_data; + gdn_sout_len = gated_delta_net_f32_nocluster_sout_len; + gdn_sout_data = (const void *)gated_delta_net_f32_nocluster_sout_data; } else { gdn_len = gated_delta_net_f32_shmem_len; gdn_data = (const void *)gated_delta_net_f32_shmem_data; + gdn_sout_len = gated_delta_net_f32_shmem_sout_len; + gdn_sout_data = (const void *)gated_delta_net_f32_shmem_sout_data; } - const uint32_t cols_per_wg = gdn_subgroup_size / lanes_per_column; - const std::array wg_denoms = {1u, 1u, cols_per_wg}; + uint32_t cols_per_sub = (ggml_vk_is_apple_agx_coopmat(device) && !gen_exact_s8) ? 4u : 1u; + if (const char * env_cps = getenv("GGML_VK_GDN_COLS_PER_SUB")) { + cols_per_sub = (uint32_t)std::max(1, atoi(env_cps)); + } + const uint32_t cols_per_sub_step = gdn_subgroup_size / lanes_per_column; + if ((S_V % (cols_per_sub_step * cols_per_sub)) != 0) { + cols_per_sub = 1u; + } + uint32_t subgroups_per_wg = 1u; + if (const char * env_nsg = getenv("GGML_VK_GDN_NSG")) { + subgroups_per_wg = (uint32_t)std::max(1, atoi(env_nsg)); + } + while (subgroups_per_wg > 1u && (S_V % (cols_per_sub_step * subgroups_per_wg * cols_per_sub)) != 0) { + subgroups_per_wg >>= 1u; + } + const uint32_t cols_per_wg = cols_per_sub_step * subgroups_per_wg; + const std::array wg_denoms = {1u, 1u, cols_per_wg * cols_per_sub}; + const uint32_t gdn_contig_v4 = (ggml_vk_is_apple_agx_coopmat(device) && !gen_exact_s8 && + S_V == 128u && lanes_per_column == 32u && + (cols_per_sub == 4u || cols_per_sub == 2u) && + !getenv("GGML_VK_DISABLE_GDN_VEC4")) + ? (getenv("GGML_VK_DISABLE_GDN_FAST_READOUT") ? 1u : 2u) + : 0u; for (uint32_t kda = 0; kda < 2; kda++) { ggml_vk_create_pipeline(device, device->pipeline_gated_delta_net[si][kda], gdn_names[si][kda], gdn_len, gdn_data, "main", 7, sizeof(vk_op_gated_delta_net_push_constants), - wg_denoms, {S_V, kda, gdn_subgroup_size, lanes_per_column}, 1, true, use_subgroup_ops, gdn_subgroup_size); + wg_denoms, {S_V, kda, gdn_subgroup_size, lanes_per_column, cols_per_sub, subgroups_per_wg, gdn_contig_v4}, 1, true, use_subgroup_ops, gdn_subgroup_size); + const std::string sout_name = std::string(gdn_names[si][kda]) + "_sout"; + ggml_vk_create_pipeline(device, device->pipeline_gated_delta_net_sout[si][kda], + sout_name.c_str(), gdn_sout_len, gdn_sout_data, "main", 8, sizeof(vk_op_gated_delta_net_push_constants), + wg_denoms, {S_V, kda, gdn_subgroup_size, lanes_per_column, cols_per_sub, subgroups_per_wg, gdn_contig_v4}, 1, true, use_subgroup_ops, gdn_subgroup_size); } } } @@ -4108,9 +4910,23 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_ssm_scan_f32_d256, "ssm_scan_256_f32", ssm_scan_f32_len, ssm_scan_f32_data, "main", 8, sizeof(vk_op_ssm_scan_push_constants), {1, 1, 1}, {256, device->subgroup_size, 16}, 1, true, true); } - ggml_vk_create_pipeline(device, device->pipeline_ssm_conv_f32, "ssm_conv_f32", ssm_conv_f32_len, ssm_conv_f32_data, "main", 4, sizeof(vk_op_ssm_conv_push_constants), {32, 16, 1}, {32, 16, 0, 0}, 1); - ggml_vk_create_pipeline(device, device->pipeline_ssm_conv_silu_f32, "ssm_conv_silu_f32", ssm_conv_f32_len, ssm_conv_f32_data, "main", 4, sizeof(vk_op_ssm_conv_push_constants), {32, 16, 1}, {32, 16, 0, 1}, 1); - ggml_vk_create_pipeline(device, device->pipeline_ssm_conv_bias_silu_f32, "ssm_conv_bias_silu_f32", ssm_conv_f32_len, ssm_conv_f32_data, "main", 4, sizeof(vk_op_ssm_conv_push_constants), {32, 16, 1}, {32, 16, 1, 1}, 1); + ggml_vk_create_pipeline(device, device->pipeline_ssm_conv_f32, "ssm_conv_f32", ssm_conv_f32_len, ssm_conv_f32_data, "main", 5, sizeof(vk_op_ssm_conv_push_constants), { 32, 16, 1}, { 32, 16, 0, 0, 0, 0}, 1, agx_no_robust); + ggml_vk_create_pipeline(device, device->pipeline_ssm_conv_silu_f32, "ssm_conv_silu_f32", ssm_conv_f32_len, ssm_conv_f32_data, "main", 5, sizeof(vk_op_ssm_conv_push_constants), { 32, 16, 1}, { 32, 16, 0, 1, 0, 0}, 1, agx_no_robust); + ggml_vk_create_pipeline(device, device->pipeline_ssm_conv_bias_silu_f32, "ssm_conv_bias_silu_f32", ssm_conv_f32_len, ssm_conv_f32_data, "main", 5, sizeof(vk_op_ssm_conv_push_constants), { 32, 16, 1}, { 32, 16, 1, 1, 0, 0}, 1, agx_no_robust); + ggml_vk_create_pipeline(device, device->pipeline_ssm_conv_1t_f32, "ssm_conv_1t_f32", ssm_conv_f32_len, ssm_conv_f32_data, "main", 5, sizeof(vk_op_ssm_conv_push_constants), {128, 1, 1}, {128, 1, 0, 0, 0, 0}, 1, agx_no_robust); + ggml_vk_create_pipeline(device, device->pipeline_ssm_conv_silu_1t_f32, "ssm_conv_silu_1t_f32", ssm_conv_f32_len, ssm_conv_f32_data, "main", 5, sizeof(vk_op_ssm_conv_push_constants), {128, 1, 1}, {128, 1, 0, 1, 0, 0}, 1, agx_no_robust); + ggml_vk_create_pipeline(device, device->pipeline_ssm_conv_sout_f32, "ssm_conv_sout_f32", ssm_conv_f32_len, ssm_conv_f32_data, "main", 5, sizeof(vk_op_ssm_conv_push_constants), { 32, 16, 1}, { 32, 16, 0, 0, 1, 0}, 1, agx_no_robust); + ggml_vk_create_pipeline(device, device->pipeline_ssm_conv_silu_sout_f32, "ssm_conv_silu_sout_f32", ssm_conv_f32_len, ssm_conv_f32_data, "main", 5, sizeof(vk_op_ssm_conv_push_constants), { 32, 16, 1}, { 32, 16, 0, 1, 1, 0}, 1, agx_no_robust); + ggml_vk_create_pipeline(device, device->pipeline_ssm_conv_concat_cpy_f32, "ssm_conv_concat_cpy_f32", ssm_conv_f32_len, ssm_conv_f32_data, "main", 5, sizeof(vk_op_ssm_conv_push_constants), {128, 1, 1}, {128, 1, 0, 0, 0, 1}, 1, agx_no_robust); + ggml_vk_create_pipeline(device, device->pipeline_ssm_conv_concat_cpy_silu_f32, "ssm_conv_concat_cpy_silu_f32", ssm_conv_f32_len, ssm_conv_f32_data, "main", 5, sizeof(vk_op_ssm_conv_push_constants), {128, 1, 1}, {128, 1, 0, 1, 0, 1}, 1, agx_no_robust); + { + const bool ssm_slide_8 = ggml_vk_is_apple_agx_coopmat(device) && !ggml_vk_exact_s8_enabled() && !getenv("GGML_VK_DISABLE_SSM_SLIDE8"); + const uint32_t mt_t_wg = ssm_slide_8 ? 4u : 8u; + const uint32_t mt_t_pt = ssm_slide_8 ? 8u : 1u; + const std::array mt_denoms = { 32u, mt_t_wg * mt_t_pt, 1u }; + ggml_vk_create_pipeline(device, device->pipeline_ssm_conv_concat_cpy_mt_f32, "ssm_conv_concat_cpy_mt_f32", ssm_conv_f32_len, ssm_conv_f32_data, "main", 5, sizeof(vk_op_ssm_conv_push_constants), mt_denoms, { 32, mt_t_wg, 0, 0, 0, 1, mt_t_pt }, 1, agx_no_robust); + ggml_vk_create_pipeline(device, device->pipeline_ssm_conv_concat_cpy_mt_silu_f32, "ssm_conv_concat_cpy_mt_silu_f32", ssm_conv_f32_len, ssm_conv_f32_data, "main", 5, sizeof(vk_op_ssm_conv_push_constants), mt_denoms, { 32, mt_t_wg, 0, 1, 0, 1, mt_t_pt }, 1, agx_no_robust); + } ggml_vk_create_pipeline(device, device->pipeline_opt_step_adamw_f32, "opt_step_adamw_f32", opt_step_adamw_f32_len, opt_step_adamw_f32_data, "main", 5, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); @@ -4423,7 +5239,8 @@ vk_device ggml_vk_get_device(size_t idx) { const char* GGML_VK_DISABLE_GRAPH_OPTIMIZE = getenv("GGML_VK_DISABLE_GRAPH_OPTIMIZE"); device->disable_graph_optimize = GGML_VK_DISABLE_GRAPH_OPTIMIZE != nullptr; - device->disable_narrow_mul_mat = getenv("GGML_VK_DISABLE_NARROW_MUL_MAT") != nullptr; + const char* GGML_VK_DISABLE_NARROW_MUL_MAT = getenv("GGML_VK_DISABLE_NARROW_MUL_MAT"); + device->disable_narrow_mul_mat = GGML_VK_DISABLE_NARROW_MUL_MAT != nullptr && strcmp(GGML_VK_DISABLE_NARROW_MUL_MAT, "0") != 0; bool fp16_storage = false; bool fp16_compute = false; @@ -4581,6 +5398,27 @@ vk_device ggml_vk_get_device(size_t idx) { device->vendor_id = device->properties.vendorID; device->driver_id = driver_props.driverID; +#if VK_HEADER_VERSION >= 287 + // Private M4 Max experiment: Borealis currently reports Mesa's vendor ID + // and a driver ID not named by the installed Vulkan headers. Match its + // advertised name and this device instead of guessing a new enum value. + if (device->vendor_id == 0x10005 && device->properties.deviceID == 0x6041 && + strcmp(driver_props.driverName.data(), "Borealis") == 0) { + const char * mode = getenv("GGML_VK_BOREALIS_TUNING"); + if (mode == nullptr) { + mode = "all"; + } + if (strcmp(mode, "off") != 0 && strcmp(mode, "tiles") != 0 && + strcmp(mode, "roll") != 0 && strcmp(mode, "all") != 0) { + throw std::invalid_argument("GGML_VK_BOREALIS_TUNING must be off, tiles, roll, or all"); + } + device->borealis_medium_tiles = strcmp(mode, "tiles") == 0 || strcmp(mode, "all") == 0; + device->borealis_roll_bk = strcmp(mode, "roll") == 0 || strcmp(mode, "all") == 0; + GGML_LOG_INFO("ggml_vulkan: Borealis M4 tuning=%s (medium_tiles=%d, roll_bk=%d)\n", + mode, device->borealis_medium_tiles, device->borealis_roll_bk); + } +#endif + device->external_memory_host = device->external_memory_host && ggml_vk_external_memory_host_usable(device->driver_id); // Implementing the async backend interfaces seems broken on older Intel HW, @@ -4633,6 +5471,8 @@ vk_device ggml_vk_get_device(size_t idx) { device->shader_core_count = amd_shader_core_properties2_props.activeComputeUnitCount; } else if (device->vendor_id == VK_VENDOR_ID_INTEL) { device->shader_core_count = ggml_vk_intel_shader_core_count(device->physical_device); + } else if (strstr(device->name.c_str(), "M4 Max") != nullptr && !ggml_vk_exact_s8_enabled()) { + device->shader_core_count = 40; } else { device->shader_core_count = 0; } @@ -5125,6 +5965,9 @@ vk_device ggml_vk_get_device(size_t idx) { if (prop.MSize == 16 && prop.NSize == 16 && prop.KSize == 16) { device->coopmat_support_16x16x16_f32acc = true; } + if (prop.MSize == 8 && prop.NSize == 8 && prop.KSize == 8) { + device->coopmat_support_8x8x8_f32acc = true; + } } else if ((vk::ComponentTypeKHR)prop.CType == vk::ComponentTypeKHR::eFloat16 && (vk::ComponentTypeKHR)prop.ResultType == vk::ComponentTypeKHR::eFloat16) { // coopmat sizes not set yet @@ -5182,8 +6025,31 @@ vk_device ggml_vk_get_device(size_t idx) { } } + if (device->driver_id == vk::DriverId::eMesaHoneykrisp || + (device->vendor_id == 0x10005 && strcmp(driver_props.driverName.data(), "Borealis") == 0)) { + device->mmv_max_cols = getenv("GGML_VK_MMV_MAX_COLS") + ? (uint32_t) std::clamp(atoi(getenv("GGML_VK_MMV_MAX_COLS")), 1, (int) mul_mat_vec_max_cols_limit) + : 32u; + device->mmv_id_max_tokens = getenv("GGML_VK_MMVID_MAX") + ? (uint32_t) std::max(1, atoi(getenv("GGML_VK_MMVID_MAX"))) + : 48u; + } + if (device->coopmat_support) { device_extensions.push_back("VK_KHR_cooperative_matrix"); + if (ggml_vk_is_apple_agx_coopmat(device) && !ggml_vk_exact_s8_enabled() && !getenv("GGML_VK_MMVID_MAX")) { + device->mmv_id_max_tokens = gen_batch_invariant() ? 8u : 4u; + } + } +#else + if (device->driver_id == vk::DriverId::eMesaHoneykrisp || + (device->vendor_id == 0x10005 && strcmp(driver_props.driverName.data(), "Borealis") == 0)) { + device->mmv_max_cols = getenv("GGML_VK_MMV_MAX_COLS") + ? (uint32_t) std::clamp(atoi(getenv("GGML_VK_MMV_MAX_COLS")), 1, (int) mul_mat_vec_max_cols_limit) + : 32u; + device->mmv_id_max_tokens = getenv("GGML_VK_MMVID_MAX") + ? (uint32_t) std::max(1, atoi(getenv("GGML_VK_MMVID_MAX"))) + : 48u; } #endif device->name = GGML_VK_NAME + std::to_string(idx); @@ -5264,15 +6130,15 @@ vk_device ggml_vk_get_device(size_t idx) { } #if VK_HEADER_VERSION >= 287 - // Honeykrisp driver for Asahi Linux doesn't report VK_VENDOR_ID_APPLE. - // Check for Honeykrisp driver and force same configuration as the VK_VENDOR_ID_APPLE case. - if (device->driver_id == vk::DriverId::eMesaHoneykrisp) { + // Mesa's Apple drivers do not report VK_VENDOR_ID_APPLE. Reuse the + // Apple tile configuration for Honeykrisp and opted-in Borealis. + if (device->driver_id == vk::DriverId::eMesaHoneykrisp || device->borealis_medium_tiles) { device->mul_mat_l[i] = false; device->mul_mat_m[i] = true; device->mul_mat_s[i] = false; device->mul_mat_id_l[i] = false; device->mul_mat_id_m[i] = true; - device->mul_mat_id_s[i] = false; + device->mul_mat_id_s[i] = getenv("GGML_VK_MMQID_S") != nullptr; // gen } #endif @@ -5282,6 +6148,29 @@ vk_device ggml_vk_get_device(size_t idx) { device->mul_mat_id_l_int[i] = device->mul_mat_id_l[i]; device->mul_mat_id_m_int[i] = device->mul_mat_id_m[i]; device->mul_mat_id_s_int[i] = device->mul_mat_id_s[i]; + if (ggml_vk_is_apple_agx_coopmat(device)) { + const char * cm1_t = getenv("GGML_VK_CM1_TILES"); + const char * cm1_id_t = getenv("GGML_VK_CM1_ID_TILES"); + if (!cm1_t || !*cm1_t) cm1_t = "sml"; + if (!cm1_id_t || !*cm1_id_t) cm1_id_t = "sm"; + device->mul_mat_s[i] = strchr(cm1_t, 's') != nullptr; + device->mul_mat_m[i] = strchr(cm1_t, 'm') != nullptr; + device->mul_mat_l[i] = strchr(cm1_t, 'l') != nullptr; + device->mul_mat_id_s[i] = strchr(cm1_id_t, 's') != nullptr; + device->mul_mat_id_m[i] = strchr(cm1_id_t, 'm') != nullptr; + device->mul_mat_id_l[i] = strchr(cm1_id_t, 'l') != nullptr; + } + } + + // Upstream narrow-N CM1 tiles ({BM, 16/32, 32, WM=16, WN=16}) and shader_core_count==0 split_k=8 + // target AMD/Intel 16x16 coopmat1; keep Apple AGX's tuned 8x8x8 s/m/l tiles and split_k=1 unless overridden. + if (GGML_VK_DISABLE_NARROW_MUL_MAT == nullptr && + (ggml_vk_is_apple_agx_coopmat(device) || device->borealis_medium_tiles +#if VK_HEADER_VERSION >= 287 + || device->driver_id == vk::DriverId::eMesaHoneykrisp +#endif + )) { + device->disable_narrow_mul_mat = true; } @@ -5344,7 +6233,8 @@ vk_device ggml_vk_get_device(size_t idx) { device->disable_descriptor_reuse = getenv("GGML_VK_DISABLE_DESCRIPTOR_REUSE") != nullptr; device->add_rms_fusion = device->subgroup_arithmetic && - device->vendor_id != VK_VENDOR_ID_INTEL; + device->vendor_id != VK_VENDOR_ID_INTEL && + getenv("GGML_VK_DISABLE_ADD_RMS") == nullptr; device->partials_binding_alignment = std::max(4u, (uint32_t)device->properties.limits.minStorageBufferOffsetAlignment); @@ -5558,6 +6448,7 @@ static void ggml_vk_print_gpu_info(size_t idx) { const bool rebar = ggml_vk_has_rebar(physical_device); integer_dot_product = integer_dot_product + && getenv("GGML_VK_DISABLE_INTEGER_DOT_PRODUCT") == nullptr && shader_integer_dot_product_props.integerDotProduct4x8BitPackedSignedAccelerated && shader_integer_dot_product_features.shaderIntegerDotProduct; @@ -5912,8 +6803,9 @@ void ggml_vk_init(ggml_backend_vk_context * ctx, size_t idx) { ctx->prealloc_size_x = 0; ctx->prealloc_size_y = 0; ctx->prealloc_size_split_k = 0; - // Fixed size of 1KB, for deterministic behavior - ctx->prealloc_size_add_rms_partials = 1024; + // Fixed size of 1KB, for deterministic behavior (gen: 1 MiB in batch-invariant mode, where the 1 KB budget is + // still applied per row so the same pairs fuse as at n=1) + ctx->prealloc_size_add_rms_partials = gen_batch_invariant() ? (1u << 20) : 1024; ctx->fence = ctx->device->device.createFence({}); ctx->almost_ready_fence = ctx->device->device.createFence({}); @@ -5981,7 +6873,7 @@ vk_pipeline ggml_vk_get_to_fp16(ggml_backend_vk_context * ctx, ggml_type type) { static vk_pipeline ggml_vk_get_dequantize_mul_mat_vec(ggml_backend_vk_context * ctx, ggml_type a_type, ggml_type b_type, uint32_t num_cols, uint32_t m, uint32_t k) { VK_LOG_DEBUG("ggml_vk_get_dequantize_mul_mat_vec()"); GGML_ASSERT(b_type == GGML_TYPE_F32 || b_type == GGML_TYPE_F16 || b_type == GGML_TYPE_Q8_1); - GGML_ASSERT(num_cols >= 1 && num_cols <= mul_mat_vec_max_cols); + GGML_ASSERT(num_cols >= 1 && num_cols <= ggml_vk_mul_mat_vec_max_cols(ctx->device)); if (b_type == GGML_TYPE_Q8_1) { switch (a_type) { @@ -6042,7 +6934,23 @@ static vk_pipeline ggml_vk_get_dequantize_mul_mat_vec(ggml_backend_vk_context * // heuristic to choose workgroup size uint32_t dmmv_wg = DMMV_WG_SIZE_SUBGROUP; - if ((ctx->device->vendor_id == VK_VENDOR_ID_NVIDIA && ctx->device->architecture != vk_device_architecture::NVIDIA_PRE_TURING) || ctx->device->vendor_id == VK_VENDOR_ID_INTEL) { + static const int gen_dmmv_large = getenv("GGML_VK_DMMV_LARGE") ? atoi(getenv("GGML_VK_DMMV_LARGE")) : 0; + if (gen_dmmv_large < 0) { + dmmv_wg = DMMV_WG_SIZE_SUBGROUP; + } else if (gen_dmmv_large > 0) { + const uint32_t max_m = gen_dmmv_large == 1 ? 8192u : (uint32_t)gen_dmmv_large; + if (a_type == GGML_TYPE_Q6_K) { + if (m < std::min(4096u, max_m) && k >= 1024) { + dmmv_wg = DMMV_WG_SIZE_LARGE; + } + } else if (m <= max_m && k >= 1024) { + dmmv_wg = DMMV_WG_SIZE_LARGE; + } + } else if (ggml_vk_is_apple_agx_coopmat(ctx->device) && !ggml_vk_exact_s8_enabled() && !gen_batch_invariant() && a_type == GGML_TYPE_Q8_0) { + if (m <= 262144 && (k >= 4096 || (m < 2048 && k >= 1024))) { + dmmv_wg = DMMV_WG_SIZE_LARGE; + } + } else if ((ctx->device->vendor_id == VK_VENDOR_ID_NVIDIA && ctx->device->architecture != vk_device_architecture::NVIDIA_PRE_TURING) || ctx->device->vendor_id == VK_VENDOR_ID_INTEL) { // Prefer larger workgroups when M is small, to spread the work out more // and keep more SMs busy. // q6_k seems to prefer small workgroup size even for "medium" values of M. @@ -6130,7 +7038,19 @@ static vk_pipeline ggml_vk_get_dequantize_mul_mat_vec_id(ggml_backend_vk_context // heuristic to choose workgroup size uint32_t dmmv_wg = DMMV_WG_SIZE_SUBGROUP; - if ((ctx->device->vendor_id == VK_VENDOR_ID_NVIDIA && ctx->device->architecture != vk_device_architecture::NVIDIA_PRE_TURING) || ctx->device->vendor_id == VK_VENDOR_ID_INTEL) { + static const int gen_dmmv_id_large = getenv("GGML_VK_DMMV_ID_LARGE") ? atoi(getenv("GGML_VK_DMMV_ID_LARGE")) : 0; + if (gen_dmmv_id_large < 0) { + dmmv_wg = DMMV_WG_SIZE_SUBGROUP; + } else if (gen_dmmv_id_large > 0) { + const uint32_t max_m = gen_dmmv_id_large == 1 ? 8192u : (uint32_t)gen_dmmv_id_large; + if (a_type == GGML_TYPE_Q6_K) { + if (m < std::min(4096u, max_m) && k >= 1024) { + dmmv_wg = DMMV_WG_SIZE_LARGE; + } + } else if (m <= max_m && k >= 1024) { + dmmv_wg = DMMV_WG_SIZE_LARGE; + } + } else if ((ctx->device->vendor_id == VK_VENDOR_ID_NVIDIA && ctx->device->architecture != vk_device_architecture::NVIDIA_PRE_TURING) || ctx->device->vendor_id == VK_VENDOR_ID_INTEL) { // Prefer larger workgroups when M is small, to spread the work out more // and keep more SMs busy. // q6_k seems to prefer small workgroup size even for "medium" values of M. @@ -6183,6 +7103,38 @@ vk_subbuffer ggml_vk_tensor_subbuffer( return vk_subbuffer{buffer, offset, size}; } +static size_t ggml_vk_tensor_byte_span(const ggml_tensor * t) { + if (!t || ggml_nelements(t) == 0) { + return 0; + } + size_t span = ggml_row_size(t->type, t->ne[0]); + for (int d = 1; d < GGML_MAX_DIMS; ++d) { + if (t->ne[d] > 1) { + span += (size_t)(t->ne[d] - 1) * t->nb[d]; + } + } + return span; +} + +static vk_subbuffer ggml_vk_strided_subbuffer( + const ggml_backend_vk_context * ctx, const ggml_tensor * tensor, bool allow_misalign = false) { + vk_subbuffer sb = ggml_vk_tensor_subbuffer(ctx, tensor, allow_misalign); + size_t raw_offset = 0; + vk_buffer host_buf = nullptr; + if (ctx->device->uma) { + ggml_vk_host_get(ctx->device, tensor->data, host_buf, raw_offset); + } + if (!host_buf) { + raw_offset = vk_tensor_offset(tensor) + tensor->view_offs; + } + const size_t misalign_bytes = raw_offset - sb.offset; + const size_t span_bytes = ggml_vk_tensor_byte_span(tensor) + misalign_bytes; + if (span_bytes > sb.size) { + sb.size = span_bytes; + } + return sb; +} + static vk_command_buffer* ggml_vk_get_or_create_cmd_buffer(vk_device& device, vk_command_pool& pool) { for (auto& cmd_buffer : pool.cmd_buffers) { if (!cmd_buffer.in_use) { @@ -6253,10 +7205,22 @@ void ggml_vk_ctx_begin(vk_device& device, vk_context& subctx) { } } +static void ggml_vk_ctx_begin_reusable(vk_device& device, vk_context& subctx) { + if (subctx->s != nullptr) { + ggml_vk_ctx_end(subctx); + } + subctx->seqs.push_back({ ggml_vk_begin_submission(device, *subctx->p, false) }); + subctx->s = subctx->seqs[subctx->seqs.size() - 1].data(); +} + vk_context ggml_vk_get_compute_ctx(ggml_backend_vk_context * ctx) { vk_context result; if (!ctx->compute_ctx.expired()) { result = ctx->compute_ctx.lock(); + } else if (ctx->recording_replay && ctx->active_replay_cmd_pool != nullptr) { + result = ggml_vk_create_context(ctx, *ctx->active_replay_cmd_pool); + ctx->compute_ctx = result; + ggml_vk_ctx_begin_reusable(ctx->device, result); } else { result = ggml_vk_create_context(ctx, ctx->compute_cmd_pool); @@ -6327,6 +7291,99 @@ void deferred_memset(void * dst, uint32_t val, size_t size, std::vectorview_src && t->view_offs == 0 && + t->type == t->view_src->type && + ggml_is_contiguous(t) && ggml_is_contiguous(t->view_src) && + ggml_nelements(t) == ggml_nelements(t->view_src)) { + t = t->view_src; + } + return t; +} + +static uint32_t ggml_vk_acquire_prealloc_y_slot( + ggml_backend_vk_context * ctx, vk_context & subctx, + vk_pipeline_struct * pipe, const ggml_tensor * src1, bool k_padded, + bool & out_cached, int avoid_slot = -1) { + const ggml_tensor * key_src1 = k_padded ? src1 : ggml_vk_prealloc_y_equiv_root(src1); + if (pipe != nullptr && key_src1 != nullptr) { + for (uint32_t s = 0; s < ctx->prealloc_y_ring_count; ++s) { + if (ctx->prealloc_y_last_pipeline_ring[s] == pipe && + ctx->prealloc_y_last_tensor_ring[s] == key_src1 && + ctx->prealloc_y_last_k_padded_ring[s] == k_padded) { + out_cached = true; + return s; + } + } + } + out_cached = false; + uint32_t slot = ctx->prealloc_y_ring_count > 1 + ? ((ctx->prealloc_y_ring_idx++) % ctx->prealloc_y_ring_count) + : 0u; + if ((int)slot == avoid_slot && ctx->prealloc_y_ring_count > 1) { + slot = (ctx->prealloc_y_ring_idx++) % ctx->prealloc_y_ring_count; + } + if (ctx->prealloc_y_slot_need_sync[slot] || (ctx->prealloc_y_ring_count == 1 && ctx->prealloc_y_need_sync)) { + ggml_vk_sync_buffers(ctx, subctx); + } + ctx->prealloc_y_last_pipeline_ring[slot] = pipe; + ctx->prealloc_y_last_tensor_ring[slot] = key_src1; + ctx->prealloc_y_last_k_padded_ring[slot] = k_padded; + ctx->prealloc_y_last_pipeline_used = pipe; + ctx->prealloc_y_last_tensor_used = key_src1; + ctx->prealloc_y_last_k_padded = k_padded; + return slot; +} + +static void ggml_vk_reserve_prealloc_y(ggml_backend_vk_context * ctx, vk_context & subctx, size_t req_bytes) { + if (req_bytes > ctx->device->properties.limits.maxStorageBufferRange) { + GGML_ABORT("Requested preallocation size is too large"); + } + if (ctx->prealloc_size_y < req_bytes || ctx->prealloc_y == nullptr) { + ctx->prealloc_size_y = std::max(ctx->prealloc_size_y, req_bytes); + ggml_vk_preallocate_buffers(ctx, subctx); + } +} + +static vk_subbuffer ggml_vk_produce_prealloc_y_f16( + ggml_backend_vk_context * ctx, vk_context & subctx, + const ggml_tensor * dst_tensor, size_t write_bytes, + bool keep_post_dispatch = false, int avoid_slot = -1) { + vk_pipeline_struct * cpy_pipe = ctx->device->pipeline_contig_cpy_f32_f16.get(); + const ggml_tensor * key_t = ggml_vk_prealloc_y_equiv_root(dst_tensor); + for (uint32_t s = 0; s < ctx->prealloc_y_ring_count; ++s) { + if (ctx->prealloc_y_last_tensor_ring[s] == key_t) { + ctx->prealloc_y_last_tensor_ring[s] = nullptr; + ctx->prealloc_y_last_pipeline_ring[s] = nullptr; + } + } + bool dummy_cached = false; + const uint32_t slot = ggml_vk_acquire_prealloc_y_slot(ctx, subctx, cpy_pipe, key_t, false, dummy_cached, avoid_slot); + GGML_ASSERT(!dummy_cached); + const size_t y_off = (size_t)slot * ctx->prealloc_y_ring_stride; + ctx->prealloc_y_need_sync = true; + ctx->prealloc_y_slot_need_sync[slot] = true; + if (keep_post_dispatch) { + ctx->prealloc_y_keep_slot = (int)slot; + } + return vk_subbuffer{ ctx->prealloc_y, y_off, write_bytes }; +} + +static uint32_t ggml_vk_acquire_prealloc_split_k_slot( + ggml_backend_vk_context * ctx, vk_context & subctx, size_t req_bytes) { + if (ctx->prealloc_split_k_ring_count > 1 && req_bytes <= ctx->prealloc_split_k_ring_stride) { + const uint32_t slot = (ctx->prealloc_split_k_ring_idx++) % ctx->prealloc_split_k_ring_count; + if (ctx->prealloc_split_k_slot_need_sync[slot]) { + ggml_vk_sync_buffers(ctx, subctx); + } + return slot; + } + if (ctx->prealloc_split_k_need_sync) { + ggml_vk_sync_buffers(ctx, subctx); + } + return 0u; +} + static uint32_t ggml_vk_guess_split_k(ggml_backend_vk_context * ctx, uint32_t m, uint32_t n, uint32_t k, bool disable_split_k, const vk_pipeline& pipeline, bool narrow_n) { VK_LOG_DEBUG("ggml_vk_guess_split_k(" << m << ", " << n << ", " << k << ", " << disable_split_k << ")"); @@ -6334,6 +7391,15 @@ static uint32_t ggml_vk_guess_split_k(ggml_backend_vk_context * ctx, uint32_t m, return 1; } + // Series 9 (S9.3): On Apple AGX coopmat, split_k > 1 forces two full 40-core GPU pipeline drains + a + // split_k_reduce dispatch per matmul (32 ms bubble across 100 alpha/beta/router projections at pp256). + if (ggml_vk_is_apple_agx_coopmat(ctx->device)) { + const uint32_t tiles = CEIL_DIV(m, pipeline->wg_denoms[0]) * CEIL_DIV(n, pipeline->wg_denoms[1]); + if (k <= 8192 || tiles >= 8) { + return 1; + } + } + const bool rdna = ctx->device->architecture == AMD_RDNA3 || ctx->device->architecture == AMD_RDNA4; uint32_t split_k = narrow_n && ctx->device->shader_core_count == 0 ? 8u : 1u; if ((narrow_n || rdna) && ctx->device->shader_core_count != 0 && k >= 2048 && n <= 32) { @@ -6345,19 +7411,21 @@ static uint32_t ggml_vk_guess_split_k(ggml_backend_vk_context * ctx, uint32_t m, } // On RDNA, also split when n <= 64 only partly fills one tile column. - if (!narrow_n && split_k == 1 && ctx->device->shader_core_count != 0 && (n >= pipeline->wg_denoms[1] || (rdna && n <= 64))) { + const uint32_t eff_cores = (ggml_vk_fast_fp16_enabled() && ggml_vk_is_apple_agx_coopmat(ctx->device) && ctx->device->shader_core_count == 0) + ? 40u : ctx->device->shader_core_count; + if (!narrow_n && split_k == 1 && eff_cores != 0 && (n >= pipeline->wg_denoms[1] || (rdna && n <= 64))) { // If k is 'large' and the SMs will fill less than halfway, use split_k. uint32_t m_tiles = CEIL_DIV(m, pipeline->wg_denoms[0]); uint32_t n_tiles = CEIL_DIV(n, pipeline->wg_denoms[1]); if (k >= 2048) { - if (m_tiles * n_tiles <= ctx->device->shader_core_count / 2) { - split_k = ctx->device->shader_core_count / (m_tiles * n_tiles); - } else if (m_tiles * n_tiles <= ctx->device->shader_core_count * 2 / 3) { + if (m_tiles * n_tiles <= eff_cores / 2) { + split_k = eff_cores / (m_tiles * n_tiles); + } else if (m_tiles * n_tiles <= eff_cores * 2 / 3) { split_k = 3; } else if ((ctx->device->architecture == AMD_RDNA3 || ctx->device->architecture == AMD_RDNA4) && - m_tiles * n_tiles > ctx->device->shader_core_count && - m_tiles * n_tiles <= ctx->device->shader_core_count * 4 / 3) { + m_tiles * n_tiles > eff_cores && + m_tiles * n_tiles <= eff_cores * 4 / 3) { // two waves, the second one at most 1/3 full split_k = 3; } @@ -6436,6 +7504,61 @@ void ggml_vk_matmul( const std::array pc2 = { (uint32_t)(m * n * batch), split_k }; ggml_vk_dispatch_pipeline(ctx, subctx, ctx->device->pipeline_matmul_split_k_reduce, { split_k_buffer, d }, pc2, { m * n * batch, 1, 1 }); ctx->prealloc_split_k_need_sync = true; + for (uint32_t s = 0; s < VK_PREALLOC_RING; ++s) { + ctx->prealloc_split_k_slot_need_sync[s] = true; + } + ctx->last_expert_count_valid = false; +} + +static void ggml_vk_matmul_fused( + ggml_backend_vk_context * ctx, vk_context& subctx, vk_pipeline& pipeline, + const vk_subbuffer& a, const vk_subbuffer& b, const vk_subbuffer& d, + const vk_subbuffer& u, const vk_subbuffer& aux0, const vk_subbuffer& aux1, + uint32_t m, uint32_t n, uint32_t k, uint32_t stride_a, uint32_t stride_b, uint32_t stride_d, + uint32_t batch_stride_a, uint32_t batch_stride_b, uint32_t batch_stride_d, + uint32_t batch, uint32_t ne02, uint32_t ne12, uint32_t broadcast2, uint32_t broadcast3, + uint32_t padded_n, uint64_t b_col_bytes, uint64_t d_col_bytes, + uint64_t aux0_col_bytes, uint64_t aux1_col_bytes, uint32_t fusion_flags, + uint32_t pair_m0 = 0, uint32_t stride_d2 = 0, uint32_t batch_stride_d2 = 0) { + const uint64_t max_n = (uint64_t)ctx->device->properties.limits.maxComputeWorkGroupCount[1] * pipeline->wg_denoms[1]; + const uint64_t n_chunks = CEIL_DIV(n, max_n); + ggml_pipeline_request_descriptor_sets(ctx, pipeline, n_chunks * CEIL_DIV(batch, ctx->device->properties.limits.maxComputeWorkGroupCount[2])); + + for (uint64_t n_base = 0; n_base < n; n_base += max_n) { + const uint32_t n_chunk = (uint32_t) std::min(n - n_base, max_n); + GGML_ASSERT(n_base * b_col_bytes < b.size && n_base * d_col_bytes < d.size); + vk_subbuffer b_chunk = b; + vk_subbuffer d_chunk = d; + vk_subbuffer aux0_chunk = aux0; + vk_subbuffer aux1_chunk = aux1; + b_chunk.offset += n_base * b_col_bytes; + b_chunk.size -= n_base * b_col_bytes; + d_chunk.offset += n_base * d_col_bytes; + d_chunk.size -= n_base * d_col_bytes; + if (aux0_col_bytes != 0) { + aux0_chunk.offset += n_base * aux0_col_bytes; + aux0_chunk.size -= n_base * aux0_col_bytes; + } + if (aux1_col_bytes != 0) { + aux1_chunk.offset += n_base * aux1_col_bytes; + aux1_chunk.size -= n_base * aux1_col_bytes; + } + + uint32_t base_work_group_z = 0; + while (base_work_group_z < batch) { + uint32_t groups_z = std::min(batch - base_work_group_z, ctx->device->properties.limits.maxComputeWorkGroupCount[2]); + const vk_mat_mat_fused_push_constants pc = { + m, n_chunk, k, stride_a, stride_b, stride_d, + batch_stride_a, batch_stride_b, batch_stride_d, + base_work_group_z, batch, k, ne02, ne12, broadcast2, broadcast3, + uint32_t(padded_n - n_base), fusion_flags, + pair_m0, stride_d2, batch_stride_d2 + }; + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, + { a, b_chunk, d_chunk, u, aux0_chunk, aux1_chunk }, pc, { m, n_chunk, groups_z }); + base_work_group_z += groups_z; + } + } } static bool ggml_vk_src1_prec_allows_q8(const ggml_tensor * dst) { @@ -6466,7 +7589,7 @@ static bool ggml_vk_prec_allows_f16acc(ggml_prec prec) { return false; } -static bool ggml_vk_get_mul_mat_mat_f16acc(ggml_backend_vk_context * ctx, ggml_type src0_type, ggml_type src1_type, ggml_prec acc_prec) { +static bool ggml_vk_get_mul_mat_mat_f16acc(const ggml_backend_vk_context * ctx, ggml_type src0_type, ggml_type src1_type, ggml_prec acc_prec) { if (src0_type == GGML_TYPE_F32 || src0_type == GGML_TYPE_BF16) return false; if (src1_type == GGML_TYPE_Q8_1) return false; const bool prec_allows_f16 = ggml_vk_prec_allows_f16acc(acc_prec); @@ -6484,7 +7607,7 @@ static bool ggml_vk_get_mul_mat_mat_f16acc(ggml_backend_vk_context * ctx, ggml_t } static const std::vector* ggml_vk_get_mul_mat_mat_pipeline_map( - ggml_backend_vk_context * ctx, ggml_type src0_type, ggml_type src1_type, ggml_prec prec, bool mul_mat_id = false) { + const ggml_backend_vk_context * ctx, ggml_type src0_type, ggml_type src1_type, ggml_prec prec, bool mul_mat_id = false) { bool f16acc = ggml_vk_get_mul_mat_mat_f16acc(ctx, src0_type, src1_type, prec); vk_matmul_pipeline_key key{src0_type, src1_type, mul_mat_id, f16acc}; auto it = ctx->device->pipeline_matmul.find(key); @@ -6541,7 +7664,27 @@ static void ggml_vk_matmul_id( ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { a, b, d, ids, expert_count_buf }, pc, elements); } -bool ggml_vk_dim01_contiguous(const ggml_tensor * tensor) { +static void ggml_vk_matmul_id_fused( + ggml_backend_vk_context * ctx, vk_context& subctx, vk_pipeline& pipeline, + const vk_subbuffer& a, const vk_subbuffer& b, const vk_subbuffer& d, + const vk_subbuffer& ids, const vk_subbuffer& expert_count_buf, + const vk_subbuffer& u, const vk_subbuffer& aux0, const vk_subbuffer& aux1, + uint32_t m, uint32_t n, uint32_t k, uint32_t stride_a, uint32_t stride_b, uint32_t stride_d, + uint32_t batch_stride_a, uint32_t batch_stride_b, uint32_t batch_stride_d, + uint32_t n_as, uint32_t nei0, uint32_t nei1, uint32_t nbi1, uint32_t ne11, + uint32_t row_ids_offset, uint32_t max_tiles, uint32_t fusion_flags) { + const vk_mat_mat_id_fused_push_constants pc = { + m, n, k, stride_a, stride_b, stride_d, + batch_stride_a, batch_stride_b, batch_stride_d, + nei0, nei1, nbi1, ne11, n_as, row_ids_offset, fusion_flags + }; + const uint32_t tiles_y = std::min(max_tiles, ctx->device->properties.limits.maxComputeWorkGroupCount[1]); + const std::array elements = { m, tiles_y * pipeline->wg_denoms[1], uint32_t(CEIL_DIV(max_tiles, tiles_y)) }; + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, + { a, b, d, ids, expert_count_buf, u, aux0, aux1 }, pc, elements); +} + +bool ggml_vk_dim01_contiguous(const ggml_tensor * tensor) { return tensor->nb[0] == ggml_type_size(tensor->type) && tensor->nb[1] == (tensor->nb[0]*tensor->ne[0])/ggml_blck_size(tensor->type) && @@ -6797,10 +7940,72 @@ static ggml_type ggml_vk_mul_mat_b_type( return f16_type; } -static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, bool disable_split_k) { +static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, bool disable_split_k, const struct ggml_cgraph * cgraph = nullptr, int node_idx = -1) { + const bool has_pair_mm = ctx->gen_pair_mm1 != nullptr; + const bool has_pair_glu = !has_pair_mm && cgraph != nullptr && ctx->num_additional_fused_ops == 2 && + cgraph->nodes[node_idx + 2]->op == GGML_OP_GLU; + const ggml_tensor * src0_up = nullptr; + if (has_pair_glu) { + const ggml_tensor * glu = cgraph->nodes[node_idx + 2]; + src0 = glu->src[0]->src[0]; + src0_up = glu->src[1]->src[0]; + } else if (has_pair_mm) { + src0_up = ctx->gen_pair_mm1->src[0]; + } + const ggml_tensor * out_dst = has_pair_mm + ? (ctx->gen_mm_dst ? ctx->gen_mm_dst : dst) + : (ctx->gen_mm_dst ? ctx->gen_mm_dst + : ((cgraph != nullptr && ctx->num_additional_fused_ops > 0) + ? cgraph->nodes[node_idx + ctx->num_additional_fused_ops] + : dst)); + const ggml_tensor * out_dst2 = has_pair_mm ? ctx->gen_pair_dst1 : nullptr; + uint32_t fusion_flags = 0; + const ggml_tensor * aux0_t = nullptr; + const ggml_tensor * aux1_t = nullptr; + uint64_t aux0_col_bytes = 0; + uint64_t aux1_col_bytes = 0; + if (has_pair_mm) { + fusion_flags |= ctx->gen_mm_flags; + aux1_t = out_dst2; + aux1_col_bytes = (uint64_t)out_dst2->nb[1]; + } else if (!has_pair_glu) { + if (ctx->gen_mm_dst != nullptr) { + fusion_flags |= ctx->gen_mm_flags; + aux0_t = ctx->gen_mm_f0; + aux1_t = ctx->gen_mm_f1; + if ((fusion_flags & MAT_VEC_FUSION_FLAGS_ROW_SCALE0) != 0) { + aux0_col_bytes = sizeof(float); + } + } else if (cgraph != nullptr && ctx->num_additional_fused_ops == 1 && + cgraph->nodes[node_idx + 1]->op == GGML_OP_GLU) { + const ggml_tensor * glu = cgraph->nodes[node_idx + 1]; + const bool is_gate = (glu->src[0] == dst); + aux0_t = is_gate ? glu->src[1] : glu->src[0]; + fusion_flags |= is_gate ? MAT_VEC_FUSION_FLAGS_GLU_A : MAT_VEC_FUSION_FLAGS_GLU_B; + aux0_col_bytes = (uint64_t)out_dst->nb[1]; + } else if (cgraph != nullptr && ctx->num_additional_fused_ops > 0) { + const ggml_tensor * add0 = cgraph->nodes[node_idx + 1]; + GGML_ASSERT(add0->op == GGML_OP_ADD); + aux0_t = (add0->src[0] == dst) ? add0->src[1] : add0->src[0]; + fusion_flags |= MAT_VEC_FUSION_FLAGS_BIAS0; + aux0_col_bytes = (uint64_t)out_dst->nb[1]; + if (ctx->num_additional_fused_ops > 1) { + const ggml_tensor * add1 = cgraph->nodes[node_idx + 2]; + GGML_ASSERT(add1->op == GGML_OP_ADD); + aux1_t = (add1->src[0] == add0) ? add1->src[1] : add1->src[0]; + fusion_flags |= MAT_VEC_FUSION_FLAGS_BIAS1; + aux1_col_bytes = (uint64_t)out_dst->nb[1]; + } + } + } + const bool use_mm_fused = has_pair_glu || (fusion_flags != 0); + if (use_mm_fused) { + disable_split_k = true; + } + VK_LOG_DEBUG("ggml_vk_mul_mat_q_f16((" << src0 << ", name=" << src0->name << ", type=" << ggml_type_name(src0->type) << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3]; std::cerr << "), (" << src1 << ", name=" << src1->name << ", type=" << ggml_type_name(src1->type) << ", ne0=" << src1->ne[0] << ", ne1=" << src1->ne[1] << ", ne2=" << src1->ne[2] << ", ne3=" << src1->ne[3] << ", nb0=" << src1->nb[0] << ", nb1=" << src1->nb[1] << ", nb2=" << src1->nb[2] << ", nb3=" << src1->nb[3]; - std::cerr << "), (" << dst << ", name=" << dst->name << ", type=" << ggml_type_name(dst->type) << ", ne0=" << dst->ne[0] << ", ne1=" << dst->ne[1] << ", ne2=" << dst->ne[2] << ", ne3=" << dst->ne[3] << ", nb0=" << dst->nb[0] << ", nb1=" << dst->nb[1] << ", nb2=" << dst->nb[2] << ", nb3=" << dst->nb[3]; + std::cerr << "), (" << out_dst << ", name=" << out_dst->name << ", type=" << ggml_type_name(out_dst->type) << ", ne0=" << out_dst->ne[0] << ", ne1=" << out_dst->ne[1] << ", ne2=" << out_dst->ne[2] << ", ne3=" << out_dst->ne[3] << ", nb0=" << out_dst->nb[0] << ", nb1=" << out_dst->nb[1] << ", nb2=" << out_dst->nb[2] << ", nb3=" << out_dst->nb[3]; std::cerr << "))"); GGML_ASSERT(ggml_vk_dim01_contiguous(src0) || src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16); // NOLINT GGML_ASSERT(ggml_vk_dim01_contiguous(src1) || src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16); // NOLINT @@ -6815,14 +8020,13 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub const uint64_t ne12 = src1->ne[2]; const uint64_t ne13 = src1->ne[3]; - const uint64_t ne21 = dst->ne[1]; - const uint32_t stride_d = dst->nb[1] / ggml_type_size(dst->type); + const uint64_t ne21 = out_dst->ne[1]; + const uint32_t stride_d = out_dst->nb[1] / ggml_type_size(out_dst->type); const uint32_t stride_batch_d = stride_d*ne21; const uint64_t r2 = ne12 / ne02; const uint64_t r3 = ne13 / ne03; - ggml_backend_vk_buffer_context * dst_buf_ctx = (ggml_backend_vk_buffer_context *)dst->buffer->context; ggml_backend_vk_buffer_context * src0_buf_ctx = (ggml_backend_vk_buffer_context *)src0->buffer->context; ggml_backend_vk_buffer_context * src1_buf_ctx = (ggml_backend_vk_buffer_context *)src1->buffer->context; @@ -6847,10 +8051,16 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub ggml_type f16_type = src0->type == GGML_TYPE_BF16 ? GGML_TYPE_BF16 : GGML_TYPE_F16; const bool is_coopmat = ctx->device->coopmat_support || ctx->device->coopmat2; - const bool can_quantize_y = (ctx->device->integer_dot_product || ctx->device->coopmat_int_support) && + const bool can_quantize_y = !use_mm_fused && (ctx->device->integer_dot_product || ctx->device->coopmat_int_support) && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && (ne11 * ne10) % 4 == 0 && ggml_vk_src1_prec_allows_q8(dst); - // coopmat shaders are fast enough that memory bandwidth reduction from f16 activations helps - const bool prefer_f16_b = is_coopmat && ggml_is_quantized(src0->type) && src1->type == GGML_TYPE_F32; + // coopmat shaders are fast enough that memory bandwidth reduction from f16 activations helps on PCIe GPUs; + // on unified-memory Apple AGX, direct F32 vec4/mat2x4 reads avoid an extra cpy_f32_f16 dispatch + barrier. + static const int gen_mm_f16_b = getenv("GGML_VK_MM_F16_B") ? atoi(getenv("GGML_VK_MM_F16_B")) : -1; + const bool allow_f32_f16_b = src0->type == GGML_TYPE_F32 && ggml_vk_is_apple_agx_coopmat(ctx->device) && + !ggml_vk_exact_s8_enabled() && !gen_batch_invariant_for_n(ne11) && ne11 >= 128; + const bool prefer_f16_b = is_coopmat && (ggml_is_quantized(src0->type) || allow_f32_f16_b) && src1->type == GGML_TYPE_F32 && + (ggml_vk_exact_s8_enabled() || gen_mm_f16_b > 0 || (gen_mm_f16_b < 0 && + (!ggml_vk_is_apple_agx_coopmat(ctx->device) || get_misalign_bytes(ctx, src1) != 0 || ne11 >= 128))); const ggml_type y_kernel_type = ggml_vk_mul_mat_b_type(ctx, src0->type, src1, f16_type, ggml_prec_get_acc(dst), false, can_quantize_y, prefer_f16_b); @@ -6872,21 +8082,47 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub GGML_ASSERT(mmp_map != nullptr); - const uint32_t kpad = quantize_y ? 0 : ggml_vk_align_size(ne10, ggml_vk_guess_matmul_pipeline_align_map(ctx, *mmp_map, ne01, ne11, false)); - const bool aligned = !quantize_y && ne10 == kpad && ne01 > 8 && ne11 > 8; + const vk_subbuffer d_U_opt = (has_pair_glu || has_pair_mm) ? ggml_vk_tensor_subbuffer(ctx, src0_up) : vk_subbuffer{}; + const size_t raw_x_offs = src0_uma ? qx_buf_offset : (vk_tensor_offset(src0) + src0->view_offs); + const size_t raw_y_offs = src1_uma ? qy_buf_offset : (vk_tensor_offset(src1) + src1->view_offs); + const bool offsets_vec8_aligned = + (qx_needs_dequant || ggml_is_quantized(src0->type) || ((raw_x_offs % 32 == 0) && (!(has_pair_glu || has_pair_mm) || (d_U_opt.offset % 32 == 0)))) && + (qy_needs_dequant || quantize_y || (raw_y_offs % 32 == 0)); - vk_pipeline pipeline = ggml_vk_guess_matmul_pipeline_map(ctx, *mmp_map, ne01, ne11, aligned, false); + const uint64_t total_m = has_pair_mm ? (ne01 + (uint64_t)ctx->gen_pair_mm1->ne[0]) : ne01; + const uint32_t sel_m = has_pair_glu ? (uint32_t)(2 * ne01) : (uint32_t)total_m; + const uint32_t map_align = ggml_vk_guess_matmul_pipeline_align_map(ctx, *mmp_map, sel_m, ne11, false); + const uint32_t kpad = quantize_y ? 0 : ggml_vk_align_size(ne10, map_align); + const bool aligned = !quantize_y && ne10 == kpad && offsets_vec8_aligned && + ((ggml_vk_is_apple_agx_coopmat(ctx->device) && !ggml_vk_exact_s8_enabled()) || (ne01 > 8 && ne11 > 8)); - if (ggml_nbytes(src0) > ctx->device->properties.limits.maxStorageBufferRange) { + vk_pipeline pipeline = ggml_vk_guess_matmul_pipeline_map(ctx, *mmp_map, sel_m, ne11, aligned, false); + + const uint64_t max_a_nbytes = std::max(ggml_nbytes(src0), src0_up ? ggml_nbytes(src0_up) : 0); + if (max_a_nbytes > ctx->device->properties.limits.maxStorageBufferRange) { pipeline = ggml_vk_get_64b_indexing_pipeline(ctx, pipeline); } + if (has_pair_glu) { + GGML_ASSERT(pipeline && pipeline->gen_pair_glu); + pipeline = pipeline->gen_pair_glu; + } else if (has_pair_mm) { + GGML_ASSERT(pipeline && pipeline->gen_pair_mm); + pipeline = pipeline->gen_pair_mm; + } else if (use_mm_fused) { + if ((fusion_flags & ~(MAT_VEC_FUSION_FLAGS_BIAS0 | MAT_VEC_FUSION_FLAGS_BIAS1 | MAT_VEC_FUSION_FLAGS_ROW_SCALE0)) == 0u && pipeline && pipeline->gen_fast_ep) { + pipeline = pipeline->gen_fast_ep; + } else { + GGML_ASSERT(pipeline && pipeline->gen_glu); + pipeline = pipeline->gen_glu; + } + } // Reserve extra storage in the N dimension for the Y matrix, so we can avoid bounds-checking uint32_t padded_n = qy_needs_dequant ? ROUNDUP_POW2(ne11, pipeline->wg_denoms[1]) : ne11; const uint64_t x_ne = ggml_nelements(src0); // 128 elements per Q8_1 x4 block const uint64_t y_ne = padded_n * ne10 * ne12 * ne13; - const uint64_t d_ne = ggml_nelements(dst); + const uint64_t d_ne = ggml_nelements(out_dst); const bool narrow_n = mmp_map->back().narrow_n_max != 0 && ne11 <= mmp_map->back().narrow_n_max; const uint32_t split_k = ggml_vk_guess_split_k(ctx, ne01, ne11, ne10, disable_split_k, pipeline, narrow_n); @@ -6954,8 +8190,9 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub } } - vk_buffer d_D = dst_buf_ctx->dev_buffer; - const uint64_t d_buf_offset = vk_tensor_offset(dst) + dst->view_offs; + const vk_subbuffer out_d_sub = ggml_vk_tensor_subbuffer(ctx, out_dst); + vk_buffer d_D = out_d_sub.buffer; + const uint64_t d_buf_offset = out_d_sub.offset; GGML_ASSERT(d_D != nullptr); GGML_ASSERT(d_D->size >= d_buf_offset + d_sz); vk_buffer d_X; @@ -7005,30 +8242,21 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub ggml_vk_dispatch_pipeline(ctx, subctx, to_fp16_vk_0, { vk_subbuffer{ d_Qx, qx_buf_offset, qx_sz }, vk_subbuffer{ d_X, 0, x_sz } }, pc, { (uint32_t)(x_ne), 1, 1}); ggml_vk_sync_buffers(ctx, subctx); } + uint32_t y_ring_slot = 0; if (y_needs_reformat) { - if (ctx->prealloc_y_last_pipeline_used != to_fp16_vk_1.get() || - ctx->prealloc_y_last_tensor_used != src1 || - ctx->prealloc_y_last_k_padded) { - if (ctx->prealloc_y_need_sync) { - ggml_vk_sync_buffers(ctx, subctx); - } - ggml_vk_cpy_to_contiguous(ctx, subctx, to_fp16_vk_1, src1, ggml_vk_subbuffer(ctx, d_Qy, qy_buf_offset), ggml_vk_subbuffer(ctx, d_Y, 0)); - ctx->prealloc_y_last_pipeline_used = to_fp16_vk_1.get(); - ctx->prealloc_y_last_tensor_used = src1; - ctx->prealloc_y_last_k_padded = false; + bool cached = false; + y_ring_slot = ggml_vk_acquire_prealloc_y_slot(ctx, subctx, to_fp16_vk_1.get(), src1, false, cached); + y_buf_offset = y_ring_slot * ctx->prealloc_y_ring_stride; + if (!cached) { + ggml_vk_cpy_to_contiguous(ctx, subctx, to_fp16_vk_1, src1, ggml_vk_subbuffer(ctx, d_Qy, qy_buf_offset), vk_subbuffer{ d_Y, y_buf_offset, y_sz }); } } if (quantize_y) { - if (ctx->prealloc_y_last_pipeline_used != to_q8_1.get() || - ctx->prealloc_y_last_tensor_used != src1 || - ctx->prealloc_y_last_k_padded) { - if (ctx->prealloc_y_need_sync) { - ggml_vk_sync_buffers(ctx, subctx); - } - ggml_vk_quantize_q8_1(ctx, subctx, ggml_vk_subbuffer(ctx, d_Qy, qy_buf_offset), ggml_vk_subbuffer(ctx, d_Y, 0), y_ne); - ctx->prealloc_y_last_pipeline_used = to_q8_1.get(); - ctx->prealloc_y_last_tensor_used = src1; - ctx->prealloc_y_last_k_padded = false; + bool cached = false; + y_ring_slot = ggml_vk_acquire_prealloc_y_slot(ctx, subctx, to_q8_1.get(), src1, false, cached); + y_buf_offset = y_ring_slot * ctx->prealloc_y_ring_stride; + if (!cached) { + ggml_vk_quantize_q8_1(ctx, subctx, ggml_vk_subbuffer(ctx, d_Qy, qy_buf_offset), vk_subbuffer{ d_Y, y_buf_offset, CEIL_DIV(y_sz, 144) * 144 }, y_ne); } } @@ -7054,21 +8282,43 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub : (uint64_t)src1->nb[1]; // compute - ggml_vk_matmul( - ctx, subctx, pipeline, - { d_X, x_buf_offset, x_range }, { d_Y, y_buf_offset, y_range }, - ggml_vk_subbuffer(ctx, d_D, d_buf_offset), { ctx->prealloc_split_k, 0, d_sz * split_k }, - ne01, ne11, ne10, - ne10, ne10, stride_d, stride_batch_x, stride_batch_y, stride_batch_d, - split_k, ne12*ne13, ne02, ne12, r2, r3, padded_n, - b_col_bytes, (uint64_t)dst->nb[1] - ); // NOLINT + if (use_mm_fused) { + const vk_subbuffer d_sub = ggml_vk_subbuffer(ctx, d_D, d_buf_offset); + const vk_subbuffer x_sub = { d_X, x_buf_offset, x_range }; + const vk_subbuffer y_sub = { d_Y, y_buf_offset, y_range }; + const vk_subbuffer u_sub = (has_pair_glu || has_pair_mm) ? d_U_opt : x_sub; + const vk_subbuffer aux0_sub = aux0_t ? ggml_vk_tensor_subbuffer(ctx, aux0_t) : d_sub; + const vk_subbuffer aux1_sub = aux1_t ? ggml_vk_tensor_subbuffer(ctx, aux1_t) : d_sub; + const uint32_t stride_d2 = has_pair_mm ? (uint32_t)(out_dst2->nb[1] / ggml_type_size(out_dst2->type)) : 0u; + const uint32_t stride_batch_d2 = has_pair_mm ? (uint32_t)(stride_d2 * out_dst2->ne[1]) : 0u; + ggml_vk_matmul_fused( + ctx, subctx, pipeline, + x_sub, y_sub, d_sub, u_sub, aux0_sub, aux1_sub, + (uint32_t)total_m, ne11, ne10, + ne10, ne10, stride_d, stride_batch_x, stride_batch_y, stride_batch_d, + ne12*ne13, ne02, ne12, r2, r3, padded_n, + b_col_bytes, (uint64_t)out_dst->nb[1], + aux0_col_bytes, aux1_col_bytes, fusion_flags, + has_pair_mm ? (uint32_t)ne01 : 0u, stride_d2, stride_batch_d2 + ); + } else { + ggml_vk_matmul( + ctx, subctx, pipeline, + { d_X, x_buf_offset, x_range }, { d_Y, y_buf_offset, y_range }, + ggml_vk_subbuffer(ctx, d_D, d_buf_offset), { ctx->prealloc_split_k, 0, d_sz * split_k }, + ne01, ne11, ne10, + ne10, ne10, stride_d, stride_batch_x, stride_batch_y, stride_batch_d, + split_k, ne12*ne13, ne02, ne12, r2, r3, padded_n, + b_col_bytes, (uint64_t)out_dst->nb[1] + ); // NOLINT + } if (x_non_contig || qx_needs_dequant) { ctx->prealloc_x_need_sync = true; } if (y_needs_reformat || quantize_y) { ctx->prealloc_y_need_sync = true; + ctx->prealloc_y_slot_need_sync[y_ring_slot] = true; } } @@ -7158,8 +8408,23 @@ static bool ggml_vk_should_use_mmvq(const vk_device& device, uint32_t m, uint32_ static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx, bool swap_inputs = false) { ggml_tensor * dst = cgraph->nodes[node_idx]; + const bool has_pair_mm = !swap_inputs && ctx->gen_pair_mm1 != nullptr; + const bool has_pair_glu = !swap_inputs && !has_pair_mm && + ((ctx->num_additional_fused_ops == 2 && cgraph->nodes[node_idx + 2]->op == GGML_OP_GLU) || + ctx->gen_lora_gate_src0 != nullptr); const ggml_tensor * src0 = dst->src[swap_inputs ? 1 : 0]; const ggml_tensor * src1 = dst->src[swap_inputs ? 0 : 1]; + const ggml_tensor * src0_up = nullptr; + if (has_pair_glu) { + if (ctx->gen_lora_gate_src0 != nullptr) { + src0 = ctx->gen_lora_gate_src0; + src0_up = ctx->gen_lora_up_src0; + } else { + const ggml_tensor * glu = cgraph->nodes[node_idx + 2]; + src0 = glu->src[0]->src[0]; + src0_up = glu->src[1]->src[0]; + } + } VK_LOG_DEBUG("ggml_vk_mul_mat_vec_q_f16((" << src0 << ", name=" << src0->name << ", type=" << src0->type << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3]; std::cerr << "), (" << src1 << ", name=" << src1->name << ", type=" << src1->type << ", ne0=" << src1->ne[0] << ", ne1=" << src1->ne[1] << ", ne2=" << src1->ne[2] << ", ne3=" << src1->ne[3] << ", nb0=" << src1->nb[0] << ", nb1=" << src1->nb[1] << ", nb2=" << src1->nb[2] << ", nb3=" << src1->nb[3]; @@ -7182,6 +8447,8 @@ static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context& const uint64_t ne21 = dst->ne[swap_inputs ? 0 : 1]; // const uint64_t ne22 = dst->ne[2]; // const uint64_t ne23 = dst->ne[3]; + const uint64_t pair_m1 = has_pair_mm ? (uint64_t)ctx->gen_pair_mm1->ne[0] : 0; + const uint64_t total_m = has_pair_mm ? (ne01 + pair_m1) : ne20; const uint64_t r2 = ne12 / ne02; const uint64_t r3 = ne13 / ne03; @@ -7199,7 +8466,7 @@ static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context& const ggml_type y_kernel_type = widen_bf16 ? GGML_TYPE_F32 : src1->type; const bool f16_f32_kernel = y_kernel_type == GGML_TYPE_F32; - bool quantize_y = ctx->device->integer_dot_product && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && !y_non_contig && (ne11 * ne10) % 4 == 0 && (swap_inputs || ggml_vk_src1_prec_allows_q8(dst)) && ggml_vk_should_use_mmvq(ctx->device, ne01, ne11, ne10, src0->type); + bool quantize_y = !has_pair_glu && !has_pair_mm && ctx->gen_lora_r == 0 && ctx->device->integer_dot_product && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && !y_non_contig && (ne11 * ne10) % 4 == 0 && (swap_inputs || ggml_vk_src1_prec_allows_q8(dst)) && ggml_vk_should_use_mmvq(ctx->device, ne01, ne11, ne10, src0->type); vk_pipeline to_fp16_vk_0 = nullptr; vk_pipeline to_fp16_vk_1 = nullptr; @@ -7212,23 +8479,164 @@ static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context& to_fp16_vk_1 = ggml_vk_get_to_fp16(ctx, src1->type); } + static const bool mma_disabled = getenv("GGML_VK_DISABLE_MMA") != nullptr; + static const int env_mma_min_n = getenv("GGML_VK_MMA_MIN_N") ? atoi(getenv("GGML_VK_MMA_MIN_N")) : 8; + static const int env_mma_max_n = getenv("GGML_VK_MMA_MAX_N") ? atoi(getenv("GGML_VK_MMA_MAX_N")) : 4096; + static const int env_mma_cfg = getenv("GGML_VK_MMA_CFG") ? atoi(getenv("GGML_VK_MMA_CFG")) : -1; + static const int env_mma_rt = getenv("GGML_VK_MMA_RT") ? atoi(getenv("GGML_VK_MMA_RT")) : -1; + static const int env_mma_f32_rt = getenv("GGML_VK_MMA_F32_RT") ? atoi(getenv("GGML_VK_MMA_F32_RT")) : -1; + static const int env_mma_f32_f16_min_n = getenv("GGML_VK_MMA_F32_F16_MIN_N") ? atoi(getenv("GGML_VK_MMA_F32_F16_MIN_N")) : 64; + static const int env_mma_bf16_min_n = getenv("GGML_VK_MMA_BF16_MIN_N") ? atoi(getenv("GGML_VK_MMA_BF16_MIN_N")) : 64; + static const uint32_t env_mma_bf16_kp = getenv("GGML_VK_MMA_BF16_KP") ? (uint32_t)atoi(getenv("GGML_VK_MMA_BF16_KP")) : 2u; + const bool use_f32_f16_mma = src0->type == GGML_TYPE_F32 && + !ggml_vk_exact_s8_enabled() && + !gen_batch_invariant_for_n(ne11) && + env_mma_f32_f16_min_n >= 0 && + (int64_t)ne11 >= env_mma_f32_f16_min_n; + const bool mma_can_bf16 = (use_f32_f16_mma || src0->type != GGML_TYPE_F32) && + !ggml_vk_exact_s8_enabled() && + !gen_batch_invariant_for_n(ne11) && + env_mma_bf16_min_n >= 0 && + (int64_t)ne11 >= env_mma_bf16_min_n && + (ne00 % (64u * env_mma_bf16_kp) == 0) && + !y_non_contig && ggml_is_contiguous(src1) && get_misalign_bytes(ctx, src1) == 0 && + (src1->type == GGML_TYPE_F16 || (src1->type == GGML_TYPE_F32 && ctx->device->pipeline_contig_cpy_f32_f16 != nullptr)); + const uint32_t mma_bf16_idx = mma_can_bf16 ? 1u : 0u; + const bool f32_force_rt2 = (src0->type == GGML_TYPE_F32) && + (env_mma_f32_rt == 2 || (env_mma_f32_rt < 0 && (!use_f32_f16_mma || total_m <= 128))); + const bool small_m_force_rt2 = (total_m <= 128 && ne11 >= 64 && ne11 <= 512); + uint32_t mma_rt_idx = 2u; + if (env_mma_rt == 1) { mma_rt_idx = 0u; } + else if (env_mma_rt == 2) { mma_rt_idx = 1u; } + else if (env_mma_rt == 4) { mma_rt_idx = 2u; } + else if (env_mma_rt == 6 && (use_f32_f16_mma || src0->type != GGML_TYPE_F32)) { mma_rt_idx = 3u; } + else if (env_mma_rt == 8 && (use_f32_f16_mma || src0->type != GGML_TYPE_F32)) { mma_rt_idx = 4u; } + else if (ne11 <= 8) { + mma_rt_idx = 0u; + } else if (ne11 <= 16 || f32_force_rt2 || small_m_force_rt2 || (total_m <= 2048 && ne11 <= 32)) { + mma_rt_idx = 1u; + } else { + mma_rt_idx = 2u; // RT=4 (BN=32, 8 accumulators = 32 half-regs, 8 KiB sh_b -> peak AGX occupancy) + } + static constexpr uint32_t k_mma_rts_sel[5] = { 1u, 2u, 4u, 6u, 8u }; + const uint32_t mma_rt = k_mma_rts_sel[mma_rt_idx]; + const uint32_t mma_k_step = ((src0->type == GGML_TYPE_Q8_0 && mma_rt_idx == 0u) || src0->type == GGML_TYPE_Q4_0) ? 32u : 64u; + const bool use_mv_mma = !mma_disabled && + ggml_vk_is_apple_agx_coopmat(ctx->device) && + !ggml_vk_exact_s8_enabled() && + !gen_batch_invariant_for_n(ne11) && + !swap_inputs && + (int64_t)ne11 >= env_mma_min_n && (int64_t)ne11 <= env_mma_max_n && + total_m >= 4 && + (y_kernel_type == GGML_TYPE_F32 || (mma_bf16_idx == 1u && y_kernel_type == GGML_TYPE_F16)) && + dst->type == GGML_TYPE_F32 && + (ne00 % mma_k_step == 0) && + (!has_pair_mm || (ne01 % 8 == 0 && ctx->gen_pair_mm1->src[0]->type == src0->type)) && + (use_f32_f16_mma ? ctx->device->pipeline_mul_mv_mma_f32_f16[mma_rt_idx][4][mma_bf16_idx] != nullptr + : ctx->device->pipeline_mul_mv_mma[src0->type][mma_rt_idx][4][mma_bf16_idx] != nullptr); + + static const int env_col_tile = getenv("GGML_VK_MMV_COL_TILE") ? atoi(getenv("GGML_VK_MMV_COL_TILE")) : 4; + const uint32_t col_tile = env_col_tile > 0 ? (uint32_t)env_col_tile : 0u; + const bool use_col_split = !use_mv_mma && col_tile > 0 && ggml_vk_is_apple_agx_coopmat(ctx->device) && + !swap_inputs && batch_n && ne12 * ne13 == 1 && ne11 > col_tile; + const uint32_t full_col_tiles = use_col_split ? (uint32_t)(ne11 / col_tile) : 0u; + const uint32_t rem_col_count = use_col_split ? (uint32_t)(ne11 % col_tile) : 0u; + const uint64_t eff_num_cols = use_col_split ? (uint64_t)col_tile : ne11; + // Check for mmq first - vk_pipeline dmmv = quantize_y ? ggml_vk_get_dequantize_mul_mat_vec(ctx, src0->type, GGML_TYPE_Q8_1, ne11, ne20, ne00) : nullptr; + vk_pipeline dmmv = nullptr; + vk_pipeline dmmv_rem = nullptr; vk_pipeline to_q8_1 = nullptr; + if (use_mv_mma) { + quantize_y = false; + const uint32_t n_steps = (uint32_t)(ne00 / mma_k_step); + const uint64_t n_col_tiles = CEIL_DIV(ne11, 8u * mma_rt) * (uint64_t)(ne12 * ne13); + const uint64_t wg_64 = CEIL_DIV(total_m, 64) * n_col_tiles; + const uint64_t wg_32 = CEIL_DIV(total_m, 32) * n_col_tiles; + uint32_t ci = 4u; + if (env_mma_cfg >= 0 && env_mma_cfg <= 7) { + ci = (uint32_t)env_mma_cfg; + if ((ci == 7u || ci == 6u || ci == 5u || ci == 3u) && (ne00 % 64 != 0)) { + ci = 4u; + } + } else if (small_m_force_rt2 && n_steps >= 8u) { + ci = (total_m >= 16) ? 2u : 1u; // NSG=4 K-split keeps all 40 cores saturated on small-M GDN/LoRA gates + } else if ((ne00 % 64 == 0) && total_m >= 64 && wg_64 >= 48) { + ci = 5u; // WG_SG=4 (128 threads), MSG=4, NT=2 (NT=1 BM=32 on _pairglu), zero barrier stragglers & peak occupancy + } else if ((ne00 % 64 == 0) && total_m >= 32 && wg_32 >= 32) { + ci = 3u; // MSG=2, NSG=1, NT=2 (BM=32 cooperative sh_b, 0 spills, fills all 40 cores!) + } else if (total_m <= 128 && n_steps >= 16u) { + ci = 0u; // NSG=8, NT=1 + } else if (total_m <= 512 && n_steps >= 8u) { + ci = 1u; // NSG=4, NT=1 + } else if (mma_rt_idx == 0u && total_m <= 2048 && n_steps >= 8u) { + ci = 2u; // NSG=4, NT=2 + } else { + ci = 4u; // NSG=1, NT=2 + } + dmmv = use_f32_f16_mma + ? ctx->device->pipeline_mul_mv_mma_f32_f16[mma_rt_idx][ci][mma_bf16_idx] + : ctx->device->pipeline_mul_mv_mma[src0->type][mma_rt_idx][ci][mma_bf16_idx]; + } else { + dmmv = quantize_y ? ggml_vk_get_dequantize_mul_mat_vec(ctx, src0->type, GGML_TYPE_Q8_1, eff_num_cols, total_m, ne00) : nullptr; + } + if (dmmv == nullptr) { // Fall back to f16 dequant mul mat - dmmv = ggml_vk_get_dequantize_mul_mat_vec(ctx, src0->type, y_kernel_type, ne11, ne20, ne00); + dmmv = ggml_vk_get_dequantize_mul_mat_vec(ctx, src0->type, y_kernel_type, eff_num_cols, total_m, ne00); quantize_y = false; } + if (rem_col_count > 0) { + dmmv_rem = quantize_y + ? ggml_vk_get_dequantize_mul_mat_vec(ctx, src0->type, GGML_TYPE_Q8_1, rem_col_count, total_m, ne00) + : ggml_vk_get_dequantize_mul_mat_vec(ctx, src0->type, y_kernel_type, rem_col_count, total_m, ne00); + if (dmmv_rem == nullptr && quantize_y) { + quantize_y = false; + dmmv = ggml_vk_get_dequantize_mul_mat_vec(ctx, src0->type, y_kernel_type, eff_num_cols, total_m, ne00); + dmmv_rem = ggml_vk_get_dequantize_mul_mat_vec(ctx, src0->type, y_kernel_type, rem_col_count, total_m, ne00); + } + } if (quantize_y) { to_q8_1 = ggml_vk_get_quantize_pipeline(ctx, GGML_TYPE_Q8_1); } - if (ggml_nbytes(src0) > ctx->device->properties.limits.maxStorageBufferRange) { - dmmv = ggml_vk_get_64b_indexing_pipeline(ctx, dmmv); - } + const uint64_t max_a_nbytes = std::max(ggml_nbytes(src0), + src0_up ? ggml_nbytes(src0_up) : (has_pair_mm ? ggml_nbytes(ctx->gen_pair_mm1->src[0]) : 0)); + const bool mv_mma_bias_only = use_mv_mma && + ((ctx->gen_mm_dst == nullptr && ctx->num_additional_fused_ops > 0 && + !(ctx->num_additional_fused_ops == 1 && cgraph->nodes[node_idx + 1]->op == GGML_OP_GLU)) || + (ctx->gen_mm_dst != nullptr && (ctx->gen_mm_flags & ~(uint32_t)MAT_VEC_FUSION_FLAGS_BIAS0) == 0)); + auto specialize_dmmv = [&](vk_pipeline p) -> vk_pipeline { + if (!p) { + return nullptr; + } + if (max_a_nbytes > ctx->device->properties.limits.maxStorageBufferRange) { + p = ggml_vk_get_64b_indexing_pipeline(ctx, p); + } + if (has_pair_glu) { + GGML_ASSERT(p && p->gen_pair_glu); + p = p->gen_pair_glu; + } else if (has_pair_mm) { + GGML_ASSERT(p && p->gen_pair_mm); + p = p->gen_pair_mm; + } else if (mv_mma_bias_only && p->gen_fast_ep) { + p = p->gen_fast_ep; + } else if ((use_mv_mma && ctx->num_additional_fused_ops > 0) || + (ctx->num_additional_fused_ops == 1 && cgraph->nodes[node_idx + 1]->op == GGML_OP_GLU) || + ctx->gen_mm_dst != nullptr) { + GGML_ASSERT(p && p->gen_glu); + p = p->gen_glu; // gen: single-op epilogue twin (GEN_GLU = 1 / FUSE_MODE = 1) + } + if (ctx->gen_lora_r > 0) { + GGML_ASSERT(p && p->gen_lora_twin); + p = p->gen_lora_twin; + } + return p; + }; + dmmv = specialize_dmmv(dmmv); + dmmv_rem = specialize_dmmv(dmmv_rem); const bool qx_needs_dequant = x_non_contig; const bool qy_needs_dequant = !quantize_y && ((src1->type != GGML_TYPE_F16 && !f16_f32_kernel) || y_non_contig); @@ -7248,6 +8656,9 @@ static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context& const uint64_t y_sz = quantize_y ? (ggml_vk_align_size(y_ne, 128) * ggml_type_size(GGML_TYPE_Q8_1) / ggml_blck_size(GGML_TYPE_Q8_1)) : (f16_f32_kernel ? sizeof(float) * y_ne : sizeof(ggml_fp16_t) * y_ne); + const bool mma_use_y_f16_ring = use_mv_mma && mma_bf16_idx == 1u && src1->type == GGML_TYPE_F32; + const uint64_t mma_y_f16_sz = sizeof(ggml_fp16_t) * y_ne; + { if ( (qx_needs_dequant && x_sz > ctx->device->properties.limits.maxStorageBufferRange) || @@ -7262,6 +8673,9 @@ static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context& ctx->prealloc_size_y = y_sz; ggml_vk_preallocate_buffers(ctx, subctx); } + if (mma_use_y_f16_ring) { + ggml_vk_reserve_prealloc_y(ctx, subctx, mma_y_f16_sz); + } // Request descriptor sets if (qx_needs_dequant) { @@ -7273,9 +8687,13 @@ static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context& if (quantize_y) { ggml_pipeline_request_descriptor_sets(ctx, to_q8_1, 1); } + if (mma_use_y_f16_ring) { + ggml_pipeline_request_descriptor_sets(ctx, ctx->device->pipeline_contig_cpy_f32_f16, 1); + } } - vk_subbuffer d_D = ggml_vk_tensor_subbuffer(ctx, cgraph->nodes[node_idx + ctx->num_additional_fused_ops]); + vk_subbuffer d_D = ggml_vk_tensor_subbuffer(ctx, has_pair_mm ? (ctx->gen_mm_dst ? ctx->gen_mm_dst : dst) : (ctx->gen_mm_dst ? ctx->gen_mm_dst : cgraph->nodes[node_idx + ctx->num_additional_fused_ops])); + vk_subbuffer d_D2 = has_pair_mm ? ggml_vk_tensor_subbuffer(ctx, ctx->gen_pair_dst1) : d_D; vk_subbuffer d_Qx = ggml_vk_tensor_subbuffer(ctx, src0); vk_subbuffer d_Qy = ggml_vk_tensor_subbuffer(ctx, src1); vk_subbuffer d_X, d_Y; @@ -7300,31 +8718,30 @@ static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context& GGML_ASSERT(x_sz == ggml_vk_align_size(ggml_type_size(src0->type) * x_ne, ctx->device->properties.limits.minStorageBufferOffsetAlignment)); ggml_vk_cpy_to_contiguous(ctx, subctx, to_fp16_vk_0, src0, d_Qx, d_X); } + uint32_t y_ring_slot = 0; if (y_non_contig) { GGML_ASSERT(y_sz == ggml_type_size(y_kernel_type) * y_ne); - if (ctx->prealloc_y_last_pipeline_used != to_fp16_vk_1.get() || - ctx->prealloc_y_last_tensor_used != src1 || - ctx->prealloc_y_last_k_padded) { - if (ctx->prealloc_y_need_sync) { - ggml_vk_sync_buffers(ctx, subctx); - } + bool cached = false; + y_ring_slot = ggml_vk_acquire_prealloc_y_slot(ctx, subctx, to_fp16_vk_1.get(), src1, false, cached); + d_Y = { ctx->prealloc_y, y_ring_slot * ctx->prealloc_y_ring_stride, y_sz }; + if (!cached) { ggml_vk_cpy_to_contiguous(ctx, subctx, to_fp16_vk_1, src1, d_Qy, d_Y); - ctx->prealloc_y_last_pipeline_used = to_fp16_vk_1.get(); - ctx->prealloc_y_last_tensor_used = src1; - ctx->prealloc_y_last_k_padded = false; + } + } + if (mma_use_y_f16_ring) { + bool cached = false; + y_ring_slot = ggml_vk_acquire_prealloc_y_slot(ctx, subctx, ctx->device->pipeline_contig_cpy_f32_f16.get(), src1, false, cached); + d_Y = { ctx->prealloc_y, y_ring_slot * ctx->prealloc_y_ring_stride, mma_y_f16_sz }; + if (!cached) { + ggml_vk_cpy_to_contiguous(ctx, subctx, ctx->device->pipeline_contig_cpy_f32_f16, src1, d_Qy, d_Y); } } if (quantize_y) { - if (ctx->prealloc_y_last_pipeline_used != to_q8_1.get() || - ctx->prealloc_y_last_tensor_used != src1 || - ctx->prealloc_y_last_k_padded) { - if (ctx->prealloc_y_need_sync) { - ggml_vk_sync_buffers(ctx, subctx); - } + bool cached = false; + y_ring_slot = ggml_vk_acquire_prealloc_y_slot(ctx, subctx, to_q8_1.get(), src1, false, cached); + d_Y = { ctx->prealloc_y, y_ring_slot * ctx->prealloc_y_ring_stride, y_sz }; + if (!cached) { ggml_vk_quantize_q8_1(ctx, subctx, d_Qy, d_Y, y_ne); - ctx->prealloc_y_last_pipeline_used = to_q8_1.get(); - ctx->prealloc_y_last_tensor_used = src1; - ctx->prealloc_y_last_k_padded = false; } } @@ -7343,10 +8760,11 @@ static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context& const uint32_t max_groups_x = ctx->device->properties.limits.maxComputeWorkGroupCount[0]; - uint32_t groups_x = ne01; + const uint64_t launch_m = has_pair_mm ? total_m : ne01; + uint32_t groups_x = (uint32_t)launch_m; uint32_t groups_z = 1; - if (ne01 > max_groups_x) { + if (launch_m > max_groups_x) { groups_z = 64; groups_x = CEIL_DIV(groups_x, groups_z); } @@ -7354,7 +8772,35 @@ static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context& uint32_t fusion_flags = 0; vk_subbuffer d_F0 = d_D; - if (ctx->num_additional_fused_ops > 0) { + vk_subbuffer d_F1 = d_D; + vk_subbuffer d_F2 = d_D; + if (has_pair_glu) { + d_F0 = ggml_vk_tensor_subbuffer(ctx, src0_up); + fusion_flags |= MAT_VEC_FUSION_FLAGS_PAIR_GLU; + } else if (has_pair_mm) { + d_F0 = ggml_vk_tensor_subbuffer(ctx, ctx->gen_pair_mm1->src[0]); + if (ctx->gen_mm_f0 != nullptr) { + d_F1 = ggml_vk_tensor_subbuffer(ctx, ctx->gen_mm_f0); + } + if (ctx->gen_mm_f1 != nullptr) { + d_F2 = ggml_vk_tensor_subbuffer(ctx, ctx->gen_mm_f1); + } + fusion_flags |= ctx->gen_mm_flags; + } else if (ctx->gen_mm_dst != nullptr) { + fusion_flags |= ctx->gen_mm_flags; + if (ctx->gen_mm_f0 != nullptr) { + d_F0 = ggml_vk_tensor_subbuffer(ctx, ctx->gen_mm_f0); + } + if (ctx->gen_mm_f1 != nullptr) { + d_F1 = ggml_vk_tensor_subbuffer(ctx, ctx->gen_mm_f1); + } + } else if (ctx->num_additional_fused_ops == 1 && cgraph->nodes[node_idx + 1]->op == GGML_OP_GLU) { + // gen: SWIGLU epilogue (GGML_GEN_FUSE=glu) + const ggml_tensor * glu = cgraph->nodes[node_idx + 1]; + const bool is_gate = glu->src[0] == dst; + d_F0 = ggml_vk_tensor_subbuffer(ctx, is_gate ? glu->src[1] : glu->src[0]); + fusion_flags |= is_gate ? MAT_VEC_FUSION_FLAGS_GLU_A : MAT_VEC_FUSION_FLAGS_GLU_B; + } else if (ctx->num_additional_fused_ops > 0) { const ggml_tensor * add = cgraph->nodes[node_idx + 1]; const ggml_tensor * bias = add->src[0] == dst ? add->src[1] : add->src[0]; @@ -7362,8 +8808,7 @@ static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context& fusion_flags |= MAT_VEC_FUSION_FLAGS_BIAS0; } - vk_subbuffer d_F1 = d_D; - if (ctx->num_additional_fused_ops == 2) { + if (!has_pair_glu && !has_pair_mm && ctx->gen_mm_dst == nullptr && ctx->num_additional_fused_ops == 2) { const ggml_tensor * add = cgraph->nodes[node_idx + 2]; const ggml_tensor * bias = add->src[0] == cgraph->nodes[node_idx + 1] ? add->src[1] : add->src[0]; @@ -7371,35 +8816,151 @@ static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context& fusion_flags |= MAT_VEC_FUSION_FLAGS_BIAS1; } - ggml_pipeline_request_descriptor_sets(ctx, dmmv, CEIL_DIV(ne12 * ne13, ctx->device->properties.limits.maxComputeWorkGroupCount[1])); - - uint32_t base_work_group_y = 0; - while (base_work_group_y < ne12 * ne13) { - - uint32_t groups_y = std::min((uint32_t)(ne12 * ne13) - base_work_group_y, ctx->device->properties.limits.maxComputeWorkGroupCount[1]); - const vk_mat_vec_push_constants pc = { - (uint32_t)ne00, (uint32_t)ne10, (uint32_t)ne10, (uint32_t)ne01, - stride_batch_x, stride_batch_y, stride_batch_d, - fusion_flags, base_work_group_y, - (uint32_t)ne02, (uint32_t)ne12, (uint32_t)r2, (uint32_t)r3, - }; - ggml_vk_dispatch_pipeline(ctx, subctx, dmmv, - { - d_X, - d_Y, - d_D, - d_F0, - d_F1, - }, - pc, { groups_x, groups_y, groups_z }); - base_work_group_y += groups_y; + vk_subbuffer d_LB0 = d_D; + vk_subbuffer d_LH0 = d_D; + vk_subbuffer d_LB1 = d_D; + vk_subbuffer d_LH1 = d_D; + const uint32_t lora_r = ctx->gen_lora_r; + const float lora_scale0 = ctx->gen_lora_scale0; + const float lora_scale1 = ctx->gen_lora_scale1; + if (lora_r > 0) { + if (ctx->gen_lora_b0 != nullptr && ctx->gen_lora_h0 != nullptr) { + d_LB0 = ggml_vk_tensor_subbuffer(ctx, ctx->gen_lora_b0); + d_LH0 = ggml_vk_tensor_subbuffer(ctx, ctx->gen_lora_h0); + fusion_flags |= MAT_VEC_FUSION_FLAGS_LORA0; + } + if (ctx->gen_lora_b1 != nullptr && ctx->gen_lora_h1 != nullptr) { + d_LB1 = ggml_vk_tensor_subbuffer(ctx, ctx->gen_lora_b1); + d_LH1 = ggml_vk_tensor_subbuffer(ctx, ctx->gen_lora_h1); + fusion_flags |= MAT_VEC_FUSION_FLAGS_LORA1; + } + } + + if (use_mv_mma) { + if (mma_bf16_idx == 1u) { + fusion_flags |= MAT_VEC_FUSION_FLAGS_B_F16; + } + const uint32_t max_wg_z = ctx->device->properties.limits.maxComputeWorkGroupCount[2]; + const uint32_t total_batches = (uint32_t)(ne12 * ne13); + const uint32_t mma_bstride_x = qx_needs_dequant ? (uint32_t)(ne00 * ne01) : (uint32_t)ggml_vk_batch_stride(src0); + const uint32_t mma_bstride_y = (y_non_contig || mma_use_y_f16_ring) ? (uint32_t)(ne10 * ne11) : (uint32_t)ggml_vk_batch_stride(src1); + const uint32_t mma_stride_d = has_pair_mm ? (uint32_t)ne01 : (uint32_t)ne20; + const uint32_t mma_bstride_d = (uint32_t)(mma_stride_d * ne21); + ggml_pipeline_request_descriptor_sets(ctx, dmmv, CEIL_DIV(total_batches, max_wg_z)); + uint32_t base_wg_z = 0; + while (base_wg_z < total_batches) { + const uint32_t cur_z = std::min(total_batches - base_wg_z, max_wg_z); + const vk_mat_mv_mma_push_constants pc = { + (uint32_t)launch_m, (uint32_t)ne11, (uint32_t)ne00, + (uint32_t)ne00, (uint32_t)ne10, mma_stride_d, + mma_bstride_x, mma_bstride_y, mma_bstride_d, + fusion_flags, base_wg_z, + (uint32_t)ne02, (uint32_t)ne12, (uint32_t)r2, (uint32_t)r3, + has_pair_mm ? (uint32_t)ne01 : 0u, + has_pair_mm ? (uint32_t)pair_m1 : 0u, + has_pair_mm ? (uint32_t)(pair_m1 * ne21) : 0u, + lora_r, lora_scale0, lora_scale1, + }; + if (lora_r > 0) { + ggml_vk_dispatch_pipeline(ctx, subctx, dmmv, + { d_X, d_Y, d_D, d_F0, d_F1, d_D2, d_F2, d_LB0, d_LH0, d_LB1, d_LH1 }, + pc, { (uint32_t)launch_m, (uint32_t)ne11, cur_z }); + } else { + ggml_vk_dispatch_pipeline(ctx, subctx, dmmv, + { d_X, d_Y, d_D, d_F0, d_F1, d_D2, d_F2 }, + pc, { (uint32_t)launch_m, (uint32_t)ne11, cur_z }); + } + base_wg_z += cur_z; + } + } else if (use_col_split) { + fusion_flags |= MAT_VEC_FUSION_FLAGS_COL_SPLIT; + const uint32_t max_wg_y = ctx->device->properties.limits.maxComputeWorkGroupCount[1]; + ggml_pipeline_request_descriptor_sets(ctx, dmmv, CEIL_DIV(full_col_tiles, max_wg_y)); + if (rem_col_count > 0) { + GGML_ASSERT(dmmv_rem != nullptr); + ggml_pipeline_request_descriptor_sets(ctx, dmmv_rem, 1); + } + uint32_t base_tile = 0; + while (base_tile < full_col_tiles) { + const uint32_t groups_y = std::min(full_col_tiles - base_tile, max_wg_y); + const vk_mat_vec_push_constants pc = { + (uint32_t)ne00, (uint32_t)ne10, (uint32_t)ne10, (uint32_t)launch_m, + stride_batch_x, stride_batch_y, stride_batch_d, + fusion_flags, base_tile * col_tile, + (uint32_t)ne02, (uint32_t)ne12, (uint32_t)r2, (uint32_t)r3, + has_pair_mm ? (uint32_t)ne01 : 0u, + has_pair_mm ? (uint32_t)pair_m1 : 0u, + has_pair_mm ? (uint32_t)(batch_n ? pair_m1 : pair_m1 * ne21) : 0u, + lora_r, lora_scale0, lora_scale1, + }; + if (lora_r > 0) { + ggml_vk_dispatch_pipeline(ctx, subctx, dmmv, + { d_X, d_Y, d_D, d_F0, d_F1, d_D2, d_F2, d_LB0, d_LH0, d_LB1, d_LH1 }, + pc, { groups_x, groups_y, groups_z }); + } else { + ggml_vk_dispatch_pipeline(ctx, subctx, dmmv, + { d_X, d_Y, d_D, d_F0, d_F1, d_D2, d_F2 }, + pc, { groups_x, groups_y, groups_z }); + } + base_tile += groups_y; + } + if (rem_col_count > 0) { + const vk_mat_vec_push_constants pc = { + (uint32_t)ne00, (uint32_t)ne10, (uint32_t)ne10, (uint32_t)launch_m, + stride_batch_x, stride_batch_y, stride_batch_d, + fusion_flags, full_col_tiles * col_tile, + (uint32_t)ne02, (uint32_t)ne12, (uint32_t)r2, (uint32_t)r3, + has_pair_mm ? (uint32_t)ne01 : 0u, + has_pair_mm ? (uint32_t)pair_m1 : 0u, + has_pair_mm ? (uint32_t)(batch_n ? pair_m1 : pair_m1 * ne21) : 0u, + lora_r, lora_scale0, lora_scale1, + }; + if (lora_r > 0) { + ggml_vk_dispatch_pipeline(ctx, subctx, dmmv_rem, + { d_X, d_Y, d_D, d_F0, d_F1, d_D2, d_F2, d_LB0, d_LH0, d_LB1, d_LH1 }, + pc, { groups_x, 1u, groups_z }); + } else { + ggml_vk_dispatch_pipeline(ctx, subctx, dmmv_rem, + { d_X, d_Y, d_D, d_F0, d_F1, d_D2, d_F2 }, + pc, { groups_x, 1u, groups_z }); + } + } + } else { + ggml_pipeline_request_descriptor_sets(ctx, dmmv, CEIL_DIV(ne12 * ne13, ctx->device->properties.limits.maxComputeWorkGroupCount[1])); + + uint32_t base_work_group_y = 0; + while (base_work_group_y < ne12 * ne13) { + + uint32_t groups_y = std::min((uint32_t)(ne12 * ne13) - base_work_group_y, ctx->device->properties.limits.maxComputeWorkGroupCount[1]); + const vk_mat_vec_push_constants pc = { + (uint32_t)ne00, (uint32_t)ne10, (uint32_t)ne10, (uint32_t)launch_m, + stride_batch_x, stride_batch_y, stride_batch_d, + fusion_flags, base_work_group_y, + (uint32_t)ne02, (uint32_t)ne12, (uint32_t)r2, (uint32_t)r3, + has_pair_mm ? (uint32_t)ne01 : 0u, + has_pair_mm ? (uint32_t)pair_m1 : 0u, + has_pair_mm ? (uint32_t)(batch_n ? pair_m1 : pair_m1 * ne21) : 0u, + lora_r, lora_scale0, lora_scale1, + }; + if (lora_r > 0) { + ggml_vk_dispatch_pipeline(ctx, subctx, dmmv, + { d_X, d_Y, d_D, d_F0, d_F1, d_D2, d_F2, d_LB0, d_LH0, d_LB1, d_LH1 }, + pc, { groups_x, groups_y, groups_z }); + } else { + ggml_vk_dispatch_pipeline(ctx, subctx, dmmv, + { d_X, d_Y, d_D, d_F0, d_F1, d_D2, d_F2 }, + pc, { groups_x, groups_y, groups_z }); + } + base_work_group_y += groups_y; + } } if (x_non_contig) { ctx->prealloc_x_need_sync = true; } - if (y_non_contig || quantize_y) { + if (y_non_contig || quantize_y || mma_use_y_f16_ring) { ctx->prealloc_y_need_sync = true; + ctx->prealloc_y_slot_need_sync[y_ring_slot] = true; } } @@ -7814,7 +9375,7 @@ void ggml_vk_mul_mat(ggml_backend_vk_context * ctx, vk_context& subctx, const st // Split_k doesn't work with M splitting. // This only supports batchsize == 1. const size_t nbytes = ggml_nbytes(src0); - const bool needs_split = dst->ne[2] == 1 && dst->ne[3] == 1 && nbytes > ctx->device->properties.limits.maxStorageBufferRange; + const bool needs_split = ctx->gen_pair_mm1 == nullptr && dst->ne[2] == 1 && dst->ne[3] == 1 && nbytes > ctx->device->properties.limits.maxStorageBufferRange; if (needs_split) { // Choose the number of rows that can fit (and divide by two, to allow for any additional offsets) const uint32_t M_split = ctx->device->properties.limits.maxStorageBufferRange / (2 * src0->nb[1]); @@ -7857,31 +9418,87 @@ void ggml_vk_mul_mat(ggml_backend_vk_context * ctx, vk_context& subctx, const st src0->ne[1] <= ctx->device->properties.limits.maxComputeWorkGroupCount[1] && src1->ne[2] <= ctx->device->properties.limits.maxComputeWorkGroupCount[2]) { ggml_vk_mul_mat_vec_nc_f16_f32(ctx, subctx, cgraph, node_idx); - // With one output row, B^T*A has the same flat output as A^T*B. + } else if ((dst->ne[1] == 1 || + ((dst->ne[1] <= (int64_t) ggml_vk_mul_mat_vec_max_cols(ctx->device) || + (ggml_vk_is_apple_agx_coopmat(ctx->device) && !ggml_vk_exact_s8_enabled() && !gen_batch_invariant_for_n(dst->ne[1]) && + dst->ne[1] <= 4096 && + ((src0->type == GGML_TYPE_F32 && dst->ne[0] <= (dst->ne[1] <= 32 ? 256 : ([]() { static const int vf = getenv("GGML_VK_MMA_F32_MAX_M") ? atoi(getenv("GGML_VK_MMA_F32_MAX_M")) : 256; return vf; })())) || + (src0->type == GGML_TYPE_Q8_0 && + (dst->ne[1] <= 32 + ? dst->ne[0] <= ([]() { static const int v = getenv("GGML_VK_MMA_Q8_MAX_M") ? atoi(getenv("GGML_VK_MMA_Q8_MAX_M")) : 65536; return v; })() + : (dst->ne[1] >= ([]() { static const int qminn = getenv("GGML_VK_MMA_Q8_MIN_N") ? atoi(getenv("GGML_VK_MMA_Q8_MIN_N")) : 8; return qminn; })() && + dst->ne[0] >= ([]() { static const int vmin = getenv("GGML_VK_MMA_Q8_MIN_M") ? atoi(getenv("GGML_VK_MMA_Q8_MIN_M")) : 0; return vmin; })() && + dst->ne[0] <= ([]() { static const int vmax = getenv("GGML_VK_MMA_Q8_MAX_M") ? atoi(getenv("GGML_VK_MMA_Q8_MAX_M")) : 65536; return vmax; })())))))) && + src1->ne[2] * src1->ne[3] == 1)) && + (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16 || ggml_is_quantized(src0->type))) { + ggml_vk_mul_mat_vec_q_f16(ctx, subctx, cgraph, node_idx); + // With one output row, B^T*A has the same flat output as A^T*B (including elementwise SIGMOID epilogue). } else if (ctx->num_additional_fused_ops == 0 && + (ctx->gen_mm_dst == nullptr || + (ctx->gen_mm_flags == MAT_VEC_FUSION_FLAGS_SIGMOID && ctx->gen_mm_f0 == nullptr && ctx->gen_mm_f1 == nullptr && + ggml_is_contiguous(ctx->gen_mm_dst) && get_misalign_bytes(ctx, ctx->gen_mm_dst) == 0)) && (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16) && (src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16 || src1->type == GGML_TYPE_BF16 || ggml_is_quantized(src1->type)) && - dst->ne[0] == 1 && dst->ne[1] > mul_mat_vec_max_cols && + dst->ne[0] == 1 && dst->ne[1] > (int64_t) ggml_vk_mul_mat_vec_max_cols(ctx->device) && src0->ne[2] == 1 && src0->ne[3] == 1 && src1->ne[2] == 1 && src1->ne[3] == 1 && ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && ggml_is_contiguous(dst) && get_misalign_bytes(ctx, src0) == 0 && get_misalign_bytes(ctx, src1) == 0 && get_misalign_bytes(ctx, dst) == 0) { ggml_vk_mul_mat_vec_q_f16(ctx, subctx, cgraph, node_idx, true); - // mul_mat_vec supports batching ne12*ne13 when ne11==1, or treating ne11 as the batch size (up to four) - // when ne12 and ne13 are one. - } else if ((dst->ne[1] == 1 || (dst->ne[1] <= ggml_vk_mul_mat_vec_max_cols(ctx->device) && src1->ne[2] * src1->ne[3] == 1)) && - (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16 || ggml_is_quantized(src0->type))) { - ggml_vk_mul_mat_vec_q_f16(ctx, subctx, cgraph, node_idx); } else { - ggml_vk_mul_mat_q_f16(ctx, subctx, src0, src1, dst, false); + ggml_vk_mul_mat_q_f16(ctx, subctx, src0, src1, dst, false, cgraph, node_idx); } } -static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst) { +static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst, const struct ggml_cgraph * cgraph = nullptr, int node_idx = -1) { + const bool has_pair_glu = cgraph != nullptr && ctx->num_additional_fused_ops == 2 && + cgraph->nodes[node_idx + 2]->op == GGML_OP_GLU; + const ggml_tensor * src0_up = nullptr; + if (has_pair_glu) { + const ggml_tensor * glu = cgraph->nodes[node_idx + 2]; + src0 = glu->src[0]->src[0]; + src0_up = glu->src[1]->src[0]; + } + const ggml_tensor * out_dst = (cgraph != nullptr && ctx->num_additional_fused_ops > 0) + ? cgraph->nodes[node_idx + ctx->num_additional_fused_ops] + : dst; + uint32_t fusion_flags = 0; + const ggml_tensor * aux0_t = nullptr; + const ggml_tensor * aux1_t = nullptr; + if (!has_pair_glu && cgraph != nullptr && ctx->num_additional_fused_ops > 0) { + const ggml_tensor * op1 = cgraph->nodes[node_idx + 1]; + if (ctx->num_additional_fused_ops == 1 && op1->op == GGML_OP_GLU) { + const bool is_gate = (op1->src[0] == dst); + aux0_t = is_gate ? op1->src[1] : op1->src[0]; + fusion_flags |= is_gate ? MAT_VEC_FUSION_FLAGS_GLU_A : MAT_VEC_FUSION_FLAGS_GLU_B; + } else { + aux0_t = op1->src[1]; + if (op1->op == GGML_OP_MUL) { + fusion_flags |= MAT_VEC_FUSION_FLAGS_SCALE0; + if (aux0_t->ne[2] == ids->ne[1] && ids->ne[1] > 1) { + fusion_flags |= MAT_VEC_FUSION_FLAGS_SCALE0_TOK; + } + } else { + GGML_ASSERT(op1->op == GGML_OP_ADD_ID); + fusion_flags |= MAT_VEC_FUSION_FLAGS_BIAS0; + } + if (ctx->num_additional_fused_ops > 1) { + const ggml_tensor * op2 = cgraph->nodes[node_idx + 2]; + GGML_ASSERT(op2->op == GGML_OP_MUL); + aux1_t = op2->src[1]; + fusion_flags |= MAT_VEC_FUSION_FLAGS_SCALE1; + if (aux1_t->ne[2] == ids->ne[1] && ids->ne[1] > 1) { + fusion_flags |= MAT_VEC_FUSION_FLAGS_SCALE1_TOK; + } + } + } + } + const bool use_mmid_fused = has_pair_glu || (fusion_flags != 0); + VK_LOG_DEBUG("ggml_vk_mul_mat_id_q_f16((" << src0 << ", name=" << src0->name << ", type=" << src0->type << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3]; std::cerr << "), (" << src1 << ", name=" << src1->name << ", type=" << src1->type << ", ne0=" << src1->ne[0] << ", ne1=" << src1->ne[1] << ", ne2=" << src1->ne[2] << ", ne3=" << src1->ne[3] << ", nb0=" << src1->nb[0] << ", nb1=" << src1->nb[1] << ", nb2=" << src1->nb[2] << ", nb3=" << src1->nb[3]; std::cerr << "), (" << ids << ", name=" << ids->name << ", type=" << ids->type << ", ne0=" << ids->ne[0] << ", ne1=" << ids->ne[1] << ", ne2=" << ids->ne[2] << ", ne3=" << ids->ne[3] << ", nb0=" << ids->nb[0] << ", nb1=" << ids->nb[1] << ", nb2=" << ids->nb[2] << ", nb3=" << ids->nb[3]; - std::cerr << "), (" << dst << ", name=" << dst->name << ", type=" << dst->type << ", ne0=" << dst->ne[0] << ", ne1=" << dst->ne[1] << ", ne2=" << dst->ne[2] << ", ne3=" << dst->ne[3] << ", nb0=" << dst->nb[0] << ", nb1=" << dst->nb[1] << ", nb2=" << dst->nb[2] << ", nb3=" << dst->nb[3] << "),)"); + std::cerr << "), (" << out_dst << ", name=" << out_dst->name << ", type=" << out_dst->type << ", ne0=" << out_dst->ne[0] << ", ne1=" << out_dst->ne[1] << ", ne2=" << out_dst->ne[2] << ", ne3=" << out_dst->ne[3] << ", nb0=" << out_dst->nb[0] << ", nb1=" << out_dst->nb[1] << ", nb2=" << out_dst->nb[2] << ", nb3=" << out_dst->nb[3] << "),)"); GGML_ASSERT(ggml_vk_dim01_contiguous(src1) || src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16); // NOLINT GGML_ASSERT(ids->type == GGML_TYPE_I32); @@ -7901,14 +9518,13 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context& const uint32_t nbi0 = ids->nb[0]; const uint32_t nbi1 = ids->nb[1]; - const uint64_t ne20 = dst->ne[0]; - const uint64_t ne21 = dst->ne[1]; - // const uint64_t ne22 = dst->ne[2]; - // const uint64_t ne23 = dst->ne[3]; + const uint64_t ne20 = out_dst->ne[0]; + const uint64_t ne21 = out_dst->ne[1]; + // const uint64_t ne22 = out_dst->ne[2]; + // const uint64_t ne23 = out_dst->ne[3]; const uint64_t n_as = ne02; - ggml_backend_vk_buffer_context * dst_buf_ctx = (ggml_backend_vk_buffer_context *)dst->buffer->context; ggml_backend_vk_buffer_context * src0_buf_ctx = (ggml_backend_vk_buffer_context *)src0->buffer->context; ggml_backend_vk_buffer_context * src1_buf_ctx = (ggml_backend_vk_buffer_context *)src1->buffer->context; @@ -7948,12 +9564,91 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context& #else const bool y_decode_vector_staging = false; #endif - const bool can_quantize_y = ctx->device->integer_dot_product && src1->type == GGML_TYPE_F32 && + const bool can_quantize_y = !use_mmid_fused && ctx->device->integer_dot_product && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && (ne11 * ne10) % 4 == 0 && ggml_vk_src1_prec_allows_q8(dst); - // Intel coopmat1 matrix cores prefer f16 activations for quantized weights - const bool prefer_f16_b = ctx->device->coopmat_support && !ctx->device->coopmat2 && - ctx->device->vendor_id == VK_VENDOR_ID_INTEL && - ggml_is_quantized(src0->type) && src1->type == GGML_TYPE_F32; + static const bool mma_id_disabled = getenv("GGML_VK_DISABLE_MMA_ID") != nullptr; + static const int env_mma_id_max_n = getenv("GGML_VK_MMA_ID_MAX_N") ? atoi(getenv("GGML_VK_MMA_ID_MAX_N")) : 2048; + static const int env_mma_id_pglu_max_n = getenv("GGML_VK_MMA_ID_PGLU_MAX_N") ? atoi(getenv("GGML_VK_MMA_ID_PGLU_MAX_N")) : -1; + static const int env_mma_id_fep_max_n = getenv("GGML_VK_MMA_ID_FASTEP_MAX_N") ? atoi(getenv("GGML_VK_MMA_ID_FASTEP_MAX_N")) : -1; + static const bool has_mma_id_rt1_n = getenv("GGML_VK_MMA_ID_RT1_MAX_N") != nullptr; + static const int env_mma_id_rt1_n = has_mma_id_rt1_n ? atoi(getenv("GGML_VK_MMA_ID_RT1_MAX_N")) : 32; + static const int env_mma_id_rt3_tok = getenv("GGML_VK_MMA_ID_RT3_MIN_TOK") ? atoi(getenv("GGML_VK_MMA_ID_RT3_MIN_TOK")) : 12; + static const int env_mma_id_rt4_tok = getenv("GGML_VK_MMA_ID_RT4_MIN_TOK") ? atoi(getenv("GGML_VK_MMA_ID_RT4_MIN_TOK")) : 21; + static const int env_mma_id_rt5_tok = getenv("GGML_VK_MMA_ID_RT5_MIN_TOK") ? atoi(getenv("GGML_VK_MMA_ID_RT5_MIN_TOK")) : 9999; + static const int env_mma_id_rt6_tok = getenv("GGML_VK_MMA_ID_RT6_MIN_TOK") ? atoi(getenv("GGML_VK_MMA_ID_RT6_MIN_TOK")) : 26; + static const int env_mma_id_rt8_tok = getenv("GGML_VK_MMA_ID_RT8_MIN_TOK") ? atoi(getenv("GGML_VK_MMA_ID_RT8_MIN_TOK")) : 9999; + static const int env_mma_id_msg = getenv("GGML_VK_MMA_ID_MSG") ? atoi(getenv("GGML_VK_MMA_ID_MSG")) : -1; + static const int env_mma_id_pglu_msg = getenv("GGML_VK_MMA_ID_PGLU_MSG") ? atoi(getenv("GGML_VK_MMA_ID_PGLU_MSG")) : 0; + static const int env_mma_id_fep_msg = getenv("GGML_VK_MMA_ID_FEP_MSG") ? atoi(getenv("GGML_VK_MMA_ID_FEP_MSG")) : 0; + static const int env_mma_id_rt = getenv("GGML_VK_MMA_ID_RT") ? atoi(getenv("GGML_VK_MMA_ID_RT")) : 0; + static const int env_mma_id_pglu_rt = getenv("GGML_VK_MMA_ID_PGLU_RT") ? atoi(getenv("GGML_VK_MMA_ID_PGLU_RT")) : 0; + static const int env_mma_id_fep_rt = getenv("GGML_VK_MMA_ID_FEP_RT") ? atoi(getenv("GGML_VK_MMA_ID_FEP_RT")) : 0; + static const int env_mma_id_f16_b = getenv("GGML_VK_MMA_ID_F16_B") ? atoi(getenv("GGML_VK_MMA_ID_F16_B") ) : 1; + static const uint32_t env_mma_id_kp = getenv("GGML_VK_MMA_ID_KP") ? (uint32_t)atoi(getenv("GGML_VK_MMA_ID_KP")) : 1u; + const bool is_fastep_cand = !has_pair_glu && use_mmid_fused && + (fusion_flags & ~(MAT_VEC_FUSION_FLAGS_SCALE0 | MAT_VEC_FUSION_FLAGS_SCALE0_TOK)) == 0u; + const int eff_mma_id_max_n = has_pair_glu ? (env_mma_id_pglu_max_n >= 0 ? env_mma_id_pglu_max_n : env_mma_id_max_n) + : is_fastep_cand ? (env_mma_id_fep_max_n >= 0 ? env_mma_id_fep_max_n : env_mma_id_max_n) + : env_mma_id_max_n; + const uint32_t mma_id_k_step = (src0->type == GGML_TYPE_Q4_0) ? 32u : 64u; + const uint32_t avg_tok_per_exp = (uint32_t)CEIL_DIV(nei0 * nei1, n_as); + const int def_mma_id_msg = env_mma_id_msg >= 0 ? env_mma_id_msg : ((nei1 >= 1024) ? 8 : -1); + const int req_msg = has_pair_glu ? (env_mma_id_pglu_msg > 0 ? env_mma_id_pglu_msg : def_mma_id_msg) + : is_fastep_cand ? (env_mma_id_fep_msg > 0 ? env_mma_id_fep_msg : def_mma_id_msg) + : def_mma_id_msg; + const bool want_msg8 = (req_msg == 8 || req_msg == 81) && ne01 >= (req_msg == 81 ? 64u : 128u) && (ne10 % 128u == 0); + const bool want_msg8_pglu = (req_msg == 8 || req_msg == 81) && has_pair_glu && ne01 >= 64 && (ne10 % 128u == 0); + const bool use_msg8_cfg = want_msg8 || want_msg8_pglu; + const int req_rt = has_pair_glu ? (env_mma_id_pglu_rt > 0 ? env_mma_id_pglu_rt : env_mma_id_rt) + : is_fastep_cand ? (env_mma_id_fep_rt > 0 ? env_mma_id_fep_rt : env_mma_id_rt) + : env_mma_id_rt; + const bool can_wide_rt = src0->type != GGML_TYPE_F32 && (ne10 % 64u == 0) && ne01 >= 32; + const uint32_t id_rt_i = (req_rt == 8 && can_wide_rt) ? 6u + : (req_rt == 6 && can_wide_rt) ? 5u + : (req_rt == 5 && can_wide_rt) ? 4u + : (req_rt == 4) ? 3u + : (req_rt == 3) ? 2u + : (req_rt == 2) ? 1u + : (req_rt == 1) ? 0u + : ((has_mma_id_rt1_n ? ((int64_t)nei1 <= env_mma_id_rt1_n) : (avg_tok_per_exp <= 4u)) ? 0u + : (can_wide_rt && (int)avg_tok_per_exp >= env_mma_id_rt8_tok) ? 6u + : (can_wide_rt && (int)avg_tok_per_exp >= env_mma_id_rt6_tok) ? 5u + : (can_wide_rt && (int)avg_tok_per_exp >= env_mma_id_rt5_tok) ? 4u + : ((int)avg_tok_per_exp >= env_mma_id_rt4_tok) ? 3u + : ((int)avg_tok_per_exp >= env_mma_id_rt3_tok) ? 2u : 1u); + static constexpr uint32_t sel_id_rt_map[7] = { 1u, 2u, 3u, 4u, 5u, 6u, 8u }; + const uint32_t sel_id_rt = sel_id_rt_map[id_rt_i]; + const uint32_t def_pglu_kp = (sel_id_rt == 3u || sel_id_rt == 4u || (use_msg8_cfg && sel_id_rt >= 3u)) ? 2u : 1u; + const uint32_t def_fep_kp = (sel_id_rt >= 5u) ? (use_msg8_cfg ? 2u : 1u) : ((sel_id_rt >= 3u) ? 2u : 4u); + const uint32_t eff_pglu_kp = getenv("GGML_VK_MMA_PGLU_KP") ? (uint32_t)atoi(getenv("GGML_VK_MMA_PGLU_KP")) : def_pglu_kp; + const uint32_t eff_fep_kp = getenv("GGML_VK_MMA_FASTEP_KP") ? (uint32_t)atoi(getenv("GGML_VK_MMA_FASTEP_KP")) : def_fep_kp; + const uint32_t eff_shb_kp = has_pair_glu ? eff_pglu_kp : (is_fastep_cand ? eff_fep_kp : env_mma_id_kp); + const bool mma_id_candidate = !mma_id_disabled && + ggml_vk_is_apple_agx_coopmat(ctx->device) && + !ggml_vk_exact_s8_enabled() && + !gen_batch_invariant_for_n(nei1) && + !x_non_contig && + src1->type == GGML_TYPE_F32 && + out_dst->type == GGML_TYPE_F32 && + (ne11 == 1 || ne11 >= nei0) && + (int64_t)nei1 <= eff_mma_id_max_n && + (ne10 % mma_id_k_step == 0) && + (get_misalign_bytes(ctx, src1) % 4 == 0) && + ctx->device->pipeline_mul_mv_mma_id[src0->type][id_rt_i][0][0] != nullptr; + const bool mma_id_f16_b = mma_id_candidate && src0->type != GGML_TYPE_F32 && + ((env_mma_id_f16_b == 1 && nei1 >= 32) || + (env_mma_id_f16_b == 2 && has_pair_glu && nei1 >= 32) || + (env_mma_id_f16_b >= 3)); + + // Intel coopmat1 prefers f16 activations; for large batches on Apple AGX, pre-staged FP16 B (cached for _pairglu!) halves scattered MoE B bandwidth. + static const int gen_mmid_f16_b = getenv("GGML_VK_MMID_F16_B") ? atoi(getenv("GGML_VK_MMID_F16_B")) : -1; + const bool prefer_f16_b = mma_id_f16_b || + (!mma_id_candidate && ctx->device->coopmat_support && !ctx->device->coopmat2 && + (gen_mmid_f16_b > 0 || (gen_mmid_f16_b < 0 && + (ctx->device->vendor_id == VK_VENDOR_ID_INTEL || + (ggml_vk_is_apple_agx_coopmat(ctx->device) && + (ggml_vk_exact_s8_enabled() ? (nei1 > 32) : (get_misalign_bytes(ctx, src1) != 0)))))) && + ggml_is_quantized(src0->type) && src1->type == GGML_TYPE_F32); const ggml_type y_kernel_type = ggml_vk_mul_mat_b_type(ctx, src0->type, src1, f16_type, ggml_prec_get_acc(dst), true, can_quantize_y, prefer_f16_b); @@ -7981,22 +9676,68 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context& y_decode_vector_staging || y_needs_k_padding); const bool qy_needs_dequant = y_needs_reformat; - const uint32_t kpad = quantize_y ? 0 : ggml_vk_align_size(ne10, ggml_vk_guess_matmul_pipeline_align_map(ctx, *mmp_map, ne01, nei1, true)); - const bool aligned = !quantize_y && ne10 == kpad && ne01 > 8 && nei1 > 8; + // gen: GGML_VK_MMID_SELECT=pairs sizes the tile by the rows an expert gets on average (tokens x used / experts), + // not by the token count: each nonempty expert gets its own tiles, so a 128-column tile for an expert with 8 rows + // computes 120 empty columns. Each output's sum over K is the same for every tile size. + static const int gen_mmid_sel_mode = []() { + const char * s = getenv("GGML_VK_MMID_SELECT"); + if (!s || !*s) return -1; + if (!strcmp(s, "pairs")) return 1; + if (!strcmp(s, "tokens")) return 0; + return -1; + }(); + const bool gen_mmid_pairs = gen_mmid_sel_mode >= 0 ? (gen_mmid_sel_mode > 0) : ggml_vk_is_apple_agx_coopmat(ctx->device); + const uint32_t gen_n_sel = gen_mmid_pairs ? (uint32_t) std::max(1, CEIL_DIV(nei0 * nei1, n_as)) : (uint32_t) (nei1); + const vk_subbuffer d_id_U_opt = has_pair_glu ? ggml_vk_tensor_subbuffer(ctx, src0_up) : vk_subbuffer{}; + const size_t raw_id_x_offs = src0_uma ? qx_buf_offset : (vk_tensor_offset(src0) + src0->view_offs); + const size_t raw_id_y_offs = src1_uma ? qy_buf_offset : (vk_tensor_offset(src1) + src1->view_offs); + const bool id_offsets_vec8_aligned = + (qx_needs_dequant || ggml_is_quantized(src0->type) || ((raw_id_x_offs % 32 == 0) && (!has_pair_glu || (d_id_U_opt.offset % 32 == 0)))) && + (qy_needs_dequant || quantize_y || (raw_id_y_offs % 32 == 0)); + const uint32_t id_sel_m = has_pair_glu ? (uint32_t)(2 * ne01) : (uint32_t)ne01; + const uint32_t kpad = quantize_y ? 0 : ggml_vk_align_size(ne10, ggml_vk_guess_matmul_pipeline_align_map(ctx, *mmp_map, id_sel_m, gen_n_sel, true)); + const bool aligned = !quantize_y && ne10 == kpad && id_offsets_vec8_aligned && + ((ggml_vk_is_apple_agx_coopmat(ctx->device) && !ggml_vk_exact_s8_enabled()) || (ne01 > 8 && nei1 > 8)); + + const bool use_mma_id = mma_id_candidate && !qx_needs_dequant && + (y_f32_kernel || (mma_id_f16_b && y_kernel_type == GGML_TYPE_F16)) && + !y_needs_k_padding; + vk_pipeline pipeline = nullptr; + if (use_mma_id) { + const uint32_t id_ci = (req_msg == 1 || (ne10 % (64u * eff_shb_kp) != 0) || ne01 < 32) + ? 1u + : ((req_msg == 2 || (req_msg < 0 && ne01 < 64)) + ? 2u + : (use_msg8_cfg ? ((req_msg == 81 && !has_pair_glu) ? 4u : 3u) : 0u)); + const uint32_t bf16_i = y_f32_kernel ? 0u : 1u; + pipeline = ctx->device->pipeline_mul_mv_mma_id[src0->type][id_rt_i][id_ci][bf16_i]; + } else { + pipeline = ggml_vk_guess_matmul_pipeline_map(ctx, *mmp_map, id_sel_m, gen_n_sel, aligned, true); + } - vk_pipeline pipeline = ggml_vk_guess_matmul_pipeline_map(ctx, *mmp_map, ne01, nei1, aligned, true); + const uint64_t max_id_a_nbytes = std::max(ggml_nbytes(src0), src0_up ? ggml_nbytes(src0_up) : 0); + if (max_id_a_nbytes > ctx->device->properties.limits.maxStorageBufferRange) { + pipeline = ggml_vk_get_64b_indexing_pipeline(ctx, pipeline); + } + if (has_pair_glu) { + GGML_ASSERT(pipeline && pipeline->gen_pair_glu); + pipeline = pipeline->gen_pair_glu; + } else if (use_mmid_fused) { + if ((fusion_flags & ~(MAT_VEC_FUSION_FLAGS_SCALE0 | MAT_VEC_FUSION_FLAGS_SCALE0_TOK)) == 0u && pipeline && pipeline->gen_fast_ep) { + pipeline = pipeline->gen_fast_ep; + } else { + GGML_ASSERT(pipeline && pipeline->gen_glu); + pipeline = pipeline->gen_glu; + } + } const uint32_t bn = pipeline->wg_denoms[1]; const uint64_t max_tiles = ggml_vk_mul_mat_id_max_tiles(n_as, nei0 * nei1, bn); const uint64_t row_ids_offset = 1 + 2 * max_tiles; const uint64_t row_map_words = row_ids_offset + max_tiles * bn; - - if (ggml_nbytes(src0) > ctx->device->properties.limits.maxStorageBufferRange) { - pipeline = ggml_vk_get_64b_indexing_pipeline(ctx, pipeline); - } const uint64_t x_ne = ggml_nelements(src0); const uint64_t y_ne = (uint64_t)y_staged_row_stride * ne11 * ne12 * ne13; - const uint64_t d_ne = ggml_nelements(dst); + const uint64_t d_ne = ggml_nelements(out_dst); const uint64_t qx_sz = ggml_type_size(src0->type) * x_ne / ggml_blck_size(src0->type); const uint64_t qy_sz = ggml_type_size(src1->type) * ggml_nelements(src1) / ggml_blck_size(src1->type); @@ -8045,6 +9786,20 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context& const size_t expert_data_size = sizeof(uint32_t) * row_map_words; + static const bool disable_pglu_f16_out = getenv("GGML_VK_DISABLE_PGLU_F16_OUT") != nullptr; + static const bool keep_pglu_f32 = getenv("GGML_VK_KEEP_PGLU_F32") != nullptr; + const bool next_mmid_wants_f16 = cgraph != nullptr && has_pair_glu && + (node_idx + 3 < cgraph->n_nodes) && + cgraph->nodes[node_idx + 3]->op == GGML_OP_MUL_MAT_ID && + cgraph->nodes[node_idx + 3]->src[1] == out_dst && + cgraph->nodes[node_idx + 3]->src[0]->type != GGML_TYPE_F32 && + cgraph->nodes[node_idx + 3]->src[0]->ne[1] >= 32 && + (out_dst->ne[0] % ((cgraph->nodes[node_idx + 3]->src[0]->type == GGML_TYPE_Q4_0) ? 32 : 64) == 0); + const bool want_pglu_f16_out = !disable_pglu_f16_out && has_pair_glu && use_mma_id && + mma_id_f16_b && next_mmid_wants_f16 && ggml_is_contiguous(out_dst) && + get_misalign_bytes(ctx, out_dst) == 0; + const size_t pglu_f16_bytes = sizeof(ggml_fp16_t) * d_ne; + { if ( (qx_needs_dequant && x_sz > ctx->device->properties.limits.maxStorageBufferRange) || @@ -8059,6 +9814,9 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context& ctx->prealloc_size_y = y_sz; ggml_vk_preallocate_buffers(ctx, subctx); } + if (want_pglu_f16_out) { + ggml_vk_reserve_prealloc_y(ctx, subctx, pglu_f16_bytes); + } if (ctx->prealloc_size_split_k < expert_data_size) { ctx->prealloc_size_split_k = expert_data_size; ggml_vk_preallocate_buffers(ctx, subctx); @@ -8078,8 +9836,9 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context& ggml_pipeline_request_descriptor_sets(ctx, count_experts, 1); } - vk_buffer d_D = dst_buf_ctx->dev_buffer; - const uint64_t d_buf_offset = vk_tensor_offset(dst) + dst->view_offs; + const vk_subbuffer out_id_d_sub = ggml_vk_tensor_subbuffer(ctx, out_dst); + vk_buffer d_D = out_id_d_sub.buffer; + const uint64_t d_buf_offset = out_id_d_sub.offset; GGML_ASSERT(d_D != nullptr); vk_buffer d_X; uint64_t x_buf_offset = 0; @@ -8122,12 +9881,22 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context& } vk_subbuffer d_ids = ggml_vk_tensor_subbuffer(ctx, ids, true); - // Count how many times each expert is used - vk_subbuffer expert_count_buf = { ctx->prealloc_split_k, 0, expert_data_size }; - if (ctx->prealloc_split_k_need_sync) { - ggml_vk_sync_buffers(ctx, subctx); - } - { + // Count how many times each expert is used (Series 9 S9.2: reuse across gate/up/down when ids come from ARGSORT >= 0) + const bool ids_non_negative = (ids->op == GGML_OP_ARGSORT) || (ids->view_src && ids->view_src->op == GGML_OP_ARGSORT); + uint32_t sk_ring_slot = 0; + vk_subbuffer expert_count_buf; + bool unbarred_stage_writes = false; + if (ids_non_negative && ctx->last_expert_count_valid && + ctx->last_expert_ids_tensor == ids && + ctx->last_expert_bn == bn && + ctx->last_expert_row_ids_offset == (uint32_t) row_ids_offset && + ctx->last_expert_n_as == (uint32_t) n_as && + ctx->last_expert_count_buf.size >= expert_data_size) { + expert_count_buf = ctx->last_expert_count_buf; + sk_ring_slot = ctx->last_expert_slot; + } else { + sk_ring_slot = ggml_vk_acquire_prealloc_split_k_slot(ctx, subctx, expert_data_size); + expert_count_buf = { ctx->prealloc_split_k, sk_ring_slot * ctx->prealloc_split_k_ring_stride, expert_data_size }; // this also zeroes the dst rows of the skipped slots (negative ids) vk_op_count_experts_push_constants pc = { (uint32_t)nei0, (uint32_t)nei1, @@ -8137,64 +9906,71 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context& (uint32_t)n_as, 0, 0, bn, (uint32_t)row_ids_offset, (uint32_t)ne01, - (uint32_t)(dst->nb[1] / sizeof(float)), - (uint32_t)(dst->nb[2] / sizeof(float)) }; + (uint32_t)(out_dst->nb[1] / sizeof(float)), + (uint32_t)(out_dst->nb[2] / sizeof(float)) }; init_pushconst_fastdiv(pc); ggml_vk_dispatch_pipeline(ctx, subctx, count_experts, { d_ids, expert_count_buf, vk_subbuffer{ d_D, d_buf_offset, d_sz } }, pc, { 1, 1, 1 }); + unbarred_stage_writes = true; + if (ids_non_negative) { + ctx->last_expert_ids_tensor = ids; + ctx->last_expert_bn = bn; + ctx->last_expert_row_ids_offset = (uint32_t) row_ids_offset; + ctx->last_expert_n_as = (uint32_t) n_as; + ctx->last_expert_slot = sk_ring_slot; + ctx->last_expert_count_buf = expert_count_buf; + ctx->last_expert_count_valid = true; + } } if (x_non_contig) { ggml_vk_cpy_to_contiguous(ctx, subctx, to_fp16_vk_0, src0, ggml_vk_subbuffer(ctx, d_Qx, qx_buf_offset), ggml_vk_subbuffer(ctx, d_X, 0)); + unbarred_stage_writes = false; } else if (qx_needs_dequant) { const std::vector pc = { (uint32_t)ne01, (uint32_t)ne10, (uint32_t)ne10, (uint32_t)ne10, (uint32_t)(ggml_nelements(src0)) }; ggml_vk_dispatch_pipeline(ctx, subctx, to_fp16_vk_0, { vk_subbuffer{ d_Qx, qx_buf_offset, qx_sz }, vk_subbuffer{ d_X, 0, x_sz } }, pc, { (uint32_t)x_ne, 1, 1}); + unbarred_stage_writes = true; } + uint32_t y_ring_slot = 0; if (y_needs_reformat) { - if (ctx->prealloc_y_last_pipeline_used != to_fp16_vk_1.get() || - ctx->prealloc_y_last_tensor_used != src1 || - ctx->prealloc_y_last_k_padded != y_needs_k_padding) { - if (ctx->prealloc_y_need_sync) { - ggml_vk_sync_buffers(ctx, subctx); - } + bool cached = false; + y_ring_slot = ggml_vk_acquire_prealloc_y_slot(ctx, subctx, to_fp16_vk_1.get(), src1, y_needs_k_padding, cached); + y_buf_offset = y_ring_slot * ctx->prealloc_y_ring_stride; + if (!cached) { if (y_needs_k_padding) { GGML_ASSERT(y_sz % 4 == 0); // Zero B padding because clamping only A can produce 0 * Inf or NaN. - subctx->s->buffer->buf.fillBuffer(d_Y->buffer, 0, y_sz, 0); + subctx->s->buffer->buf.fillBuffer(d_Y->buffer, y_buf_offset, y_sz, 0); ggml_vk_sync_buffers(ctx, subctx); const ggml_tensor y_staged_dst = make_y_staged_dst(); const uint32_t y_staged_dst_type_size = ggml_type_size(y_staged_dst.type); ggml_vk_cpy_to_strided( ctx, subctx, to_fp16_vk_1, src1, - ggml_vk_subbuffer(ctx, d_Qy, qy_buf_offset), ggml_vk_subbuffer(ctx, d_Y, 0), + ggml_vk_subbuffer(ctx, d_Qy, qy_buf_offset), vk_subbuffer{ d_Y, y_buf_offset, y_sz }, (uint32_t)(y_staged_dst.nb[0] / y_staged_dst_type_size), (uint32_t)(y_staged_dst.nb[1] / y_staged_dst_type_size), (uint32_t)(y_staged_dst.nb[2] / y_staged_dst_type_size), (uint32_t)(y_staged_dst.nb[3] / y_staged_dst_type_size)); } else { - ggml_vk_cpy_to_contiguous(ctx, subctx, to_fp16_vk_1, src1, ggml_vk_subbuffer(ctx, d_Qy, qy_buf_offset), ggml_vk_subbuffer(ctx, d_Y, 0)); + ggml_vk_cpy_to_contiguous(ctx, subctx, to_fp16_vk_1, src1, ggml_vk_subbuffer(ctx, d_Qy, qy_buf_offset), vk_subbuffer{ d_Y, y_buf_offset, y_sz }); } - ctx->prealloc_y_last_pipeline_used = to_fp16_vk_1.get(); - ctx->prealloc_y_last_tensor_used = src1; - ctx->prealloc_y_last_k_padded = y_needs_k_padding; + unbarred_stage_writes = false; } } if (quantize_y) { - if (ctx->prealloc_y_last_pipeline_used != to_q8_1.get() || - ctx->prealloc_y_last_tensor_used != src1 || - ctx->prealloc_y_last_k_padded) { - if (ctx->prealloc_y_need_sync) { - ggml_vk_sync_buffers(ctx, subctx); - } - ggml_vk_quantize_q8_1(ctx, subctx, ggml_vk_subbuffer(ctx, d_Qy, qy_buf_offset), ggml_vk_subbuffer(ctx, d_Y, 0), y_ne); - ctx->prealloc_y_last_pipeline_used = to_q8_1.get(); - ctx->prealloc_y_last_tensor_used = src1; - ctx->prealloc_y_last_k_padded = false; + bool cached = false; + y_ring_slot = ggml_vk_acquire_prealloc_y_slot(ctx, subctx, to_q8_1.get(), src1, false, cached); + y_buf_offset = y_ring_slot * ctx->prealloc_y_ring_stride; + if (!cached) { + ggml_vk_quantize_q8_1(ctx, subctx, ggml_vk_subbuffer(ctx, d_Qy, qy_buf_offset), vk_subbuffer{ d_Y, y_buf_offset, CEIL_DIV(y_sz, 144) * 144 }, y_ne); + unbarred_stage_writes = false; } } - ggml_vk_sync_buffers(ctx, subctx); + if (unbarred_stage_writes) { + ggml_vk_sync_buffers(ctx, subctx); + } const uint64_t x_range = qx_needs_dequant ? x_sz : ggml_nbytes(src0); const uint64_t y_range = (qy_needs_dequant || quantize_y) ? y_sz : ggml_nbytes(src1); @@ -8212,29 +9988,67 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context& } // compute - ggml_vk_matmul_id( - ctx, subctx, pipeline, - { d_X, x_buf_offset, x_range }, { d_Y, y_buf_offset, y_range }, - { d_D, d_buf_offset, d_sz }, std::move(d_ids), expert_count_buf, - ne01, ne21, ne10, ne10, stride_b_y, ne01, - stride_batch_x, stride_batch_y, ne20*ne21, - n_as, nei0, nei1, nbi1 / ggml_type_size(ids->type), ne11, row_ids_offset, max_tiles - ); // NOLINT + if (use_mmid_fused || use_mma_id) { + const int in_y_slot = (y_needs_reformat || quantize_y) ? (int)y_ring_slot : -1; + const bool do_pglu_f16_out = want_pglu_f16_out && (in_y_slot < 0 || ctx->prealloc_y_ring_count >= 2); + uint32_t mma_fusion_flags = fusion_flags | ((use_mma_id && !y_f32_kernel) ? MAT_VEC_FUSION_FLAGS_B_F16 : 0u); + const uint32_t mma_stride_b_y = (use_mma_id && ne11 == 1) ? 0u : stride_b_y; + const vk_subbuffer x_sub = { d_X, x_buf_offset, x_range }; + const vk_subbuffer y_sub = { d_Y, y_buf_offset, y_range }; + const vk_subbuffer d_sub = { d_D, d_buf_offset, d_sz }; + const vk_subbuffer u_sub = has_pair_glu ? d_id_U_opt : x_sub; + const vk_subbuffer aux0_sub = aux0_t ? ggml_vk_tensor_subbuffer(ctx, aux0_t) : d_sub; + vk_subbuffer aux1_sub = aux1_t ? ggml_vk_tensor_subbuffer(ctx, aux1_t) : d_sub; + if (do_pglu_f16_out) { + aux1_sub = ggml_vk_produce_prealloc_y_f16(ctx, subctx, out_dst, pglu_f16_bytes, false, in_y_slot); + GGML_ASSERT(in_y_slot < 0 || aux1_sub.offset != y_buf_offset); + mma_fusion_flags |= MAT_VEC_FUSION_FLAGS_D_F16_AUX1; + if (ctx->gen_pair_glu_sole_mmid_next && ids_non_negative && !keep_pglu_f32) { + mma_fusion_flags |= MAT_VEC_FUSION_FLAGS_SKIP_F32_D; + } + } + ggml_vk_matmul_id_fused( + ctx, subctx, pipeline, + x_sub, y_sub, d_sub, d_ids, expert_count_buf, + u_sub, aux0_sub, aux1_sub, + ne01, ne21, ne10, ne10, mma_stride_b_y, ne01, + stride_batch_x, stride_batch_y, ne20*ne21, + n_as, nei0, nei1, nbi1 / ggml_type_size(ids->type), ne11, row_ids_offset, max_tiles, mma_fusion_flags + ); + } else { + ggml_vk_matmul_id( + ctx, subctx, pipeline, + { d_X, x_buf_offset, x_range }, { d_Y, y_buf_offset, y_range }, + { d_D, d_buf_offset, d_sz }, std::move(d_ids), expert_count_buf, + ne01, ne21, ne10, ne10, stride_b_y, ne01, + stride_batch_x, stride_batch_y, ne20*ne21, + n_as, nei0, nei1, nbi1 / ggml_type_size(ids->type), ne11, row_ids_offset, max_tiles + ); // NOLINT + } if (x_non_contig || qx_needs_dequant) { ctx->prealloc_x_need_sync = true; } if (y_needs_reformat || quantize_y) { ctx->prealloc_y_need_sync = true; + ctx->prealloc_y_slot_need_sync[y_ring_slot] = true; } ctx->prealloc_split_k_need_sync = true; + ctx->prealloc_split_k_slot_need_sync[sk_ring_slot] = true; } static void ggml_vk_mul_mat_vec_id_q_f16(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx) { ggml_tensor * dst = cgraph->nodes[node_idx]; + const bool has_pair_glu = ctx->num_additional_fused_ops == 2 && cgraph->nodes[node_idx + 2]->op == GGML_OP_GLU; ggml_tensor * src0 = dst->src[0]; ggml_tensor * src1 = dst->src[1]; ggml_tensor * ids = dst->src[2]; + const ggml_tensor * src0_up = nullptr; + if (has_pair_glu) { + const ggml_tensor * glu = cgraph->nodes[node_idx + 2]; + src0 = glu->src[0]->src[0]; + src0_up = glu->src[1]->src[0]; + } VK_LOG_DEBUG("ggml_vk_mul_mat_vec_id_q_f16((" << src0 << ", name=" << src0->name << ", type=" << src0->type << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3]; std::cerr << "), (" << src1 << ", name=" << src1->name << ", type=" << src1->type << ", ne0=" << src1->ne[0] << ", ne1=" << src1->ne[1] << ", ne2=" << src1->ne[2] << ", ne3=" << src1->ne[3] << ", nb0=" << src1->nb[0] << ", nb1=" << src1->nb[1] << ", nb2=" << src1->nb[2] << ", nb3=" << src1->nb[3]; std::cerr << "), (" << ids << ", name=" << ids->name << ", type=" << ids->type << ", ne0=" << ids->ne[0] << ", ne1=" << ids->ne[1] << ", ne2=" << ids->ne[2] << ", ne3=" << ids->ne[3] << ", nb0=" << ids->nb[0] << ", nb1=" << ids->nb[1] << ", nb2=" << ids->nb[2] << ", nb3=" << ids->nb[3]; @@ -8267,7 +10081,7 @@ static void ggml_vk_mul_mat_vec_id_q_f16(ggml_backend_vk_context * ctx, vk_conte const bool y_non_contig = !ggml_vk_dim01_contiguous(src1); const bool f16_f32_kernel = src1->type == GGML_TYPE_F32; - bool quantize_y = ctx->device->integer_dot_product && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && !y_non_contig && (ne11 * ne10) % 4 == 0 && ggml_vk_src1_prec_allows_q8(dst) && ggml_vk_should_use_mmvq(ctx->device, ne01, ne12, ne10, src0->type); + bool quantize_y = !has_pair_glu && ctx->device->integer_dot_product && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && !y_non_contig && (ne11 * ne10) % 4 == 0 && ggml_vk_src1_prec_allows_q8(dst) && ggml_vk_should_use_mmvq(ctx->device, ne01, ne12, ne10, src0->type); vk_pipeline to_fp16_vk_0 = nullptr; vk_pipeline to_fp16_vk_1 = nullptr; @@ -8297,9 +10111,17 @@ static void ggml_vk_mul_mat_vec_id_q_f16(ggml_backend_vk_context * ctx, vk_conte const bool qx_needs_dequant = x_non_contig; const bool qy_needs_dequant = !quantize_y && ((src1->type != GGML_TYPE_F16 && !f16_f32_kernel) || y_non_contig); - if (ggml_nbytes(src0) > ctx->device->properties.limits.maxStorageBufferRange) { + const uint64_t max_a_nbytes = std::max(ggml_nbytes(src0), src0_up ? ggml_nbytes(src0_up) : 0); + if (max_a_nbytes > ctx->device->properties.limits.maxStorageBufferRange) { dmmv = ggml_vk_get_64b_indexing_pipeline(ctx, dmmv); } + if (has_pair_glu) { + GGML_ASSERT(dmmv && dmmv->gen_pair_glu); + dmmv = dmmv->gen_pair_glu; + } else if ((ctx->num_additional_fused_ops == 1 && cgraph->nodes[node_idx + 1]->op == GGML_OP_GLU) || ctx->gen_mm_dst != nullptr) { + GGML_ASSERT(dmmv && dmmv->gen_glu); + dmmv = dmmv->gen_glu; // gen: single-op epilogue twin (GEN_GLU = 1) + } // Not implemented GGML_ASSERT(y_non_contig || !qy_needs_dequant); // NOLINT @@ -8315,6 +10137,12 @@ static void ggml_vk_mul_mat_vec_id_q_f16(ggml_backend_vk_context * ctx, vk_conte const uint64_t y_sz = quantize_y ? (ggml_vk_align_size(y_ne, 128) * ggml_type_size(GGML_TYPE_Q8_1) / ggml_blck_size(GGML_TYPE_Q8_1)) : (f16_f32_kernel ? sizeof(float) * y_ne : sizeof(ggml_fp16_t) * y_ne); + static const bool mmvid_toky_enabled = + (!getenv("GGML_VK_DISABLE_MMVID_TOKY") || atoi(getenv("GGML_VK_DISABLE_MMVID_TOKY")) == 0) && + (!getenv("GGML_VK_MMVID_ONE") || atoi(getenv("GGML_VK_MMVID_ONE")) != 0); + const bool use_toky = mmvid_toky_enabled && nei1 > 1 && + (nei0 * nei1) <= ctx->device->properties.limits.maxComputeWorkGroupCount[1]; + { if ( (qx_needs_dequant && x_sz > ctx->device->properties.limits.maxStorageBufferRange) || @@ -8340,10 +10168,10 @@ static void ggml_vk_mul_mat_vec_id_q_f16(ggml_backend_vk_context * ctx, vk_conte if (quantize_y) { ggml_pipeline_request_descriptor_sets(ctx, to_q8_1, 1); } - ggml_pipeline_request_descriptor_sets(ctx, dmmv, nei1); + ggml_pipeline_request_descriptor_sets(ctx, dmmv, use_toky ? 1 : nei1); } - vk_subbuffer d_D = ggml_vk_tensor_subbuffer(ctx, cgraph->nodes[node_idx + ctx->num_additional_fused_ops]); + vk_subbuffer d_D = ggml_vk_tensor_subbuffer(ctx, ctx->gen_mm_dst ? ctx->gen_mm_dst : cgraph->nodes[node_idx + ctx->num_additional_fused_ops]); vk_subbuffer d_Qx = ggml_vk_tensor_subbuffer(ctx, src0); vk_subbuffer d_Qy = ggml_vk_tensor_subbuffer(ctx, src1); vk_subbuffer d_ids = ggml_vk_tensor_subbuffer(ctx, ids); @@ -8371,31 +10199,22 @@ static void ggml_vk_mul_mat_vec_id_q_f16(ggml_backend_vk_context * ctx, vk_conte GGML_ASSERT(x_sz == ggml_vk_align_size(ggml_type_size(src0->type) * x_ne, ctx->device->properties.limits.minStorageBufferOffsetAlignment)); ggml_vk_cpy_to_contiguous(ctx, subctx, to_fp16_vk_0, src0, d_Qx, d_X); } + uint32_t y_ring_slot = 0; if (y_non_contig) { GGML_ASSERT(y_sz == ggml_type_size(src1->type) * y_ne); - if (ctx->prealloc_y_last_pipeline_used != to_fp16_vk_1.get() || - ctx->prealloc_y_last_tensor_used != src1 || - ctx->prealloc_y_last_k_padded) { - if (ctx->prealloc_y_need_sync) { - ggml_vk_sync_buffers(ctx, subctx); - } + bool cached = false; + y_ring_slot = ggml_vk_acquire_prealloc_y_slot(ctx, subctx, to_fp16_vk_1.get(), src1, false, cached); + d_Y = { ctx->prealloc_y, y_ring_slot * ctx->prealloc_y_ring_stride, y_sz }; + if (!cached) { ggml_vk_cpy_to_contiguous(ctx, subctx, to_fp16_vk_1, src1, d_Qy, d_Y); - ctx->prealloc_y_last_pipeline_used = to_fp16_vk_1.get(); - ctx->prealloc_y_last_tensor_used = src1; - ctx->prealloc_y_last_k_padded = false; } } if (quantize_y) { - if (ctx->prealloc_y_last_pipeline_used != to_q8_1.get() || - ctx->prealloc_y_last_tensor_used != src1 || - ctx->prealloc_y_last_k_padded) { - if (ctx->prealloc_y_need_sync) { - ggml_vk_sync_buffers(ctx, subctx); - } + bool cached = false; + y_ring_slot = ggml_vk_acquire_prealloc_y_slot(ctx, subctx, to_q8_1.get(), src1, false, cached); + d_Y = { ctx->prealloc_y, y_ring_slot * ctx->prealloc_y_ring_stride, y_sz }; + if (!cached) { ggml_vk_quantize_q8_1(ctx, subctx, d_Qy, d_Y, y_ne); - ctx->prealloc_y_last_pipeline_used = to_q8_1.get(); - ctx->prealloc_y_last_tensor_used = src1; - ctx->prealloc_y_last_k_padded = false; } } @@ -8418,13 +10237,25 @@ static void ggml_vk_mul_mat_vec_id_q_f16(ggml_backend_vk_context * ctx, vk_conte uint32_t fusion_flags = 0; - if (ctx->num_additional_fused_ops > 0) { + if (has_pair_glu) { + d_F0 = ggml_vk_tensor_subbuffer(ctx, src0_up); + fusion_flags |= MAT_VEC_FUSION_FLAGS_PAIR_GLU; + } else if (ctx->num_additional_fused_ops == 1 && cgraph->nodes[node_idx + 1]->op == GGML_OP_GLU) { + // gen: SWIGLU epilogue (GGML_GEN_FUSE=glu) + const ggml_tensor * glu = cgraph->nodes[node_idx + 1]; + const bool is_gate = glu->src[0] == dst; + d_F0 = ggml_vk_tensor_subbuffer(ctx, is_gate ? glu->src[1] : glu->src[0]); + fusion_flags |= is_gate ? MAT_VEC_FUSION_FLAGS_GLU_A : MAT_VEC_FUSION_FLAGS_GLU_B; + } else if (ctx->num_additional_fused_ops > 0) { const ggml_tensor * bias = cgraph->nodes[node_idx + 1]->src[1]; d_F0 = ggml_vk_tensor_subbuffer(ctx, bias); if (cgraph->nodes[node_idx + 1]->op == GGML_OP_MUL) { fusion_flags |= MAT_VEC_FUSION_FLAGS_SCALE0; + if (bias->ne[2] == (int64_t) nei1 && nei1 > 1) { + fusion_flags |= MAT_VEC_FUSION_FLAGS_SCALE0_TOK; + } } else { GGML_ASSERT(cgraph->nodes[node_idx + 1]->op == GGML_OP_ADD_ID); fusion_flags |= MAT_VEC_FUSION_FLAGS_BIAS0; @@ -8432,20 +10263,22 @@ static void ggml_vk_mul_mat_vec_id_q_f16(ggml_backend_vk_context * ctx, vk_conte } vk_subbuffer d_F1 = d_D; - if (ctx->num_additional_fused_ops > 1) { + if (!has_pair_glu && ctx->num_additional_fused_ops > 1) { const ggml_tensor * scale = cgraph->nodes[node_idx + 2]->src[1]; d_F1 = ggml_vk_tensor_subbuffer(ctx, scale); fusion_flags |= MAT_VEC_FUSION_FLAGS_SCALE1; + if (scale->ne[2] == (int64_t) nei1 && nei1 > 1) { + fusion_flags |= MAT_VEC_FUSION_FLAGS_SCALE1_TOK; + } } - // Loop over the batch dimension - for (uint32_t expert_i1 = 0; expert_i1 < nei1; ++expert_i1) { + if (use_toky) { const vk_mat_vec_id_push_constants pc = { (uint32_t)ne00, (uint32_t)ne10, (uint32_t)ne10, (uint32_t)ne01, stride_batch_x, stride_batch_y, (uint32_t)(ne20 * ne21), - fusion_flags, - (uint32_t)nei0, (uint32_t)ne11, expert_i1, nbi1 + fusion_flags | MAT_VEC_FUSION_FLAGS_GEN_TOKY, + (uint32_t)nei0, (uint32_t)ne11, 0u, nbi1 }; ggml_vk_dispatch_pipeline(ctx, subctx, dmmv, { @@ -8456,7 +10289,27 @@ static void ggml_vk_mul_mat_vec_id_q_f16(ggml_backend_vk_context * ctx, vk_conte d_F1, d_ids, }, - pc, { groups_x, (uint32_t)nei0, groups_z }); + pc, { groups_x, (uint32_t)(nei0 * nei1), groups_z }); + } else { + // Loop over the batch dimension + for (uint32_t expert_i1 = 0; expert_i1 < nei1; ++expert_i1) { + const vk_mat_vec_id_push_constants pc = { + (uint32_t)ne00, (uint32_t)ne10, (uint32_t)ne10, (uint32_t)ne01, + stride_batch_x, stride_batch_y, (uint32_t)(ne20 * ne21), + fusion_flags, + (uint32_t)nei0, (uint32_t)ne11, expert_i1, nbi1 + }; + ggml_vk_dispatch_pipeline(ctx, subctx, dmmv, + { + d_X, + d_Y, + d_D, + d_F0, + d_F1, + d_ids, + }, + pc, { groups_x, (uint32_t)nei0, groups_z }); + } } if (x_non_contig) { @@ -8464,17 +10317,18 @@ static void ggml_vk_mul_mat_vec_id_q_f16(ggml_backend_vk_context * ctx, vk_conte } if (y_non_contig || quantize_y) { ctx->prealloc_y_need_sync = true; + ctx->prealloc_y_slot_need_sync[y_ring_slot] = true; } } -static bool ggml_vk_use_mul_mat_vec_id(const ggml_tensor * dst) { +static bool ggml_vk_use_mul_mat_vec_id(const ggml_tensor * dst, uint32_t max_tokens) { const ggml_tensor * src0 = dst->src[0]; const ggml_tensor * ids = dst->src[2]; - return ids->ne[1] <= 8 && (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type)); + return ids->ne[1] <= (int64_t) max_tokens && (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type)); } -bool ggml_vk_use_mul_mat_vec_id(const struct ggml_cgraph * cgraph, int node_idx) { - return ggml_vk_use_mul_mat_vec_id(cgraph->nodes[node_idx]); +bool ggml_vk_use_mul_mat_vec_id(const ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx) { + return ggml_vk_use_mul_mat_vec_id(cgraph->nodes[node_idx], ctx->device->mmv_id_max_tokens); } void ggml_vk_mul_mat_id(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx) { @@ -8483,10 +10337,10 @@ void ggml_vk_mul_mat_id(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src1 = dst->src[1]; ggml_tensor * src2 = dst->src[2]; VK_LOG_DEBUG("ggml_vk_mul_mat_id(" << src0 << ", " << src1 << ", " << src2 << ", " << dst << ")"); - if (ggml_vk_use_mul_mat_vec_id(cgraph, node_idx)) { + if (ggml_vk_use_mul_mat_vec_id(ctx, cgraph, node_idx)) { ggml_vk_mul_mat_vec_id_q_f16(ctx, subctx, cgraph, node_idx); } else { - ggml_vk_mul_mat_id_q_f16(ctx, subctx, src0, src1, src2, dst); + ggml_vk_mul_mat_id_q_f16(ctx, subctx, src0, src1, src2, dst, cgraph, node_idx); } } @@ -8544,7 +10398,6 @@ bool ggml_vk_flash_attn_scalar_shmem_support(const vk_device& device, const vk_f } bool ggml_vk_flash_attn_coopmat_shmem_support(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool f32acc, ggml_type k_type, ggml_type v_type) { - GGML_UNUSED(v_type); // Needs to be kept up to date on shader changes const uint32_t Br = params.block_rows; const uint32_t Bc = params.block_cols; @@ -8559,6 +10412,25 @@ bool ggml_vk_flash_attn_coopmat_shmem_support(const vk_device& device, const vk_ const uint32_t acctype = f32acc ? 4 : 2; const uint32_t f16vec4 = 8; + if (!device->coopmat_support_16x16x16_f32acc && device->coopmat_support_8x8x8_f32acc && + k_type == GGML_TYPE_F16 && v_type == GGML_TYPE_F16) { + // Series 13 COOPMAT_8X8 F16 layout (no iq_shmem, no pvsh, compact 1 KB kvsh[128], WG=128 or WG=256) + const uint32_t num_sg = params.workgroup_size / params.subgroup_size; + const uint32_t tmpsh = num_sg * sizeof(float); + const uint32_t sh_eMf = Br * sizeof(float); + const uint32_t slope = Br * acctype; + const uint32_t Qf_T = hsk_pad * Br * sizeof(ggml_fp16_t); + const uint32_t Psh_s = Br * (Bc + 8) * sizeof(ggml_fp16_t); + const uint32_t sfstride = (hsk <= 128) ? (Br / 4 + 1) : (Br / 4); + const uint32_t ostride = 16u; + const uint32_t sfsh = std::max(Bc * sfstride, Br * ostride) * (4 * acctype); + const uint32_t kvsh = 128 * f16vec4; + const uint32_t total_size = tmpsh + sh_eMf + slope + Qf_T + Psh_s + sfsh + kvsh; + const bool supported = total_size <= device->properties.limits.maxComputeSharedMemorySize; + VK_LOG_DEBUG("ggml_vk_flash_attn_coopmat_shmem_support_8x8(WG=" << params.workgroup_size << ", HSK=" << hsk << ", HSV=" << hsv << ", total_size=" << total_size << ", supported=" << supported); + return supported; + } + const uint32_t tmpsh = (Bc / MatBc) * sizeof(float); // DATA_A_IQ4_NL is compiled into the FA shaders unconditionally, so its shared table is always allocated. const uint32_t iq_shmem = 16 * sizeof(ggml_fp16_t); @@ -8581,7 +10453,8 @@ bool ggml_vk_flash_attn_coopmat_shmem_support(const vk_device& device, const vk_ uint32_t pvsh = 0; if (Br <= MatBr) { - const uint32_t pvsh_elem_size = (k_type == GGML_TYPE_BF16) ? 16u : f16vec4; + // BF16 / 8x8 PVMat accumulator is f32 (no f16/bf16 accumulator support), so pvsh is vec4 (16 bytes) + const uint32_t pvsh_elem_size = (k_type == GGML_TYPE_BF16 || (!device->coopmat_support_16x16x16_f32acc && device->coopmat_support_8x8x8_f32acc)) ? 16u : f16vec4; const uint32_t osh_stride = params.row_split * MatBr / 4; pvsh = Br * osh_stride * pvsh_elem_size; } @@ -8806,8 +10679,17 @@ void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const aligned = false; } - // Only use mask opt when the mask is fairly large. This hasn't been tuned extensively. - bool use_mask_opt = mask && !use_sparse && nem1 >= 32 && nem0 * nem1 > 32768 && nem0 >= tuning_params.block_cols * 16 + // Only use mask opt when the mask is fairly large. On Apple AGX 8x8 coopmat (with graph-cached mask_opt), + // enable down to KV >= 128 (nem0 >= 2 * Bc) and nem1 >= 16 when gqa_ratio == 1. + static const int env_fa_mo_min_kv = getenv("GGML_VK_FA_MASK_OPT_MIN_KV") ? atoi(getenv("GGML_VK_FA_MASK_OPT_MIN_KV")) : 128; + const bool agx_fast_mask_opt = ggml_vk_is_apple_agx_coopmat(ctx->device) && + !ggml_vk_exact_s8_enabled() && + !gen_batch_invariant_for_n(N) && + env_fa_mo_min_kv > 0; + bool use_mask_opt = mask && !use_sparse && gqa_ratio == 1 && + (agx_fast_mask_opt + ? (nem1 >= 16 && (int64_t)nem0 * nem1 >= 4096 && (int)nem0 >= env_fa_mo_min_kv && nem0 >= tuning_params.block_cols * 2) + : (nem1 >= 32 && nem0 * nem1 > 32768 && nem0 >= tuning_params.block_cols * 16)) && (ctx->device->architecture != vk_device_architecture::AMD_GCN || HSK > 256 || HSV > 256); vk_fa_pipeline_state fa_pipeline_state = get_fa_pipeline_state(ctx->device, tuning_params, HSK, HSV, aligned, f32acc, mask != nullptr, use_mask_opt, logit_softcap != 0, use_sparse, k_type_eff, v_type_eff); @@ -8937,10 +10819,14 @@ void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const // Match the shader's per-split block count so no split is empty. const uint32_t per_blocks = CEIL_DIV(total_blocks, split_k); split_k = CEIL_DIV(total_blocks, per_blocks); - } else if (gqa_ratio > 1 && workgroups_x <= Br) { - split_k = shader_core_count * 2 / (workgroups_x * workgroups_y * workgroups_z); + } else if (gqa_ratio > 1 && (workgroups_x <= Br || (gen_batch_invariant() && workgroups_x <= 8))) { + // gen: in batch-invariant mode also above Br query tokens (gqa ratio 2 has Br 2), so k <= 8 keeps n=1's split + // gen: GGML_VK_BATCH_INVARIANT=1 sizes the KV split as for one query token, so a token's attention sums in the + // same order whether it is evaluated alone or in a small verification batch + const uint32_t wx_split = gen_batch_invariant() ? 1u : workgroups_x; + split_k = shader_core_count * 2 / (wx_split * workgroups_y * workgroups_z); } else if (gqa_ratio <= 1) { - uint32_t total_wgs_no_split = Tr * workgroups_y * workgroups_z; + uint32_t total_wgs_no_split = (gen_batch_invariant() && N <= 8 ? 1u : Tr) * workgroups_y * workgroups_z; if (total_wgs_no_split < shader_core_count * 2) { split_k = shader_core_count * 2 / total_wgs_no_split; } @@ -9000,13 +10886,11 @@ void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const assert(pipeline_fa_mask_opt); ggml_pipeline_request_descriptor_sets(ctx, pipeline_fa_mask_opt, 1); - if (ctx->prealloc_size_y < mask_opt_size) { - ctx->prealloc_size_y = mask_opt_size; + const uint64_t req_mo_size = std::max(mask_opt_size, 65536ULL); + if (ctx->prealloc_mask_opt == nullptr || ctx->prealloc_size_mask_opt < req_mo_size) { + ctx->prealloc_size_mask_opt = req_mo_size; ggml_vk_preallocate_buffers(ctx, subctx); } - if (ctx->prealloc_y_need_sync) { - ggml_vk_sync_buffers(ctx, subctx); - } } // Sparse index scratch reuses prealloc_y (mutually exclusive with mask opt). @@ -9038,12 +10922,12 @@ void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, dst); vk_subbuffer mask_buf = mask ? ggml_vk_tensor_subbuffer(ctx, mask) : q_buf; vk_subbuffer sinks_buf = sinks ? ggml_vk_tensor_subbuffer(ctx, sinks) : q_buf; - vk_subbuffer mask_opt_buf = use_mask_opt ? ggml_vk_subbuffer(ctx, ctx->prealloc_y, 0) : q_buf; + vk_subbuffer mask_opt_buf = use_mask_opt ? ggml_vk_subbuffer(ctx, ctx->prealloc_mask_opt, 0) : q_buf; vk_subbuffer sparse_buf = use_sparse ? ggml_vk_subbuffer(ctx, ctx->prealloc_y, 0) : q_buf; - if (use_mask_opt || use_sparse) { - // the mask opt bits and the sparse index list overwrite a matmul input converted into prealloc_y - ctx->prealloc_y_last_pipeline_used = nullptr; - ctx->prealloc_y_last_tensor_used = nullptr; + if (use_sparse) { + // the sparse index list overwrites a matmul input converted into prealloc_y + ggml_vk_invalidate_prealloc_y_ring(ctx); + ctx->prealloc_y_slot_need_sync[0] = true; } if (use_dequant_kv) { @@ -9106,22 +10990,58 @@ void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const if (use_mask_opt) { - const vk_op_flash_attn_mask_opt_push_constants opt_pc = { - nem0, - nem1, - nem2, - (uint32_t)(mask->nb[1] / sizeof(ggml_fp16_t)), - (uint32_t)(mask->nb[2] / sizeof(ggml_fp16_t)), - (uint32_t)(mask->nb[3] / sizeof(ggml_fp16_t)), - mask_opt_num_dwords, - mask_opt_num_dwords * CEIL_DIV(nem1, Br), - mask_opt_num_dwords * CEIL_DIV(nem1, Br) * nem2, - }; + static const int env_fa_mo_cache = getenv("GGML_VK_FA_MASK_OPT_CACHE") ? atoi(getenv("GGML_VK_FA_MASK_OPT_CACHE")) : 1; + const VkBuffer cur_m_vk_buf = mask_buf.buffer ? (VkBuffer)mask_buf.buffer->buffer : VK_NULL_HANDLE; + const bool can_reuse_mo = (env_fa_mo_cache != 0) && + !ggml_vk_exact_s8_enabled() && + ctx->last_mask_opt_valid && + ctx->last_mask_opt_tensor == mask && + ctx->last_mask_opt_vk_buf == cur_m_vk_buf && + ctx->last_mask_opt_offset == mask_buf.offset && + ctx->last_mask_opt_nem0 == nem0 && + ctx->last_mask_opt_nem1 == nem1 && + ctx->last_mask_opt_nem2 == nem2 && + ctx->last_mask_opt_nem3 == nem3 && + ctx->last_mask_opt_nb1 == mask->nb[1] && + ctx->last_mask_opt_nb2 == mask->nb[2] && + ctx->last_mask_opt_nb3 == mask->nb[3] && + ctx->last_mask_opt_br == Br && + ctx->last_mask_opt_bc == Bc; + if (!can_reuse_mo) { + if (ctx->prealloc_mask_opt_need_sync) { + ggml_vk_sync_buffers(ctx, subctx); + } + const vk_op_flash_attn_mask_opt_push_constants opt_pc = { + nem0, + nem1, + nem2, + (uint32_t)(mask->nb[1] / sizeof(ggml_fp16_t)), + (uint32_t)(mask->nb[2] / sizeof(ggml_fp16_t)), + (uint32_t)(mask->nb[3] / sizeof(ggml_fp16_t)), + mask_opt_num_dwords, + mask_opt_num_dwords * CEIL_DIV(nem1, Br), + mask_opt_num_dwords * CEIL_DIV(nem1, Br) * nem2, + }; - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline_fa_mask_opt, - { mask_buf, mask_opt_buf }, opt_pc, - { mask_opt_num_dwords, CEIL_DIV(nem1, Br), nem2 * nem3 }); - ggml_vk_sync_buffers(ctx, subctx); + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline_fa_mask_opt, + { mask_buf, mask_opt_buf }, opt_pc, + { mask_opt_num_dwords, CEIL_DIV(nem1, Br), nem2 * nem3 }); + ggml_vk_sync_buffers(ctx, subctx); + + ctx->last_mask_opt_tensor = mask; + ctx->last_mask_opt_vk_buf = cur_m_vk_buf; + ctx->last_mask_opt_offset = mask_buf.offset; + ctx->last_mask_opt_nem0 = nem0; + ctx->last_mask_opt_nem1 = nem1; + ctx->last_mask_opt_nem2 = nem2; + ctx->last_mask_opt_nem3 = nem3; + ctx->last_mask_opt_nb1 = mask->nb[1]; + ctx->last_mask_opt_nb2 = mask->nb[2]; + ctx->last_mask_opt_nb3 = mask->nb[3]; + ctx->last_mask_opt_br = Br; + ctx->last_mask_opt_bc = Bc; + ctx->last_mask_opt_valid = true; + } } if (use_sparse) @@ -9236,6 +11156,10 @@ void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const {split_k_buf, sinks_buf, dst_buf}, pc2, { (uint32_t)ne1, HSV, (uint32_t)(ne2 * ne3) }); ctx->prealloc_split_k_need_sync = true; + for (uint32_t s = 0; s < VK_PREALLOC_RING; ++s) { + ctx->prealloc_split_k_slot_need_sync[s] = true; + } + ctx->last_expert_count_valid = false; } else { if (gqa_ratio > 1) { // When using gqa, we want one actual workgroup per batch, so cancel out wg_denoms @@ -9249,8 +11173,14 @@ void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const if (use_dequant_kv) { ctx->prealloc_x_need_sync = true; } - if (use_mask_opt || use_sparse) { + if (use_mask_opt) { + ctx->prealloc_mask_opt_need_sync = true; + } + if (use_sparse) { ctx->prealloc_y_need_sync = true; + for (uint32_t s = 0; s < VK_PREALLOC_RING; ++s) { + ctx->prealloc_y_slot_need_sync[s] = true; + } } } @@ -9340,7 +11270,8 @@ static vk_conv_shapes ggml_vk_conv3d_select_shape(ggml_backend_vk_context * ctx, return CONV_SHAPE_256x128; } -static vk_pipeline ggml_vk_rms_norm_pipeline(const std::array & pipelines, int64_t ncols) { +template +static vk_pipeline ggml_vk_rms_norm_pipeline(const std::array & pipelines, int64_t ncols) { if (pipelines[0] && ncols <= 128) { return pipelines[0]; } @@ -9370,6 +11301,22 @@ static bool ggml_vk_concat_is_transposed(const vk_device & device, const ggml_te return src0->ne[1] == src1->ne[1] && dst->ne[1] == src1->ne[1]; } +static int ggml_vk_rms_norm_fast_v4_idx(const std::array & pipelines, int64_t ncols) { + if (ncols == 128 && pipelines[3]) { + return 3; // BLOCK_SIZE = 32 (1 subgroup, 1 vec4/lane, 0 barriers) + } + if ((ncols == 512 || ncols == 2048) && pipelines[0]) { + return 0; // BLOCK_SIZE = 128 (1 or 4 vec4s/lane, 4 subgroups) + } + if ((ncols == 1024 || ncols == 4096) && pipelines[1]) { + return 1; // BLOCK_SIZE = 256 + } + if (ncols == 8192 && pipelines[2]) { + return 2; // BLOCK_SIZE = 512 + } + return -1; +} + static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * dst, ggml_op op) { switch (op) { case GGML_OP_GET_ROWS: @@ -9414,6 +11361,9 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const switch (op) { case GGML_OP_ADD: { + if (ctx->s10_add_rms_dst != nullptr) { + return ctx->device->pipeline_multi_add_rms_mul[ctx->s10_add_rms_num_adds - 1][ctx->s10_add_rms_tree_mode][ctx->s10_add_rms_weight != nullptr ? 1 : 0]; + } if (ctx->num_additional_fused_ops > 0) { if (ctx->do_add_rms_partials) { return ctx->device->pipeline_multi_add_rms[ctx->num_additional_fused_ops]; @@ -9506,6 +11456,14 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const return nullptr; case GGML_OP_SCALE: if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + static const bool disable_s10_fuse = getenv("GGML_VK_DISABLE_S10_FUSE") != nullptr && atoi(getenv("GGML_VK_DISABLE_S10_FUSE")) != 0; + if (!disable_s10_fuse && ctx->device->pipeline_scale_zero_f32 != nullptr) { + const float sc = ggml_get_op_params_f32(dst, 0); + const float bi = ggml_get_op_params_f32(dst, 1); + if (sc == 0.0f && !std::signbit(sc) && bi == 0.0f && !std::signbit(bi)) { + return ctx->device->pipeline_scale_zero_f32; + } + } return ctx->device->pipeline_scale_f32; } return nullptr; @@ -9622,6 +11580,9 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const if (ctx->do_add_rms_partials) { return ctx->fused_rms_norm_mode == RMS_NORM_MUL ? ctx->device->pipeline_rms_norm_mul_partials_f32 : ctx->device->pipeline_rms_norm_partials_f32; } + if (ctx->fused_rms_norm_mode == RMS_NORM_SCALE) { + return ggml_vk_rms_norm_pipeline(ctx->device->pipeline_rms_norm_scale_f32, src0->ne[0]); + } return ggml_vk_rms_norm_pipeline(ctx->fused_rms_norm_mode == RMS_NORM_MUL ? ctx->device->pipeline_rms_norm_mul_f32 : ctx->device->pipeline_rms_norm_f32, src0->ne[0]); } return nullptr; @@ -9927,6 +11888,9 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const case 128: si = 3; break; default: return nullptr; } + if (ctx->gen_gdn_state_out != nullptr) { + return ctx->device->pipeline_gated_delta_net_sout[si][kda]; + } return ctx->device->pipeline_gated_delta_net[si][kda]; } return nullptr; @@ -9942,6 +11906,23 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const return nullptr; case GGML_OP_SSM_CONV: if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + const bool apply_silu = (dst->op == GGML_OP_UNARY); + if (ctx->s10_ssm_conv_mode == 1 || ctx->s10_ssm_conv_mode == 3) { + if (dst->ne[1] > 8 && !ggml_vk_exact_s8_enabled()) { + return apply_silu ? ctx->device->pipeline_ssm_conv_concat_cpy_mt_silu_f32 + : ctx->device->pipeline_ssm_conv_concat_cpy_mt_f32; + } + return apply_silu ? ctx->device->pipeline_ssm_conv_concat_cpy_silu_f32 + : ctx->device->pipeline_ssm_conv_concat_cpy_f32; + } + if (ctx->s10_ssm_conv_mode == 2) { + return apply_silu ? ctx->device->pipeline_ssm_conv_silu_sout_f32 + : ctx->device->pipeline_ssm_conv_sout_f32; + } + if (dst->ne[1] == 1 && ctx->num_additional_fused_ops <= 1) { + return (ctx->num_additional_fused_ops == 1) ? ctx->device->pipeline_ssm_conv_silu_1t_f32 + : ctx->device->pipeline_ssm_conv_1t_f32; + } switch (ctx->num_additional_fused_ops) { case 0: return ctx->device->pipeline_ssm_conv_f32; case 1: return ctx->device->pipeline_ssm_conv_silu_f32; @@ -10212,8 +12193,8 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, co break; case GGML_OP_RMS_NORM: if (ctx->do_add_rms_partials) { - // Run one element per thread, 128 threads per workgroup - elements = { (uint32_t)CEIL_DIV(ne00, 128), 1, 1 }; + // Run one element per thread, 128 threads per workgroup (gen: one row of workgroups per row) + elements = { (uint32_t)CEIL_DIV(ne00, 128), (uint32_t)ne01, 1 }; } else { elements = { (uint32_t)ne01, (uint32_t)ne02, (uint32_t)ne03 }; elements[1] = std::min(elements[1], ctx->device->properties.limits.maxComputeWorkGroupCount[1]); @@ -10609,15 +12590,16 @@ void ggml_vk_acc(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_t } void ggml_vk_multi_add(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_cgraph * cgraph, int node_idx) { + static constexpr uint32_t MULTI_ADD_BINDINGS = 12; const ggml_tensor *first_node = cgraph->nodes[node_idx]; const ggml_tensor *dst = cgraph->nodes[node_idx + ctx->num_additional_fused_ops]; // Make a list of all the tensors used by the op. // Last element of the list is the dest tensor. - const ggml_tensor *tensors[MAX_PARAMETER_COUNT]; + const ggml_tensor *tensors[MULTI_ADD_BINDINGS]; uint32_t num_srcs = ctx->num_additional_fused_ops + 2; uint32_t num_tensors = num_srcs + 1; - GGML_ASSERT(num_tensors + ctx->do_add_rms_partials <= MAX_PARAMETER_COUNT); + GGML_ASSERT(num_tensors + ctx->do_add_rms_partials <= MULTI_ADD_BINDINGS); tensors[0] = first_node->src[0]; tensors[1] = first_node->src[1]; @@ -10655,10 +12637,10 @@ void ggml_vk_multi_add(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_c ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); - ggml_backend_vk_buffer_context * buf_ctx[MAX_PARAMETER_COUNT]; - vk_buffer buf[MAX_PARAMETER_COUNT]; - size_t offset[MAX_PARAMETER_COUNT]; - bool uma[MAX_PARAMETER_COUNT]; + ggml_backend_vk_buffer_context * buf_ctx[MULTI_ADD_BINDINGS]; + vk_buffer buf[MULTI_ADD_BINDINGS]; + size_t offset[MULTI_ADD_BINDINGS]; + bool uma[MULTI_ADD_BINDINGS]; for (uint32_t i = 0; i < num_tensors; ++i) { buf_ctx[i] = (ggml_backend_vk_buffer_context *)tensors[i]->buffer->context; @@ -10677,7 +12659,7 @@ void ggml_vk_multi_add(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_c GGML_ASSERT(buf[i] != nullptr); } // If any remaining descriptors are unused, just point them at src[0] - for (uint32_t i = num_tensors; i < MAX_PARAMETER_COUNT; ++i) { + for (uint32_t i = num_tensors; i < MULTI_ADD_BINDINGS; ++i) { buf[i] = buf[0]; offset[i] = 0; } @@ -10697,7 +12679,7 @@ void ggml_vk_multi_add(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_c elements = { ne, 1, 1 }; } - static_assert(MAX_PARAMETER_COUNT == 12); + static_assert(MULTI_ADD_BINDINGS <= MAX_PARAMETER_COUNT); ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { ggml_vk_subbuffer(ctx, buf[0], offset[0]), @@ -10715,6 +12697,74 @@ void ggml_vk_multi_add(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_c }, pc, elements); } +static void ggml_vk_multi_add_rms_mul(ggml_backend_vk_context * ctx, vk_context & subctx, ggml_cgraph * cgraph, int node_idx) { + GGML_UNUSED(cgraph); + GGML_UNUSED(node_idx); + const ggml_tensor * dst = ctx->s10_add_rms_dst; + const uint32_t num_srcs = ctx->s10_add_rms_num_adds + 1; + GGML_ASSERT(dst != nullptr && num_srcs >= 2 && num_srcs <= 10); + + vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, ctx->s10_add_rms_srcs[0], ctx->s10_add_rms_srcs[1], nullptr, dst, GGML_OP_ADD); + if (pipeline == nullptr) { + std::cerr << "ggml_vulkan: Error: Missing multi_add_rms_mul" << std::endl; + GGML_ABORT("fatal error"); + } + + const uint32_t ne0 = (uint32_t)dst->ne[0]; + const uint32_t nrows = (uint32_t)ggml_nrows(dst); + static const bool disable_f16_dual = getenv("GGML_VK_DISABLE_F16_DUAL") != nullptr; + const bool want_f16_dual = !disable_f16_dual && + ctx->s10_add_rms_tree_mode == 2u && + nrows >= 32u && + !gen_batch_invariant_for_n(nrows) && + ggml_is_contiguous(dst) && + get_misalign_bytes(ctx, dst) == 0; + const size_t req_y_bytes = sizeof(ggml_fp16_t) * (size_t)ROUNDUP_POW2(nrows, 128u) * ne0; + const size_t wr_y_bytes = sizeof(ggml_fp16_t) * (size_t)nrows * ne0; + if (want_f16_dual) { + ggml_vk_reserve_prealloc_y(ctx, subctx, req_y_bytes); + } + + ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); + + vk_op_multi_add_rms_push_constants pc {}; + pc.ne0 = ne0; + pc.nrows = nrows; + pc.eps = ctx->s10_add_rms_eps; + pc.write_add_dst = ctx->s10_add_rms_write_add | (want_f16_dual ? 2u : 0u); + for (uint32_t s = 0; s < num_srcs; ++s) { + pc.nb1[s] = (uint32_t)(ctx->s10_add_rms_srcs[s]->nb[1] / sizeof(float)); + } + + vk_subbuffer sbuf[14]; + for (uint32_t s = 0; s < num_srcs; ++s) { + sbuf[s] = ggml_vk_tensor_subbuffer(ctx, ctx->s10_add_rms_srcs[s]); + } + for (uint32_t s = num_srcs; s < 10; ++s) { + sbuf[s] = sbuf[0]; + } + sbuf[10] = (ctx->s10_add_rms_write_add && ctx->s10_add_rms_add_dst) + ? ggml_vk_tensor_subbuffer(ctx, ctx->s10_add_rms_add_dst) + : sbuf[0]; + sbuf[11] = ctx->s10_add_rms_weight + ? ggml_vk_tensor_subbuffer(ctx, ctx->s10_add_rms_weight) + : sbuf[0]; + sbuf[12] = ggml_vk_tensor_subbuffer(ctx, dst); + sbuf[13] = want_f16_dual + ? ggml_vk_produce_prealloc_y_f16(ctx, subctx, dst, wr_y_bytes, true) + : sbuf[12]; + + const uint32_t wg_x = std::min(nrows, ctx->device->properties.limits.maxComputeWorkGroupCount[0]); + const uint32_t wg_y = CEIL_DIV(nrows, wg_x); + const std::array elements = { wg_x, wg_y, 1 }; + + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, + { + sbuf[0], sbuf[1], sbuf[2], sbuf[3], sbuf[4], sbuf[5], sbuf[6], + sbuf[7], sbuf[8], sbuf[9], sbuf[10], sbuf[11], sbuf[12], sbuf[13], + }, pc, elements); +} + void ggml_vk_add(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { const uint32_t src0_type_size = ggml_type_size(src0->type); const uint32_t src1_type_size = ggml_type_size(src1->type); @@ -10823,6 +12873,40 @@ void ggml_vk_unary_mul(ggml_backend_vk_context * ctx, vk_context& subctx, const }, pipeline); } +static void ggml_vk_cont_unary_mul(ggml_backend_vk_context * ctx, vk_context & subctx, const struct ggml_cgraph * cgraph, int node_idx) { + const ggml_tensor * cont = cgraph->nodes[node_idx]; + const ggml_tensor * unary = cgraph->nodes[node_idx + ctx->num_additional_fused_ops - 1]; + ggml_tensor * mul = cgraph->nodes[node_idx + ctx->num_additional_fused_ops]; + + const ggml_tensor * src0 = cont->src[0]; + const ggml_tensor * src1 = (mul->src[0] == unary) ? mul->src[1] : mul->src[0]; + + const bool f16 = src0->type == GGML_TYPE_F16; + const int oi = ggml_vk_unary_mul_op_index(ggml_get_unary_op(unary)); + if (oi < 0) { + GGML_ABORT("fatal error"); + } + vk_pipeline pipeline = ctx->device->pipeline_unary_mul[oi][f16][1][0]; + + const uint32_t ts = ggml_type_size(src0->type); + const uint32_t d0 = (uint32_t) src0->ne[0]; + const uint32_t d1 = (uint32_t) src0->ne[1]; + const uint32_t d2 = (uint32_t) src0->ne[2]; + const uint32_t d3 = (uint32_t) src0->ne[3]; + const uint32_t s1 = d0; + const uint32_t s2 = d0 * d1; + const uint32_t s3 = d0 * d1 * d2; + + ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, nullptr, mul, GGML_OP_UNARY, { + (uint32_t) ggml_nelements(mul), + d0, d1, d2, d3, (uint32_t) (src0->nb[0] / ts), (uint32_t) (src0->nb[1] / ts), (uint32_t) (src0->nb[2] / ts), (uint32_t) (src0->nb[3] / ts), + d0, d1, d2, d3, 1u, s1, s2, s3, + d0, d1, d2, d3, 1u, s1, s2, s3, + 0, + 0.0f, 0.0f, 0, + }, pipeline); +} + void ggml_vk_div(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { const uint32_t src0_type_size = ggml_type_size(src0->type); const uint32_t src1_type_size = ggml_type_size(src1->type); @@ -11056,6 +13140,9 @@ void ggml_vk_gated_delta_net(ggml_backend_vk_context * ctx, vk_context& subctx, const uint32_t rq3 = (uint32_t)(src_v->ne[3] / src_q->ne[3]); const float scale = 1.0f / sqrtf((float)S_v); + const uint32_t sout_nb2 = (ctx->gen_gdn_state_out != nullptr && K > 1) + ? (uint32_t)(ctx->gen_gdn_state_out->nb[2] / sizeof(float)) : 0u; + const uint32_t pc_K = K | (ctx->gen_gdn_zero_state ? 0x80000000u : 0u); const vk_op_gated_delta_net_push_constants pc = { H, n_tokens, n_seqs, s_off, sq1, sq2, sq3, @@ -11063,9 +13150,19 @@ void ggml_vk_gated_delta_net(ggml_backend_vk_context * ctx, vk_context& subctx, sb1, sb2, sb3, neq1, rq3, scale, - K + pc_K, + sout_nb2 }; + if (ctx->gen_gdn_state_out != nullptr) { + // gen: the final state / rollback snapshots are written straight into the CPY destination + vk_subbuffer state_out_buf = ggml_vk_strided_subbuffer(ctx, ctx->gen_gdn_state_out); + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, + {src_buf[0], src_buf[1], src_buf[2], src_buf[3], src_buf[4], src_buf[5], dst_buf, state_out_buf}, + pc, { H, n_seqs, S_v }); + return; + } + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, {src_buf[0], src_buf[1], src_buf[2], src_buf[3], src_buf[4], src_buf[5], dst_buf}, pc, { H, n_seqs, S_v }); @@ -11129,36 +13226,73 @@ void ggml_vk_ssm_scan(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_te } void ggml_vk_ssm_conv(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx) { - ggml_tensor * conv = cgraph->nodes[node_idx]; - const ggml_tensor * src0 = conv->src[0]; + const int mode = ctx->s10_ssm_conv_mode; + const bool is_sliding = (mode == 1 || mode == 3); + const ggml_tensor * conv = (mode != 0) ? ctx->s10_ssm_conv_node : cgraph->nodes[node_idx]; + const ggml_tensor * src0 = (mode == 3) ? ctx->s10_ssm_old_st : ((mode == 1) ? cgraph->nodes[node_idx]->src[0] : conv->src[0]); const ggml_tensor * src1 = conv->src[1]; - // Pick the destination tensor (last node in the fused chain) and the optional bias. - // Fusion modes: 0 = ssm_conv, 1 = ssm_conv+silu, 2 = ssm_conv+add(bias)+silu. - ggml_tensor * dst = conv; + ggml_tensor * dst = cgraph->nodes[node_idx + ctx->num_additional_fused_ops]; const ggml_tensor * bias = nullptr; - if (ctx->num_additional_fused_ops == 1) { - dst = cgraph->nodes[node_idx + 1]; // silu - } else if (ctx->num_additional_fused_ops == 2) { - ggml_tensor * add = cgraph->nodes[node_idx + 1]; - bias = (add->src[0] == conv) ? add->src[1] : add->src[0]; - dst = cgraph->nodes[node_idx + 2]; // silu + if (mode == 0) { + if (ctx->num_additional_fused_ops == 1) { + dst = cgraph->nodes[node_idx + 1]; // silu + } else if (ctx->num_additional_fused_ops == 2) { + ggml_tensor * add = cgraph->nodes[node_idx + 1]; + bias = (add->src[0] == conv) ? add->src[1] : add->src[0]; + dst = cgraph->nodes[node_idx + 2]; // silu + } } - // The shader always declares 4 bindings; bind src0 as a dummy when bias isn't fused. - const ggml_tensor * src2 = bias ? bias : src0; + const ggml_tensor * src2 = is_sliding ? ctx->s10_ssm_qkv : (bias ? bias : src0); + const ggml_tensor * state_out = (mode != 0) ? ctx->s10_ssm_state_out : dst; + + vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, src0, src1, src2, dst, GGML_OP_SSM_CONV); + if (pipeline == nullptr) { + std::cerr << "ggml_vulkan: Error: Missing ssm_conv pipeline" << std::endl; + GGML_ABORT("fatal error"); + } + ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); - ggml_vk_op_f32(ctx, subctx, src0, src1, src2, nullptr, dst, GGML_OP_SSM_CONV, { + const uint32_t nc = (uint32_t)src1->ne[0]; + const uint32_t ncs = (uint32_t)conv->src[0]->ne[0]; + const uint32_t nr = (uint32_t)conv->src[0]->ne[1]; + const uint32_t n_t = (uint32_t)dst->ne[1]; + const uint32_t n_s = (uint32_t)dst->ne[2]; + + const uint32_t st_nb1 = (mode == 3) ? (uint32_t)state_out->nb[2] : ((mode != 0) ? (uint32_t)((nc - 1) * sizeof(float)) : 0u); + const uint32_t st_nb2 = (mode == 3) ? (uint32_t)state_out->nb[3] : ((mode != 0) ? (uint32_t)((nc - 1) * nr * sizeof(float)) : 0u); + const uint32_t qkv_nb0 = is_sliding ? (uint32_t)ctx->s10_ssm_qkv->nb[0] : 0u; + const uint32_t qkv_nb1 = is_sliding ? (uint32_t)ctx->s10_ssm_qkv->nb[1] : 0u; + const uint32_t qkv_nb2 = is_sliding ? (uint32_t)ctx->s10_ssm_qkv->nb[2] : 0u; + const uint32_t st_k = ((mode == 3) ? ctx->s10_ssm_st_k : 0u) | ((is_sliding && ctx->s10_ssm_zero_state) ? 0x80000000u : 0u); + const uint32_t st_slot_nb = (mode == 3) ? ctx->s10_ssm_st_slot_nb : 0u; + + vk_op_ssm_conv_push_constants pc = { (uint32_t)src0->nb[1], (uint32_t)src0->nb[2], (uint32_t)src1->nb[1], (uint32_t)dst->nb[0], (uint32_t)dst->nb[1], (uint32_t)dst->nb[2], - (uint32_t)src1->ne[0], - (uint32_t)src0->ne[0], - (uint32_t)src0->ne[1], - (uint32_t)dst->ne[1], - (uint32_t)dst->ne[2], - }); + nc, ncs, nr, n_t, n_s, + st_nb1, st_nb2, + qkv_nb0, qkv_nb1, qkv_nb2, + st_k, st_slot_nb, + }; + + const std::array elements = { + nr, + (is_sliding && (n_t <= 8 || ggml_vk_exact_s8_enabled())) ? 1u : n_t, + n_s, + }; + + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, + { + ggml_vk_strided_subbuffer(ctx, src0), + ggml_vk_tensor_subbuffer(ctx, src1), + ggml_vk_strided_subbuffer(ctx, src2), + ggml_vk_tensor_subbuffer(ctx, dst), + ggml_vk_strided_subbuffer(ctx, state_out), + }, pc, elements); } static void ggml_vk_op_f32_opt_step_adamw(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst, const vk_op_push_constants&& pc) { @@ -11487,6 +13621,13 @@ uint32_t ggml_vk_rms_partials_size(ggml_backend_vk_context * ctx, const ggml_ten return num_bytes; } +// gen (GGML_VK_BATCH_INVARIANT=1): the partials of a multi-row ADD+RMS_NORM, one row's partials after another; the +// per-row values are exactly those of the same row evaluated alone +static uint32_t gen_rms_partials_size_rows(ggml_backend_vk_context * ctx, const ggml_tensor *node) { + const uint32_t num_partials = ggml_vk_rms_num_partials(ctx, node) * (uint32_t) ggml_nrows(node); + return ROUNDUP_POW2(num_partials * sizeof(uint32_t), ctx->device->partials_binding_alignment); +} + static vk_op_rope_push_constants ggml_vk_make_rope_constants(const ggml_tensor *dst, const ggml_tensor *src0, const bool has_ff, bool backprop, const uint32_t set_rows_stride) { const int n_dims = ((const int32_t *) dst->op_params)[1]; const int mode = ((const int32_t *) dst->op_params)[2]; @@ -11536,7 +13677,8 @@ static vk_op_rope_push_constants ggml_vk_make_rope_constants(const ggml_tensor * static void ggml_vk_rms_norm_finish(ggml_backend_vk_context * ctx, const ggml_tensor * src0) { if (ctx->do_add_rms_partials_offset_calculation) { - ctx->prealloc_size_add_rms_partials_offset += ggml_vk_rms_partials_size(ctx, src0); + ctx->prealloc_size_add_rms_partials_offset += gen_batch_invariant() ? gen_rms_partials_size_rows(ctx, src0) : ggml_vk_rms_partials_size(ctx, src0); + ctx->gen_n1_partials_offset += ggml_vk_rms_partials_size(ctx, src0); ctx->do_add_rms_partials = false; ctx->do_add_rms_partials_offset_calculation = false; } @@ -11571,15 +13713,105 @@ void ggml_vk_rms_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const s return; } - if (ctx->fused_rms_norm_mode == RMS_NORM_MUL_ADD || ctx->fused_rms_norm_mode == RMS_NORM_MUL_ADD_MUL) { + if (ctx->fused_rms_norm_mode == RMS_NORM_MUL_SILU_GATE) { + // gen: nodes RMS_NORM, MUL(w), [RESHAPE/VIEW(z)], UNARY silu(z), MUL -> one dispatch + GGML_ASSERT(ctx->num_additional_fused_ops == 3 || ctx->num_additional_fused_ops == 4); ggml_tensor * mul = cgraph->nodes[node_idx + 1]; - ggml_tensor * add = cgraph->nodes[node_idx + 2]; + ggml_tensor * silu = cgraph->nodes[node_idx + ctx->num_additional_fused_ops - 1]; + ggml_tensor * dst = cgraph->nodes[node_idx + ctx->num_additional_fused_ops]; const ggml_tensor * weight = mul->src[0] == rms ? mul->src[1] : mul->src[0]; - const ggml_tensor * residual = add->src[0] == mul ? add->src[1] : add->src[0]; - const bool do_post_multiply = ctx->fused_rms_norm_mode == RMS_NORM_MUL_ADD_MUL; - GGML_ASSERT(ctx->num_additional_fused_ops == (do_post_multiply ? 3 : 2)); - ggml_tensor * dst = do_post_multiply ? cgraph->nodes[node_idx + 3] : add; - const ggml_tensor * post_scale = do_post_multiply ? + const ggml_tensor * z = silu->src[0]; + vk_op_binary_push_constants pc = ggml_vk_rms_norm_push_constants(src0, weight, dst, op_params[0], 0); + init_pushconst_tensor_offsets(ctx, pc, src0, weight, nullptr, nullptr, dst); + init_pushconst_fastdiv(pc); + + static const bool disable_fast_rms = getenv("GGML_VK_DISABLE_FAST_RMS") != nullptr; + static const bool disable_f16_dual = getenv("GGML_VK_DISABLE_F16_DUAL") != nullptr; + static const bool keep_silu_f32 = getenv("GGML_VK_KEEP_SILU_F32") != nullptr; + const int fast_idx = (!disable_fast_rms && ggml_vk_is_apple_agx_coopmat(ctx->device) && !ggml_vk_exact_s8_enabled()) + ? ggml_vk_rms_norm_fast_v4_idx(ctx->device->pipeline_rms_norm_mul_silu_f32, src0->ne[0]) + : -1; + const bool v4_ok = fast_idx >= 0 && + ((pc.misalign_offsets >> 16) % 4 == 0) && + (((pc.misalign_offsets >> 8) & 0xFF) % 4 == 0) && + ((pc.misalign_offsets & 0xFF) == 0) && + (src0->ne[1] == 1 || pc.nb01 % 4 == 0) && + (src0->ne[2] == 1 || pc.nb02 % 4 == 0) && + (src0->ne[3] == 1 || pc.nb03 % 4 == 0) && + (weight->ne[0] % 4 == 0) && + (weight->ne[1] == 1 || pc.nb11 % 4 == 0) && + (weight->ne[2] == 1 || pc.nb12 % 4 == 0) && + (weight->ne[3] == 1 || pc.nb13 % 4 == 0) && + get_misalign_bytes(ctx, dst) == 0 && + get_misalign_bytes(ctx, z) == 0; + const uint64_t tok_dim = dst->ne[2] > 1 ? (uint64_t)(dst->ne[2] * dst->ne[3]) : (uint64_t)dst->ne[1]; + const uint64_t feat_dim = ggml_nelements(dst) / std::max(1, tok_dim); + const bool want_f16_dual = v4_ok && !disable_f16_dual && + tok_dim >= 128 && !gen_batch_invariant_for_n(tok_dim) && + ggml_is_contiguous(dst); + const size_t req_y_bytes = sizeof(ggml_fp16_t) * (size_t)ROUNDUP_POW2(tok_dim, 128ULL) * feat_dim; + const size_t wr_y_bytes = sizeof(ggml_fp16_t) * (size_t)ggml_nelements(dst); + if (want_f16_dual) { + ggml_vk_reserve_prealloc_y(ctx, subctx, req_y_bytes); + } + uint32_t rms_flags = 0u; + if (v4_ok) { + rms_flags |= 2u; + if (want_f16_dual) { + rms_flags |= 4u; + if (ctx->gen_rms_silu_sole_mm_next && !keep_silu_f32) { + rms_flags |= 8u; + } + } + } + pc.param3 = (int)rms_flags; + vk_pipeline pipeline = v4_ok + ? ctx->device->pipeline_rms_norm_mul_silu_f32[fast_idx] + : ggml_vk_rms_norm_pipeline(ctx->device->pipeline_rms_norm_mul_silu_f32, src0->ne[0]); + ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); + vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, dst, true); + if (pipeline->parameter_count == 5) { + vk_subbuffer f16_buf = want_f16_dual + ? ggml_vk_produce_prealloc_y_f16(ctx, subctx, dst, wr_y_bytes, false) + : dst_buf; + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, + { + ggml_vk_tensor_subbuffer(ctx, src0, true), + ggml_vk_tensor_subbuffer(ctx, weight, true), + dst_buf, + ggml_vk_tensor_subbuffer(ctx, z), + f16_buf, + }, pc, { + (uint32_t)src0->ne[1], + std::min((uint32_t)src0->ne[2], ctx->device->properties.limits.maxComputeWorkGroupCount[1]), + std::min((uint32_t)src0->ne[3], ctx->device->properties.limits.maxComputeWorkGroupCount[2]), + }); + } else { + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, + { + ggml_vk_tensor_subbuffer(ctx, src0, true), + ggml_vk_tensor_subbuffer(ctx, weight, true), + dst_buf, + ggml_vk_tensor_subbuffer(ctx, z), + }, pc, { + (uint32_t)src0->ne[1], + std::min((uint32_t)src0->ne[2], ctx->device->properties.limits.maxComputeWorkGroupCount[1]), + std::min((uint32_t)src0->ne[3], ctx->device->properties.limits.maxComputeWorkGroupCount[2]), + }); + } + ggml_vk_rms_norm_finish(ctx, src0); + return; + } + + if (ctx->fused_rms_norm_mode == RMS_NORM_MUL_ADD || ctx->fused_rms_norm_mode == RMS_NORM_MUL_ADD_MUL) { + ggml_tensor * mul = cgraph->nodes[node_idx + 1]; + ggml_tensor * add = cgraph->nodes[node_idx + 2]; + const ggml_tensor * weight = mul->src[0] == rms ? mul->src[1] : mul->src[0]; + const ggml_tensor * residual = add->src[0] == mul ? add->src[1] : add->src[0]; + const bool do_post_multiply = ctx->fused_rms_norm_mode == RMS_NORM_MUL_ADD_MUL; + GGML_ASSERT(ctx->num_additional_fused_ops == (do_post_multiply ? 3 : 2)); + ggml_tensor * dst = do_post_multiply ? cgraph->nodes[node_idx + 3] : add; + const ggml_tensor * post_scale = do_post_multiply ? (dst->src[0] == add ? dst->src[1] : dst->src[0]) : src0; const uint32_t num_partials = ctx->do_add_rms_partials ? ggml_vk_rms_num_partials(ctx, dst) : 0; @@ -11604,7 +13836,7 @@ void ggml_vk_rms_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const s ggml_vk_subbuffer(ctx, ctx->prealloc_add_rms_partials, ctx->prealloc_size_add_rms_partials_offset), ggml_vk_tensor_subbuffer(ctx, residual), ggml_vk_tensor_subbuffer(ctx, post_scale), - }, pc, { (uint32_t)CEIL_DIV(src0->ne[0], 128), 1, 1 }); + }, pc, { (uint32_t)CEIL_DIV(src0->ne[0], 128), (uint32_t)src0->ne[1], 1 }); } else { ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { @@ -11623,6 +13855,53 @@ void ggml_vk_rms_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const s return; } + if (ctx->fused_rms_norm_mode == RMS_NORM_SCALE) { + ggml_tensor * dst0 = const_cast(ctx->gen_rms_scale_dst); + ggml_tensor * dst1 = const_cast(ctx->gen_rms_scale_dst2); + vk_op_binary_push_constants pc = ggml_vk_rms_norm_push_constants(src0, src0, dst0, op_params[0], 0); + init_pushconst_tensor_offsets(ctx, pc, src0, src0, nullptr, nullptr, dst0); + init_pushconst_fastdiv(pc); + pc.param2 = ggml_get_op_params_f32(dst0, 0); + uint32_t p3_bits = dst1 ? (1u | (((uint32_t)(get_misalign_bytes(ctx, dst1) / sizeof(float))) << 8)) : 0u; + static const bool disable_fast_rms = getenv("GGML_VK_DISABLE_FAST_RMS") != nullptr; + const int fast_idx = (!disable_fast_rms && ggml_vk_is_apple_agx_coopmat(ctx->device) && !ggml_vk_exact_s8_enabled()) + ? ggml_vk_rms_norm_fast_v4_idx(ctx->device->pipeline_rms_norm_scale_f32, src0->ne[0]) + : -1; + const bool v4_ok = fast_idx >= 0 && + ((pc.misalign_offsets >> 16) % 4 == 0) && + ((pc.misalign_offsets & 0xFF) % 4 == 0) && + ((dst1 != nullptr || src0->ne[1] > 1) ? (pc.nb01 % 4 == 0) : true) && + (src0->ne[2] == 1 || pc.nb02 % 4 == 0) && + (src0->ne[3] == 1 || pc.nb03 % 4 == 0) && + (!dst1 || (get_misalign_bytes(ctx, dst1) % 16 == 0)); + if (v4_ok) { + p3_bits |= 2u; + } + pc.param3 = (int)p3_bits; + vk_pipeline pipeline = v4_ok + ? ctx->device->pipeline_rms_norm_scale_f32[fast_idx] + : ggml_vk_rms_norm_pipeline(ctx->device->pipeline_rms_norm_scale_f32, src0->ne[0]); + ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); + vk_subbuffer s0_buf = ggml_vk_strided_subbuffer(ctx, src0, true); + if (dst1 != nullptr && ctx->gen_rms_scale_rsrc2 != nullptr) { + vk_subbuffer s1_buf = ggml_vk_strided_subbuffer(ctx, ctx->gen_rms_scale_rsrc2, true); + const size_t end1 = s1_buf.offset + s1_buf.size; + if (end1 > s0_buf.offset + s0_buf.size) { + s0_buf.size = end1 - s0_buf.offset; + } + } + vk_subbuffer d0_buf = ggml_vk_tensor_subbuffer(ctx, dst0, true); + vk_subbuffer d1_buf = dst1 ? ggml_vk_tensor_subbuffer(ctx, dst1, true) : d0_buf; + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, + { s0_buf, s0_buf, d0_buf, d1_buf }, pc, { + (uint32_t)(dst1 ? 2 * src0->ne[1] : src0->ne[1]), + std::min((uint32_t)src0->ne[2], ctx->device->properties.limits.maxComputeWorkGroupCount[1]), + std::min((uint32_t)src0->ne[3], ctx->device->properties.limits.maxComputeWorkGroupCount[2]), + }); + ggml_vk_rms_norm_finish(ctx, src0); + return; + } + ggml_tensor * dst; const ggml_tensor * src1; @@ -11720,7 +13999,75 @@ void ggml_vk_rms_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const s ggml_vk_subbuffer(ctx, buf[6], offset[6]), }, pc, elements); } else { - GGML_ASSERT(ctx->fused_rms_norm_mode == RMS_NORM_MUL || ctx->fused_rms_norm_mode == RMS_NORM_COUNT); + GGML_ASSERT(ctx->fused_rms_norm_mode == RMS_NORM_MUL || ctx->fused_rms_norm_mode == RMS_NORM_SCALE || ctx->fused_rms_norm_mode == RMS_NORM_COUNT); + static const bool disable_fast_rms = getenv("GGML_VK_DISABLE_FAST_RMS") != nullptr; + static const bool disable_f16_dual = getenv("GGML_VK_DISABLE_F16_DUAL") != nullptr; + const auto & pipe_arr = (ctx->fused_rms_norm_mode == RMS_NORM_MUL) + ? ctx->device->pipeline_rms_norm_mul_f32 + : ctx->device->pipeline_rms_norm_f32; + const int fast_idx = (!ctx->do_add_rms_partials && !disable_fast_rms && + ggml_vk_is_apple_agx_coopmat(ctx->device) && !ggml_vk_exact_s8_enabled() && + src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) + ? ggml_vk_rms_norm_fast_v4_idx(pipe_arr, src0->ne[0]) + : -1; + if (fast_idx >= 0) { + vk_op_binary_push_constants pc = bin; + init_pushconst_tensor_offsets(ctx, pc, src0, src1, nullptr, nullptr, dst); + init_pushconst_fastdiv(pc); + const bool v4_ok = + ((pc.misalign_offsets >> 16) % 4 == 0) && + (((pc.misalign_offsets >> 8) & 0xFF) % 4 == 0) && + ((pc.misalign_offsets & 0xFF) % 4 == 0) && + (src0->ne[1] == 1 || pc.nb01 % 4 == 0) && + (src0->ne[2] == 1 || pc.nb02 % 4 == 0) && + (src0->ne[3] == 1 || pc.nb03 % 4 == 0) && + (src1->ne[0] % 4 == 0) && + (src1->ne[1] == 1 || pc.nb11 % 4 == 0) && + (src1->ne[2] == 1 || pc.nb12 % 4 == 0) && + (src1->ne[3] == 1 || pc.nb13 % 4 == 0); + if (v4_ok) { + const int next_i = node_idx + ctx->num_additional_fused_ops + 1; + const bool has_mm_consumer = cgraph != nullptr && next_i < cgraph->n_nodes && + (cgraph->nodes[next_i]->op == GGML_OP_MUL_MAT || + cgraph->nodes[next_i]->op == GGML_OP_MUL_MAT_ID || + cgraph->nodes[next_i]->op == GGML_OP_RESHAPE || + cgraph->nodes[next_i]->op == GGML_OP_VIEW); + const uint32_t nrows = (uint32_t)src0->ne[1]; + const bool want_f16_dual = !disable_f16_dual && + ctx->fused_rms_norm_mode == RMS_NORM_MUL && + has_mm_consumer && + src0->ne[2] == 1 && src0->ne[3] == 1 && + nrows >= 32u && + !gen_batch_invariant_for_n(nrows) && + ggml_is_contiguous(dst) && + get_misalign_bytes(ctx, dst) == 0; + const size_t req_y_bytes = sizeof(ggml_fp16_t) * (size_t)ROUNDUP_POW2(nrows, 128u) * (size_t)src0->ne[0]; + const size_t wr_y_bytes = sizeof(ggml_fp16_t) * (size_t)nrows * (size_t)src0->ne[0]; + if (want_f16_dual) { + ggml_vk_reserve_prealloc_y(ctx, subctx, req_y_bytes); + } + pc.param3 = (int)(2u | (want_f16_dual ? 4u : 0u)); + vk_pipeline pipeline = pipe_arr[fast_idx]; + ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); + vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, dst, true); + vk_subbuffer f16_buf = want_f16_dual + ? ggml_vk_produce_prealloc_y_f16(ctx, subctx, dst, wr_y_bytes, false) + : dst_buf; + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, + { + ggml_vk_tensor_subbuffer(ctx, src0, true), + ggml_vk_tensor_subbuffer(ctx, src1, true), + dst_buf, + f16_buf, + }, pc, { + (uint32_t)src0->ne[1], + std::min((uint32_t)src0->ne[2], ctx->device->properties.limits.maxComputeWorkGroupCount[1]), + std::min((uint32_t)src0->ne[3], ctx->device->properties.limits.maxComputeWorkGroupCount[2]), + }); + ggml_vk_rms_norm_finish(ctx, src0); + return; + } + } ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_RMS_NORM, std::move(bin)); } @@ -11865,8 +14212,8 @@ void ggml_vk_soft_max(ggml_backend_vk_context * ctx, vk_context& subctx, const g vk_subbuffer buf_y = { ctx->prealloc_y, 0, tmp_size }; // the partial results overwrite a matmul input converted into prealloc_y - ctx->prealloc_y_last_pipeline_used = nullptr; - ctx->prealloc_y_last_tensor_used = nullptr; + ggml_vk_invalidate_prealloc_y_ring(ctx); + ctx->prealloc_y_slot_need_sync[0] = true; std::array elements = { num_wgs, nrows_x, 1 }; @@ -11886,6 +14233,9 @@ void ggml_vk_soft_max(ggml_backend_vk_context * ctx, vk_context& subctx, const g ctx->prealloc_x_need_sync = true; ctx->prealloc_y_need_sync = true; + for (uint32_t s = 0; s < VK_PREALLOC_RING; ++s) { + ctx->prealloc_y_slot_need_sync[s] = true; + } } } @@ -12331,6 +14681,10 @@ void ggml_vk_cumsum(ggml_backend_vk_context * ctx, vk_context& subctx, const ggm ggml_vk_dispatch_pipeline(ctx, subctx, pipeline2, {src_buf, dst_buf, temp_buf}, pc, elements); ctx->prealloc_split_k_need_sync = true; + for (uint32_t s = 0; s < VK_PREALLOC_RING; ++s) { + ctx->prealloc_split_k_slot_need_sync[s] = true; + } + ctx->last_expert_count_valid = false; } static std::array ggml_vk_nrows_elements(uint32_t nr) { @@ -12921,7 +15275,12 @@ void ggml_vk_preallocate_buffers(ggml_backend_vk_context * ctx, vk_context subct #endif if (subctx) { + if (ctx->recording_replay) { + ctx->replay_abort = true; + ctx->recording_replay = false; + } // Submit and wait for any pending work before reallocating the buffers + ggml_vk_ensure_fence_free(ctx); ggml_vk_ctx_end(subctx); ggml_vk_submit(subctx, {}); ctx->submit_pending = true; @@ -12939,24 +15298,47 @@ void ggml_vk_preallocate_buffers(ggml_backend_vk_context * ctx, vk_context subct } ctx->prealloc_x = ggml_vk_create_buffer_device(ctx->device, ctx->prealloc_size_x); } - if (ctx->prealloc_y == nullptr || (ctx->prealloc_size_y > 0 && ctx->prealloc_y->size < ctx->prealloc_size_y)) { - VK_LOG_MEMORY("ggml_vk_preallocate_buffers(y_size: " << ctx->prealloc_size_y << ")"); + const size_t y_stride = ctx->prealloc_size_y > 0 ? ROUNDUP_POW2(ctx->prealloc_size_y, 4096ULL) : 0; + const uint32_t y_ring = (y_stride > 0 && y_stride * VK_PREALLOC_RING <= ctx->device->properties.limits.maxStorageBufferRange && + y_stride * VK_PREALLOC_RING <= 512ULL * 1024 * 1024) ? VK_PREALLOC_RING : 1u; + const size_t y_total = y_ring > 1 ? y_stride * y_ring : ctx->prealloc_size_y; + if (ctx->prealloc_y == nullptr || (y_total > 0 && ctx->prealloc_y->size < y_total)) { + VK_LOG_MEMORY("ggml_vk_preallocate_buffers(y_size: " << y_total << ")"); // Resize buffer if (ctx->prealloc_y != nullptr) { ggml_vk_destroy_buffer(ctx->prealloc_y); } - ctx->prealloc_y = ggml_vk_create_buffer_device(ctx->device, ctx->prealloc_size_y); - ctx->prealloc_y_last_pipeline_used = nullptr; - ctx->prealloc_y_last_tensor_used = nullptr; - ctx->prealloc_y_last_k_padded = false; - } - if (ctx->prealloc_split_k == nullptr || (ctx->prealloc_size_split_k > 0 && ctx->prealloc_split_k->size < ctx->prealloc_size_split_k)) { - VK_LOG_MEMORY("ggml_vk_preallocate_buffers(split_k_size: " << ctx->prealloc_size_split_k << ")"); + ctx->prealloc_y = ggml_vk_create_buffer_device(ctx->device, y_total); + ctx->prealloc_y_ring_stride = y_stride; + ctx->prealloc_y_ring_count = y_ring; + ctx->prealloc_y_ring_idx = 0; + ggml_vk_invalidate_prealloc_y_ring(ctx); + } else if (y_total > 0) { + if (ctx->prealloc_y_ring_stride != y_stride || ctx->prealloc_y_ring_count != y_ring) { + ctx->prealloc_y_ring_idx = 0; + ggml_vk_invalidate_prealloc_y_ring(ctx); + } + ctx->prealloc_y_ring_stride = y_stride; + ctx->prealloc_y_ring_count = y_ring; + } + const size_t sk_stride = ctx->prealloc_size_split_k > 0 ? ROUNDUP_POW2(ctx->prealloc_size_split_k, 4096ULL) : 0; + const uint32_t sk_ring = (sk_stride > 0 && sk_stride * VK_PREALLOC_RING <= ctx->device->properties.limits.maxStorageBufferRange && + sk_stride * VK_PREALLOC_RING <= 128ULL * 1024 * 1024) ? VK_PREALLOC_RING : 1u; + const size_t sk_total = sk_ring > 1 ? sk_stride * sk_ring : ctx->prealloc_size_split_k; + if (ctx->prealloc_split_k == nullptr || (sk_total > 0 && ctx->prealloc_split_k->size < sk_total)) { + VK_LOG_MEMORY("ggml_vk_preallocate_buffers(split_k_size: " << sk_total << ")"); // Resize buffer if (ctx->prealloc_split_k != nullptr) { ggml_vk_destroy_buffer(ctx->prealloc_split_k); } - ctx->prealloc_split_k = ggml_vk_create_buffer_device(ctx->device, ctx->prealloc_size_split_k); + ctx->prealloc_split_k = ggml_vk_create_buffer_device(ctx->device, sk_total); + ctx->prealloc_split_k_ring_stride = sk_stride; + ctx->prealloc_split_k_ring_count = sk_ring; + ctx->prealloc_split_k_ring_idx = 0; + ctx->last_expert_count_valid = false; + } else if (sk_total > 0) { + ctx->prealloc_split_k_ring_stride = sk_stride; + ctx->prealloc_split_k_ring_count = sk_ring; } if (ctx->prealloc_add_rms_partials == nullptr || (ctx->prealloc_size_add_rms_partials > 0 && ctx->prealloc_add_rms_partials->size < ctx->prealloc_size_add_rms_partials)) { VK_LOG_MEMORY("ggml_vk_preallocate_buffers(add_partials_size: " << ctx->prealloc_add_rms_partials << ")"); @@ -12966,6 +15348,15 @@ void ggml_vk_preallocate_buffers(ggml_backend_vk_context * ctx, vk_context subct } ctx->prealloc_add_rms_partials = ggml_vk_create_buffer_device(ctx->device, ctx->prealloc_size_add_rms_partials); } + if (ctx->prealloc_size_mask_opt > 0 && (ctx->prealloc_mask_opt == nullptr || ctx->prealloc_mask_opt->size < ctx->prealloc_size_mask_opt)) { + VK_LOG_MEMORY("ggml_vk_preallocate_buffers(mask_opt_size: " << ctx->prealloc_size_mask_opt << ")"); + if (ctx->prealloc_mask_opt != nullptr) { + ggml_vk_destroy_buffer(ctx->prealloc_mask_opt); + } + ctx->prealloc_mask_opt = ggml_vk_create_buffer_device(ctx->device, ctx->prealloc_size_mask_opt); + ctx->last_mask_opt_valid = false; + ctx->prealloc_mask_opt_need_sync = false; + } } bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, int node_idx, ggml_tensor *node_begin, int node_idx_begin, bool last_node, bool almost_ready, bool submit){ @@ -12985,16 +15376,25 @@ bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, in ggml_tensor * src2 = node->src[2]; ggml_tensor * src3 = node->src[3]; - if (node->op == GGML_OP_ADD) { + if (node->op == GGML_OP_ADD && ctx->s10_add_rms_dst == nullptr) { int next_node_idx = node_idx + 1 + ctx->num_additional_fused_ops; if (next_node_idx < cgraph->n_nodes && cgraph->nodes[next_node_idx]->op == GGML_OP_RMS_NORM && cgraph->nodes[next_node_idx]->src[0] == cgraph->nodes[next_node_idx - 1] && - ggml_nrows(cgraph->nodes[next_node_idx]) == 1 && + (ggml_nrows(cgraph->nodes[next_node_idx]) == 1 || + (gen_batch_invariant() && ggml_nrows(cgraph->nodes[next_node_idx]) <= 64 && + ggml_is_contiguous(cgraph->nodes[node_idx]) && ggml_is_contiguous(cgraph->nodes[next_node_idx]) && + cgraph->nodes[node_idx]->ne[0] % ctx->device->pipeline_add_rms[0][0][0]->wg_denoms[0] == 0)) && ctx->device->add_rms_fusion && !ctx->device->disable_fusion) { uint32_t size = ggml_vk_rms_partials_size(ctx, cgraph->nodes[node_idx]); ctx->do_add_rms_partials_offset_calculation = true; - if (ctx->prealloc_size_add_rms_partials_offset + size <= ctx->prealloc_size_add_rms_partials) { + // gen: in batch-invariant mode the same ADD+RMS pairs fuse as for one row (budget counted per row, as at + // n=1), the real buffer holds every row's partials + const bool fits = gen_batch_invariant() ? + ctx->gen_n1_partials_offset + size <= 1024 && + ctx->prealloc_size_add_rms_partials_offset + gen_rms_partials_size_rows(ctx, cgraph->nodes[node_idx]) <= ctx->prealloc_size_add_rms_partials : + ctx->prealloc_size_add_rms_partials_offset + size <= ctx->prealloc_size_add_rms_partials; + if (fits) { ctx->do_add_rms_partials = true; ggml_vk_count_fusion(ctx->device, "ADD_RMS_NORM_PARTIALS"); } @@ -13021,7 +15421,7 @@ bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, in return false; } auto n_base = vk_tensor_offset(node) + node->view_offs; - auto n_size = ggml_nbytes(node); + auto n_size = std::max(ggml_nbytes(node), ggml_vk_tensor_byte_span(node)); ggml_backend_vk_buffer_context * a_buf_ctx = (ggml_backend_vk_buffer_context *)node->buffer->context; vk_buffer a_buf = a_buf_ctx->dev_buffer; for (auto &other : unsynced_nodes) { @@ -13029,7 +15429,7 @@ bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, in vk_buffer o_buf = o_buf_ctx->dev_buffer; if (a_buf == o_buf) { auto o_base = vk_tensor_offset(other) + other->view_offs; - auto o_size = ggml_nbytes(other); + auto o_size = std::max(ggml_nbytes(other), ggml_vk_tensor_byte_span(other)); if ((o_base <= n_base && n_base < o_base + o_size) || (n_base <= o_base && o_base < n_base + n_size)) { @@ -13040,6 +15440,24 @@ bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, in return false; }; + auto is_intra_elided = [&](const ggml_tensor * t, int32_t limit_i) -> bool { + const ggml_tensor * base = t; + while (base && base->view_src != nullptr) { + base = base->view_src; + } + if (!base || base->op == GGML_OP_NONE) { + return false; + } + for (int32_t m = 0; m < limit_i; ++m) { + if ((ctx->fused_ops_write_mask & (1 << m)) == 0 && + cgraph->nodes[node_idx + m] == base && + base->view_src == nullptr) { + return true; + } + } + return false; + }; + // For all fused ops, check if the destination node or any of the source // nodes require synchronization. for (int32_t i = 0; i < ctx->num_additional_fused_ops + 1 && !need_sync; ++i) { @@ -13052,7 +15470,9 @@ bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, in } } for (uint32_t j = 0; j < GGML_MAX_SRC; ++j) { - if (!cur_node->src[j]) { + if (!cur_node->src[j] || is_intra_elided(cur_node->src[j], i) || + ((ctx->s10_ssm_conv_mode == 3 || (ctx->s10_ssm_zero_state && ctx->s10_ssm_conv_mode == 1)) && i == 0 && j == 0) || + (ctx->gen_gdn_zero_state && cur_node->op == GGML_OP_GATED_DELTA_NET && i == 0 && j == 5)) { continue; } if (overlaps_unsynced(cur_node->src[j], ctx->unsynced_nodes_written)) { @@ -13082,9 +15502,28 @@ bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, in // Multiple outputs could be written, e.g. in topk_moe. Add them all to the list. if (ctx->fused_ops_write_mask & (1 << i)) { ctx->unsynced_nodes_written.push_back(cur_node); + for (uint32_t s = 0; s < ctx->prealloc_y_ring_count; ++s) { + const ggml_tensor * cy = ctx->prealloc_y_last_tensor_ring[s]; + if (cy && cy->buffer == cur_node->buffer && ggml_vk_tensors_overlap(cur_node, cy, false)) { + ctx->prealloc_y_last_tensor_ring[s] = nullptr; + ctx->prealloc_y_last_pipeline_ring[s] = nullptr; + } + } + if (ctx->last_expert_count_valid && ctx->last_expert_ids_tensor && + ctx->last_expert_ids_tensor->buffer == cur_node->buffer && + ggml_vk_tensors_overlap(cur_node, ctx->last_expert_ids_tensor, false)) { + ctx->last_expert_count_valid = false; + } + if (ctx->last_mask_opt_valid && ctx->last_mask_opt_tensor && + ctx->last_mask_opt_tensor->buffer == cur_node->buffer && + ggml_vk_tensors_overlap(cur_node, ctx->last_mask_opt_tensor, false)) { + ctx->last_mask_opt_valid = false; + } } for (uint32_t j = 0; j < GGML_MAX_SRC; ++j) { - if (!cur_node->src[j]) { + if (!cur_node->src[j] || is_intra_elided(cur_node->src[j], i) || + ((ctx->s10_ssm_conv_mode == 3 || (ctx->s10_ssm_zero_state && ctx->s10_ssm_conv_mode == 1)) && i == 0 && j == 0) || + (ctx->gen_gdn_zero_state && cur_node->op == GGML_OP_GATED_DELTA_NET && i == 0 && j == 5)) { continue; } ctx->unsynced_nodes_read.push_back(cur_node->src[j]); @@ -13136,7 +15575,9 @@ bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, in break; case GGML_OP_ADD: - if (ctx->num_additional_fused_ops) { + if (ctx->s10_add_rms_dst != nullptr) { + ggml_vk_multi_add_rms_mul(ctx, compute_ctx, cgraph, node_idx); + } else if (ctx->num_additional_fused_ops) { ggml_vk_multi_add(ctx, compute_ctx, cgraph, node_idx); } else { ggml_vk_add(ctx, compute_ctx, src0, src1, node); @@ -13166,7 +15607,11 @@ bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, in break; case GGML_OP_CONCAT: - ggml_vk_concat(ctx, compute_ctx, src0, src1, node); + if (ctx->s10_ssm_conv_mode == 1) { + ggml_vk_ssm_conv(ctx, compute_ctx, cgraph, node_idx); + } else { + ggml_vk_concat(ctx, compute_ctx, src0, src1, node); + } break; case GGML_OP_UPSCALE: @@ -13234,10 +15679,21 @@ bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, in ggml_vk_roll(ctx, compute_ctx, src0, node); break; - case GGML_OP_CPY: case GGML_OP_CONT: - case GGML_OP_DUP: + if (ctx->num_additional_fused_ops >= 2 && + cgraph->nodes[node_idx + ctx->num_additional_fused_ops]->op == GGML_OP_MUL) { + ggml_vk_cont_unary_mul(ctx, compute_ctx, cgraph, node_idx); + break; + } ggml_vk_cpy(ctx, compute_ctx, src0, node); + break; + case GGML_OP_CPY: + case GGML_OP_DUP: + if (node->op == GGML_OP_CPY && ctx->s10_ssm_conv_mode == 2) { + ggml_vk_ssm_conv(ctx, compute_ctx, cgraph, node_idx); + } else { + ggml_vk_cpy(ctx, compute_ctx, src0, node); + } break; case GGML_OP_SET_ROWS: @@ -13526,7 +15982,7 @@ bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, in last_node = true; #endif - if (submit || last_node) { + if ((submit || last_node) && !ctx->recording_replay) { ggml_vk_ctx_end(compute_ctx); // TODO probably it'd be better to pass a exit_node flag to ggml_vk_compute_forward @@ -13567,11 +16023,18 @@ void ggml_vk_compute_forward(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph memset(mset.dst, mset.val, mset.n); } + ggml_vk_ensure_fence_free(ctx); + const bool is_exit = (tensor_idx == subctx->exit_tensor_idx); + const bool need_transfer_sem = ctx->device->async_use_transfer_queue && + ctx->transfer_semaphore_last_submitted < ctx->transfer_semaphore.value; if (ctx->device->serialize_submissions) { ggml_vk_submit(subctx, ctx->fence); } else if (almost_ready && !ctx->almost_ready_fence_pending) { ggml_vk_submit(subctx, ctx->almost_ready_fence); ctx->almost_ready_fence_pending = true; + } else if (is_exit && !need_transfer_sem && subctx->out_memcpys.empty()) { + ggml_vk_submit(subctx, ctx->fence); + ctx->fence_submitted = true; } else { ggml_vk_submit(subctx, {}); } @@ -13594,15 +16057,40 @@ void ggml_vk_compute_forward(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph } } +static void ggml_vk_destroy_replay_entry(vk_device & device, vk_graph_replay_entry & e) { + for (auto & pool : e.descriptor_pools) { + device->device.destroyDescriptorPool(pool); + } + e.descriptor_pools.clear(); + e.descriptor_sets.clear(); + e.descriptor_set_bindings.clear(); + if (e.cmd_pool.pool) { + e.cmd_pool.destroy(device->device); + } + e.cmd_buf = vk::CommandBuffer{}; + e.memsets.clear(); + e.sig_words.clear(); + e.ready = false; +} + void ggml_vk_graph_cleanup(ggml_backend_vk_context * ctx) { VK_LOG_DEBUG("ggml_vk_graph_cleanup()"); - ctx->prealloc_y_last_pipeline_used = {}; - ctx->prealloc_y_last_tensor_used = nullptr; - ctx->prealloc_y_last_k_padded = false; + ggml_vk_invalidate_prealloc_y_ring(ctx); + ctx->prealloc_y_ring_idx = 0; + ctx->prealloc_split_k_ring_idx = 0; + ctx->last_expert_count_valid = false; + ctx->last_expert_ids_tensor = nullptr; + ctx->last_mask_opt_valid = false; + ctx->last_mask_opt_tensor = nullptr; ctx->unsynced_nodes_written.clear(); ctx->unsynced_nodes_read.clear(); ctx->prealloc_x_need_sync = ctx->prealloc_y_need_sync = ctx->prealloc_split_k_need_sync = false; + ctx->prealloc_mask_opt_need_sync = false; + for (uint32_t s = 0; s < VK_PREALLOC_RING; ++s) { + ctx->prealloc_y_slot_need_sync[s] = false; + ctx->prealloc_split_k_slot_need_sync[s] = false; + } ggml_vk_command_pool_cleanup(ctx->device, ctx->compute_cmd_pool); if (ctx->device->async_use_transfer_queue) { @@ -13646,6 +16134,7 @@ void ggml_vk_cleanup(ggml_backend_vk_context * ctx) { ggml_vk_destroy_buffer(ctx->prealloc_y); ggml_vk_destroy_buffer(ctx->prealloc_split_k); ggml_vk_destroy_buffer(ctx->prealloc_add_rms_partials); + ggml_vk_destroy_buffer(ctx->prealloc_mask_opt); ggml_vk_destroy_buffer(ctx->sync_staging); ctx->prealloc_y_last_pipeline_used = nullptr; @@ -13655,6 +16144,7 @@ void ggml_vk_cleanup(ggml_backend_vk_context * ctx) { ctx->prealloc_size_x = 0; ctx->prealloc_size_y = 0; ctx->prealloc_size_split_k = 0; + ctx->prealloc_size_mask_opt = 0; for (auto& event : ctx->gc.events) { ctx->device->device.destroyEvent(event); @@ -13669,6 +16159,11 @@ void ggml_vk_cleanup(ggml_backend_vk_context * ctx) { ctx->device->device.destroyFence(ctx->fence); ctx->device->device.destroyFence(ctx->almost_ready_fence); + for (auto & re : ctx->replay_cache) { + ggml_vk_destroy_replay_entry(ctx->device, re); + } + ctx->replay_cache.clear(); + for (auto& pool : ctx->descriptor_pools) { ctx->device->device.destroyDescriptorPool(pool); } @@ -14017,7 +16512,27 @@ static void ggml_backend_vk_set_tensor_2d_async(ggml_backend_t backend, ggml_ten return; } + if (!ctx->uma_out_memcpys.empty()) { + ggml_vk_synchronize(ctx); + } + ggml_backend_vk_buffer_context * buf_ctx = (ggml_backend_vk_buffer_context *)tensor->buffer->context; + vk_buffer buf = buf_ctx->dev_buffer; + auto dst_offset = vk_tensor_offset(tensor) + tensor->view_offs + offset; + + // Series 9: On UMA when the backend is idle (between graph evaluations), write directly into + // coherent unified memory on the CPU instead of allocating sync_staging + submitting a GPU copyBuffer + fence wait. + const bool transfer_idle = !ctx->device->async_use_transfer_queue || + ctx->transfer_semaphore_last_submitted == ctx->transfer_semaphore.value; + const bool backend_idle = ctx->compute_ctx.expired() && ctx->transfer_ctx.expired() && + !ctx->submit_pending && !ctx->fence_submitted && !ctx->almost_ready_fence_pending && transfer_idle; + const bool dst_host_coherent = + (buf->memory_property_flags & (vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent)) == + (vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent); + if (ctx->device->uma && backend_idle && dst_host_coherent && buf->ptr != nullptr) { + ggml_vk_buffer_write_2d(buf, dst_offset, data, stride_data, stride_tensor, size, n_copies); + return; + } vk_context cpy_ctx; @@ -14027,10 +16542,6 @@ static void ggml_backend_vk_set_tensor_2d_async(ggml_backend_t backend, ggml_ten cpy_ctx = ggml_vk_get_compute_ctx(ctx); } - vk_buffer buf = buf_ctx->dev_buffer; - - auto dst_offset = vk_tensor_offset(tensor) + tensor->view_offs + offset; - bool ret = ggml_vk_buffer_write_2d_async(cpy_ctx, buf, dst_offset, data, stride_data, stride_tensor, size, n_copies); if (!ret) { @@ -14081,12 +16592,27 @@ static void ggml_backend_vk_get_tensor_2d_async(ggml_backend_t backend, const gg } ggml_backend_vk_buffer_context * buf_ctx = (ggml_backend_vk_buffer_context *)tensor->buffer->context; - - vk_context compute_ctx = ggml_vk_get_compute_ctx(ctx); - vk_buffer buf = buf_ctx->dev_buffer; - auto src_offset = vk_tensor_offset(tensor) + tensor->view_offs + offset; + + // Series 9: On cached coherent UMA (Apple Silicon), defer the host memcpy until after ctx->fence + // signals in ggml_vk_synchronize instead of recording/submitting an extra GPU copyBuffer + fence round-trip. + const auto cached_coherent = vk::MemoryPropertyFlagBits::eHostVisible | + vk::MemoryPropertyFlagBits::eHostCoherent | + vk::MemoryPropertyFlagBits::eHostCached; + if (ctx->device->uma && buf->ptr != nullptr && (buf->memory_property_flags & cached_coherent) == cached_coherent) { + const uint8_t * base_src = (const uint8_t *) buf->ptr + src_offset; + if (size == stride_tensor && size == stride_data) { + deferred_memcpy(data, base_src, size * n_copies, &ctx->uma_out_memcpys); + } else { + for (size_t i = 0; i < n_copies; ++i) { + deferred_memcpy((uint8_t *) data + i * stride_data, base_src + i * stride_tensor, size, &ctx->uma_out_memcpys); + } + } + return; + } + + vk_context compute_ctx = ggml_vk_get_compute_ctx(ctx); bool ret = ggml_vk_buffer_read_2d_async(compute_ctx, buf, src_offset, data, stride_tensor, stride_data, size, n_copies); if (!ret) { @@ -14135,6 +16661,10 @@ static bool ggml_backend_vk_cpy_tensor_async(ggml_backend_t backend_src, ggml_ba return true; } + if (!ctx->uma_out_memcpys.empty()) { + ggml_vk_synchronize(ctx); + } + if (dst->buffer->buft != ggml_backend_vk_get_default_buffer_type(backend_dst)) { return false; } @@ -14172,7 +16702,7 @@ static bool ggml_backend_vk_cpy_tensor_async(ggml_backend_t backend_src, ggml_ba const bool transfer_idle = !ctx->device->async_use_transfer_queue || ctx->transfer_semaphore_last_submitted == ctx->transfer_semaphore.value; const bool backend_idle = ctx->compute_ctx.expired() && ctx->transfer_ctx.expired() && - !ctx->submit_pending && !ctx->almost_ready_fence_pending && transfer_idle; + !ctx->submit_pending && !ctx->fence_submitted && !ctx->almost_ready_fence_pending && transfer_idle; const bool dst_host_coherent = (dst_buf->memory_property_flags & (vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent)) == (vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent); @@ -14209,6 +16739,9 @@ void ggml_vk_synchronize(ggml_backend_vk_context * ctx) { vk_context compute_ctx; vk_command_buffer* cmd_buf = nullptr; if (do_transfer) { + if (ctx->fence_submitted) { + ggml_vk_wait_for_fence(ctx); + } compute_ctx = ctx->compute_ctx.lock(); if (compute_ctx->s) { cmd_buf = compute_ctx->s->buffer; @@ -14219,11 +16752,21 @@ void ggml_vk_synchronize(ggml_backend_vk_context * ctx) { for (auto& cpy : compute_ctx->in_memcpys) { memcpy(cpy.dst, cpy.src, cpy.n); } + for (auto& mset : compute_ctx->memsets) { + memset(mset.dst, mset.val, mset.n); + } + const bool need_transfer_sem_submit = ctx->device->async_use_transfer_queue && + ctx->transfer_semaphore_last_submitted < ctx->transfer_semaphore.value; if (ctx->device->serialize_submissions) { ggml_vk_submit(compute_ctx, ctx->fence); VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "synchronize waitForFences", ctx->device); ctx->device->device.resetFences({ ctx->fence }); + ctx->fence_submitted = false; + } else if (!need_transfer_sem_submit && !compute_ctx->seqs.empty()) { + // Series 9: Signal ctx->fence directly on compute_ctx submit instead of issuing a second empty vkQueueSubmit + ggml_vk_submit(compute_ctx, ctx->fence); + ctx->fence_submitted = true; } else { ggml_vk_submit(compute_ctx, {}); } @@ -14234,6 +16777,9 @@ void ggml_vk_synchronize(ggml_backend_vk_context * ctx) { if (ctx->device->serialize_submissions) { ctx->submit_pending = false; } else if (ctx->device->async_use_transfer_queue && ctx->transfer_semaphore_last_submitted < ctx->transfer_semaphore.value) { + if (ctx->fence_submitted) { + ggml_vk_wait_for_fence(ctx); + } vk::TimelineSemaphoreSubmitInfo tl_info{ 1, &ctx->transfer_semaphore.value, 0, nullptr, @@ -14246,9 +16792,11 @@ void ggml_vk_synchronize(ggml_backend_vk_context * ctx) { }; si.setPNext(&tl_info); ctx->device->compute_queue->handle->submit({ si }, ctx->fence); + ctx->fence_submitted = true; ctx->transfer_semaphore_last_submitted = ctx->transfer_semaphore.value; - } else { + } else if (!ctx->fence_submitted) { ctx->device->compute_queue->handle->submit({}, ctx->fence); + ctx->fence_submitted = true; } if (!ctx->device->serialize_submissions) { ggml_vk_wait_for_fence(ctx); @@ -14264,16 +16812,33 @@ void ggml_vk_synchronize(ggml_backend_vk_context * ctx) { for (auto& cpy : compute_ctx->out_memcpys) { memcpy(cpy.dst, cpy.src, cpy.n); } + compute_ctx->in_memcpys.clear(); + compute_ctx->out_memcpys.clear(); + compute_ctx->memsets.clear(); ctx->compute_ctx.reset(); } + + if (!ctx->uma_out_memcpys.empty()) { + for (auto & cpy : ctx->uma_out_memcpys) { + memcpy(cpy.dst, cpy.src, cpy.n); + } + ctx->uma_out_memcpys.clear(); + } } static void ggml_backend_vk_synchronize(ggml_backend_t backend) { VK_LOG_DEBUG("ggml_backend_vk_synchronize()"); ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context; + static const bool gen_stats = getenv("GGML_VK_GRAPH_STATS") != nullptr && atoi(getenv("GGML_VK_GRAPH_STATS")) != 0; + const int64_t gen_t_start = gen_stats ? ggml_time_us() : 0; + ggml_vk_synchronize(ctx); + if (gen_stats) { + fprintf(stderr, "ggml_vk_sync_stats: wait_us=%lld\n", (long long) (ggml_time_us() - gen_t_start)); + } + ggml_vk_graph_cleanup(ctx); } @@ -14331,6 +16896,86 @@ static bool ggml_vk_can_fuse_hc_post_gate(const struct ggml_cgraph * cgraph, int ggml_are_same_shape(scale_in->src[0], scale_out); } +static bool gen_uses_dmmv_noswap(const ggml_backend_vk_context * ctx, const ggml_tensor * mm); + +static bool gen_cm1_fused_type_supported(ggml_type t) { + switch (t) { + case GGML_TYPE_F32: + case GGML_TYPE_F16: + case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: + case GGML_TYPE_Q8_0: + case GGML_TYPE_Q2_K: + case GGML_TYPE_Q3_K: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: + case GGML_TYPE_TQ1_0: + case GGML_TYPE_TQ2_0: + case GGML_TYPE_IQ4_XS: + case GGML_TYPE_IQ4_NL: + return true; + default: + return false; + } +} + +static bool gen_uses_cm1_fused_mm(const ggml_backend_vk_context * ctx, const ggml_tensor * mm) { + if (!mm || mm->op != GGML_OP_MUL_MAT || mm->type != GGML_TYPE_F32) { + return false; + } + if (!ctx->device->coopmat_support || ctx->device->coopmat2 || !ctx->device->coopmat_acc_f32_support || + ggml_vk_exact_s8_enabled() || gen_batch_invariant_for_n(mm->ne[1])) { + return false; + } + const ggml_tensor * src0 = mm->src[0]; + const ggml_tensor * src1 = mm->src[1]; + if (!src0 || !src1 || !gen_cm1_fused_type_supported(src0->type) || + (src1->type != GGML_TYPE_F32 && src1->type != GGML_TYPE_F16)) { + return false; + } + if (!ggml_vk_dim01_contiguous(src0) || + (src0->type == GGML_TYPE_F32 && !ggml_vk_dim01_contiguous(src1))) { + return false; + } + if ((mm->ne[2] == 1 && mm->ne[3] == 1 && ggml_nbytes(src0) > ctx->device->properties.limits.maxStorageBufferRange) || + ggml_vk_can_use_fwht(ctx, src1, const_cast(mm)) || + ggml_vk_get_mul_mat_mat_f16acc(ctx, src0->type, src1->type, ggml_prec_get_acc(mm))) { + return false; + } + const ggml_type b_type = ggml_vk_dim01_contiguous(src1) ? src1->type : GGML_TYPE_F16; + return ggml_vk_get_mul_mat_mat_pipeline_map(const_cast(ctx), src0->type, b_type, ggml_prec_get_acc(mm), false) != nullptr; +} + +static bool gen_uses_cm1_fused_mmid(const ggml_backend_vk_context * ctx, const ggml_tensor * mmid) { + if (!mmid || mmid->op != GGML_OP_MUL_MAT_ID || mmid->type != GGML_TYPE_F32) { + return false; + } + if (!ctx->device->coopmat_support || ctx->device->coopmat2 || !ctx->device->coopmat_acc_f32_support || + ggml_vk_exact_s8_enabled() || gen_batch_invariant_for_n(mmid->src[2] ? mmid->src[2]->ne[1] : 0)) { + return false; + } + const ggml_tensor * src0 = mmid->src[0]; + const ggml_tensor * src1 = mmid->src[1]; + const ggml_tensor * ids = mmid->src[2]; + if (!src0 || !src1 || !ids || ids->type != GGML_TYPE_I32 || + !gen_cm1_fused_type_supported(src0->type) || + (src1->type != GGML_TYPE_F32 && src1->type != GGML_TYPE_F16)) { + return false; + } + if (!ggml_vk_dim01_contiguous(src0) || + (src0->type == GGML_TYPE_F32 && !ggml_vk_dim01_contiguous(src1)) || + ggml_vk_get_mul_mat_mat_f16acc(ctx, src0->type, src1->type, ggml_prec_get_acc(mmid))) { + return false; + } + const ggml_type b_type = ggml_vk_dim01_contiguous(src1) ? src1->type : GGML_TYPE_F16; + return ggml_vk_get_mul_mat_mat_pipeline_map(const_cast(ctx), src0->type, b_type, ggml_prec_get_acc(mmid), true) != nullptr; +} + bool ggml_vk_can_fuse(const ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx, std::initializer_list ops) { if (ops.size() == 2 && ops.begin()[0] == GGML_OP_UNARY && ops.begin()[1] == GGML_OP_MUL) { return ggml_vk_can_fuse_unary_mul_pair(cgraph, node_idx); @@ -14402,11 +17047,19 @@ bool ggml_vk_can_fuse(const ggml_backend_vk_context * ctx, const struct ggml_cgr } } - auto const &mm_add_ok = [&](const ggml_tensor *mul, const ggml_tensor *add) { + auto const &mm_add_ok = [&](const ggml_tensor *root_mm, const ggml_tensor *mul, const ggml_tensor *add) { const ggml_tensor *bias = add->src[0] == mul ? add->src[1] : add->src[0]; - // mat-vec only - if (ggml_nrows(mul) != 1) { + // mat-vec, or multi-column mat-vec / cm1 matmul when not in exact-s8 / batch-invariant<=8 mode + const bool ok_rows = + ggml_nrows(mul) == 1 || + (gen_batch_invariant_for_n(mul->ne[1]) + ? (mul->ne[2] == 1 && mul->ne[3] == 1 && mul->ne[1] <= 8 && + mul->src[1] && mul->src[1]->ne[2] * mul->src[1]->ne[3] == 1) + : (!ggml_vk_exact_s8_enabled() && + ((gen_uses_dmmv_noswap(ctx, root_mm) && ggml_is_contiguous(root_mm)) || + gen_uses_cm1_fused_mm(ctx, root_mm)))); + if (!ok_rows) { return false; } // shaders assume the types match @@ -14415,11 +17068,13 @@ bool ggml_vk_can_fuse(const ggml_backend_vk_context * ctx, const struct ggml_cgr } // shaders reuse the D shape for bias if (!ggml_are_same_shape(mul, bias) || - !ggml_are_same_stride(mul, bias)) { + !ggml_are_same_stride(mul, bias) || + !ggml_are_same_shape(mul, add) || + !ggml_are_same_stride(mul, add)) { return false; } - // unaligned bias isn't handled - if (get_misalign_bytes(ctx, bias) != 0) { + // unaligned bias/dst isn't handled + if (get_misalign_bytes(ctx, bias) != 0 || get_misalign_bytes(ctx, add) != 0) { return false; } return true; @@ -14430,14 +17085,14 @@ bool ggml_vk_can_fuse(const ggml_backend_vk_context * ctx, const struct ggml_cgr const ggml_tensor *mul = cgraph->nodes[node_idx]; const ggml_tensor *add = cgraph->nodes[node_idx + 1]; - if (!mm_add_ok(mul, add)) { + if (!mm_add_ok(mul, mul, add)) { return false; } if (ops.size() == 3) { if (ops.begin()[2] != GGML_OP_ADD) { return false; } - if (!mm_add_ok(add, cgraph->nodes[node_idx + 2])) { + if (!mm_add_ok(mul, add, cgraph->nodes[node_idx + 2])) { return false; } } @@ -14449,8 +17104,9 @@ bool ggml_vk_can_fuse(const ggml_backend_vk_context * ctx, const struct ggml_cgr if (mmid != mul->src[0]) { return false; } - // mat-vec only - if (!ggml_vk_use_mul_mat_vec_id(cgraph, node_idx)) { + // mat-vec or cm1 fused matmul_id + if (!ggml_vk_use_mul_mat_vec_id(ctx, cgraph, node_idx) && + !gen_uses_cm1_fused_mmid(ctx, cgraph->nodes[node_idx])) { return false; } // shaders assume the types match @@ -14461,14 +17117,14 @@ bool ggml_vk_can_fuse(const ggml_backend_vk_context * ctx, const struct ggml_cgr if (!ggml_is_contiguous(scale)) { return false; } - // unaligned bias isn't handled - if (get_misalign_bytes(ctx, scale) != 0) { + // unaligned bias/dst isn't handled + if (get_misalign_bytes(ctx, scale) != 0 || get_misalign_bytes(ctx, mul) != 0) { return false; } - // shader only indexes by expert index + // shader indexes by expert slot, or (slot, token) via SCALE0_TOK / SCALE1_TOK if (scale->ne[0] != 1 || scale->ne[1] != mul->ne[1] || - scale->ne[2] != 1 || + (scale->ne[2] != 1 && scale->ne[2] != mul->ne[2]) || scale->ne[3] != 1) { return false; } @@ -14484,8 +17140,9 @@ bool ggml_vk_can_fuse(const ggml_backend_vk_context * ctx, const struct ggml_cgr if (mul != add->src[0]) { return false; } - // mat-vec only - if (!ggml_vk_use_mul_mat_vec_id(cgraph, node_idx)) { + // mat-vec or cm1 fused matmul_id + if (!ggml_vk_use_mul_mat_vec_id(ctx, cgraph, node_idx) && + !gen_uses_cm1_fused_mmid(ctx, mul)) { return false; } // shaders assume the types match @@ -14500,8 +17157,8 @@ bool ggml_vk_can_fuse(const ggml_backend_vk_context * ctx, const struct ggml_cgr if (mul->src[2] != add->src[2]) { return false; } - // unaligned bias isn't handled - if (get_misalign_bytes(ctx, bias) != 0) { + // unaligned bias/dst isn't handled + if (get_misalign_bytes(ctx, bias) != 0 || get_misalign_bytes(ctx, add) != 0) { return false; } @@ -14921,6 +17578,25 @@ bool ggml_vk_tensors_overlap(const ggml_tensor * a, const ggml_tensor * b, bool return false; } +static bool ggml_vk_tensors_overlap_strided(const ggml_tensor * a, const ggml_tensor * b) { + if (!a || !b || !a->buffer || !b->buffer) { + return false; + } + ggml_backend_vk_buffer_context * a_buf_ctx = (ggml_backend_vk_buffer_context *)a->buffer->context; + ggml_backend_vk_buffer_context * b_buf_ctx = (ggml_backend_vk_buffer_context *)b->buffer->context; + if (a_buf_ctx->dev_buffer != b_buf_ctx->dev_buffer) { + return false; + } + const size_t a_base = vk_tensor_offset(a) + a->view_offs; + const size_t a_size = std::max(ggml_nbytes(a), ggml_vk_tensor_byte_span(a)); + const size_t b_base = vk_tensor_offset(b) + b->view_offs; + const size_t b_size = std::max(ggml_nbytes(b), ggml_vk_tensor_byte_span(b)); + return (b_base <= a_base && a_base < b_base + b_size) || + (a_base <= b_base && b_base < a_base + a_size); +} + +static bool gen_fuse_enabled(const char * what); + bool ggml_vk_can_fuse_rms_norm_mul_rope(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx) { const ggml_tensor *rms = cgraph->nodes[node_idx + 0]; @@ -14936,8 +17612,9 @@ bool ggml_vk_can_fuse_rms_norm_mul_rope(ggml_backend_vk_context * ctx, const str return false; } - // only norm/neox are handled in the shader - if (mode != GGML_ROPE_TYPE_NEOX && mode != GGML_ROPE_TYPE_NORMAL) { + // norm/neox always; mrope/imrope when GGML_GEN_FUSE=mrope + if (mode != GGML_ROPE_TYPE_NEOX && mode != GGML_ROPE_TYPE_NORMAL && + !(gen_fuse_enabled("mrope") && (mode == GGML_ROPE_TYPE_MROPE || mode == GGML_ROPE_TYPE_IMROPE))) { return false; } @@ -15018,40 +17695,1926 @@ static int32_t find_first_set(uint32_t x) { return ret; } -static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) { - VK_LOG_DEBUG("ggml_backend_vk_graph_compute(" << cgraph->n_nodes << " nodes)"); - ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context; - - ctx->device->diag_cgraph = nullptr; - ctx->device->diag_prev_start = -1; - ctx->device->diag_prev_end = -1; - - if (vk_instance.debug_utils_support) { - ctx->device->debug_cmdbuf_idx = 0; +// gen: GGML_GEN_FUSE=glu,l2 (or all; default all unless none/0) enables the extra fusions +static bool gen_fuse_enabled(const char * what) { + static const std::string v = getenv("GGML_GEN_FUSE") ? getenv("GGML_GEN_FUSE") : "all"; + if (v.empty() || v == "none" || v == "0") { + return false; } - - // queue scope, so it encloses every submit this evaluation makes. - // closed when the function returns - ggml_vk_debug_label queue_dbg(ctx->device->compute_queue->handle.get(), "ggml_backend_vk_graph_compute"); - - ctx->prealloc_size_add_rms_partials_offset = 0; - ctx->do_add_rms_partials = false; - ctx->do_add_rms_partials_offset_calculation = false; - - int last_node = cgraph->n_nodes - 1; - - // If the last op in the cgraph isn't backend GPU, the command buffer doesn't get closed properly - while (last_node > 0 && (ggml_vk_is_empty(cgraph->nodes[last_node]) || ((cgraph->nodes[last_node]->flags & GGML_TENSOR_FLAG_COMPUTE) == 0))) { - last_node -= 1; + if (v == "all") { + return true; } + return ("," + v + ",").find("," + std::string(what) + ",") != std::string::npos; +} - // Reserve tensor context space for all nodes - ctx->tensor_ctxs.resize(cgraph->n_nodes); - - bool first_node_in_batch = true; // true if next node will be first node in a batch - int submit_node_idx = 0; // index to first node in a batch - - ggml_vk_submit_transfer_ctx(ctx); +static bool gen_uses_dmmv_noswap(const ggml_backend_vk_context * ctx, const ggml_tensor * mm) { + if (mm->op != GGML_OP_MUL_MAT || mm->type != GGML_TYPE_F32 || mm->ne[2] != 1 || mm->ne[3] != 1) { + return false; + } + const ggml_tensor * src0 = mm->src[0]; + const ggml_tensor * src1 = mm->src[1]; + if (!(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16 || ggml_is_quantized(src0->type))) { + return false; + } + if (src1->type != GGML_TYPE_F32 && src1->type != GGML_TYPE_F16) { + return false; + } + if (ggml_nbytes(src0) > ctx->device->properties.limits.maxStorageBufferRange) { + return false; + } + if (ggml_vk_can_use_fwht(ctx, src1, const_cast(mm))) { + return false; + } + if (!ggml_is_contiguous(src0) || ggml_is_permuted(src0) || ggml_is_permuted(src1) || ggml_is_transposed(src1)) { + return false; + } + static const int env_mma_q8_min_n_fn = getenv("GGML_VK_MMA_Q8_MIN_N") ? atoi(getenv("GGML_VK_MMA_Q8_MIN_N")) : 8; + static const int env_mma_q8_min_m_fn = getenv("GGML_VK_MMA_Q8_MIN_M") ? atoi(getenv("GGML_VK_MMA_Q8_MIN_M")) : 0; + static const int env_mma_q8_max_m_s_fn = getenv("GGML_VK_MMA_Q8_MAX_M") ? atoi(getenv("GGML_VK_MMA_Q8_MAX_M")) : 65536; + static const int env_mma_q8_max_m_l_fn = getenv("GGML_VK_MMA_Q8_MAX_M") ? atoi(getenv("GGML_VK_MMA_Q8_MAX_M")) : 65536; + static const int env_mma_f32_max_m_fn = getenv("GGML_VK_MMA_F32_MAX_M") ? atoi(getenv("GGML_VK_MMA_F32_MAX_M")) : 256; + return src1->ne[2] * src1->ne[3] == 1 && + (mm->ne[1] <= (int64_t) ggml_vk_mul_mat_vec_max_cols(ctx->device) || + (ggml_vk_is_apple_agx_coopmat(ctx->device) && !ggml_vk_exact_s8_enabled() && !gen_batch_invariant_for_n(mm->ne[1]) && + mm->ne[1] <= 4096 && + ((src0->type == GGML_TYPE_F32 && mm->ne[0] <= (mm->ne[1] <= 32 ? 256 : env_mma_f32_max_m_fn)) || + (src0->type == GGML_TYPE_Q8_0 && (mm->ne[1] <= 32 + ? (mm->ne[0] <= env_mma_q8_max_m_s_fn) + : (mm->ne[1] >= env_mma_q8_min_n_fn && mm->ne[0] >= env_mma_q8_min_m_fn && mm->ne[0] <= env_mma_q8_max_m_l_fn)))))); +} + +static bool gen_supports_pair_type(ggml_type t) { + switch (t) { + case GGML_TYPE_F32: + case GGML_TYPE_F16: + case GGML_TYPE_BF16: + case GGML_TYPE_Q8_0: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: + return true; + default: + return false; + } +} + +static bool gen_can_fuse_gate_up_glu(const ggml_backend_vk_context * ctx, const ggml_cgraph * cgraph, int i) { + if (!gen_fuse_enabled("gateup") || i + 2 >= cgraph->n_nodes) { + return false; + } + const ggml_tensor * n0 = cgraph->nodes[i]; + const ggml_tensor * n1 = cgraph->nodes[i + 1]; + const ggml_tensor * glu = cgraph->nodes[i + 2]; + if ((n0->op != GGML_OP_MUL_MAT && n0->op != GGML_OP_MUL_MAT_ID) || n1->op != n0->op || glu->op != GGML_OP_GLU || n0 == n1) { + return false; + } + if (!ggml_can_fuse_subgraph(cgraph, i, { n0->op, n1->op, GGML_OP_GLU }, { i + 2 })) { + return false; + } + if (ggml_get_glu_op(glu) != GGML_GLU_OP_SWIGLU || glu->src[1] == nullptr || ggml_get_op_params_i32(glu, 1) != 0) { + return false; + } + if (!((glu->src[0] == n0 && glu->src[1] == n1) || (glu->src[0] == n1 && glu->src[1] == n0))) { + return false; + } + if (n0->src[1] != n1->src[1] || (n0->op == GGML_OP_MUL_MAT_ID && n0->src[2] != n1->src[2])) { + return false; + } + const ggml_tensor * w0 = n0->src[0]; + const ggml_tensor * w1 = n1->src[0]; + if (w0->type != w1->type || (w0->ne[0] & 3) != 0) { + return false; + } + if (!ggml_are_same_shape(w0, w1) || !ggml_are_same_stride(w0, w1) || + !ggml_is_contiguous(w0) || !ggml_is_contiguous(w1) || + !w0->buffer || !w1->buffer || get_misalign_bytes(ctx, w0) != 0 || get_misalign_bytes(ctx, w1) != 0) { + return false; + } + if (!n0->src[1]->buffer || get_misalign_bytes(ctx, n0->src[1]) != 0) { + return false; + } + if (n0->type != GGML_TYPE_F32 || n1->type != GGML_TYPE_F32 || glu->type != GGML_TYPE_F32 || + !ggml_are_same_shape(n0, n1) || !ggml_are_same_shape(n0, glu) || + !ggml_are_same_stride(n0, n1) || !ggml_are_same_stride(n0, glu) || + !ggml_is_contiguous(n0) || !ggml_is_contiguous(glu) || !glu->buffer || get_misalign_bytes(ctx, glu) != 0) { + return false; + } + if (n0->op == GGML_OP_MUL_MAT) { + if (gen_uses_dmmv_noswap(ctx, n0) && gen_uses_dmmv_noswap(ctx, n1)) { + if (gen_supports_pair_type(w0->type)) { + return true; + } + return ggml_vk_is_apple_agx_coopmat(ctx->device) && !ggml_vk_exact_s8_enabled() && !gen_batch_invariant_for_n(n0->ne[1]) && + n0->ne[1] >= 2 && (w0->type == GGML_TYPE_IQ4_XS || w0->type == GGML_TYPE_IQ4_NL) && (w0->ne[0] % 64 == 0); + } + return gen_uses_cm1_fused_mm(ctx, n0) && gen_uses_cm1_fused_mm(ctx, n1); + } + if (ggml_vk_use_mul_mat_vec_id(ctx, cgraph, i) && ggml_vk_use_mul_mat_vec_id(ctx, cgraph, i + 1)) { + return gen_supports_pair_type(w0->type); + } + return gen_uses_cm1_fused_mmid(ctx, n0) && gen_uses_cm1_fused_mmid(ctx, n1); +} + +static bool gen_can_fuse_mm_glu(const ggml_backend_vk_context * ctx, const ggml_cgraph * cgraph, int i) { + if (!gen_fuse_enabled("glu") || i + 1 >= cgraph->n_nodes) { + return false; + } + const ggml_tensor * mm = cgraph->nodes[i]; + const ggml_tensor * glu = cgraph->nodes[i + 1]; + if ((mm->op != GGML_OP_MUL_MAT && mm->op != GGML_OP_MUL_MAT_ID) || glu->op != GGML_OP_GLU) { + return false; + } + if (!ggml_can_fuse(cgraph, i, { mm->op, GGML_OP_GLU })) { + return false; + } + if (ggml_get_glu_op(glu) != GGML_GLU_OP_SWIGLU || glu->src[1] == nullptr || ggml_get_op_params_i32(glu, 1) != 0) { + return false; + } + if (glu->src[0] != mm && glu->src[1] != mm) { + return false; + } + const ggml_tensor * other = glu->src[0] == mm ? glu->src[1] : glu->src[0]; + if (other == mm || mm->type != GGML_TYPE_F32 || other->type != GGML_TYPE_F32 || glu->type != GGML_TYPE_F32) { + return false; + } + if (!ggml_are_same_shape(mm, other) || !ggml_are_same_stride(mm, other) || + !ggml_are_same_shape(mm, glu) || !ggml_are_same_stride(mm, glu) || !ggml_is_contiguous(mm)) { + return false; + } + if (get_misalign_bytes(ctx, other) != 0 || get_misalign_bytes(ctx, glu) != 0) { + return false; + } + if (mm->op == GGML_OP_MUL_MAT) { + return gen_uses_dmmv_noswap(ctx, mm) || gen_uses_cm1_fused_mm(ctx, mm); + } + return ggml_vk_use_mul_mat_vec_id(ctx, cgraph, i) || gen_uses_cm1_fused_mmid(ctx, mm); +} + +struct gen_mm_epilogue { + const ggml_tensor * dst {}; + const ggml_tensor * f0 {}; + const ggml_tensor * f1 {}; + uint32_t flags {}; + int skip[3] { -1, -1, -1 }; + const char * name {}; +}; + +struct vk_cgraph_use_index { + struct slot_t { + const ggml_tensor * t; + uint32_t epoch; + int32_t first_node; + uint32_t head_edge; + }; + std::vector slots; + std::vector edge_node; + std::vector edge_next; + uint32_t epoch = 1; + size_t mask = 0; + + uint32_t find_or_insert(const ggml_tensor * t) { + uintptr_t p = (uintptr_t) t; + size_t s = ((p >> 5) * 11400714819323198485ULL) & mask; + while (slots[s].epoch == epoch) { + if (slots[s].t == t) { + return (uint32_t) s; + } + s = (s + 1) & mask; + } + slots[s].t = t; + slots[s].epoch = epoch; + slots[s].first_node = -1; + slots[s].head_edge = UINT32_MAX; + return (uint32_t) s; + } + + int32_t find_slot(const ggml_tensor * t) const { + if (!t || slots.empty()) { + return -1; + } + uintptr_t p = (uintptr_t) t; + size_t s = ((p >> 5) * 11400714819323198485ULL) & mask; + while (slots[s].epoch == epoch) { + if (slots[s].t == t) { + return (int32_t) s; + } + s = (s + 1) & mask; + } + return -1; + } + + void add_reader(const ggml_tensor * t, int32_t k) { + uint32_t s = find_or_insert(t); + uint32_t head = slots[s].head_edge; + if (head != UINT32_MAX && edge_node[head] == k) { + return; + } + uint32_t eid = (uint32_t) edge_node.size(); + edge_node.push_back(k); + edge_next.push_back(head); + slots[s].head_edge = eid; + } + + void build(const ggml_cgraph * g) { + const int n_nodes = g->n_nodes; + size_t cap = 4096; + const size_t min_cap = (size_t) (n_nodes + 64) * 8; + while (cap < min_cap) { + cap <<= 1; + } + if (slots.size() < cap) { + slots.assign(cap, slot_t{ nullptr, 0, -1, UINT32_MAX }); + epoch = 1; + } else if (++epoch == 0) { + for (auto & sl : slots) { + sl.epoch = 0; + } + epoch = 1; + } + mask = slots.size() - 1; + edge_node.clear(); + edge_next.clear(); + edge_node.reserve((size_t) n_nodes * 4); + edge_next.reserve((size_t) n_nodes * 4); + + for (int32_t k = 0; k < n_nodes; ++k) { + const ggml_tensor * n = g->nodes[k]; + uint32_t ns = find_or_insert(n); + if (slots[ns].first_node < 0) { + slots[ns].first_node = k; + } + for (int s = 0; s < GGML_MAX_SRC; ++s) { + const ggml_tensor * src = n->src[s]; + if (!src) { + continue; + } + add_reader(src, k); + if (src->view_src && src->view_src != src) { + add_reader(src->view_src, k); + } + } + } + } + + int32_t first_node_idx(const ggml_tensor * t) const { + int32_t s = find_slot(t); + return s >= 0 ? slots[s].first_node : -1; + } + + template + bool any_reader(const ggml_tensor * t, Pred && pred) const { + int32_t s = find_slot(t); + if (s < 0) { + return false; + } + for (uint32_t e = slots[s].head_edge; e != UINT32_MAX; e = edge_next[e]) { + if (pred(edge_node[e])) { + return true; + } + } + return false; + } +}; + +static bool gen_find_mm_epilogue(const ggml_backend_vk_context * ctx, const ggml_cgraph * g, int i, int last_node, + const std::vector & gen_skip, const vk_cgraph_use_index & use_idx, gen_mm_epilogue & out) { + const bool do_mmgate = gen_fuse_enabled("mmgate"); + const bool do_shexp = gen_fuse_enabled("shexp"); + if (!do_mmgate && !do_shexp) { + return false; + } + const ggml_tensor * mm = g->nodes[i]; + if ((!gen_uses_dmmv_noswap(ctx, mm) && !gen_uses_cm1_fused_mm(ctx, mm)) || !ggml_is_contiguous(mm) || !mm->buffer || + get_misalign_bytes(ctx, mm) != 0 || (mm->flags & GGML_TENSOR_FLAG_OUTPUT)) { + return false; + } + auto same_bytes_of_mm = [&](const ggml_tensor * t) { + return t == mm || ((t->op == GGML_OP_RESHAPE || t->op == GGML_OP_VIEW) && + t->view_src == mm && t->view_offs == 0 && t->type == GGML_TYPE_F32 && + ggml_is_contiguous(t) && ggml_nbytes(t) == ggml_nbytes(mm)); + }; + auto check_valid = [&](const ggml_tensor * dst_out, const ggml_tensor * f0, const ggml_tensor * f1, + int final_idx, const int (&skips)[3], const ggml_tensor * const (&elided)[4]) { + if (!dst_out || dst_out->type != GGML_TYPE_F32 || !ggml_is_contiguous(dst_out) || + ggml_nbytes(dst_out) != ggml_nbytes(mm) || !dst_out->buffer || get_misalign_bytes(ctx, dst_out) != 0) { + return false; + } + for (const ggml_tensor * e : elided) { + if (e && (e->flags & GGML_TENSOR_FLAG_OUTPUT)) { + return false; + } + } + auto is_skipped = [&](int idx) { + return idx == skips[0] || idx == skips[1] || idx == skips[2] || (!gen_skip.empty() && gen_skip[idx]); + }; + for (const ggml_tensor * e : elided) { + if (e && use_idx.any_reader(e, [&](int k) { + return k != i && !is_skipped(k) && !ggml_op_is_empty(g->nodes[k]->op); + })) { + return false; + } + } + if (ggml_vk_tensors_overlap(mm->src[0], dst_out, false) || + ggml_vk_tensors_overlap(mm->src[1], dst_out, false) || + (f0 && ggml_vk_tensors_overlap(f0, dst_out, false)) || + (f1 && ggml_vk_tensors_overlap(f1, dst_out, false))) { + return false; + } + for (int k = i + 1; k < final_idx; k++) { + const ggml_tensor * n = g->nodes[k]; + if (is_skipped(k) || ggml_op_is_empty(n->op)) { + continue; + } + if (n->buffer && ggml_vk_tensors_overlap(n, dst_out, false)) { + return false; + } + if ((f0 && n->buffer && ggml_vk_tensors_overlap(n, f0, false)) || + (f1 && n->buffer && ggml_vk_tensors_overlap(n, f1, false))) { + return false; + } + for (int s = 0; s < GGML_MAX_SRC; s++) { + const ggml_tensor * src = n->src[s]; + if (src && src->buffer && ggml_vk_tensors_overlap(src, dst_out, false)) { + return false; + } + } + } + return true; + }; + + const int hi = std::min(last_node, i + 32); + if (do_mmgate) { + for (int j = i + 1; j < hi; j++) { + if (!gen_skip.empty() && gen_skip[j]) { + continue; + } + const ggml_tensor * n = g->nodes[j]; + if (n->op == GGML_OP_UNARY && ggml_get_unary_op(n) == GGML_UNARY_OP_SIGMOID && same_bytes_of_mm(n->src[0])) { + const int skips[3] = { j, -1, -1 }; + const ggml_tensor * elided[4] = { mm, n->src[0], nullptr, nullptr }; + if (check_valid(n, nullptr, nullptr, j, skips, elided)) { + out = { n, nullptr, nullptr, MAT_VEC_FUSION_FLAGS_SIGMOID, { j, -1, -1 }, "GEN_MUL_MAT_SIGMOID" }; + return true; + } + break; + } + if (n->op == GGML_OP_ADD && j + 2 < last_node && (same_bytes_of_mm(n->src[0]) || same_bytes_of_mm(n->src[1]))) { + const ggml_tensor * view_t = same_bytes_of_mm(n->src[0]) ? n->src[0] : n->src[1]; + const ggml_tensor * dt = same_bytes_of_mm(n->src[0]) ? n->src[1] : n->src[0]; + const ggml_tensor * sp = g->nodes[j + 1]; + const ggml_tensor * mul = g->nodes[j + 2]; + const ggml_tensor * a_w = (mul->op == GGML_OP_MUL) ? (mul->src[0] == sp ? mul->src[1] : (mul->src[1] == sp ? mul->src[0] : nullptr)) : nullptr; + if (dt && a_w && dt->op == GGML_OP_NONE && a_w->op == GGML_OP_NONE && + dt->type == GGML_TYPE_F32 && a_w->type == GGML_TYPE_F32 && + ggml_is_contiguous(dt) && ggml_is_contiguous(a_w) && + dt->ne[0] == mm->ne[0] && ggml_nrows(dt) == 1 && + a_w->ne[0] == mm->ne[0] && ggml_nrows(a_w) == 1 && + dt->buffer && a_w->buffer && + get_misalign_bytes(ctx, dt) == 0 && get_misalign_bytes(ctx, a_w) == 0 && + n->type == GGML_TYPE_F32 && ggml_is_contiguous(n) && + sp->op == GGML_OP_UNARY && ggml_get_unary_op(sp) == GGML_UNARY_OP_SOFTPLUS && sp->src[0] == n && + sp->type == GGML_TYPE_F32 && ggml_is_contiguous(sp)) { + const int skips[3] = { j, j + 1, j + 2 }; + const ggml_tensor * elided[4] = { mm, view_t, n, sp }; + if (check_valid(mul, dt, a_w, j + 2, skips, elided)) { + out = { mul, dt, a_w, MAT_VEC_FUSION_FLAGS_SOFTPLUS_GATE, { j, j + 1, j + 2 }, "GEN_MUL_MAT_SOFTPLUS_GATE" }; + return true; + } + } + break; + } + } + } + if (do_shexp) { + for (int j = i + 1; j < hi; j++) { + if (!gen_skip.empty() && gen_skip[j]) { + continue; + } + const ggml_tensor * mul = g->nodes[j]; + if (mul->op == GGML_OP_MUL && (mul->src[0] == mm || mul->src[1] == mm)) { + const ggml_tensor * gate = (mul->src[0] == mm) ? mul->src[1] : mul->src[0]; + bool gate_ready = gate && gate->op == GGML_OP_NONE; + if (gate && !gate_ready) { + const int32_t gfirst = use_idx.first_node_idx(gate); + if (gfirst >= 0 && gfirst < i) { + gate_ready = true; + } + if (!gate_ready && !gen_skip.empty()) { + for (int k = i + 1; k < j; k++) { + if (g->nodes[k] == gate && gen_skip[k]) { + gate_ready = true; + break; + } + } + } + } + if (gate_ready && gate != mm && gate->type == GGML_TYPE_F32 && ggml_is_contiguous(gate) && + gate->ne[0] == 1 && gate->ne[1] == mm->ne[1] && gate->ne[2] == 1 && gate->ne[3] == 1 && + gate->buffer && get_misalign_bytes(ctx, gate) == 0 && + ggml_are_same_shape(mul, mm) && ggml_are_same_stride(mul, mm)) { + const int skips[3] = { j, -1, -1 }; + const ggml_tensor * elided[4] = { mm, nullptr, nullptr, nullptr }; + if (check_valid(mul, gate, nullptr, j, skips, elided)) { + out = { mul, gate, nullptr, MAT_VEC_FUSION_FLAGS_ROW_SCALE0, { j, -1, -1 }, "GEN_MUL_MAT_ROW_SCALE0" }; + return true; + } + } + break; + } + } + } + return false; +} + +static int gen_find_mm_deferred_scale0(const ggml_backend_vk_context * ctx, const ggml_cgraph * g, int i, int last_node, + const vk_cgraph_use_index & use_idx) { + if (!gen_fuse_enabled("shexp")) { + return -1; + } + const ggml_tensor * mm = g->nodes[i]; + if ((!gen_uses_dmmv_noswap(ctx, mm) && !gen_uses_cm1_fused_mm(ctx, mm)) || !ggml_is_contiguous(mm) || !mm->buffer || + get_misalign_bytes(ctx, mm) != 0 || (mm->flags & GGML_TENSOR_FLAG_OUTPUT)) { + return -1; + } + int mul_idx = -1; + for (int j = i + 1; j < std::min(last_node, i + 8); j++) { + const ggml_tensor * n = g->nodes[j]; + if (n->op == GGML_OP_MUL && (n->src[0] == mm || n->src[1] == mm)) { + mul_idx = j; + break; + } + } + if (mul_idx < 0) { + return -1; + } + const ggml_tensor * mul = g->nodes[mul_idx]; + const ggml_tensor * gate = (mul->src[0] == mm) ? mul->src[1] : mul->src[0]; + if (!gate || gate == mm || gate->type != GGML_TYPE_F32 || !ggml_is_contiguous(gate) || + gate->ne[0] != 1 || gate->ne[1] != mm->ne[1] || gate->ne[2] != 1 || gate->ne[3] != 1 || + !gate->buffer || get_misalign_bytes(ctx, gate) != 0 || + mul->type != GGML_TYPE_F32 || !ggml_is_contiguous(mul) || !mul->buffer || get_misalign_bytes(ctx, mul) != 0 || + !ggml_are_same_shape(mul, mm) || !ggml_are_same_stride(mul, mm)) { + return -1; + } + if (use_idx.any_reader(mm, [&](int k) { return k != mul_idx; })) { + return -1; + } + if (ggml_vk_tensors_overlap(mm->src[0], mul, false) || + ggml_vk_tensors_overlap(mm->src[1], mul, false) || + ggml_vk_tensors_overlap(gate, mul, false)) { + return -1; + } + for (int k = i + 1; k < mul_idx; k++) { + const ggml_tensor * n = g->nodes[k]; + if (!ggml_op_is_empty(n->op) && n->buffer) { + if ((mm->src[0]->buffer && ggml_vk_tensors_overlap(n, mm->src[0], false)) || + (mm->src[1]->buffer && ggml_vk_tensors_overlap(n, mm->src[1], false))) { + return -1; + } + } + } + return mul_idx; +} + +static bool s10_fuse_disabled() { + static const bool v = getenv("GGML_VK_DISABLE_S10_FUSE") != nullptr && atoi(getenv("GGML_VK_DISABLE_S10_FUSE")) != 0; + return v; +} + +static bool s10_pair_gdn_disabled() { + static const bool v = getenv("GGML_VK_DISABLE_S10_PAIR_GDN") != nullptr && atoi(getenv("GGML_VK_DISABLE_S10_PAIR_GDN")) != 0; + return s10_fuse_disabled() || v; +} + +static bool s10_ssm_conv_disabled() { + static const bool v = getenv("GGML_VK_DISABLE_S10_SSM_CONV") != nullptr && atoi(getenv("GGML_VK_DISABLE_S10_SSM_CONV")) != 0; + return s10_fuse_disabled() || v; +} + +static bool s10_add_rms_disabled() { + static const bool v = getenv("GGML_VK_DISABLE_S10_ADD_RMS") != nullptr && atoi(getenv("GGML_VK_DISABLE_S10_ADD_RMS")) != 0; + return s10_fuse_disabled() || v; +} + +static int gen_can_fuse_pair_mm(ggml_backend_vk_context * ctx, const ggml_cgraph * cgraph, int i, int last_node, + const std::vector & gen_skip, const vk_cgraph_use_index & use_idx) { + ctx->gen_pair_mul0_rel = 0; + if (!gen_fuse_enabled("pairmm") || i + 1 >= cgraph->n_nodes) { + return 0; + } + if (!gen_skip.empty() && gen_skip[i]) { + return 0; + } + ggml_tensor * mm0 = cgraph->nodes[i]; + if (mm0->op != GGML_OP_MUL_MAT || (mm0->ne[0] & 7) != 0) { + return 0; + } + const bool mm0_dmmv = gen_uses_dmmv_noswap(ctx, mm0); + const bool mm0_cm1 = !mm0_dmmv && mm0->ne[1] > 1 && mm0->ne[2] == 1 && mm0->ne[3] == 1 && + mm0->src[1] && mm0->src[1]->ne[2] * mm0->src[1]->ne[3] == 1 && + (mm0->ne[0] % 128 == 0) && gen_uses_cm1_fused_mm(ctx, mm0); + if (!mm0_dmmv && !mm0_cm1) { + return 0; + } + const ggml_tensor * w0 = mm0->src[0]; + if (!gen_supports_pair_type(w0->type) || (w0->ne[0] & 3) != 0 || w0->ne[2] != 1 || w0->ne[3] != 1 || + !ggml_is_contiguous(w0) || !w0->buffer || get_misalign_bytes(ctx, w0) != 0 || + !mm0->src[1]->buffer || get_misalign_bytes(ctx, mm0->src[1]) != 0 || + !mm0->buffer || !ggml_is_contiguous(mm0) || get_misalign_bytes(ctx, mm0) != 0) { + return 0; + } + + auto is_zero_offs_contig_view = [](const ggml_tensor * v, const ggml_tensor * base) -> bool { + return v && (v->op == GGML_OP_RESHAPE || v->op == GGML_OP_VIEW) && + v->view_src == base && v->view_offs == 0 && v->type == GGML_TYPE_F32 && + ggml_is_contiguous(v) && ggml_nbytes(v) == ggml_nbytes(base); + }; + + // S10.3: 6/7/8-node GDN [mm0, [reshape0], add(dt), softplus, mul(a_w), mm1, [reshape1], sigmoid] + if (mm0_dmmv && !s10_pair_gdn_disabled() && gen_fuse_enabled("mmgate") && i + 5 < cgraph->n_nodes && !(mm0->flags & GGML_TENSOR_FLAG_OUTPUT)) { + int pos = i + 1; + const ggml_tensor * view0 = mm0; + if (pos < cgraph->n_nodes && is_zero_offs_contig_view(cgraph->nodes[pos], mm0)) { + view0 = cgraph->nodes[pos]; + pos++; + } + if (pos + 4 < cgraph->n_nodes) { + ggml_tensor * add0 = cgraph->nodes[pos]; + ggml_tensor * sp0 = cgraph->nodes[pos + 1]; + ggml_tensor * mul0 = cgraph->nodes[pos + 2]; + ggml_tensor * mm1 = cgraph->nodes[pos + 3]; + const int mul0_idx = pos + 2; + const int mm1_idx = pos + 3; + int pos1 = mm1_idx + 1; + const ggml_tensor * view1 = mm1; + if (pos1 < cgraph->n_nodes && is_zero_offs_contig_view(cgraph->nodes[pos1], mm1)) { + view1 = cgraph->nodes[pos1]; + pos1++; + } + if (pos1 < cgraph->n_nodes && + add0->op == GGML_OP_ADD && (add0->src[0] == view0 || add0->src[1] == view0) && + sp0->op == GGML_OP_UNARY && ggml_get_unary_op(sp0) == GGML_UNARY_OP_SOFTPLUS && sp0->src[0] == add0 && + mul0->op == GGML_OP_MUL && (mul0->src[0] == sp0 || mul0->src[1] == sp0) && + mm1->op == GGML_OP_MUL_MAT && mm1 != mm0 && mm1->src[1] == mm0->src[1] && + mm0->ne[1] == mm1->ne[1] && gen_uses_dmmv_noswap(ctx, mm1)) { + ggml_tensor * sig1 = cgraph->nodes[pos1]; + const int sig1_idx = pos1; + const ggml_tensor * w1 = mm1->src[0]; + const ggml_tensor * dt = (add0->src[0] == view0) ? add0->src[1] : add0->src[0]; + const ggml_tensor * a_w = (mul0->src[0] == sp0) ? mul0->src[1] : mul0->src[0]; + bool any_skip = false; + if (!gen_skip.empty()) { + for (int k = i; k <= sig1_idx; ++k) { + any_skip = any_skip || (gen_skip[k] != 0); + } + } + if (!any_skip && + w1->type == w0->type && w1->ne[0] == w0->ne[0] && w1->ne[2] == 1 && w1->ne[3] == 1 && + ggml_is_contiguous(w1) && w1->buffer && get_misalign_bytes(ctx, w1) == 0 && + dt && a_w && dt->op == GGML_OP_NONE && a_w->op == GGML_OP_NONE && + dt->type == GGML_TYPE_F32 && a_w->type == GGML_TYPE_F32 && + ggml_is_contiguous(dt) && ggml_is_contiguous(a_w) && + dt->ne[0] == mm0->ne[0] && ggml_nrows(dt) == 1 && + a_w->ne[0] == mm0->ne[0] && ggml_nrows(a_w) == 1 && + dt->buffer && a_w->buffer && get_misalign_bytes(ctx, dt) == 0 && get_misalign_bytes(ctx, a_w) == 0 && + add0->type == GGML_TYPE_F32 && ggml_is_contiguous(add0) && + sp0->type == GGML_TYPE_F32 && ggml_is_contiguous(sp0) && + mul0->type == GGML_TYPE_F32 && ggml_is_contiguous(mul0) && ggml_nbytes(mul0) == ggml_nbytes(mm0) && + mul0->buffer && get_misalign_bytes(ctx, mul0) == 0 && + mm1->type == GGML_TYPE_F32 && ggml_is_contiguous(mm1) && !(mm1->flags & GGML_TENSOR_FLAG_OUTPUT) && + sig1->op == GGML_OP_UNARY && ggml_get_unary_op(sig1) == GGML_UNARY_OP_SIGMOID && + sig1->src[0] == view1 && sig1->type == GGML_TYPE_F32 && + ggml_is_contiguous(sig1) && ggml_nbytes(sig1) == ggml_nbytes(mm1) && + sig1->buffer && get_misalign_bytes(ctx, sig1) == 0) { + const ggml_tensor * elided[6] = { + mm0, (view0 != mm0 ? view0 : nullptr), add0, sp0, mm1, (view1 != mm1 ? view1 : nullptr) + }; + bool elided_ok = true; + for (const ggml_tensor * e : elided) { + if (!e) { + continue; + } + if ((e->flags & GGML_TENSOR_FLAG_OUTPUT) || + use_idx.any_reader(e, [&](int k) { + return (k < i || k > sig1_idx) && !ggml_op_is_empty(cgraph->nodes[k]->op); + })) { + elided_ok = false; + break; + } + } + const ggml_tensor * inputs[5] = { w0, mm0->src[1], w1, dt, a_w }; + for (const ggml_tensor * inp : inputs) { + if (ggml_vk_tensors_overlap(inp, mul0, false) || ggml_vk_tensors_overlap(inp, sig1, false)) { + elided_ok = false; + } + } + if (ggml_vk_tensors_overlap(mul0, sig1, false)) { + elided_ok = false; + } + if (elided_ok) { + ctx->gen_pair_mm1 = mm1; + ctx->gen_pair_dst1 = sig1; + ctx->gen_mm_dst = mul0; + ctx->gen_mm_f0 = dt; + ctx->gen_mm_f1 = a_w; + ctx->gen_mm_flags = MAT_VEC_FUSION_FLAGS_PAIR_MM | MAT_VEC_FUSION_FLAGS_SOFTPLUS_GATE | MAT_VEC_FUSION_FLAGS_PAIR_MM_SIGMOID1; + ctx->gen_pair_mul0_rel = mul0_idx - i; + return sig1_idx - i; + } + } + } + } + } + + if (!gen_skip.empty() && gen_skip[i + 1]) { + return 0; + } + ggml_tensor * mm1 = cgraph->nodes[i + 1]; + if (mm1->op != GGML_OP_MUL_MAT || mm0 == mm1 || mm0->src[1] != mm1->src[1] || + mm0->ne[1] != mm1->ne[1]) { + return 0; + } + static const bool unfuse_pair_mm1 = getenv("GGML_VK_UNFUSE_PAIR_MM1") != nullptr; + static const bool disable_unfuse_dmmv_mm1 = getenv("GGML_VK_DISABLE_UNFUSE_DMMV_MM1") != nullptr; + if (!mm0_dmmv && unfuse_pair_mm1 && mm1->ne[0] == 1 && mm0->ne[0] >= 64 && (mm0->ne[0] % 64 == 0) && mm0->ne[1] > 32) { + return 0; + } + if (mm0_dmmv && !disable_unfuse_dmmv_mm1 && ggml_vk_is_apple_agx_coopmat(ctx->device) && !ggml_vk_exact_s8_enabled() && + mm1->ne[0] == 1 && mm0->ne[0] >= 64 && (mm0->ne[0] % 64 == 0) && mm0->ne[1] >= 64) { + return 0; + } + const bool pair_path_ok = mm0_dmmv + ? gen_uses_dmmv_noswap(ctx, mm1) + : (mm1->ne[2] == 1 && mm1->ne[3] == 1 && + mm1->src[0] && ggml_nbytes(mm1->src[0]) <= ctx->device->properties.limits.maxStorageBufferRange && + gen_uses_cm1_fused_mm(ctx, mm1)); + if (!pair_path_ok) { + return 0; + } + const ggml_tensor * w1 = mm1->src[0]; + if (w0->type != w1->type || w0->ne[0] != w1->ne[0] || w1->ne[2] != 1 || w1->ne[3] != 1 || + !ggml_is_contiguous(w1) || !w1->buffer || get_misalign_bytes(ctx, w1) != 0) { + return 0; + } + // Check 3/4-node [mm0, mm1, [reshape1], sigmoid(mm1)] first (e.g. [ffn_gate_inp, ffn_gate_inp_shexp, sigmoid]) + gen_mm_epilogue tmp_ep {}; + if (gen_find_mm_epilogue(ctx, cgraph, i, last_node, gen_skip, use_idx, tmp_ep) || + gen_find_mm_deferred_scale0(ctx, cgraph, i, last_node, use_idx) >= 0) { + return 0; + } + if (i + 2 < cgraph->n_nodes && !(mm1->flags & GGML_TENSOR_FLAG_OUTPUT)) { + int s_pos = i + 2; + const ggml_tensor * view1 = mm1; + if (s_pos + 1 < cgraph->n_nodes && is_zero_offs_contig_view(cgraph->nodes[s_pos], mm1)) { + view1 = cgraph->nodes[s_pos]; + s_pos++; + } + if (s_pos < cgraph->n_nodes && + (gen_skip.empty() || (!gen_skip[i + 2] && !gen_skip[s_pos]))) { + ggml_tensor * sig1 = cgraph->nodes[s_pos]; + if (sig1->op == GGML_OP_UNARY && ggml_get_unary_op(sig1) == GGML_UNARY_OP_SIGMOID && + sig1->src[0] == view1 && sig1->type == GGML_TYPE_F32 && + ggml_nbytes(sig1) == ggml_nbytes(mm1) && ggml_is_contiguous(sig1) && + sig1->buffer && get_misalign_bytes(ctx, sig1) == 0 && + !(view1 != mm1 && (view1->flags & GGML_TENSOR_FLAG_OUTPUT)) && + !ggml_vk_tensors_overlap(mm0, sig1, false) && + !ggml_vk_tensors_overlap(w0, sig1, false) && + !ggml_vk_tensors_overlap(w1, sig1, false) && + !ggml_vk_tensors_overlap(mm0->src[1], sig1, false)) { + auto out_of_span = [&](int k) { return (k < i || k > s_pos) && !ggml_op_is_empty(cgraph->nodes[k]->op); }; + const bool mm1_only_here = !use_idx.any_reader(mm1, out_of_span) && + !(view1 != mm1 && use_idx.any_reader(view1, out_of_span)); + if (mm1_only_here && + !ggml_vk_tensors_overlap(w0, mm0, false) && + !ggml_vk_tensors_overlap(w1, mm0, false) && + !ggml_vk_tensors_overlap(mm0->src[1], mm0, false)) { + ctx->gen_pair_mm1 = mm1; + ctx->gen_pair_dst1 = sig1; + ctx->gen_mm_flags = MAT_VEC_FUSION_FLAGS_PAIR_MM | MAT_VEC_FUSION_FLAGS_PAIR_MM_SIGMOID1; + return s_pos - i; + } + } + } + } + // Protect downstream fusions on mm1 (glu, bias add, mmgate, shexp) + if (gen_can_fuse_mm_glu(ctx, cgraph, i + 1) || + ggml_vk_can_fuse(ctx, cgraph, i + 1, { GGML_OP_MUL_MAT, GGML_OP_ADD }) || + gen_find_mm_epilogue(ctx, cgraph, i + 1, last_node, gen_skip, use_idx, tmp_ep) || + gen_find_mm_deferred_scale0(ctx, cgraph, i + 1, last_node, use_idx) >= 0) { + return 0; + } + // In [Q, K, V] with matching types, leave Q standalone and let i + 1 pair [K, V] + if (i + 2 < cgraph->n_nodes) { + const ggml_tensor * mm2 = cgraph->nodes[i + 2]; + if (mm2->op == GGML_OP_MUL_MAT && mm2->src[1] == mm0->src[1] && + mm2->src[0]->type == w1->type && mm0->ne[0] > mm1->ne[0] && mm1->ne[0] == mm2->ne[0]) { + return 0; + } + } + if (!mm1->buffer || !ggml_is_contiguous(mm1) || get_misalign_bytes(ctx, mm1) != 0 || + !ggml_can_fuse_subgraph(cgraph, i, { GGML_OP_MUL_MAT, GGML_OP_MUL_MAT }, { i, i + 1 }) || + ggml_vk_tensors_overlap(mm0, mm1, false)) { + return 0; + } + ctx->gen_pair_mm1 = mm1; + ctx->gen_pair_dst1 = mm1; + ctx->gen_mm_flags = MAT_VEC_FUSION_FLAGS_PAIR_MM; + return 1; +} + +static bool gen_lora_fuse_disabled() { + static const bool v = getenv("GGML_VK_DISABLE_LORA_FUSE") != nullptr && atoi(getenv("GGML_VK_DISABLE_LORA_FUSE")) != 0; + return v; +} + +struct vk_lora_tail_match { + ggml_tensor * mm {}; + const ggml_tensor * b {}; + const ggml_tensor * h {}; + ggml_tensor * lb {}; + ggml_tensor * ls {}; + ggml_tensor * add {}; + uint32_t r {}; + float scale {}; +}; + +static bool gen_match_lora_tail4(const ggml_backend_vk_context * ctx, const ggml_cgraph * cgraph, int s, + const std::vector & gen_skip, vk_lora_tail_match & out) { + if (s < 0 || s + 3 >= cgraph->n_nodes) { + return false; + } + for (int t = 0; t < 4; ++t) { + if (!gen_skip.empty() && gen_skip[s + t]) { + return false; + } + } + ggml_tensor * mm = cgraph->nodes[s + 0]; + ggml_tensor * lb = cgraph->nodes[s + 1]; + ggml_tensor * ls = cgraph->nodes[s + 2]; + ggml_tensor * add = cgraph->nodes[s + 3]; + if (mm->op != GGML_OP_MUL_MAT || lb->op != GGML_OP_MUL_MAT || + ls->op != GGML_OP_SCALE || add->op != GGML_OP_ADD) { + return false; + } + if (!gen_uses_dmmv_noswap(ctx, mm)) { + return false; + } + const ggml_tensor * w = mm->src[0]; + const ggml_tensor * x = mm->src[1]; + const ggml_tensor * b = lb->src[0]; + const ggml_tensor * h = lb->src[1]; + if (!w || !x || !b || !h || b->op != GGML_OP_NONE || w->op != GGML_OP_NONE) { + return false; + } + if (!gen_supports_pair_type(w->type) || (w->ne[0] & 3) != 0 || w->ne[2] != 1 || w->ne[3] != 1 || + !ggml_is_contiguous(w) || !w->buffer || get_misalign_bytes(ctx, w) != 0 || + !x->buffer || get_misalign_bytes(ctx, x) != 0) { + return false; + } + const int64_t r = b->ne[0]; + if (r <= 0 || r > 64 || b->type != GGML_TYPE_F32 || h->type != GGML_TYPE_F32 || + b->ne[1] != mm->ne[0] || b->ne[2] != 1 || b->ne[3] != 1 || + h->ne[0] != r || h->ne[1] != mm->ne[1] || h->ne[2] != 1 || h->ne[3] != 1 || + !ggml_is_contiguous(b) || !ggml_is_contiguous(h) || + !b->buffer || !h->buffer || get_misalign_bytes(ctx, b) != 0 || get_misalign_bytes(ctx, h) != 0) { + return false; + } + if (ls->src[0] != lb || ggml_get_op_params_f32(ls, 1) != 0.0f) { + return false; + } + if (!((add->src[0] == mm && add->src[1] == ls) || (add->src[0] == ls && add->src[1] == mm))) { + return false; + } + if (mm->type != GGML_TYPE_F32 || lb->type != GGML_TYPE_F32 || ls->type != GGML_TYPE_F32 || add->type != GGML_TYPE_F32 || + !ggml_are_same_shape(mm, lb) || !ggml_are_same_shape(mm, ls) || !ggml_are_same_shape(mm, add) || + !ggml_are_same_stride(mm, add) || !ggml_is_contiguous(mm) || !ggml_is_contiguous(add) || + !add->buffer || get_misalign_bytes(ctx, add) != 0) { + return false; + } + out = { mm, b, h, lb, ls, add, (uint32_t)r, ggml_get_op_params_f32(ls, 0) }; + return true; +} + +static int gen_can_fuse_lora(ggml_backend_vk_context * ctx, const ggml_cgraph * cgraph, int i, + const std::vector & gen_skip, const vk_cgraph_use_index & use_idx, + const char *& out_fusion_name, bool * op_srcs_fused_elementwise) { + if (gen_lora_fuse_disabled() || !ggml_vk_is_apple_agx_coopmat(ctx->device) || ggml_vk_exact_s8_enabled()) { + return 0; + } + vk_lora_tail_match t0 {}; + if (!gen_match_lora_tail4(ctx, cgraph, i, gen_skip, t0)) { + return 0; + } + + auto elided_ok_in = [&](const ggml_tensor * const * elided, int n_elided, int end_idx) -> bool { + for (int idx = 0; idx < n_elided; ++idx) { + const ggml_tensor * e = elided[idx]; + if (!e) { continue; } + if ((e->flags & GGML_TENSOR_FLAG_OUTPUT) || + use_idx.any_reader(e, [&](int k) { + return (k < i || k > end_idx) && !ggml_op_is_empty(cgraph->nodes[k]->op); + })) { + return false; + } + } + return true; + }; + + auto h_outside_span = [&](const ggml_tensor * h, int end_idx) -> bool { + for (int k = i; k <= end_idx; ++k) { + if (cgraph->nodes[k] == h) { + return false; + } + } + return true; + }; + + // Check paired [tail0, tail1, [GLU]] + vk_lora_tail_match t1 {}; + if (gen_match_lora_tail4(ctx, cgraph, i + 4, gen_skip, t1) && + t0.mm != t1.mm && t0.mm->src[1] == t1.mm->src[1] && + t0.mm->ne[1] == t1.mm->ne[1] && t0.r == t1.r && + t0.mm->src[0]->type == t1.mm->src[0]->type && + t0.mm->src[0]->ne[0] == t1.mm->src[0]->ne[0]) { + + // 1. 9-node [tail0, tail1, SWIGLU] -> GEN_MUL_MAT_GATE_UP_GLU + LORA0|LORA1 + if (gen_fuse_enabled("gateup") && i + 8 < cgraph->n_nodes && (gen_skip.empty() || !gen_skip[i + 8])) { + ggml_tensor * glu = cgraph->nodes[i + 8]; + if (glu->op == GGML_OP_GLU && ggml_get_glu_op(glu) == GGML_GLU_OP_SWIGLU && + glu->src[1] != nullptr && ggml_get_op_params_i32(glu, 1) == 0 && + ((glu->src[0] == t0.add && glu->src[1] == t1.add) || (glu->src[0] == t1.add && glu->src[1] == t0.add)) && + ggml_are_same_shape(t0.mm->src[0], t1.mm->src[0]) && + ggml_are_same_stride(t0.mm->src[0], t1.mm->src[0]) && + glu->type == GGML_TYPE_F32 && ggml_are_same_shape(t0.mm, glu) && + ggml_are_same_stride(t0.mm, glu) && ggml_is_contiguous(glu) && + glu->buffer && get_misalign_bytes(ctx, glu) == 0 && + h_outside_span(t0.h, i + 8) && h_outside_span(t1.h, i + 8)) { + const ggml_tensor * elided[8] = { t0.mm, t0.lb, t0.ls, t0.add, t1.mm, t1.lb, t1.ls, t1.add }; + const ggml_tensor * inps[7] = { t0.mm->src[0], t1.mm->src[0], t0.mm->src[1], t0.b, t0.h, t1.b, t1.h }; + bool ov_ok = elided_ok_in(elided, 8, i + 8); + for (const ggml_tensor * inp : inps) { + if (ggml_vk_tensors_overlap(inp, glu, false)) { ov_ok = false; } + } + if (ov_ok) { + const bool t0_is_gate = (glu->src[0] == t0.add); + const vk_lora_tail_match & tg = t0_is_gate ? t0 : t1; + const vk_lora_tail_match & tu = t0_is_gate ? t1 : t0; + ctx->gen_lora_gate_src0 = tg.mm->src[0]; + ctx->gen_lora_up_src0 = tu.mm->src[0]; + ctx->gen_lora_b0 = tg.b; + ctx->gen_lora_h0 = tg.h; + ctx->gen_lora_scale0 = tg.scale; + ctx->gen_lora_b1 = tu.b; + ctx->gen_lora_h1 = tu.h; + ctx->gen_lora_scale1 = tu.scale; + ctx->gen_lora_r = tg.r; + std::fill_n(op_srcs_fused_elementwise, 9, false); + op_srcs_fused_elementwise[8] = true; + out_fusion_name = "GEN_LORA_GATE_UP_GLU"; + return 8; + } + } + } + + // 2. 8-node [tail0, tail1] -> GEN_PAIR_MM + LORA0|LORA1 (provided tail1 doesn't feed an immediate ADD/GLU) + const bool t1_has_res_add = (i + 8 < cgraph->n_nodes) && + cgraph->nodes[i + 8]->op == GGML_OP_ADD && + (cgraph->nodes[i + 8]->src[0] == t1.add || cgraph->nodes[i + 8]->src[1] == t1.add); + const bool t0_has_res_add = cgraph->nodes[i + 4]->op == GGML_OP_ADD && + (cgraph->nodes[i + 4]->src[0] == t0.add || cgraph->nodes[i + 4]->src[1] == t0.add); + if (gen_fuse_enabled("pairmm") && !t0_has_res_add && !t1_has_res_add && + (t0.mm->ne[0] & 7) == 0 && + h_outside_span(t0.h, i + 7) && h_outside_span(t1.h, i + 7)) { + const ggml_tensor * elided[6] = { t0.mm, t0.lb, t0.ls, t1.mm, t1.lb, t1.ls }; + const ggml_tensor * inps[7] = { t0.mm->src[0], t1.mm->src[0], t0.mm->src[1], t0.b, t0.h, t1.b, t1.h }; + bool ov_ok = elided_ok_in(elided, 6, i + 7) && !ggml_vk_tensors_overlap(t0.add, t1.add, false); + for (const ggml_tensor * inp : inps) { + if (ggml_vk_tensors_overlap(inp, t0.add, false) || ggml_vk_tensors_overlap(inp, t1.add, false)) { + ov_ok = false; + } + } + if (ov_ok) { + ctx->gen_mm_dst = t0.add; + ctx->gen_pair_mm1 = t1.mm; + ctx->gen_pair_dst1 = t1.add; + ctx->gen_mm_flags = MAT_VEC_FUSION_FLAGS_PAIR_MM; + ctx->gen_pair_mul0_rel = 3; + ctx->fused_ops_write_mask |= (1u << 3); + ctx->gen_lora_b0 = t0.b; + ctx->gen_lora_h0 = t0.h; + ctx->gen_lora_scale0 = t0.scale; + ctx->gen_lora_b1 = t1.b; + ctx->gen_lora_h1 = t1.h; + ctx->gen_lora_scale1 = t1.scale; + ctx->gen_lora_r = t0.r; + std::fill_n(op_srcs_fused_elementwise, 8, false); + out_fusion_name = "GEN_LORA_PAIR_MM"; + return 7; + } + } + } + + // 3. 5-node [tail0, ADD(t0.add, residual)] -> MUL_MAT_ADD + LORA0 + if (i + 4 < cgraph->n_nodes && (gen_skip.empty() || !gen_skip[i + 4])) { + ggml_tensor * radd = cgraph->nodes[i + 4]; + if (radd->op == GGML_OP_ADD && (radd->src[0] == t0.add || radd->src[1] == t0.add)) { + const ggml_tensor * bias = (radd->src[0] == t0.add) ? radd->src[1] : radd->src[0]; + if (bias && bias != t0.add && bias->type == GGML_TYPE_F32 && radd->type == GGML_TYPE_F32 && + ggml_are_same_shape(t0.add, bias) && ggml_are_same_stride(t0.add, bias) && + ggml_are_same_shape(t0.add, radd) && ggml_are_same_stride(t0.add, radd) && + ggml_is_contiguous(bias) && ggml_is_contiguous(radd) && + bias->buffer && radd->buffer && + get_misalign_bytes(ctx, bias) == 0 && get_misalign_bytes(ctx, radd) == 0 && + h_outside_span(t0.h, i + 4) && h_outside_span(bias, i + 4)) { + const ggml_tensor * elided[4] = { t0.mm, t0.lb, t0.ls, t0.add }; + const ggml_tensor * inps[4] = { t0.mm->src[0], t0.mm->src[1], t0.b, t0.h }; + bool ov_ok = elided_ok_in(elided, 4, i + 4) && !ggml_vk_tensors_overlap(bias, radd, true); + for (const ggml_tensor * inp : inps) { + if (ggml_vk_tensors_overlap(inp, radd, false)) { ov_ok = false; } + } + if (ov_ok) { + ctx->gen_mm_dst = radd; + ctx->gen_mm_f0 = bias; + ctx->gen_mm_flags = MAT_VEC_FUSION_FLAGS_BIAS0; + ctx->gen_lora_b0 = t0.b; + ctx->gen_lora_h0 = t0.h; + ctx->gen_lora_scale0 = t0.scale; + ctx->gen_lora_r = t0.r; + std::fill_n(op_srcs_fused_elementwise, 4, false); + op_srcs_fused_elementwise[4] = true; + out_fusion_name = "GEN_LORA_MUL_MAT_ADD"; + return 4; + } + } + } + } + + // 4. 4-node [tail0] -> MUL_MAT + LORA0 + if (h_outside_span(t0.h, i + 3)) { + const ggml_tensor * elided[3] = { t0.mm, t0.lb, t0.ls }; + const ggml_tensor * inps[4] = { t0.mm->src[0], t0.mm->src[1], t0.b, t0.h }; + bool ov_ok = elided_ok_in(elided, 3, i + 3); + for (const ggml_tensor * inp : inps) { + if (ggml_vk_tensors_overlap(inp, t0.add, false)) { ov_ok = false; } + } + if (ov_ok) { + ctx->gen_mm_dst = t0.add; + ctx->gen_mm_flags = 0; + ctx->gen_lora_b0 = t0.b; + ctx->gen_lora_h0 = t0.h; + ctx->gen_lora_scale0 = t0.scale; + ctx->gen_lora_r = t0.r; + std::fill_n(op_srcs_fused_elementwise, 4, false); + out_fusion_name = "GEN_LORA_MUL_MAT"; + return 3; + } + } + + return 0; +} + +// gen (GGML_GEN_FUSE=gate): RMS_NORM, MUL(weight), [RESHAPE/VIEW(z)], UNARY silu(z), MUL(that, silu) in 4 or 5 consecutive nodes, +// z contiguous with dst's shape, all f32; the rms rows contiguous (as RMS_NORM_MUL requires) +static int gen_can_fuse_rms_mul_silu(const ggml_backend_vk_context * ctx, const ggml_cgraph * cgraph, int i) { + if (!gen_fuse_enabled("gate") || i + 3 >= cgraph->n_nodes || ctx->do_add_rms_partials) { + return 0; + } + const ggml_tensor * rms = cgraph->nodes[i], * mul = cgraph->nodes[i + 1]; + if (rms->op != GGML_OP_RMS_NORM || mul->op != GGML_OP_MUL) { + return 0; + } + int silu_off = 2; + const ggml_tensor * maybe_view = cgraph->nodes[i + 2]; + if ((maybe_view->op == GGML_OP_RESHAPE || maybe_view->op == GGML_OP_VIEW) && i + 4 < cgraph->n_nodes) { + silu_off = 3; + } + const ggml_tensor * silu = cgraph->nodes[i + silu_off]; + const ggml_tensor * out = cgraph->nodes[i + silu_off + 1]; + if (silu->op != GGML_OP_UNARY || out->op != GGML_OP_MUL || ggml_get_unary_op(silu) != GGML_UNARY_OP_SILU) { + return 0; + } + if (silu_off == 3 && silu->src[0] != maybe_view) { + return 0; + } + if (!((out->src[0] == mul && out->src[1] == silu) || (out->src[0] == silu && out->src[1] == mul))) { + return 0; + } + if (mul->src[0] != rms && mul->src[1] != rms) { + return 0; + } + const ggml_tensor * w = mul->src[0] == rms ? mul->src[1] : mul->src[0]; + const ggml_tensor * z = silu->src[0]; + const ggml_tensor * gen_ts[6] = { rms->src[0], w, z, mul, silu, out }; + for (const ggml_tensor * t : gen_ts) { + if (t->type != GGML_TYPE_F32) { + return 0; + } + } + if (!ggml_are_same_shape(mul, rms) || !ggml_are_same_shape(out, mul) || !ggml_are_same_shape(z, out) || + !ggml_is_contiguous(z) || !ggml_is_contiguous(out) || !ggml_is_contiguous_rows(rms->src[0]) || + !ggml_is_contiguous_rows(w) || w->ne[0] != rms->ne[0] || get_misalign_bytes(ctx, z) != 0) { + return 0; + } + if (mul->src[1] == rms && !ggml_are_same_shape(mul->src[0], rms)) { + return 0; + } + if (silu_off == 3) { + const int outs[1] = { i + 4 }; + const int idxs[5] = { i, i + 1, i + 2, i + 3, i + 4 }; + const ggml_op ops[5] = { GGML_OP_RMS_NORM, GGML_OP_MUL, maybe_view->op, GGML_OP_UNARY, GGML_OP_MUL }; + return ggml_can_fuse_subgraph_ext(cgraph, idxs, 5, ops, outs, 1) ? 4 : 0; + } + const int outs[1] = { i + 3 }; + const int idxs[4] = { i, i + 1, i + 2, i + 3 }; + const ggml_op ops[4] = { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_UNARY, GGML_OP_MUL }; + return ggml_can_fuse_subgraph_ext(cgraph, idxs, 4, ops, outs, 1) ? 3 : 0; +} + +// gen (GGML_GEN_FUSE=s8): CONT(gate_view), [empty view(s)], UNARY, MUL(other, unary) -> single strided unary_mul dispatch +static int gen_can_fuse_cont_unary_mul(const ggml_backend_vk_context * ctx, const ggml_cgraph * cgraph, int i) { + GGML_UNUSED(ctx); + if (!gen_fuse_enabled("s8") || i + 2 >= cgraph->n_nodes) { + return 0; + } + const ggml_tensor * cont = cgraph->nodes[i]; + if (cont->op != GGML_OP_CONT) { + return 0; + } + const ggml_tensor * gate_view = cont->src[0]; + if (!gate_view || (gate_view->type != GGML_TYPE_F32 && gate_view->type != GGML_TYPE_F16) || + cont->type != gate_view->type || !ggml_is_contiguous_1(gate_view) || !gate_view->buffer) { + return 0; + } + const size_t ts = ggml_type_size(gate_view->type); + for (int d = 0; d < GGML_MAX_DIMS; ++d) { + if (gate_view->nb[d] % ts != 0) { + return 0; + } + } + int u_off = 1; + while (u_off <= 2 && i + u_off + 1 < cgraph->n_nodes && ggml_op_is_empty(cgraph->nodes[i + u_off]->op) && + cgraph->nodes[i + u_off]->op != GGML_OP_NONE) { + u_off++; + } + if (i + u_off + 1 >= cgraph->n_nodes) { + return 0; + } + const ggml_tensor * unary = cgraph->nodes[i + u_off]; + const ggml_tensor * mul = cgraph->nodes[i + u_off + 1]; + if (unary->op != GGML_OP_UNARY || mul->op != GGML_OP_MUL) { + return 0; + } + if (ggml_vk_unary_mul_op_index(ggml_get_unary_op(unary)) < 0) { + return 0; + } + const bool unary_reads_cont = unary->src[0] == cont || + ((unary->src[0]->op == GGML_OP_RESHAPE || unary->src[0]->op == GGML_OP_VIEW) && + unary->src[0]->view_src == cont && unary->src[0]->view_offs == 0 && + ggml_is_contiguous(unary->src[0]) && ggml_nelements(unary->src[0]) == ggml_nelements(cont)); + if (!unary_reads_cont || (mul->src[0] != unary && mul->src[1] != unary)) { + return 0; + } + const ggml_tensor * other = (mul->src[0] == unary) ? mul->src[1] : mul->src[0]; + if (!other || !other->buffer || !mul->buffer || + unary->type != gate_view->type || other->type != gate_view->type || mul->type != gate_view->type || + !ggml_is_contiguous(cont) || !ggml_is_contiguous(other) || !ggml_is_contiguous(mul) || + ggml_nelements(cont) != ggml_nelements(gate_view) || + ggml_nelements(other) != ggml_nelements(gate_view) || + ggml_nelements(mul) != ggml_nelements(gate_view)) { + return 0; + } + int idxs[5]; + ggml_op ops[5]; + const int n_nodes = u_off + 2; + for (int k = 0; k < n_nodes; ++k) { + idxs[k] = i + k; + ops[k] = cgraph->nodes[i + k]->op; + } + const int outs[1] = { i + n_nodes - 1 }; + return ggml_can_fuse_subgraph_ext(cgraph, idxs, n_nodes, ops, outs, 1) ? (n_nodes - 1) : 0; +} + +// gen (GGML_GEN_FUSE=l2): the SCALE (bias 0) that is the only consumer of RMS_NORM node i, a few nodes later. +// The RMS_NORM is deferred and runs fused at the SCALE's position (its own output is never written). +static int gen_find_rms_norm_scale(const ggml_backend_vk_context * ctx, const ggml_cgraph * cgraph, int i, int last_node, + const vk_cgraph_use_index & use_idx) { + if (!gen_fuse_enabled("l2") || ctx->do_add_rms_partials) { + return -1; + } + const ggml_tensor * rms = cgraph->nodes[i]; + if (rms->op != GGML_OP_RMS_NORM || rms->type != GGML_TYPE_F32 || rms->src[0]->type != GGML_TYPE_F32 || + (rms->flags & GGML_TENSOR_FLAG_OUTPUT)) { + return -1; + } + int sc_idx = -1; + for (int j = i + 1; j < std::min(last_node, i + 24); j++) { + const ggml_tensor * n = cgraph->nodes[j]; + if (n->op == GGML_OP_SCALE && n->src[0] == rms) { + sc_idx = j; + break; + } + } + if (sc_idx < 0) { + return -1; + } + const ggml_tensor * sc = cgraph->nodes[sc_idx]; + if (sc->type != GGML_TYPE_F32 || ggml_get_op_params_f32(sc, 1) != 0.0f || !ggml_is_contiguous(sc) || + !ggml_are_same_shape(rms, sc) || !sc->buffer) { + return -1; + } + // rms_norm's output is elided: the scale must be its only reader + if (use_idx.any_reader(rms, [&](int k) { return k != sc_idx; })) { + return -1; + } + if (rms->src[0]->buffer && ggml_vk_tensors_overlap(rms->src[0], sc, true)) { + return -1; + } + // the fused kernel runs at the SCALE's position: nothing in between may write the RMS_NORM input + for (int k = i + 1; k < sc_idx; k++) { + const ggml_tensor * n = cgraph->nodes[k]; + if (!ggml_op_is_empty(n->op) && n->buffer && rms->src[0]->buffer && ggml_vk_tensors_overlap(n, rms->src[0], false)) { + return -1; + } + } + return sc_idx; +} + +// Series 10 (S10.3): +// Mode 1: CONCAT(old_state, qkv) -> VIEW -> CPY(state_out) -> SSM_CONV -> [UNARY(SILU)] (5 -> 1 dispatch) +// Mode 2: CPY(state_out) -> SSM_CONV -> [UNARY(SILU)] (3 -> 1 dispatch, when n_t > max_sliding_nt) +static int s10_can_fuse_ssm_conv(ggml_backend_vk_context * ctx, const ggml_cgraph * cgraph, int i, + const std::vector & gen_skip, const vk_cgraph_use_index & use_idx) { + static const int max_sliding_nt = getenv("GGML_VK_SSM_SLIDING_MAX_NT") ? atoi(getenv("GGML_VK_SSM_SLIDING_MAX_NT")) : (ggml_vk_exact_s8_enabled() ? 32 : 65536); + if (s10_ssm_conv_disabled()) { + return 0; + } + const ggml_tensor * n0 = cgraph->nodes[i]; + + // Mode 1: CONCAT -> VIEW -> CPY -> SSM_CONV -> [SILU] + if (n0->op == GGML_OP_CONCAT && i + 3 < cgraph->n_nodes) { + const ggml_tensor * concat = n0; + const ggml_tensor * view = cgraph->nodes[i + 1]; + const ggml_tensor * cpy = cgraph->nodes[i + 2]; + const ggml_tensor * conv = cgraph->nodes[i + 3]; + if (view->op != GGML_OP_VIEW || cpy->op != GGML_OP_CPY || conv->op != GGML_OP_SSM_CONV) { + return 0; + } + if (!gen_skip.empty() && (gen_skip[i] || gen_skip[i + 1] || gen_skip[i + 2] || gen_skip[i + 3])) { + return 0; + } + if (ggml_get_op_params_i32(concat, 0) != 0 || conv->src[0] != concat || + concat->type != GGML_TYPE_F32 || conv->type != GGML_TYPE_F32 || + !ggml_is_contiguous(concat) || !ggml_is_contiguous(conv) || + !conv->buffer || get_misalign_bytes(ctx, conv) != 0) { + return 0; + } + const ggml_tensor * c_w = conv->src[1]; + if (!c_w || c_w->type != GGML_TYPE_F32 || !ggml_is_contiguous_rows(c_w) || + !c_w->buffer || get_misalign_bytes(ctx, c_w) != 0) { + return 0; + } + const int64_t nc = c_w->ne[0]; + const int64_t nr = c_w->ne[1]; + const int64_t n_t = conv->ne[1]; + const int64_t n_s = conv->ne[2]; + if (nc != 4 || conv->ne[0] != nr || conv->ne[3] != 1 || n_t < 1 || n_t > max_sliding_nt || n_s < 1 || + concat->ne[0] != (nc - 1) + n_t || concat->ne[1] != nr || concat->ne[2] != n_s || concat->ne[3] != 1) { + return 0; + } + const ggml_tensor * old_st = concat->src[0]; + const ggml_tensor * qkv = concat->src[1]; + if (!old_st || !qkv || old_st->type != GGML_TYPE_F32 || qkv->type != GGML_TYPE_F32 || + !old_st->buffer || !qkv->buffer || + get_misalign_bytes(ctx, old_st) != 0 || get_misalign_bytes(ctx, qkv) != 0 || + old_st->ne[0] != nc - 1 || old_st->ne[1] != nr || old_st->ne[2] != n_s || old_st->ne[3] != 1 || + old_st->nb[0] != sizeof(float) || (old_st->nb[1] % sizeof(float)) != 0 || (old_st->nb[2] % sizeof(float)) != 0 || + qkv->ne[0] != n_t || qkv->ne[1] != nr || qkv->ne[2] != n_s || qkv->ne[3] != 1 || + (qkv->nb[0] % sizeof(float)) != 0 || (qkv->nb[1] % sizeof(float)) != 0 || (qkv->nb[2] % sizeof(float)) != 0) { + return 0; + } + if (view->view_src != concat || view->type != GGML_TYPE_F32 || + view->ne[0] != nc - 1 || view->ne[1] != nr || view->ne[2] != n_s || view->ne[3] != 1 || + view->nb[0] != sizeof(float) || view->nb[1] != concat->nb[1] || view->nb[2] != concat->nb[2] || + view->view_offs != (size_t)n_t * sizeof(float) || + cpy->src[0] != view || cpy->type != GGML_TYPE_F32 || !ggml_is_contiguous(cpy) || + !cpy->buffer || get_misalign_bytes(ctx, cpy) != 0 || + ggml_nbytes(cpy) != (size_t)(nc - 1) * (size_t)nr * (size_t)n_s * sizeof(float)) { + return 0; + } + int end_idx = i + 3; + const ggml_tensor * final_dst = conv; + if (i + 4 < cgraph->n_nodes && (gen_skip.empty() || !gen_skip[i + 4]) && !(conv->flags & GGML_TENSOR_FLAG_OUTPUT)) { + const ggml_tensor * u = cgraph->nodes[i + 4]; + if (u->op == GGML_OP_UNARY && ggml_get_unary_op(u) == GGML_UNARY_OP_SILU && + u->src[0] == conv && u->type == GGML_TYPE_F32 && + ggml_are_same_shape(u, conv) && ggml_are_same_stride(u, conv) && + u->buffer && get_misalign_bytes(ctx, u) == 0) { + const bool conv_only_here = !use_idx.any_reader(conv, [&](int k) { + return k != i + 4 && !ggml_op_is_empty(cgraph->nodes[k]->op); + }); + if (conv_only_here) { + end_idx = i + 4; + final_dst = u; + } + } + } + if ((concat->flags & GGML_TENSOR_FLAG_OUTPUT) || (view->flags & GGML_TENSOR_FLAG_OUTPUT)) { + return 0; + } + auto out_of_ssm_span = [&](int k) { return (k < i || k > end_idx) && !ggml_op_is_empty(cgraph->nodes[k]->op); }; + if (use_idx.any_reader(concat, out_of_ssm_span) || use_idx.any_reader(view, out_of_ssm_span)) { + return 0; + } + // old_st may alias cpy iff exact same contiguous [nc-1, nr, n_s] slice (per-channel register shift) + if (ggml_vk_tensors_overlap(old_st, cpy, false)) { + const bool exact_same = + old_st->buffer == cpy->buffer && + vk_tensor_offset(old_st) + old_st->view_offs == vk_tensor_offset(cpy) + cpy->view_offs && + ggml_nbytes(old_st) == ggml_nbytes(cpy) && + ggml_is_contiguous(old_st); + if (!exact_same) { + return 0; + } + } + if (ggml_vk_tensors_overlap(qkv, cpy, false) || + ggml_vk_tensors_overlap(c_w, cpy, false) || + ggml_vk_tensors_overlap(cpy, final_dst, false) || + ggml_vk_tensors_overlap(old_st, final_dst, false) || + ggml_vk_tensors_overlap(qkv, final_dst, false) || + ggml_vk_tensors_overlap(c_w, final_dst, false)) { + return 0; + } + ctx->s10_ssm_conv_mode = 1; + ctx->s10_ssm_conv_node = conv; + ctx->s10_ssm_state_out = cpy; + ctx->s10_ssm_qkv = qkv; + ctx->fused_ops_write_mask |= (1u << 2); + return end_idx - i; + } + + // Mode 2: CPY -> SSM_CONV -> [SILU] + if (n0->op == GGML_OP_CPY && i + 1 < cgraph->n_nodes) { + const ggml_tensor * cpy = n0; + const ggml_tensor * conv = cgraph->nodes[i + 1]; + if (conv->op != GGML_OP_SSM_CONV) { + return 0; + } + if (!gen_skip.empty() && (gen_skip[i] || gen_skip[i + 1])) { + return 0; + } + const ggml_tensor * view = cpy->src[0]; + const ggml_tensor * src0 = conv->src[0]; + const ggml_tensor * c_w = conv->src[1]; + if (!view || !src0 || !c_w || view->op != GGML_OP_VIEW || view->view_src != src0 || + src0->type != GGML_TYPE_F32 || c_w->type != GGML_TYPE_F32 || conv->type != GGML_TYPE_F32 || + !ggml_is_contiguous_rows(src0) || !ggml_is_contiguous_rows(c_w) || !ggml_is_contiguous(conv) || + !src0->buffer || !c_w->buffer || !conv->buffer || + get_misalign_bytes(ctx, src0) != 0 || get_misalign_bytes(ctx, c_w) != 0 || get_misalign_bytes(ctx, conv) != 0) { + return 0; + } + const int64_t nc = c_w->ne[0]; + const int64_t nr = c_w->ne[1]; + const int64_t n_t = conv->ne[1]; + const int64_t n_s = conv->ne[2]; + if (nc < 2 || conv->ne[0] != nr || conv->ne[3] != 1 || n_t < 1 || n_s < 1 || + src0->ne[0] != (nc - 1) + n_t || src0->ne[1] != nr || src0->ne[2] != n_s || src0->ne[3] != 1 || + view->type != GGML_TYPE_F32 || view->ne[0] != nc - 1 || view->ne[1] != nr || view->ne[2] != n_s || view->ne[3] != 1 || + view->nb[0] != sizeof(float) || view->nb[1] != src0->nb[1] || view->nb[2] != src0->nb[2] || + view->view_offs != (size_t)n_t * sizeof(float) || + cpy->type != GGML_TYPE_F32 || !ggml_is_contiguous(cpy) || !cpy->buffer || get_misalign_bytes(ctx, cpy) != 0 || + ggml_nbytes(cpy) != (size_t)(nc - 1) * (size_t)nr * (size_t)n_s * sizeof(float)) { + return 0; + } + int end_idx = i + 1; + const ggml_tensor * final_dst = conv; + if (i + 2 < cgraph->n_nodes && (gen_skip.empty() || !gen_skip[i + 2]) && !(conv->flags & GGML_TENSOR_FLAG_OUTPUT)) { + const ggml_tensor * u = cgraph->nodes[i + 2]; + if (u->op == GGML_OP_UNARY && ggml_get_unary_op(u) == GGML_UNARY_OP_SILU && + u->src[0] == conv && u->type == GGML_TYPE_F32 && + ggml_are_same_shape(u, conv) && ggml_are_same_stride(u, conv) && + u->buffer && get_misalign_bytes(ctx, u) == 0) { + const bool conv_only_here = !use_idx.any_reader(conv, [&](int k) { + return k != i + 2 && !ggml_op_is_empty(cgraph->nodes[k]->op); + }); + if (conv_only_here) { + end_idx = i + 2; + final_dst = u; + } + } + } + if (ggml_vk_tensors_overlap(src0, cpy, false) || + ggml_vk_tensors_overlap(c_w, cpy, false) || + ggml_vk_tensors_overlap(cpy, final_dst, false) || + ggml_vk_tensors_overlap(src0, final_dst, false) || + ggml_vk_tensors_overlap(c_w, final_dst, false)) { + return 0; + } + ctx->s10_ssm_conv_mode = 2; + ctx->s10_ssm_conv_node = conv; + ctx->s10_ssm_state_out = cpy; + ctx->s10_ssm_qkv = nullptr; + ctx->fused_ops_write_mask |= 1u; + return end_idx - i; + } + return 0; +} + +// Series 10 (S10.3): 1..9 ADD -> ... -> RMS_NORM -> [MUL(weight)] in a single row-per-WG dispatch +static uint32_t s10_can_fuse_multi_add_rms_mul(ggml_backend_vk_context * ctx, const ggml_cgraph * cgraph, int i, + int last_node, std::vector & gen_skip, + const vk_cgraph_use_index & use_idx, int & out_final_idx) { + out_final_idx = -1; + if (s10_add_rms_disabled() || !ctx->device->multi_add || ctx->device->pipeline_multi_add_rms_mul[0][0][0] == nullptr) { + return 0; + } + const ggml_tensor * first_add = cgraph->nodes[i]; + if (first_add->op != GGML_OP_ADD || first_add->type != GGML_TYPE_F32 || + !first_add->buffer || get_misalign_bytes(ctx, first_add) != 0) { + return 0; + } + const int64_t ne0 = first_add->ne[0]; + const int64_t nrows = ggml_nrows(first_add); + if (ne0 <= 0 || ne0 > 4096 || nrows <= 0) { + return 0; + } + + auto valid_add_src = [&](const ggml_tensor * t) -> bool { + if (!t || t->type != GGML_TYPE_F32 || !t->buffer || get_misalign_bytes(ctx, t) != 0 || + !ggml_are_same_shape(first_add, t) || t->nb[0] != sizeof(float)) { + return false; + } + if (nrows == 1) { + return true; + } + return (t->ne[2] == 1 && t->ne[3] == 1 && (t->nb[1] % sizeof(float)) == 0) || ggml_is_contiguous(t); + }; + + if (!valid_add_src(first_add->src[0]) || !valid_add_src(first_add->src[1])) { + return 0; + } + + int32_t num_adds = 1; + while (i + num_adds < cgraph->n_nodes && num_adds < MAX_FUSED_ADDS && + (gen_skip.empty() || !gen_skip[i + num_adds])) { + const ggml_tensor * prev = cgraph->nodes[i + num_adds - 1]; + const ggml_tensor * na = cgraph->nodes[i + num_adds]; + if (na->op != GGML_OP_ADD || na->type != GGML_TYPE_F32 || !na->buffer || get_misalign_bytes(ctx, na) != 0 || + (na->src[0] != prev && na->src[1] != prev) || + !valid_add_src(na->src[0]) || !valid_add_src(na->src[1])) { + break; + } + num_adds++; + } + ggml_op adds[MAX_FUSED_ADDS]; + for (int32_t k = 0; k < num_adds; ++k) { + adds[k] = GGML_OP_ADD; + } + while (num_adds > 1 && !ggml_can_fuse(cgraph, i, adds, num_adds)) { + num_adds--; + } + + const int add_end_idx = i + num_adds - 1; + if (num_adds == 1 && i > 0) { + const ggml_tensor * prev_node = cgraph->nodes[i - 1]; + if ((prev_node->op == GGML_OP_MUL_MAT || prev_node->op == GGML_OP_MUL_MAT_ID) && + (first_add->src[0] == prev_node || first_add->src[1] == prev_node)) { + return 0; + } + } + const ggml_tensor * last_add = cgraph->nodes[add_end_idx]; + + int rms_idx = -1; + const int search_hi = std::min(last_node + 1, add_end_idx + 17); + for (int j = add_end_idx + 1; j < search_hi; ++j) { + if (!gen_skip.empty() && gen_skip[j]) { + continue; + } + const ggml_tensor * n = cgraph->nodes[j]; + if (n->op == GGML_OP_RMS_NORM && n->src[0] == last_add) { + rms_idx = j; + break; + } + } + if (rms_idx < 0) { + return 0; + } + const ggml_tensor * rms = cgraph->nodes[rms_idx]; + if (rms->type != GGML_TYPE_F32 || !ggml_are_same_shape(rms, last_add) || + !ggml_is_contiguous(rms) || !rms->buffer || get_misalign_bytes(ctx, rms) != 0) { + return 0; + } + + const bool was_adjacent_cand = + (rms_idx == add_end_idx + 1) && + (nrows == 1 || + (gen_batch_invariant() && nrows <= 64 && + ggml_is_contiguous(first_add) && ggml_is_contiguous(rms) && + (ne0 % ctx->device->pipeline_add_rms[0][0][0]->wg_denoms[0]) == 0)) && + ctx->device->add_rms_fusion && !ctx->device->disable_fusion; + + bool fits = false; + if (was_adjacent_cand) { + const uint32_t psize = ggml_vk_rms_partials_size(ctx, first_add); + fits = gen_batch_invariant() + ? (ctx->gen_n1_partials_offset + psize <= 1024 && + ctx->prealloc_size_add_rms_partials_offset + gen_rms_partials_size_rows(ctx, first_add) <= ctx->prealloc_size_add_rms_partials) + : (ctx->prealloc_size_add_rms_partials_offset + psize <= ctx->prealloc_size_add_rms_partials); + } + const uint32_t tree_mode = (was_adjacent_cand && fits) ? 1u : 0u; + if (tree_mode == 1u) { + if (ne0 < 512 || ne0 > 4096 || (ne0 % 512) != 0) { + return 0; + } + } else { + const bool rms_subgroups = (ctx->device->vendor_id == VK_VENDOR_ID_INTEL || ctx->device->vendor_id == VK_VENDOR_ID_NVIDIA || ctx->device->vendor_id == VK_VENDOR_ID_AMD) && ctx->device->subgroup_arithmetic && ctx->device->subgroup_require_full_support; + if (rms_subgroups || ne0 <= 256 || ne0 > 4096) { + return 0; + } + } + + // Preserve specialized downstream RMS_NORM fusions (rope, set_rows, add, silu_gate, scale) + if (!fits) { + if (gen_can_fuse_rms_mul_silu(ctx, cgraph, rms_idx) > 0 || + gen_find_rms_norm_scale(ctx, cgraph, rms_idx, last_node, use_idx) >= 0) { + return 0; + } + } + if ((rms_idx + 2 < cgraph->n_nodes && + cgraph->nodes[rms_idx + 1]->op == GGML_OP_MUL && + cgraph->nodes[rms_idx + 2]->op == GGML_OP_ROPE) || + (rms_idx + 2 < cgraph->n_nodes && + cgraph->nodes[rms_idx + 1]->op == GGML_OP_VIEW && + cgraph->nodes[rms_idx + 2]->op == GGML_OP_SET_ROWS) || + ggml_vk_can_fuse(ctx, cgraph, rms_idx, rms_norm_mul_add_pattern)) { + return 0; + } + + int mul_idx = -1; + const ggml_tensor * weight = nullptr; + const ggml_tensor * final_dst = rms; + int final_idx = rms_idx; + + if (rms_idx + 1 <= last_node && (gen_skip.empty() || !gen_skip[rms_idx + 1]) && + !(rms->flags & GGML_TENSOR_FLAG_OUTPUT)) { + const ggml_tensor * m = cgraph->nodes[rms_idx + 1]; + if (m->op == GGML_OP_MUL && (m->src[0] == rms || m->src[1] == rms)) { + const ggml_tensor * w = (m->src[0] == rms) ? m->src[1] : m->src[0]; + if (m->type == GGML_TYPE_F32 && ggml_are_same_shape(m, rms) && ggml_is_contiguous(m) && + m->buffer && get_misalign_bytes(ctx, m) == 0 && + w && w != rms && w->type == GGML_TYPE_F32 && ggml_is_contiguous(w) && + w->ne[0] == ne0 && ggml_nrows(w) == 1 && w->buffer && get_misalign_bytes(ctx, w) == 0) { + bool weight_ready = (w->op == GGML_OP_NONE); + if (!weight_ready) { + const int32_t wfirst = use_idx.first_node_idx(w); + weight_ready = (wfirst >= 0 && wfirst < i); + } + const bool rms_only_here = weight_ready && !use_idx.any_reader(rms, [&](int k) { + return k != rms_idx + 1 && !ggml_op_is_empty(cgraph->nodes[k]->op); + }); + if (rms_only_here) { + mul_idx = rms_idx + 1; + weight = w; + final_dst = m; + final_idx = mul_idx; + } + } + } + } + + const bool write_add = ((last_add->flags & GGML_TENSOR_FLAG_OUTPUT) != 0) || + use_idx.any_reader(last_add, [&](int k) { + return (k < i || k > add_end_idx) && k != rms_idx && !ggml_op_is_empty(cgraph->nodes[k]->op); + }); + if (write_add && (!ggml_is_contiguous(last_add) || !last_add->buffer || get_misalign_bytes(ctx, last_add) != 0)) { + return 0; + } + + const ggml_tensor * srcs[10] {}; + srcs[0] = first_add->src[0]; + srcs[1] = first_add->src[1]; + for (int32_t k = 0; k < num_adds - 1; ++k) { + const ggml_tensor * prev = cgraph->nodes[i + k]; + const ggml_tensor * next = cgraph->nodes[i + k + 1]; + srcs[k + 2] = (next->src[0] == prev) ? next->src[1] : next->src[0]; + } + + if (write_add && ggml_vk_tensors_overlap(last_add, final_dst, false)) { + return 0; + } + if (weight && (ggml_vk_tensors_overlap(weight, final_dst, false) || + (write_add && ggml_vk_tensors_overlap(weight, last_add, false)))) { + return 0; + } + for (int32_t s = 0; s <= num_adds; ++s) { + if (ggml_vk_tensors_overlap(srcs[s], final_dst, true) || + (write_add && ggml_vk_tensors_overlap(srcs[s], last_add, true))) { + return 0; + } + } + + // Intervening ops in (add_end_idx .. final_idx) run AFTER our fused dispatch at i + for (int k = add_end_idx + 1; k < final_idx; ++k) { + if (k == rms_idx || k == mul_idx || (!gen_skip.empty() && gen_skip[k])) { + continue; + } + const ggml_tensor * mid = cgraph->nodes[k]; + if (ggml_op_is_empty(mid->op)) { + continue; + } + if (mid->buffer) { + if (ggml_vk_tensors_overlap(mid, final_dst, false) || + (write_add && ggml_vk_tensors_overlap(mid, last_add, false)) || + (weight && ggml_vk_tensors_overlap(mid, weight, false))) { + return 0; + } + } + for (int s = 0; s < GGML_MAX_SRC; ++s) { + const ggml_tensor * msrc = mid->src[s]; + if (msrc && msrc->buffer && ggml_vk_tensors_overlap(msrc, final_dst, false)) { + return 0; + } + } + } + + if (was_adjacent_cand) { + ctx->prealloc_size_add_rms_partials_offset += gen_batch_invariant() + ? gen_rms_partials_size_rows(ctx, first_add) + : ggml_vk_rms_partials_size(ctx, first_add); + ctx->gen_n1_partials_offset += ggml_vk_rms_partials_size(ctx, first_add); + } + + uint32_t eff_tree_mode = tree_mode; + if (eff_tree_mode == 0u && ggml_vk_is_apple_agx_coopmat(ctx->device) && !ggml_vk_exact_s8_enabled() && + !getenv("GGML_VK_DISABLE_S14_ADD_RMS") && (ne0 % 16 == 0) && + ctx->device->pipeline_multi_add_rms_mul[num_adds - 1][2][weight != nullptr ? 1 : 0] != nullptr) { + bool v4_ok = (get_misalign_bytes(ctx, final_dst) == 0) && + (!write_add || get_misalign_bytes(ctx, last_add) == 0) && + (!weight || get_misalign_bytes(ctx, weight) == 0); + for (int32_t s = 0; s <= num_adds && v4_ok; ++s) { + if ((srcs[s]->nb[1] % 16 != 0) || (get_misalign_bytes(ctx, srcs[s]) != 0)) { + v4_ok = false; + } + } + if (v4_ok) { + eff_tree_mode = 2u; + } + } + + for (int32_t s = 0; s <= num_adds; ++s) { + ctx->s10_add_rms_srcs[s] = srcs[s]; + } + ctx->s10_add_rms_add_dst = last_add; + ctx->s10_add_rms_weight = weight; + ctx->s10_add_rms_dst = final_dst; + ctx->s10_add_rms_num_adds = (uint32_t)num_adds; + ctx->s10_add_rms_tree_mode = eff_tree_mode; + ctx->s10_add_rms_write_add = write_add ? 1u : 0u; + memcpy(&ctx->s10_add_rms_eps, rms->op_params, sizeof(float)); + + gen_skip[rms_idx] = 1; + if (mul_idx >= 0) { + gen_skip[mul_idx] = 1; + } + out_final_idx = final_idx; + return (uint32_t)num_adds; +} + +// gen (GGML_GEN_GDN_INPLACE): find the CPY that stores GATED_DELTA_NET node i's final/rollback state (K >= 1) and +// nothing else reads, so the GDN dispatch can write the state straight to the CPY destination. +static int gen_find_gdn_state_cpy(const ggml_backend_vk_context * ctx, const ggml_cgraph * g, int i, int last_node, const vk_cgraph_use_index & use_idx) { + const ggml_tensor * gdn = g->nodes[i]; + const int32_t K = ggml_get_op_params_i32(gdn, 0); + if (K < 1 || gdn->type != GGML_TYPE_F32 || !gdn->buffer || (gdn->flags & GGML_TENSOR_FLAG_OUTPUT)) { + return -1; + } + const ggml_tensor * v = gdn->src[2]; + const size_t S_v = v->ne[0], H = v->ne[1], n_tokens = v->ne[2], n_seqs = v->ne[3]; + const size_t D = S_v * S_v * H; + const size_t n_written = std::min(n_tokens, (size_t) K); + if (n_written == 0) { + return -1; + } + const size_t s_off_bytes = S_v * H * n_tokens * n_seqs * sizeof(float); + const size_t snap_bytes = D * n_seqs * sizeof(float); + const size_t state_bytes = n_written * snap_bytes; + int cpy = -1; + for (int j = i + 1; j < std::min(last_node, i + 32); j++) { + const ggml_tensor * n = g->nodes[j]; + if (n->op != GGML_OP_CPY || !n->src[0] || n->src[0]->view_src != gdn || n->src[0]->view_offs != s_off_bytes || + n->src[0]->type != GGML_TYPE_F32 || n->type != GGML_TYPE_F32 || + ggml_nbytes(n->src[0]) != state_bytes || !ggml_is_contiguous(n->src[0]) || + !n->buffer || get_misalign_bytes(ctx, n) != 0) { + continue; + } + const bool dst_layout_ok = (K == 1) + ? (ggml_nbytes(n) == state_bytes && ggml_is_contiguous(n)) + : (n->ne[0] == (int64_t) D && n->ne[1] == (int64_t) n_seqs && + n->ne[2] == (int64_t) n_written && n->ne[3] == 1 && + n->nb[0] == sizeof(float) && + (n_seqs == 1 || n->nb[1] == D * sizeof(float)) && + (n_written == 1 || (n->nb[2] % sizeof(float)) == 0)); + if (dst_layout_ok) { + cpy = j; + break; + } + } + if (cpy < 0) { + return -1; + } + // nothing else may read the state part of the GDN result + if (use_idx.any_reader(gdn, [&](int k) { + const ggml_tensor * n = g->nodes[k]; + if (k == cpy || ggml_op_is_empty(n->op)) { + return false; + } + for (int s = 0; s < GGML_MAX_SRC; s++) { + const ggml_tensor * src = n->src[s]; + if (src == nullptr) { + continue; + } + if (src == gdn && k != i) { + return true; + } + if (src->view_src == gdn && src->view_offs + ggml_nbytes(src) > s_off_bytes) { + return true; + } + } + return false; + })) { + return -1; + } + // the destination must not overlap q, k, v, g, beta or intervening ops; it may be the K==1 state input in place or disjoint + const ggml_tensor * dst = g->nodes[cpy]; + for (int m_idx = i + 1; m_idx < cpy; ++m_idx) { + const ggml_tensor * mid = g->nodes[m_idx]; + if (ggml_op_is_empty(mid->op)) { + continue; + } + if (mid->buffer && ggml_vk_tensors_overlap_strided(mid, dst)) { + return -1; + } + for (int s = 0; s < GGML_MAX_SRC; ++s) { + if (mid->src[s] && mid->src[s]->buffer && ggml_vk_tensors_overlap_strided(mid->src[s], dst)) { + return -1; + } + } + } + for (int s = 0; s < 5; s++) { + if (gdn->src[s] && gdn->src[s]->buffer && ggml_vk_tensors_overlap_strided(gdn->src[s], dst)) { + return -1; + } + } + const ggml_tensor * st = gdn->src[5]; + if (st->buffer && ggml_vk_tensors_overlap_strided(st, dst)) { + const bool same = K == 1 && st->buffer == dst->buffer && + vk_tensor_offset(st) + st->view_offs == vk_tensor_offset(dst) + dst->view_offs && + ggml_nbytes(st) == snap_bytes && ggml_is_contiguous(st) && ggml_is_contiguous(dst); + if (!same) { + return -1; + } + } + return cpy; +} + +static bool ggml_vk_build_replay_sig( + const ggml_cgraph * cgraph, + std::vector & out_words, + uint64_t & out_h1, + uint64_t & out_h2) { + if (cgraph->n_nodes <= 0) { + return false; + } + struct seen_table_t { + std::vector ptrs; + std::vector epochs; + std::vector ids; + uint32_t epoch = 1; + }; + static thread_local seen_table_t tbl; + size_t cap = 4096; + const size_t min_cap = (size_t) (cgraph->n_nodes + 256) * 8; + while (cap < min_cap) { + cap <<= 1; + } + if (tbl.ptrs.size() < cap) { + tbl.ptrs.assign(cap, nullptr); + tbl.epochs.assign(cap, 0); + tbl.ids.assign(cap, 0); + tbl.epoch = 1; + } else if (++tbl.epoch == 0) { + std::fill(tbl.epochs.begin(), tbl.epochs.end(), 0); + tbl.epoch = 1; + } + const size_t mask = tbl.ptrs.size() - 1; + const uint32_t ep = tbl.epoch; + + out_words.clear(); + out_words.reserve((size_t) cgraph->n_nodes * 18); + out_words.push_back((uint64_t) cgraph->n_nodes); + + uint32_t next_id = 0; + auto intern_tensor = [&](auto & self, const ggml_tensor * t, uint32_t & out_id) -> bool { + if (t == nullptr) { + out_id = 0; + return true; + } + uintptr_t p = (uintptr_t) t; + size_t slot = ((p >> 5) * 11400714819323198485ULL) & mask; + while (tbl.epochs[slot] == ep) { + if (tbl.ptrs[slot] == t) { + out_id = tbl.ids[slot]; + return true; + } + slot = (slot + 1) & mask; + } + uint32_t vs_id = 0; + if (t->view_src != nullptr) { + if (!self(self, t->view_src, vs_id)) { + return false; + } + } + ggml_backend_buffer_t b = t->buffer ? t->buffer : (t->view_src ? t->view_src->buffer : nullptr); + uint64_t buf_handle = 0; + uint64_t byte_off = (uint64_t) t->view_offs; + if (b != nullptr) { + if (!ggml_backend_buffer_is_vk(b)) { + return false; + } + vk_buffer db = ((ggml_backend_vk_buffer_context *) b->context)->dev_buffer; + if (!db) { + return false; + } + buf_handle = (uint64_t) (VkBuffer) db->buffer; + byte_off = vk_tensor_offset(t) + t->view_offs; + } else if (t->data != nullptr && !ggml_op_is_empty(t->op)) { + return false; + } + + const uint32_t id = ++next_id; + // Re-probe in case recursive view_src inserted into tbl + slot = ((p >> 5) * 11400714819323198485ULL) & mask; + while (tbl.epochs[slot] == ep) { + slot = (slot + 1) & mask; + } + tbl.ptrs[slot] = t; + tbl.epochs[slot] = ep; + tbl.ids[slot] = id; + out_id = id; + + out_words.push_back(((uint64_t) id << 32) | 0x54454e53ULL); + out_words.push_back((uint64_t) t->op | ((uint64_t) t->type << 16) | ((uint64_t) (uint32_t) t->flags << 32)); + out_words.push_back((uint64_t) t->ne[0]); + out_words.push_back((uint64_t) t->ne[1]); + out_words.push_back((uint64_t) t->ne[2]); + out_words.push_back((uint64_t) t->ne[3]); + out_words.push_back((uint64_t) t->nb[0]); + out_words.push_back((uint64_t) t->nb[1]); + out_words.push_back((uint64_t) t->nb[2]); + out_words.push_back((uint64_t) t->nb[3]); + out_words.push_back(buf_handle); + out_words.push_back(byte_off); + out_words.push_back((uint64_t) vs_id); + if (t->op != GGML_OP_NONE) { + uint64_t opw[8]; + static_assert(sizeof(t->op_params) == sizeof(opw), "op_params size"); + memcpy(opw, t->op_params, sizeof(opw)); + for (int k = 0; k < 8; ++k) { + out_words.push_back(opw[k]); + } + } + return true; + }; + + for (int i = 0; i < cgraph->n_nodes; ++i) { + const ggml_tensor * n = cgraph->nodes[i]; + uint32_t nid = 0; + if (!intern_tensor(intern_tensor, n, nid)) { + return false; + } + uint32_t sids[GGML_MAX_SRC]; + for (int s = 0; s < GGML_MAX_SRC; ++s) { + if (!intern_tensor(intern_tensor, n->src[s], sids[s])) { + return false; + } + } + out_words.push_back(nid); + for (int s = 0; s < GGML_MAX_SRC; s += 2) { + const uint64_t lo = sids[s]; + const uint64_t hi = (s + 1 < GGML_MAX_SRC) ? sids[s + 1] : 0u; + out_words.push_back(lo | (hi << 32)); + } + } + + uint64_t h1 = 0x9e3779b97f4a7c15ULL ^ out_words.size(); + uint64_t h2 = 0xbf58476d1ce4e5b9ULL ^ (out_words.size() * 31ULL); + for (uint64_t w : out_words) { + h1 ^= w + 0x9e3779b97f4a7c15ULL + (h1 << 6) + (h1 >> 2); + h2 = (h2 ^ w) * 0x94d049bb133111ebULL; + } + out_h1 = h1; + out_h2 = h2; + return true; +} + +static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) { + VK_LOG_DEBUG("ggml_backend_vk_graph_compute(" << cgraph->n_nodes << " nodes)"); + ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context; + + if (!ctx->uma_out_memcpys.empty()) { + ggml_vk_synchronize(ctx); + } + + ctx->device->diag_cgraph = nullptr; + ctx->device->diag_prev_start = -1; + ctx->device->diag_prev_end = -1; + + if (vk_instance.debug_utils_support) { + ctx->device->debug_cmdbuf_idx = 0; + } + + // queue scope, so it encloses every submit this evaluation makes. + // closed when the function returns + ggml_vk_debug_label queue_dbg(ctx->device->compute_queue->handle.get(), "ggml_backend_vk_graph_compute"); + + ctx->prealloc_size_add_rms_partials_offset = 0; + ctx->gen_n1_partials_offset = 0; + ctx->do_add_rms_partials = false; + ctx->do_add_rms_partials_offset_calculation = false; + + // gen instrumentation (env-gated, off by default): + // GGML_VK_GRAPH_STATS=1 one line per graph: nodes, dispatches, syncs, submits, CPU recording time + // GGML_VK_FLOPS_PER_SUBMIT=N override the flops-based submit threshold (0 disables it) + // GGML_VK_DUP_OPS=A,B|ALL record matching non-in-place nodes twice (timing ablation; results unchanged) + struct gen_env_t { + bool stats = false; + int64_t flops_per_submit = -1; + bool dup_all = false; + std::vector dup; + gen_env_t() { + const char * s = getenv("GGML_VK_GRAPH_STATS"); + stats = s != nullptr && atoi(s) != 0; + const char * f = getenv("GGML_VK_FLOPS_PER_SUBMIT"); + flops_per_submit = f != nullptr ? std::stoll(f) : -1; + const char * d = getenv("GGML_VK_DUP_OPS"); + if (d != nullptr) { + std::string list(d); + size_t pos = 0; + while (pos <= list.size()) { + size_t comma = list.find(',', pos); + if (comma == std::string::npos) { + comma = list.size(); + } + std::string item = list.substr(pos, comma - pos); + if (item == "ALL") { + dup_all = true; + } else if (!item.empty()) { + dup.push_back(item); + } + pos = comma + 1; + } + } + } + }; + static const gen_env_t gen_env; + const int64_t gen_t_start = gen_env.stats ? ggml_time_us() : 0; + ctx->stat_dispatches = 0; + ctx->stat_syncs = 0; + uint32_t gen_dups = 0; + auto const gen_dup_match = [&](const ggml_tensor * n) -> bool { + if (n->op == GGML_OP_ADD || n->op == GGML_OP_NONE || ggml_vk_is_empty(const_cast(n))) { + return false; // ADD may carry rms partials state; views record nothing + } + for (uint32_t s = 0; s < GGML_MAX_SRC; ++s) { + if (n->src[s] && n->src[s]->buffer && n->buffer && ggml_vk_tensors_overlap(n->src[s], n, false)) { + return false; // in-place: a second run would not be idempotent + } + } + if (gen_env.dup_all) { + return true; + } + const char * name = ggml_op_desc(n); + for (const std::string & d : gen_env.dup) { + if (d == name) { + return true; + } + } + return false; + }; + + int last_node = cgraph->n_nodes - 1; + + // If the last op in the cgraph isn't backend GPU, the command buffer doesn't get closed properly + while (last_node > 0 && (ggml_vk_is_empty(cgraph->nodes[last_node]) || ((cgraph->nodes[last_node]->flags & GGML_TENSOR_FLAG_COMPUTE) == 0))) { + last_node -= 1; + } + + // Reserve tensor context space for all nodes + ctx->tensor_ctxs.resize(cgraph->n_nodes); + + bool first_node_in_batch = true; // true if next node will be first node in a batch + int submit_node_idx = 0; // index to first node in a batch + + ggml_vk_submit_transfer_ctx(ctx); vk_context compute_ctx; if (vk_perf_logger_enabled) { @@ -15084,9 +19647,191 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg ggml_vk_sync_buffers(ctx, compute_ctx); } - ctx->prealloc_y_last_pipeline_used = nullptr; - ctx->prealloc_y_last_tensor_used = nullptr; - ctx->prealloc_y_last_k_padded = false; + ggml_vk_invalidate_prealloc_y_ring(ctx); + ctx->prealloc_y_ring_idx = 0; + ctx->prealloc_split_k_ring_idx = 0; + ctx->last_expert_count_valid = false; + ctx->last_expert_ids_tensor = nullptr; + ctx->last_mask_opt_valid = false; + ctx->last_mask_opt_tensor = nullptr; + + // Series 9 (S9.1): Single-Front reusable command buffer LRU cache (opt-in via GGML_VK_SINGLE_FRONT=1; geo FIRST=32 overlaps CPU+GPU better) + static const int s9_single_front_env = getenv("GGML_VK_SINGLE_FRONT") ? atoi(getenv("GGML_VK_SINGLE_FRONT")) : 0; + const bool single_front_enabled = (s9_single_front_env > 0); + vk_graph_replay_entry * rec_entry = nullptr; + auto pre_buf_h = [](const vk_buffer & b) -> uint64_t { return b ? (uint64_t) (VkBuffer) b->buffer : 0ULL; }; + auto pre_buf_sz = [](const vk_buffer & b) -> size_t { return b ? b->size : 0; }; + + if (single_front_enabled && !vk_perf_logger_enabled && !gen_env.dup_all && gen_env.dup.empty() && + !ctx->device->serialize_submissions && !vk_instance.debug_utils_support && + ctx->compute_ctx.expired()) { + const bool throttle_sig = (ctx->replay_miss_n_nodes == cgraph->n_nodes && + ctx->replay_consec_misses >= 16 && + (ctx->replay_consec_misses & 31u) != 0); + if (throttle_sig) { + ++ctx->replay_consec_misses; + } else { + static thread_local std::vector cur_sig; + uint64_t h1 = 0, h2 = 0; + if (ggml_vk_build_replay_sig(cgraph, cur_sig, h1, h2)) { + const uint64_t cur_destroy = ctx->device->buffer_destroy_count.load(std::memory_order_acquire); + if ((ctx->prealloc_size_add_rms_partials && + (ctx->prealloc_add_rms_partials == nullptr || ctx->prealloc_add_rms_partials->size < ctx->prealloc_size_add_rms_partials)) || + (ctx->prealloc_size_mask_opt && + (ctx->prealloc_mask_opt == nullptr || ctx->prealloc_mask_opt->size < ctx->prealloc_size_mask_opt))) { + ggml_vk_preallocate_buffers(ctx, nullptr); + } + const uint64_t cur_xh = pre_buf_h(ctx->prealloc_x); + const uint64_t cur_yh = pre_buf_h(ctx->prealloc_y); + const uint64_t cur_skh = pre_buf_h(ctx->prealloc_split_k); + const uint64_t cur_ah = pre_buf_h(ctx->prealloc_add_rms_partials); + const uint64_t cur_moh = pre_buf_h(ctx->prealloc_mask_opt); + const size_t cur_xsz = pre_buf_sz(ctx->prealloc_x); + const size_t cur_ysz = pre_buf_sz(ctx->prealloc_y); + const size_t cur_sks = pre_buf_sz(ctx->prealloc_split_k); + const size_t cur_asz = pre_buf_sz(ctx->prealloc_add_rms_partials); + const size_t cur_mos = pre_buf_sz(ctx->prealloc_mask_opt); + + vk_graph_replay_entry * hit = nullptr; + for (auto & e : ctx->replay_cache) { + if (e.hash1 == h1 && e.hash2 == h2 && e.sig_words == cur_sig) { + hit = &e; + break; + } + } + if (hit != nullptr && hit->ready && + hit->destroy_count == cur_destroy && + hit->pre_x_handle == cur_xh && hit->pre_x_size == cur_xsz && + hit->pre_y_handle == cur_yh && hit->pre_y_size == cur_ysz && + hit->pre_sk_handle == cur_skh && hit->pre_sk_size == cur_sks && + hit->pre_arp_handle == cur_ah && hit->pre_arp_size == cur_asz && + hit->pre_mo_handle == cur_moh && hit->pre_mo_size == cur_mos) { + ctx->replay_consec_misses = 0; + hit->lru_tick = ++ctx->replay_lru_clock; + ggml_vk_ensure_fence_free(ctx); + for (const auto & m : hit->memsets) { + memset(m.dst, m.val, m.n); + } + if (ctx->device->async_use_transfer_queue && ctx->transfer_semaphore_last_submitted < ctx->transfer_semaphore.value) { + vk::TimelineSemaphoreSubmitInfo tl_info{ 1, &ctx->transfer_semaphore.value, 0, nullptr }; + vk::PipelineStageFlags stage = ctx->device->transfer_queue->stage_flags; + vk::SubmitInfo si{ 1, &ctx->transfer_semaphore.s, &stage, 1, &hit->cmd_buf, 0, nullptr }; + si.setPNext(&tl_info); + ctx->device->compute_queue->handle->submit({ si }, ctx->fence); + ctx->transfer_semaphore_last_submitted = ctx->transfer_semaphore.value; + } else { + vk::SubmitInfo si{ 0, nullptr, nullptr, 1, &hit->cmd_buf, 0, nullptr }; + ctx->device->compute_queue->handle->submit({ si }, ctx->fence); + } + ctx->fence_submitted = true; + ctx->submit_pending = true; + ctx->active_replay_entry = hit; + ctx->replay_submit_us = ggml_time_us(); + ctx->last_total_flops = hit->total_flops; + if (gen_env.stats) { + fprintf(stderr, "ggml_vk_graph_stats: nodes=%d dispatches=%llu syncs=%llu submits=1 dups=0 replay=1 cpu_record_us=%lld\n", + cgraph->n_nodes, (unsigned long long) hit->stat_dispatches, (unsigned long long) hit->stat_syncs, + (long long) (ggml_time_us() - gen_t_start)); + } + if (!ctx->device->support_async) { + ggml_vk_synchronize(ctx); + } + return GGML_STATUS_SUCCESS; + } + + bool start_record = false; + if (hit != nullptr) { + ctx->replay_consec_misses = 0; + if (!hit->ready && hit->stable_runs >= 1 && hit->destroy_count == cur_destroy) { + start_record = true; + } else { + ggml_vk_synchronize(ctx); + if (hit->cmd_pool.pool) { + ggml_vk_command_pool_cleanup(ctx->device, hit->cmd_pool); + } + hit->ready = false; + hit->destroy_count = cur_destroy; + hit->stable_runs = 1; + hit->lru_tick = ++ctx->replay_lru_clock; + } + } else { + bool in_probation = false; + for (uint32_t p = 0; p < 8; ++p) { + if (ctx->replay_probation_h1[p] == h1 && + ctx->replay_probation_h2[p] == h2) { + in_probation = true; + break; + } + } + if (!in_probation) { + const uint32_t slot = ctx->replay_probation_clock++ & 7u; + ctx->replay_probation_h1[slot] = h1; + ctx->replay_probation_h2[slot] = h2; + ctx->replay_probation_destroy[slot] = cur_destroy; + if (ctx->replay_miss_n_nodes == cgraph->n_nodes) { + ++ctx->replay_consec_misses; + } else { + ctx->replay_miss_n_nodes = cgraph->n_nodes; + ctx->replay_consec_misses = 1; + } + } else { + ctx->replay_consec_misses = 0; + if (ctx->replay_cache.size() >= VK_REPLAY_CACHE_MAX) { + ggml_vk_synchronize(ctx); + size_t victim = 0; + for (size_t k = 1; k < ctx->replay_cache.size(); ++k) { + if (ctx->replay_cache[k].lru_tick < ctx->replay_cache[victim].lru_tick) { + victim = k; + } + } + ggml_vk_destroy_replay_entry(ctx->device, ctx->replay_cache[victim]); + hit = &ctx->replay_cache[victim]; + } else { + ctx->replay_cache.emplace_back(); + hit = &ctx->replay_cache.back(); + } + hit->hash1 = h1; + hit->hash2 = h2; + hit->sig_words = cur_sig; + hit->destroy_count = cur_destroy; + hit->stable_runs = 1; + hit->ready = false; + start_record = true; + } + } + + if (start_record && hit != nullptr) { + ggml_vk_ensure_fence_free(ctx); + if (!hit->cmd_pool.pool) { + hit->cmd_pool.cmd_buffers.clear(); + hit->cmd_pool.q = ctx->device->compute_queue.get(); + hit->cmd_pool.pool = ctx->device->device.createCommandPool({ + vk::CommandPoolCreateFlags(VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT), + ctx->device->compute_queue->queue_family_index + }); + } else { + ggml_vk_command_pool_cleanup(ctx->device, hit->cmd_pool); + } + hit->lru_tick = ++ctx->replay_lru_clock; + hit->pre_x_handle = cur_xh; hit->pre_x_size = cur_xsz; + hit->pre_y_handle = cur_yh; hit->pre_y_size = cur_ysz; + hit->pre_sk_handle = cur_skh; hit->pre_sk_size = cur_sks; + hit->pre_arp_handle = cur_ah; hit->pre_arp_size = cur_asz; + hit->pre_mo_handle = cur_moh; hit->pre_mo_size = cur_mos; + rec_entry = hit; + ctx->recording_replay = true; + ctx->replay_abort = false; + ctx->active_replay_cmd_pool = &hit->cmd_pool; + std::swap(ctx->descriptor_pools, hit->descriptor_pools); + std::swap(ctx->descriptor_sets, hit->descriptor_sets); + std::swap(ctx->descriptor_set_bindings, hit->descriptor_set_bindings); + std::swap(ctx->descriptor_set_bindings_destroy_count, hit->descriptor_destroy_count); + ctx->descriptor_set_idx = 0; + ctx->pipeline_descriptor_set_requirements = 0; + } + } + } + } if (ctx->prealloc_size_add_rms_partials) { ggml_vk_preallocate_buffers(ctx, nullptr); @@ -15117,6 +19862,44 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg // small graphs gain nothing from splitting, each submit costs CPU time uint64_t flops_per_submit = ctx->last_total_flops < 1'000'000'000ULL ? 0 : std::clamp(ctx->last_total_flops / 40u, std::min(100'000'000ULL, flops_cap), flops_cap); + if (gen_env.flops_per_submit >= 0) { + flops_per_submit = (uint64_t) gen_env.flops_per_submit; + } + + // gen: submit policy (GGML_VK_SUBMIT_MODE): + // default upstream: every max_nodes_per_submit nodes and on the flops threshold + // one a single submit per graph (no flops/node/almost-ready submits) + // geo first submit after GGML_VK_SUBMIT_FIRST nodes (default 32), each later batch + // GGML_VK_SUBMIT_GROWTH times larger (default 4); no flops-based submits + static const int gen_submit_mode = [] { + const char * m = getenv("GGML_VK_SUBMIT_MODE"); + if (m == nullptr) return -1; + if (strcmp(m, "default") == 0) return 0; + if (strcmp(m, "one") == 0) return 1; + if (strcmp(m, "geo") == 0) return 2; + GGML_ABORT("GGML_VK_SUBMIT_MODE must be default, one or geo"); + }(); + const int eff_submit_mode = gen_submit_mode >= 0 + ? gen_submit_mode + : (ggml_vk_is_apple_agx_coopmat(ctx->device) ? 2 : 0); + static const uint32_t gen_submit_first = getenv("GGML_VK_SUBMIT_FIRST") ? (uint32_t) std::max(1, atoi(getenv("GGML_VK_SUBMIT_FIRST"))) : 96u; + static const uint32_t gen_submit_growth = getenv("GGML_VK_SUBMIT_GROWTH") ? (uint32_t) std::max(1, atoi(getenv("GGML_VK_SUBMIT_GROWTH"))) : 4096u; + uint32_t gen_max_nodes = ctx->device->max_nodes_per_submit; + if (eff_submit_mode == 1 || ctx->recording_replay) { + flops_per_submit = 0; + gen_max_nodes = UINT32_MAX; + } else if (eff_submit_mode == 2) { + bool has_mm_id = false; + for (int ni = 0; ni < cgraph->n_nodes; ++ni) { + if (cgraph->nodes[ni] && cgraph->nodes[ni]->op == GGML_OP_MUL_MAT_ID) { + has_mm_id = true; + break; + } + } + static const uint64_t env_submit_flops = getenv("GGML_VK_SUBMIT_FLOPS") ? strtoull(getenv("GGML_VK_SUBMIT_FLOPS"), nullptr, 10) : 25000000000000ULL; + flops_per_submit = (has_mm_id && !getenv("GGML_VK_SUBMIT_FLOPS")) ? 0 : env_submit_flops; + gen_max_nodes = gen_submit_first; + } auto const submit_after = [&](int start, int end) { if (ctx->device->serialize_submissions) { @@ -15142,13 +19925,425 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg first_node_in_batch = true; submitted_nodes = 0; batch_flops = 0; - if (submit_count < 3) { + if (eff_submit_mode != 2 && submit_count < 3) { flops_per_submit *= 2; } + if (eff_submit_mode == 2) { + gen_max_nodes = gen_max_nodes > UINT32_MAX / gen_submit_growth ? UINT32_MAX : gen_max_nodes * gen_submit_growth; + } submit_count++; }; + static const bool gen_gdn_inplace = getenv("GGML_GEN_GDN_INPLACE") ? (atoi(getenv("GGML_GEN_GDN_INPLACE")) != 0) : true; + static const int s10_max_sliding_nt = getenv("GGML_VK_SSM_SLIDING_MAX_NT") ? atoi(getenv("GGML_VK_SSM_SLIDING_MAX_NT")) : (ggml_vk_exact_s8_enabled() ? 32 : 65536); + struct s10_rb_info { + bool active = false; + int num_extra = 0; + int sr_idx = -1; + const ggml_tensor * conv = nullptr; + const ggml_tensor * state_out = nullptr; + const ggml_tensor * old_st = nullptr; + const ggml_tensor * qkv = nullptr; + uint32_t st_k = 0; + uint32_t st_slot_nb = 0; + }; + struct s10_pair_rms_info { + int rms0_idx = -1; + int sc0_idx = -1; + int rms1_idx = -1; + }; + std::vector s10_rb; + std::vector s10_pair_rms; + std::vector s12_zero_sc; + std::vector gen_deferred_rms; + std::vector gen_deferred_mm; + std::vector gen_skip; + static thread_local vk_cgraph_use_index use_idx; + if (!ctx->device->disable_fusion || gen_gdn_inplace || gen_fuse_enabled("mmgate") || gen_fuse_enabled("shexp")) { + gen_skip.assign(cgraph->n_nodes, 0); + use_idx.build(cgraph); + + if (!ctx->device->disable_fusion && !s10_ssm_conv_disabled()) { + for (int conv_idx = 1; conv_idx < cgraph->n_nodes; ++conv_idx) { + const ggml_tensor * conv = cgraph->nodes[conv_idx]; + if (conv->op != GGML_OP_SSM_CONV || !conv->src[0] || conv->src[0]->op != GGML_OP_CONCAT || gen_skip[conv_idx]) { + continue; + } + const ggml_tensor * concat = conv->src[0]; + int concat_idx = -1; + for (int k = conv_idx - 1; k >= std::max(0, conv_idx - 24); --k) { + if (cgraph->nodes[k] == concat) { + concat_idx = k; + break; + } + } + if (concat_idx < 0 || concat_idx >= last_node || gen_skip[concat_idx]) { + continue; + } + int sr_idx = -1; + for (int k = concat_idx + 1; k < conv_idx; ++k) { + const ggml_tensor * n = cgraph->nodes[k]; + if (n->op == GGML_OP_SET_ROWS && n->src[0] && n->src[0]->op == GGML_OP_VIEW && n->src[0]->view_src == concat) { + sr_idx = k; + break; + } + } + if (sr_idx < 0 || sr_idx >= last_node || gen_skip[sr_idx]) { + continue; + } + if (ggml_get_op_params_i32(concat, 0) != 0 || + concat->type != GGML_TYPE_F32 || conv->type != GGML_TYPE_F32 || + !ggml_is_contiguous(concat) || !ggml_is_contiguous(conv) || + !conv->buffer || get_misalign_bytes(ctx, conv) != 0) { + continue; + } + const ggml_tensor * c_w = conv->src[1]; + if (!c_w || c_w->type != GGML_TYPE_F32 || !ggml_is_contiguous_rows(c_w) || + !c_w->buffer || get_misalign_bytes(ctx, c_w) != 0) { + continue; + } + const int64_t nc = c_w->ne[0]; + const int64_t nr = c_w->ne[1]; + const int64_t n_t = conv->ne[1]; + const int64_t n_s = conv->ne[2]; + if (nc != 4 || conv->ne[0] != nr || conv->ne[3] != 1 || n_t < 1 || n_t > s10_max_sliding_nt || n_s < 1 || + concat->ne[0] != (nc - 1) + n_t || concat->ne[1] != nr || concat->ne[2] != n_s || concat->ne[3] != 1) { + continue; + } + const ggml_tensor * old_st = concat->src[0]; + const ggml_tensor * qkv = concat->src[1]; + if (!old_st || !qkv || old_st->type != GGML_TYPE_F32 || qkv->type != GGML_TYPE_F32 || + !old_st->buffer || !qkv->buffer || + get_misalign_bytes(ctx, old_st) != 0 || get_misalign_bytes(ctx, qkv) != 0 || + old_st->ne[0] != nc - 1 || old_st->ne[1] != nr || old_st->ne[2] != n_s || old_st->ne[3] != 1 || + old_st->nb[0] != sizeof(float) || (old_st->nb[1] % sizeof(float)) != 0 || (old_st->nb[2] % sizeof(float)) != 0 || + qkv->ne[0] != n_t || qkv->ne[1] != nr || qkv->ne[2] != n_s || qkv->ne[3] != 1 || + (qkv->nb[0] % sizeof(float)) != 0 || (qkv->nb[1] % sizeof(float)) != 0 || (qkv->nb[2] % sizeof(float)) != 0) { + continue; + } + const ggml_tensor * sr = cgraph->nodes[sr_idx]; + const int64_t K = sr->ne[1]; + const int64_t Kw = std::min(n_t, K); + if (K < 1 || sr->type != GGML_TYPE_F32 || !sr->buffer || get_misalign_bytes(ctx, sr) != 0 || + sr->ne[0] != nc - 1 || sr->ne[2] != nr || sr->ne[3] != n_s || + sr->nb[0] != sizeof(float) || (sr->nb[1] % sizeof(float)) != 0 || + (sr->nb[2] % sizeof(float)) != 0 || (sr->nb[3] % sizeof(float)) != 0) { + continue; + } + const ggml_tensor * windows = sr->src[0]; + const ggml_tensor * slot_order = sr->src[1]; + if (!windows || windows->op != GGML_OP_VIEW || windows->view_src != concat || windows->type != GGML_TYPE_F32 || + windows->ne[0] != nc - 1 || windows->ne[1] != Kw || windows->ne[2] != nr || windows->ne[3] != n_s || + windows->nb[0] != sizeof(float) || windows->nb[1] != sizeof(float) || + windows->nb[2] != concat->nb[1] || windows->nb[3] != concat->nb[2] || + windows->view_offs != (size_t)(n_t - (Kw - 1)) * sizeof(float) || + !slot_order || slot_order->op != GGML_OP_VIEW || slot_order->type != GGML_TYPE_I32 || + slot_order->ne[0] != Kw || ggml_nrows(slot_order) != 1 || slot_order->nb[0] != sizeof(int32_t) || + slot_order->view_offs != (size_t)(K - Kw) * sizeof(int32_t)) { + continue; + } + const ggml_tensor * cast_i32 = slot_order->view_src; + const ggml_tensor * sc = cast_i32 ? cast_i32->src[0] : nullptr; + const ggml_tensor * ar = sc ? sc->src[0] : nullptr; + if (!cast_i32 || cast_i32->op != GGML_OP_CPY || cast_i32->type != GGML_TYPE_I32 || cast_i32->src[1] != cast_i32 || + cast_i32->ne[0] != K || ggml_nrows(cast_i32) != 1 || !ggml_is_contiguous(cast_i32) || + !sc || sc->op != GGML_OP_SCALE || sc->type != GGML_TYPE_F32 || + sc->ne[0] != K || ggml_nrows(sc) != 1 || + ggml_get_op_params_f32(sc, 0) != -1.0f || ggml_get_op_params_f32(sc, 1) != (float)(K - 1) || + !ar || ar->op != GGML_OP_ARANGE || ar->type != GGML_TYPE_F32 || + ar->ne[0] != K || ggml_nrows(ar) != 1 || + ggml_get_op_params_f32(ar, 0) != 0.0f || ggml_get_op_params_f32(ar, 1) != (float)K || ggml_get_op_params_f32(ar, 2) != 1.0f) { + continue; + } + if ((concat->flags & GGML_TENSOR_FLAG_OUTPUT) || (windows->flags & GGML_TENSOR_FLAG_OUTPUT) || (sr->flags & GGML_TENSOR_FLAG_OUTPUT)) { + continue; + } + if (use_idx.any_reader(concat, [&](int k) { return k != sr_idx && k != conv_idx && !ggml_op_is_empty(cgraph->nodes[k]->op); }) || + use_idx.any_reader(windows, [&](int k) { return k != sr_idx && !ggml_op_is_empty(cgraph->nodes[k]->op); }) || + use_idx.any_reader(sr, [&](int k) { return !ggml_op_is_empty(cgraph->nodes[k]->op); })) { + continue; + } + int end_idx = conv_idx; + const ggml_tensor * final_dst = conv; + if (conv_idx + 1 <= last_node && !gen_skip[conv_idx + 1] && !(conv->flags & GGML_TENSOR_FLAG_OUTPUT)) { + const ggml_tensor * u = cgraph->nodes[conv_idx + 1]; + if (u->op == GGML_OP_UNARY && ggml_get_unary_op(u) == GGML_UNARY_OP_SILU && + u->src[0] == conv && u->type == GGML_TYPE_F32 && + ggml_are_same_shape(u, conv) && ggml_are_same_stride(u, conv) && + u->buffer && get_misalign_bytes(ctx, u) == 0 && + !use_idx.any_reader(conv, [&](int k) { return k != conv_idx + 1 && !ggml_op_is_empty(cgraph->nodes[k]->op); })) { + end_idx = conv_idx + 1; + final_dst = u; + } + } + if (ggml_vk_tensors_overlap_strided(old_st, sr) || + ggml_vk_tensors_overlap_strided(qkv, sr) || + ggml_vk_tensors_overlap_strided(c_w, sr) || + ggml_vk_tensors_overlap_strided(sr, final_dst) || + ggml_vk_tensors_overlap_strided(old_st, final_dst) || + ggml_vk_tensors_overlap_strided(qkv, final_dst) || + ggml_vk_tensors_overlap_strided(c_w, final_dst)) { + continue; + } + int cast_idx = -1, sc_idx = -1, ar_idx = -1; + for (int k = std::max(0, concat_idx - 8); k < sr_idx; ++k) { + if (cgraph->nodes[k] == cast_i32) cast_idx = k; + else if (cgraph->nodes[k] == sc) sc_idx = k; + else if (cgraph->nodes[k] == ar) ar_idx = k; + } + bool mid_hazard = false; + for (int m_idx = concat_idx + 1; m_idx < conv_idx; ++m_idx) { + if (m_idx == sr_idx || m_idx == cast_idx || m_idx == sc_idx || m_idx == ar_idx || gen_skip[m_idx]) { + continue; + } + const ggml_tensor * mid = cgraph->nodes[m_idx]; + if (ggml_op_is_empty(mid->op)) { + continue; + } + if (mid->buffer && (ggml_vk_tensors_overlap_strided(mid, old_st) || + ggml_vk_tensors_overlap_strided(mid, qkv) || + ggml_vk_tensors_overlap_strided(mid, sr))) { + mid_hazard = true; + break; + } + for (int s = 0; s < GGML_MAX_SRC; ++s) { + if (mid->src[s] && mid->src[s]->buffer && ggml_vk_tensors_overlap_strided(mid->src[s], sr)) { + mid_hazard = true; + break; + } + } + if (mid_hazard) { + break; + } + } + if (mid_hazard) { + continue; + } + if (s10_rb.empty()) { + s10_rb.resize(cgraph->n_nodes); + } + s10_rb[conv_idx] = { + true, + end_idx - conv_idx, + sr_idx, + conv, + sr, + old_st, + qkv, + (uint32_t) K, + (uint32_t) sr->nb[1], + }; + gen_skip[concat_idx] = 1; + gen_skip[sr_idx] = 1; + if (cast_idx >= 0 && cast_idx < last_node && !(cast_i32->flags & GGML_TENSOR_FLAG_OUTPUT) && + !use_idx.any_reader(slot_order, [&](int k) { return k != sr_idx && !ggml_op_is_empty(cgraph->nodes[k]->op); }) && + !use_idx.any_reader(cast_i32, [&](int k) { return k != sr_idx && !ggml_op_is_empty(cgraph->nodes[k]->op); })) { + gen_skip[cast_idx] = 1; + if (sc_idx >= 0 && sc_idx < last_node && !(sc->flags & GGML_TENSOR_FLAG_OUTPUT) && + !use_idx.any_reader(sc, [&](int k) { return k != cast_idx && !ggml_op_is_empty(cgraph->nodes[k]->op); })) { + gen_skip[sc_idx] = 1; + if (ar_idx >= 0 && ar_idx < last_node && !(ar->flags & GGML_TENSOR_FLAG_OUTPUT) && + !use_idx.any_reader(ar, [&](int k) { return k != sc_idx && !ggml_op_is_empty(cgraph->nodes[k]->op); })) { + gen_skip[ar_idx] = 1; + } + } + } + } + } + + if (!ctx->device->disable_fusion && gen_fuse_enabled("l2") && !ctx->do_add_rms_partials) { + for (int rms0_idx = 0; rms0_idx < cgraph->n_nodes; ++rms0_idx) { + if (gen_skip[rms0_idx] || cgraph->nodes[rms0_idx]->op != GGML_OP_RMS_NORM) { + continue; + } + const int sc0_idx = gen_find_rms_norm_scale(ctx, cgraph, rms0_idx, last_node, use_idx); + if (sc0_idx < 0 || sc0_idx >= last_node || gen_skip[sc0_idx]) { + continue; + } + int rms1_idx = -1, sc1_idx = -1; + for (int j = rms0_idx + 1; j <= std::min(cgraph->n_nodes - 1, sc0_idx + 6); ++j) { + if (j == sc0_idx || gen_skip[j] || cgraph->nodes[j]->op != GGML_OP_RMS_NORM) { + continue; + } + const int cand_sc1 = gen_find_rms_norm_scale(ctx, cgraph, j, last_node, use_idx); + if (cand_sc1 > sc0_idx && cand_sc1 <= sc0_idx + 8 && !gen_skip[cand_sc1]) { + rms1_idx = j; + sc1_idx = cand_sc1; + break; + } + } + if (rms1_idx < 0 || sc1_idx < 0) { + continue; + } + const ggml_tensor * rms0 = cgraph->nodes[rms0_idx]; + const ggml_tensor * sc0 = cgraph->nodes[sc0_idx]; + const ggml_tensor * rms1 = cgraph->nodes[rms1_idx]; + const ggml_tensor * sc1 = cgraph->nodes[sc1_idx]; + const ggml_tensor * r0 = rms0->src[0]; + const ggml_tensor * r1 = rms1->src[0]; + if (!r0 || !r1 || !r0->buffer || r0->buffer != r1->buffer || + !ggml_are_same_shape(r0, r1) || !ggml_are_same_stride(r0, r1) || + r0->nb[0] != sizeof(float) || (r0->nb[1] % sizeof(float)) != 0 || + (r0->nb[2] % sizeof(float)) != 0 || (r0->nb[3] % sizeof(float)) != 0 || + (get_misalign_bytes(ctx, r0) % sizeof(float)) != 0 || + (vk_tensor_offset(r1) + r1->view_offs) != (vk_tensor_offset(r0) + r0->view_offs) + (size_t)r0->ne[1] * r0->nb[1] || + 2 * (uint64_t)r0->ne[1] > ctx->device->properties.limits.maxComputeWorkGroupCount[0] || + ggml_get_op_params_f32(rms0, 0) != ggml_get_op_params_f32(rms1, 0) || + ggml_get_op_params_f32(sc0, 0) != ggml_get_op_params_f32(sc1, 0) || + (get_misalign_bytes(ctx, sc0) % sizeof(float)) != 0 || + (get_misalign_bytes(ctx, sc1) % sizeof(float)) != 0 || + (get_misalign_bytes(ctx, sc1) / sizeof(float)) > 0xFFFFFFu || + ggml_vk_tensors_overlap_strided(sc0, sc1) || + ggml_vk_tensors_overlap_strided(r0, sc0) || + ggml_vk_tensors_overlap_strided(r0, sc1) || + ggml_vk_tensors_overlap_strided(r1, sc0) || + ggml_vk_tensors_overlap_strided(r1, sc1)) { + continue; + } + if ((sc0->flags & GGML_TENSOR_FLAG_OUTPUT) || + use_idx.any_reader(sc0, [&](int k) { return k > sc0_idx && k < sc1_idx && k != rms1_idx && !ggml_op_is_empty(cgraph->nodes[k]->op); })) { + continue; + } + bool mid_hazard = false; + for (int m_idx = rms0_idx + 1; m_idx < sc1_idx; ++m_idx) { + if (m_idx == sc0_idx || m_idx == rms1_idx || gen_skip[m_idx]) { + continue; + } + const ggml_tensor * mid = cgraph->nodes[m_idx]; + if (ggml_op_is_empty(mid->op)) { + continue; + } + if (mid->buffer && (ggml_vk_tensors_overlap_strided(mid, r0) || + ggml_vk_tensors_overlap_strided(mid, r1) || + (m_idx > sc0_idx && ggml_vk_tensors_overlap_strided(mid, sc0)))) { + mid_hazard = true; + break; + } + if (m_idx > sc0_idx) { + for (int s = 0; s < GGML_MAX_SRC; ++s) { + if (mid->src[s] && mid->src[s]->buffer && ggml_vk_tensors_overlap_strided(mid->src[s], sc0)) { + mid_hazard = true; + break; + } + } + } + if (mid_hazard) { + break; + } + } + if (mid_hazard) { + continue; + } + if (s10_pair_rms.empty()) { + s10_pair_rms.resize(cgraph->n_nodes); + } + s10_pair_rms[sc1_idx] = { rms0_idx, sc0_idx, rms1_idx }; + gen_skip[rms0_idx] = 1; + gen_skip[sc0_idx] = 1; + gen_skip[rms1_idx] = 1; + rms0_idx = sc1_idx; + } + } + + static const bool s12_zero_state_disabled = + getenv("GGML_VK_DISABLE_ZERO_STATE") != nullptr && atoi(getenv("GGML_VK_DISABLE_ZERO_STATE")) != 0; + if (!ctx->device->disable_fusion && !ggml_vk_exact_s8_enabled() && !s12_zero_state_disabled) { + auto same_vk_slice = [&](const ggml_tensor * a, const ggml_tensor * b) -> bool { + if (!a || !b || a->type != GGML_TYPE_F32 || b->type != GGML_TYPE_F32 || + !ggml_is_contiguous(a) || !ggml_is_contiguous(b) || + !a->buffer || !b->buffer || + !ggml_backend_buffer_is_vk(a->buffer) || !ggml_backend_buffer_is_vk(b->buffer)) { + return false; + } + const auto * ab = (const ggml_backend_vk_buffer_context *) a->buffer->context; + const auto * bb = (const ggml_backend_vk_buffer_context *) b->buffer->context; + if (!ab || !bb || ab->dev_buffer != bb->dev_buffer) { + return false; + } + const size_t nb = ggml_nbytes(a); + return nb > 0 && nb == ggml_nbytes(b) && + vk_tensor_offset(a) + a->view_offs == vk_tensor_offset(b) + b->view_offs; + }; + auto find_prior_zero_scale = [&](int consumer_idx, const ggml_tensor * st_in, const ggml_tensor * st_out) -> int { + if (!same_vk_slice(st_in, st_out)) { + return -1; + } + for (int k = consumer_idx - 1; k >= 0; --k) { + if (gen_skip[k]) { + continue; + } + const ggml_tensor * m = cgraph->nodes[k]; + if (ggml_op_is_empty(m->op)) { + continue; + } + if (m->op == GGML_OP_SCALE && k < last_node && + !(m->flags & GGML_TENSOR_FLAG_OUTPUT) && + ggml_get_op_params_f32(m, 0) == 0.0f && + ggml_get_op_params_f32(m, 1) == 0.0f && + same_vk_slice(m, st_in) && + !use_idx.any_reader(m, [&](int r) { + return r != consumer_idx && !gen_skip[r] && !ggml_op_is_empty(cgraph->nodes[r]->op); + })) { + return k; + } + if (m->buffer && ggml_vk_tensors_overlap_strided(m, st_in)) { + return -1; + } + for (int s = 0; s < GGML_MAX_SRC; ++s) { + if (m->src[s] && m->src[s]->buffer && ggml_vk_tensors_overlap_strided(m->src[s], st_in)) { + return -1; + } + } + } + return -1; + }; + for (int idx = 0; idx < cgraph->n_nodes; ++idx) { + if (gen_skip[idx]) { + continue; + } + int sc_idx = -1; + if (!s10_rb.empty() && s10_rb[idx].active && s10_rb[idx].conv->ne[1] >= (int64_t) s10_rb[idx].st_k) { + sc_idx = find_prior_zero_scale(idx, s10_rb[idx].old_st, s10_rb[idx].state_out); + } else if (cgraph->nodes[idx]->op == GGML_OP_CONCAT && idx + 3 < cgraph->n_nodes) { + const uint32_t saved_wmask = ctx->fused_ops_write_mask; + const int n_ssm = s10_can_fuse_ssm_conv(ctx, cgraph, idx, gen_skip, use_idx); + const bool is_mode1 = n_ssm > 0 && ctx->s10_ssm_conv_mode == 1; + ctx->s10_ssm_conv_mode = 0; + ctx->s10_ssm_conv_node = nullptr; + ctx->s10_ssm_state_out = nullptr; + ctx->s10_ssm_old_st = nullptr; + ctx->s10_ssm_qkv = nullptr; + ctx->s10_ssm_st_k = 0; + ctx->s10_ssm_st_slot_nb = 0; + ctx->fused_ops_write_mask = saved_wmask; + if (is_mode1) { + sc_idx = find_prior_zero_scale(idx, cgraph->nodes[idx]->src[0], cgraph->nodes[idx + 2]); + } + } else if (gen_gdn_inplace && cgraph->nodes[idx]->op == GGML_OP_GATED_DELTA_NET) { + const ggml_tensor * gdn = cgraph->nodes[idx]; + const int32_t gdn_k = ggml_get_op_params_i32(gdn, 0); + if (gdn_k >= 1 && gdn->src[2] && gdn->src[2]->ne[2] >= gdn_k) { + const int cpy_idx = gen_find_gdn_state_cpy(ctx, cgraph, idx, last_node, use_idx); + if (cpy_idx >= 0) { + sc_idx = find_prior_zero_scale(idx, gdn->src[5], cgraph->nodes[cpy_idx]); + } + } + } + if (sc_idx >= 0) { + if (s12_zero_sc.empty()) { + s12_zero_sc.assign(cgraph->n_nodes, -1); + } + s12_zero_sc[idx] = sc_idx; + gen_skip[sc_idx] = 1; + } + } + } + } + for (int i = 0; i < cgraph->n_nodes; i++) { + if (!gen_skip.empty() && gen_skip[i]) { + continue; // gen: a state CPY or fused epilogue already done by an earlier dispatch + } if (first_node_in_batch) { submit_node_idx = i; } @@ -15175,7 +20370,8 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg // the fused result in an elementwise-way. This affects whether the memory for // the src is allowed to overlap the memory for the destination. // The array is sized to handle the largest fusion (asserted later). - bool op_srcs_fused_elementwise[13]; + bool op_srcs_fused_elementwise[16]; + int s10_add_rms_final_idx = -1; ctx->fused_topk_moe_mode = TOPK_MOE_COUNT; ctx->fused_topk_moe_scale = false; @@ -15183,12 +20379,76 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg ctx->fused_hc_post_gate = false; ctx->fused_rms_norm_mode = RMS_NORM_COUNT; const char *fusion_string {}; - if (!ctx->device->disable_fusion) { - uint32_t num_adds = ggml_vk_fuse_multi_add(ctx, cgraph, i); - if (num_adds) { + if (!ctx->device->disable_fusion && !ggml_op_is_empty(cgraph->nodes[i]->op)) { + uint32_t s10_adds = s10_can_fuse_multi_add_rms_mul(ctx, cgraph, i, last_node, gen_skip, use_idx, s10_add_rms_final_idx); + uint32_t num_adds = s10_adds ? 0 : ggml_vk_fuse_multi_add(ctx, cgraph, i); + if (s10_adds) { + ctx->num_additional_fused_ops = s10_adds - 1; + fusion_string = ctx->s10_add_rms_weight != nullptr ? "S10_MULTI_ADD_RMS_MUL" : "S10_MULTI_ADD_RMS"; + std::fill_n(op_srcs_fused_elementwise, ctx->num_additional_fused_ops + 1, true); + } else if (num_adds) { ctx->num_additional_fused_ops = num_adds - 1; fusion_string = "MULTI_ADD"; std::fill_n(op_srcs_fused_elementwise, ctx->num_additional_fused_ops + 1, true); + } else if (int n_gate_ops = gen_can_fuse_rms_mul_silu(ctx, cgraph, i)) { + ctx->num_additional_fused_ops = n_gate_ops; + ctx->fused_rms_norm_mode = RMS_NORM_MUL_SILU_GATE; + fusion_string = "GEN_RMS_NORM_MUL_SILU"; + std::fill_n(op_srcs_fused_elementwise, n_gate_ops + 1, true); + const ggml_tensor * g_out = cgraph->nodes[i + n_gate_ops]; + const int r_idx = i + n_gate_ops + 1; + int mm_idx = -1; + if (r_idx < cgraph->n_nodes && !(g_out->flags & GGML_TENSOR_FLAG_OUTPUT)) { + if (cgraph->nodes[r_idx]->op == GGML_OP_MUL_MAT && cgraph->nodes[r_idx]->src[1] == g_out) { + mm_idx = r_idx; + } else if (cgraph->nodes[r_idx]->op == GGML_OP_RESHAPE && + !(cgraph->nodes[r_idx]->flags & GGML_TENSOR_FLAG_OUTPUT) && + cgraph->nodes[r_idx]->view_src == g_out && + cgraph->nodes[r_idx]->view_offs == 0 && + ggml_is_contiguous(cgraph->nodes[r_idx]) && + ggml_nelements(cgraph->nodes[r_idx]) == ggml_nelements(g_out) && + r_idx + 1 < cgraph->n_nodes && + cgraph->nodes[r_idx + 1]->op == GGML_OP_MUL_MAT && + cgraph->nodes[r_idx + 1]->src[1] == cgraph->nodes[r_idx]) { + mm_idx = r_idx + 1; + } + } + if (mm_idx >= 0) { + const ggml_tensor * mm_node = cgraph->nodes[mm_idx]; + const bool sole_ok = ggml_is_quantized(mm_node->src[0]->type) && + mm_node->src[1]->ne[1] >= 128 && + !use_idx.any_reader(g_out, [&](int k) { return k != r_idx && k != mm_idx; }) && + (r_idx == mm_idx || !use_idx.any_reader(cgraph->nodes[r_idx], [&](int k) { return k != mm_idx; })); + ctx->gen_rms_silu_sole_mm_next = sole_ok; + } + } else if (int n_lora_ops = gen_can_fuse_lora(ctx, cgraph, i, gen_skip, use_idx, fusion_string, op_srcs_fused_elementwise)) { + ctx->num_additional_fused_ops = n_lora_ops; + } else if (gen_can_fuse_gate_up_glu(ctx, cgraph, i)) { + ctx->num_additional_fused_ops = 2; + fusion_string = "GEN_MUL_MAT_GATE_UP_GLU"; + op_srcs_fused_elementwise[0] = false; + op_srcs_fused_elementwise[1] = false; + op_srcs_fused_elementwise[2] = true; + const ggml_tensor * glu_node = cgraph->nodes[i + 2]; + ctx->gen_pair_glu_sole_mmid_next = + cgraph->nodes[i]->op == GGML_OP_MUL_MAT_ID && + !(glu_node->flags & GGML_TENSOR_FLAG_OUTPUT) && + (i + 3 < cgraph->n_nodes) && + cgraph->nodes[i + 3]->op == GGML_OP_MUL_MAT_ID && + cgraph->nodes[i + 3]->src[1] == glu_node && + cgraph->nodes[i + 3]->src[2] == cgraph->nodes[i]->src[2] && + !use_idx.any_reader(glu_node, [&](int k) { return k != i + 3; }); + } else if (int n_pair_ops = gen_can_fuse_pair_mm(ctx, cgraph, i, last_node, gen_skip, use_idx)) { + ctx->num_additional_fused_ops = n_pair_ops; + ctx->fused_ops_write_mask |= (1u << ctx->gen_pair_mul0_rel); + fusion_string = ctx->gen_pair_mul0_rel > 0 ? "S10_PAIR_MM_GDN_GATES" + : (n_pair_ops >= 2 ? "GEN_PAIR_MM_SIGMOID" : "GEN_PAIR_MM"); + std::fill_n(op_srcs_fused_elementwise, n_pair_ops + 1, false); + } else if (gen_can_fuse_mm_glu(ctx, cgraph, i)) { + ctx->num_additional_fused_ops = 1; + fusion_string = "GEN_MUL_MAT_GLU"; + op_srcs_fused_elementwise[0] = false; + op_srcs_fused_elementwise[1] = true; } else if (ggml_vk_can_fuse(ctx, cgraph, i, { GGML_OP_MUL_MAT, GGML_OP_ADD, GGML_OP_ADD })) { ctx->num_additional_fused_ops = 2; fusion_string = "MUL_MAT_ADD_ADD"; @@ -15269,7 +20529,8 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg // they are overwritten, and one workgroup per row. So close enough. op_srcs_fused_elementwise[0] = true; op_srcs_fused_elementwise[1] = true; - } else if (ggml_vk_can_fuse(ctx, cgraph, i, { GGML_OP_UNARY, GGML_OP_MUL })) { + } else if ((gen_deferred_mm.empty() || gen_deferred_mm[i + 1] < 0) && + ggml_vk_can_fuse(ctx, cgraph, i, { GGML_OP_UNARY, GGML_OP_MUL })) { ctx->num_additional_fused_ops = 1; switch (ggml_get_unary_op(cgraph->nodes[i])) { case GGML_UNARY_OP_GELU: fusion_string = "GELU_MUL"; break; @@ -15279,6 +20540,28 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg } op_srcs_fused_elementwise[0] = true; op_srcs_fused_elementwise[1] = true; + } else if (int n_cum_ops = gen_can_fuse_cont_unary_mul(ctx, cgraph, i)) { + ctx->num_additional_fused_ops = n_cum_ops; + fusion_string = "GEN_CONT_UNARY_MUL"; + op_srcs_fused_elementwise[0] = false; + for (int k = 1; k <= n_cum_ops; ++k) { + op_srcs_fused_elementwise[k] = true; + } + } else if (!s10_rb.empty() && s10_rb[i].active) { + ctx->num_additional_fused_ops = s10_rb[i].num_extra; + ctx->s10_ssm_conv_mode = 3; + ctx->s10_ssm_conv_node = s10_rb[i].conv; + ctx->s10_ssm_state_out = s10_rb[i].state_out; + ctx->s10_ssm_old_st = s10_rb[i].old_st; + ctx->s10_ssm_qkv = s10_rb[i].qkv; + ctx->s10_ssm_st_k = s10_rb[i].st_k; + ctx->s10_ssm_st_slot_nb = s10_rb[i].st_slot_nb; + fusion_string = "S10_SSM_CONV_SLIDING_RB"; + std::fill_n(op_srcs_fused_elementwise, s10_rb[i].num_extra + 1, false); + } else if (int n_ssm_ops = s10_can_fuse_ssm_conv(ctx, cgraph, i, gen_skip, use_idx)) { + ctx->num_additional_fused_ops = n_ssm_ops; + fusion_string = ctx->s10_ssm_conv_mode == 1 ? "S10_SSM_CONV_SLIDING" : "S10_SSM_CONV_SOUT"; + std::fill_n(op_srcs_fused_elementwise, n_ssm_ops + 1, false); } else if (ggml_vk_can_fuse_ssm_conv(ctx, cgraph, i, 2)) { ctx->num_additional_fused_ops = 2; fusion_string = "SSM_CONV_BIAS_SILU"; @@ -15371,10 +20654,13 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg } GGML_ASSERT(ctx->num_additional_fused_ops < (int)(sizeof(op_srcs_fused_elementwise) / sizeof(op_srcs_fused_elementwise[0]))); ctx->fused_ops_write_mask |= 1 << ctx->num_additional_fused_ops; + if (ctx->s10_add_rms_dst != nullptr && !ctx->s10_add_rms_write_add) { + ctx->fused_ops_write_mask = 0; + } // Check whether fusion would overwrite src operands while they're still in use. // If so, disable fusion. - if (ctx->num_additional_fused_ops) { + if (ctx->num_additional_fused_ops && ctx->s10_ssm_conv_mode == 0 && ctx->s10_add_rms_dst == nullptr && ctx->gen_pair_mm1 == nullptr) { // There are up to two output nodes - topk_moe has two. uint32_t bits = ctx->fused_ops_write_mask & ~(1 << ctx->num_additional_fused_ops); ggml_tensor *output_nodes[2] {}; @@ -15396,6 +20682,12 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg // the destination and the src is not an intermediate node that's being // elided, then disable fusion. for (int k = 0; k <= ctx->num_additional_fused_ops; ++k) { + if ((ctx->fused_rms_norm_mode == RMS_NORM_MUL_SILU_GATE || + (cgraph->nodes[i]->op == GGML_OP_CONT && + cgraph->nodes[i + ctx->num_additional_fused_ops]->op == GGML_OP_MUL)) && + ggml_op_is_empty(cgraph->nodes[i + k]->op)) { + continue; + } for (uint32_t s = 0; s < GGML_MAX_SRC; ++s) { ggml_tensor *src = cgraph->nodes[i + k]->src[s]; if (!src || src->op == GGML_OP_NONE) { @@ -15424,24 +20716,404 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg ctx->fused_topk_qsa = false; ctx->fused_hc_post_gate = false; ctx->fused_rms_norm_mode = RMS_NORM_COUNT; + ctx->gen_pair_mm1 = nullptr; + ctx->gen_pair_dst1 = nullptr; + ctx->gen_mm_dst = nullptr; + ctx->gen_mm_f0 = nullptr; + ctx->gen_mm_f1 = nullptr; + ctx->gen_mm_flags = 0; + ctx->gen_lora_b0 = nullptr; + ctx->gen_lora_h0 = nullptr; + ctx->gen_lora_b1 = nullptr; + ctx->gen_lora_h1 = nullptr; + ctx->gen_lora_gate_src0 = nullptr; + ctx->gen_lora_up_src0 = nullptr; + ctx->gen_lora_r = 0; + ctx->gen_lora_scale0 = 0.0f; + ctx->gen_lora_scale1 = 0.0f; + ctx->gen_pair_glu_sole_mmid_next = false; + ctx->gen_rms_silu_sole_mm_next = false; fusion_string = nullptr; } } // Signal the almost_ready fence when the graph is mostly complete (< 20% remaining) - bool almost_ready = (cgraph->n_nodes - i) < cgraph->n_nodes / 5; - bool submit = (submitted_nodes >= ctx->device->max_nodes_per_submit) || + const bool is_last_in_graph = (i + ctx->num_additional_fused_ops >= last_node) || (s10_add_rms_final_idx >= last_node); + bool almost_ready = !ctx->recording_replay && eff_submit_mode == 0 && (cgraph->n_nodes - i) < cgraph->n_nodes / 5; + bool submit = !ctx->recording_replay && ( + (submitted_nodes >= gen_max_nodes) || (flops_per_submit != 0 && batch_flops >= flops_per_submit) || - (i + ctx->num_additional_fused_ops >= last_node) || - (almost_ready && !ctx->almost_ready_fence_pending); + is_last_in_graph || + (almost_ready && !ctx->almost_ready_fence_pending)); + + int gen_state_cpy = -1; + if (gen_gdn_inplace && cgraph->nodes[i]->op == GGML_OP_GATED_DELTA_NET && ctx->num_additional_fused_ops == 0) { + gen_state_cpy = gen_find_gdn_state_cpy(ctx, cgraph, i, last_node, use_idx); + } + if (gen_state_cpy < 0 && cgraph->nodes[i]->op == GGML_OP_RMS_NORM && ctx->num_additional_fused_ops == 0 && + ctx->fused_rms_norm_mode == RMS_NORM_COUNT) { + const int sc_idx = gen_find_rms_norm_scale(ctx, cgraph, i, last_node, use_idx); + if (sc_idx >= 0) { + // defer: this RMS_NORM runs fused when the loop reaches its SCALE + if (gen_deferred_rms.empty()) { + gen_deferred_rms.assign(cgraph->n_nodes, -1); + } + gen_deferred_rms[sc_idx] = i; + ctx->num_additional_fused_ops = 0; + ctx->fused_ops_write_mask = 0; + continue; + } + } + int gen_rms_idx = -1; + if (gen_state_cpy < 0 && !s10_pair_rms.empty() && s10_pair_rms[i].rms0_idx >= 0) { + const auto & pr = s10_pair_rms[i]; + gen_rms_idx = pr.rms0_idx; + const ggml_tensor * sc0 = cgraph->nodes[pr.sc0_idx]; + const ggml_tensor * sc1 = cgraph->nodes[i]; + const ggml_tensor * r0 = cgraph->nodes[pr.rms0_idx]->src[0]; + const ggml_tensor * r1 = cgraph->nodes[pr.rms1_idx]->src[0]; + bool need_sync = false; + for (const ggml_tensor * o : ctx->unsynced_nodes_read) { + need_sync = need_sync || ggml_vk_tensors_overlap_strided(o, sc0) || ggml_vk_tensors_overlap_strided(o, sc1); + } + for (const ggml_tensor * o : ctx->unsynced_nodes_written) { + need_sync = need_sync || + ggml_vk_tensors_overlap_strided(o, sc0) || + ggml_vk_tensors_overlap_strided(o, sc1) || + ggml_vk_tensors_overlap_strided(o, r0) || + ggml_vk_tensors_overlap_strided(o, r1); + } + vk_context gen_cctx = ggml_vk_get_compute_ctx(ctx); + if (need_sync) { + ctx->unsynced_nodes_written.clear(); + ctx->unsynced_nodes_read.clear(); + ggml_vk_sync_buffers(ctx, gen_cctx); + } + ctx->tensor_ctxs[pr.sc0_idx] = gen_cctx; + ctx->tensor_ctxs[i] = gen_cctx; + ctx->num_additional_fused_ops = 0; + ctx->fused_ops_write_mask = 0; + ctx->fused_rms_norm_mode = RMS_NORM_SCALE; + ctx->gen_rms_scale_dst = sc0; + ctx->gen_rms_scale_dst2 = sc1; + ctx->gen_rms_scale_rsrc2 = r1; + fusion_string = "S10_PAIR_RMS_NORM_SCALE"; + } else if (gen_state_cpy < 0 && !gen_deferred_rms.empty() && gen_deferred_rms[i] >= 0) { + gen_rms_idx = gen_deferred_rms[i]; + const ggml_tensor * sc = cgraph->nodes[i]; + const ggml_tensor * rsrc = cgraph->nodes[gen_rms_idx]->src[0]; + bool need_sync = false; + for (const ggml_tensor * o : ctx->unsynced_nodes_read) { + need_sync = need_sync || ggml_vk_tensors_overlap_strided(o, sc); + } + for (const ggml_tensor * o : ctx->unsynced_nodes_written) { + need_sync = need_sync || ggml_vk_tensors_overlap_strided(o, sc) || + ggml_vk_tensors_overlap_strided(o, rsrc); + } + vk_context gen_cctx = ggml_vk_get_compute_ctx(ctx); + if (need_sync) { + ctx->unsynced_nodes_written.clear(); + ctx->unsynced_nodes_read.clear(); + ggml_vk_sync_buffers(ctx, gen_cctx); + } + ctx->tensor_ctxs[i] = gen_cctx; + ctx->num_additional_fused_ops = 0; + ctx->fused_ops_write_mask = 0; + ctx->fused_rms_norm_mode = RMS_NORM_SCALE; + ctx->gen_rms_scale_dst = sc; + } + if (gen_state_cpy >= 0) { + const ggml_tensor * cpy = cgraph->nodes[gen_state_cpy]; + bool need_sync = false; + for (const ggml_tensor * o : ctx->unsynced_nodes_read) { + need_sync = need_sync || ggml_vk_tensors_overlap_strided(o, cpy); + } + for (const ggml_tensor * o : ctx->unsynced_nodes_written) { + need_sync = need_sync || ggml_vk_tensors_overlap_strided(o, cpy); + } + vk_context sync_ctx = ggml_vk_get_compute_ctx(ctx); + if (need_sync) { + ctx->unsynced_nodes_written.clear(); + ctx->unsynced_nodes_read.clear(); + ggml_vk_sync_buffers(ctx, sync_ctx); + } + ctx->tensor_ctxs[gen_state_cpy] = sync_ctx; + if (cgraph->nodes[i]->op == GGML_OP_GATED_DELTA_NET) { + ctx->gen_gdn_state_out = cpy; + } + gen_skip[gen_state_cpy] = 1; + } + + if (!s12_zero_sc.empty() && s12_zero_sc[i] >= 0) { + ctx->tensor_ctxs[s12_zero_sc[i]] = ggml_vk_get_compute_ctx(ctx); + if (gen_state_cpy >= 0 && cgraph->nodes[i]->op == GGML_OP_GATED_DELTA_NET) { + ctx->gen_gdn_zero_state = true; + } else if (ctx->s10_ssm_conv_mode == 1 || ctx->s10_ssm_conv_mode == 3) { + ctx->s10_ssm_zero_state = true; + } + GGML_ASSERT(ctx->gen_gdn_zero_state || ctx->s10_ssm_zero_state); + } - bool enqueued = ggml_vk_build_graph(ctx, cgraph, i, cgraph->nodes[submit_node_idx], submit_node_idx, i + ctx->num_additional_fused_ops >= last_node, almost_ready, submit); + if (ctx->s10_ssm_conv_mode == 3) { + const ggml_tensor * sout = ctx->s10_ssm_state_out; + const ggml_tensor * ost = ctx->s10_ssm_zero_state ? nullptr : ctx->s10_ssm_old_st; + const ggml_tensor * qkv = ctx->s10_ssm_qkv; + bool need_sync = false; + for (const ggml_tensor * o : ctx->unsynced_nodes_read) { + need_sync = need_sync || (sout && ggml_vk_tensors_overlap_strided(o, sout)); + } + for (const ggml_tensor * o : ctx->unsynced_nodes_written) { + need_sync = need_sync || + (sout && ggml_vk_tensors_overlap_strided(o, sout)) || + (ost && ggml_vk_tensors_overlap_strided(o, ost)) || + (qkv && ggml_vk_tensors_overlap_strided(o, qkv)); + } + vk_context sync_ctx = ggml_vk_get_compute_ctx(ctx); + if (need_sync) { + ctx->unsynced_nodes_written.clear(); + ctx->unsynced_nodes_read.clear(); + ggml_vk_sync_buffers(ctx, sync_ctx); + } + if (!s10_rb.empty() && s10_rb[i].sr_idx >= 0) { + ctx->tensor_ctxs[s10_rb[i].sr_idx] = sync_ctx; + } + } + + if (ctx->s10_add_rms_dst != nullptr) { + const ggml_tensor * fdst = ctx->s10_add_rms_dst; + const ggml_tensor * fw = ctx->s10_add_rms_weight; + bool need_sync = false; + for (const ggml_tensor * o : ctx->unsynced_nodes_read) { + need_sync = need_sync || (o->buffer == fdst->buffer && ggml_vk_tensors_overlap(o, fdst, false)); + } + for (const ggml_tensor * o : ctx->unsynced_nodes_written) { + need_sync = need_sync || + (o->buffer == fdst->buffer && ggml_vk_tensors_overlap(o, fdst, false)) || + (fw && o->buffer == fw->buffer && ggml_vk_tensors_overlap(o, fw, false)); + } + if (need_sync) { + vk_context sync_ctx = ggml_vk_get_compute_ctx(ctx); + ctx->unsynced_nodes_written.clear(); + ctx->unsynced_nodes_read.clear(); + ggml_vk_sync_buffers(ctx, sync_ctx); + } + } + + gen_mm_epilogue mm_ep {}; + int gen_mm_idx = -1; + if (!ctx->device->disable_fusion && gen_state_cpy < 0 && gen_rms_idx < 0 && + !gen_deferred_mm.empty() && gen_deferred_mm[i] >= 0) { + gen_mm_idx = gen_deferred_mm[i]; + const ggml_tensor * mul = cgraph->nodes[i]; + const ggml_tensor * mm = cgraph->nodes[gen_mm_idx]; + const ggml_tensor * gate = (mul->src[0] == mm) ? mul->src[1] : mul->src[0]; + bool need_sync = false; + for (const ggml_tensor * o : ctx->unsynced_nodes_read) { + need_sync = need_sync || (o->buffer == mul->buffer && ggml_vk_tensors_overlap(o, mul, false)); + } + for (const ggml_tensor * o : ctx->unsynced_nodes_written) { + need_sync = need_sync || (o->buffer == mul->buffer && ggml_vk_tensors_overlap(o, mul, false)) || + (o->buffer == gate->buffer && ggml_vk_tensors_overlap(o, gate, false)) || + (mm->src[0]->buffer && o->buffer == mm->src[0]->buffer && ggml_vk_tensors_overlap(o, mm->src[0], false)) || + (mm->src[1]->buffer && o->buffer == mm->src[1]->buffer && ggml_vk_tensors_overlap(o, mm->src[1], false)); + } + vk_context gen_cctx = ggml_vk_get_compute_ctx(ctx); + if (need_sync) { + ctx->unsynced_nodes_written.clear(); + ctx->unsynced_nodes_read.clear(); + ggml_vk_sync_buffers(ctx, gen_cctx); + } + ctx->tensor_ctxs[i] = gen_cctx; + ctx->num_additional_fused_ops = 0; + ctx->fused_ops_write_mask = 0; + mm_ep = { mul, gate, nullptr, MAT_VEC_FUSION_FLAGS_ROW_SCALE0, { -1, -1, -1 }, "GEN_MUL_MAT_ROW_SCALE0" }; + ctx->gen_mm_dst = mm_ep.dst; + ctx->gen_mm_f0 = mm_ep.f0; + ctx->gen_mm_flags = mm_ep.flags; + fusion_string = mm_ep.name; + } else if (!ctx->device->disable_fusion && gen_state_cpy < 0 && gen_rms_idx < 0 && + cgraph->nodes[i]->op == GGML_OP_MUL_MAT && ctx->num_additional_fused_ops == 0) { + if (gen_find_mm_epilogue(ctx, cgraph, i, last_node, gen_skip, use_idx, mm_ep)) { + bool need_sync = false; + for (const ggml_tensor * o : ctx->unsynced_nodes_read) { + need_sync = need_sync || (o->buffer == mm_ep.dst->buffer && ggml_vk_tensors_overlap(o, mm_ep.dst, false)); + } + for (const ggml_tensor * o : ctx->unsynced_nodes_written) { + need_sync = need_sync || (o->buffer == mm_ep.dst->buffer && ggml_vk_tensors_overlap(o, mm_ep.dst, false)) || + (mm_ep.f0 && o->buffer == mm_ep.f0->buffer && ggml_vk_tensors_overlap(o, mm_ep.f0, false)) || + (mm_ep.f1 && o->buffer == mm_ep.f1->buffer && ggml_vk_tensors_overlap(o, mm_ep.f1, false)); + } + if (need_sync) { + vk_context sync_ctx = ggml_vk_get_compute_ctx(ctx); + ctx->unsynced_nodes_written.clear(); + ctx->unsynced_nodes_read.clear(); + ggml_vk_sync_buffers(ctx, sync_ctx); + } + ctx->fused_ops_write_mask = 0; + ctx->gen_mm_dst = mm_ep.dst; + ctx->gen_mm_f0 = mm_ep.f0; + ctx->gen_mm_f1 = mm_ep.f1; + ctx->gen_mm_flags = mm_ep.flags; + fusion_string = mm_ep.name; + for (int sk : mm_ep.skip) { + if (sk >= 0) { + gen_skip[sk] = 1; + } + } + } else { + const int def_mul = gen_find_mm_deferred_scale0(ctx, cgraph, i, last_node, use_idx); + if (def_mul >= 0) { + if (gen_deferred_mm.empty()) { + gen_deferred_mm.assign(cgraph->n_nodes, -1); + } + gen_deferred_mm[def_mul] = i; + ctx->num_additional_fused_ops = 0; + ctx->fused_ops_write_mask = 0; + continue; + } + } + } + + if (gen_state_cpy < 0 && gen_rms_idx < 0 && gen_mm_idx < 0 && mm_ep.dst == nullptr && + ctx->gen_pair_mm1 == nullptr && ctx->s10_add_rms_dst == nullptr && ctx->s10_ssm_conv_mode == 0 && + (gen_env.dup_all || !gen_env.dup.empty()) && gen_dup_match(cgraph->nodes[i]) && + !(cgraph->nodes[i]->op == GGML_OP_RMS_NORM && ctx->do_add_rms_partials)) { + if (ggml_vk_build_graph(ctx, cgraph, i, cgraph->nodes[submit_node_idx], submit_node_idx, false, false, false)) { + gen_dups++; + } + } + const int exec_idx = gen_rms_idx >= 0 ? gen_rms_idx : (gen_mm_idx >= 0 ? gen_mm_idx : i); + bool enqueued = ggml_vk_build_graph(ctx, cgraph, exec_idx, cgraph->nodes[submit_node_idx], submit_node_idx, is_last_in_graph, almost_ready, submit); + if (ctx->gen_pair_mm1 != nullptr) { + ctx->gen_pair_mm1 = nullptr; + ctx->gen_pair_dst1 = nullptr; + ctx->gen_pair_mul0_rel = 0; + ctx->gen_mm_dst = nullptr; + ctx->gen_mm_f0 = nullptr; + ctx->gen_mm_f1 = nullptr; + ctx->gen_mm_flags = 0; + } + if (ctx->gen_lora_r != 0) { + ctx->gen_mm_dst = nullptr; + ctx->gen_mm_f0 = nullptr; + ctx->gen_mm_f1 = nullptr; + ctx->gen_mm_flags = 0; + ctx->gen_lora_b0 = nullptr; + ctx->gen_lora_h0 = nullptr; + ctx->gen_lora_b1 = nullptr; + ctx->gen_lora_h1 = nullptr; + ctx->gen_lora_gate_src0 = nullptr; + ctx->gen_lora_up_src0 = nullptr; + ctx->gen_lora_r = 0; + ctx->gen_lora_scale0 = 0.0f; + ctx->gen_lora_scale1 = 0.0f; + } + if (ctx->s10_ssm_conv_mode != 0) { + if (ctx->s10_ssm_conv_mode == 3) { + if (ctx->s10_ssm_state_out != nullptr) { + ctx->unsynced_nodes_written.push_back(ctx->s10_ssm_state_out); + } + if (ctx->s10_ssm_old_st != nullptr && !ctx->s10_ssm_zero_state) { + ctx->unsynced_nodes_read.push_back(ctx->s10_ssm_old_st); + } + if (ctx->s10_ssm_qkv != nullptr) { + ctx->unsynced_nodes_read.push_back(ctx->s10_ssm_qkv); + } + } + ctx->s10_ssm_conv_mode = 0; + ctx->s10_ssm_conv_node = nullptr; + ctx->s10_ssm_state_out = nullptr; + ctx->s10_ssm_old_st = nullptr; + ctx->s10_ssm_qkv = nullptr; + ctx->s10_ssm_st_k = 0; + ctx->s10_ssm_st_slot_nb = 0; + } + ctx->s10_ssm_zero_state = false; + ctx->gen_gdn_zero_state = false; + ctx->gen_pair_glu_sole_mmid_next = false; + ctx->gen_rms_silu_sole_mm_next = false; + if (ctx->s10_add_rms_dst != nullptr) { + const ggml_tensor * fdst = ctx->s10_add_rms_dst; + ctx->unsynced_nodes_written.push_back(fdst); + for (uint32_t s = 0; s < ctx->prealloc_y_ring_count; ++s) { + if ((int)s == ctx->prealloc_y_keep_slot) { + continue; + } + const ggml_tensor * cy = ctx->prealloc_y_last_tensor_ring[s]; + if (cy && cy->buffer == fdst->buffer && ggml_vk_tensors_overlap(fdst, cy, false)) { + ctx->prealloc_y_last_tensor_ring[s] = nullptr; + ctx->prealloc_y_last_pipeline_ring[s] = nullptr; + } + } + ctx->prealloc_y_keep_slot = -1; + if (ctx->s10_add_rms_weight != nullptr) { + ctx->unsynced_nodes_read.push_back(ctx->s10_add_rms_weight); + } + ctx->s10_add_rms_dst = nullptr; + ctx->s10_add_rms_weight = nullptr; + ctx->s10_add_rms_add_dst = nullptr; + ctx->s10_add_rms_num_adds = 0; + ctx->s10_add_rms_tree_mode = 0; + ctx->s10_add_rms_write_add = 0; + } + auto invalidate_y_ring = [&](const ggml_tensor * wdst) { + if (!wdst || !wdst->buffer) { + return; + } + for (uint32_t s = 0; s < ctx->prealloc_y_ring_count; ++s) { + if ((int)s == ctx->prealloc_y_keep_slot) { + continue; + } + const ggml_tensor * cy = ctx->prealloc_y_last_tensor_ring[s]; + if (cy && cy->buffer == wdst->buffer && ggml_vk_tensors_overlap(wdst, cy, false)) { + ctx->prealloc_y_last_tensor_ring[s] = nullptr; + ctx->prealloc_y_last_pipeline_ring[s] = nullptr; + } + } + }; + if (gen_rms_idx >= 0) { + if (ctx->gen_rms_scale_dst != nullptr) { + ctx->unsynced_nodes_written.push_back(ctx->gen_rms_scale_dst); + invalidate_y_ring(ctx->gen_rms_scale_dst); + } + if (ctx->gen_rms_scale_dst2 != nullptr) { + ctx->unsynced_nodes_written.push_back(ctx->gen_rms_scale_dst2); + invalidate_y_ring(ctx->gen_rms_scale_dst2); + } + if (ctx->gen_rms_scale_rsrc2 != nullptr) { + ctx->unsynced_nodes_read.push_back(ctx->gen_rms_scale_rsrc2); + } + ctx->gen_rms_scale_dst = nullptr; + ctx->gen_rms_scale_dst2 = nullptr; + ctx->gen_rms_scale_rsrc2 = nullptr; + } + if (gen_state_cpy >= 0) { + ctx->gen_gdn_state_out = nullptr; + ctx->gen_rms_scale_dst = nullptr; + ctx->unsynced_nodes_written.push_back(cgraph->nodes[gen_state_cpy]); + invalidate_y_ring(cgraph->nodes[gen_state_cpy]); + } + if (mm_ep.dst != nullptr) { + ctx->gen_mm_dst = nullptr; + ctx->gen_mm_f0 = nullptr; + ctx->gen_mm_f1 = nullptr; + ctx->gen_mm_flags = 0; + ctx->unsynced_nodes_written.push_back(mm_ep.dst); + invalidate_y_ring(mm_ep.dst); + if (mm_ep.f0 != nullptr) { + ctx->unsynced_nodes_read.push_back(mm_ep.f0); + } + if (mm_ep.f1 != nullptr) { + ctx->unsynced_nodes_read.push_back(mm_ep.f1); + } + } + ctx->prealloc_y_keep_slot = -1; if (vk_perf_logger_enabled && enqueued) { compute_ctx = ggml_vk_get_compute_ctx(ctx); if (!vk_perf_logger_concurrent) { // track a single node/fusion for the current query - ctx->query_nodes[ctx->query_idx] = cgraph->nodes[i]; + ctx->query_nodes[ctx->query_idx] = cgraph->nodes[exec_idx]; ctx->query_fusion_names[ctx->query_idx] = fusion_string; compute_ctx->s->buffer->buf.writeTimestamp(vk::PipelineStageFlagBits::eAllCommands, ctx->query_pool, ctx->query_idx++); ggml_vk_sync_buffers(ctx, compute_ctx); @@ -15473,15 +21145,101 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg } if (submit && enqueued) { - submit_after(submit_node_idx, i + (int)ctx->num_additional_fused_ops); + submit_after(submit_node_idx, std::max(i + (int)ctx->num_additional_fused_ops, s10_add_rms_final_idx)); } i += ctx->num_additional_fused_ops; ctx->num_additional_fused_ops = 0; ctx->fused_ops_write_mask = 0; } + if (rec_entry != nullptr) { + std::swap(ctx->descriptor_pools, rec_entry->descriptor_pools); + std::swap(ctx->descriptor_sets, rec_entry->descriptor_sets); + std::swap(ctx->descriptor_set_bindings, rec_entry->descriptor_set_bindings); + std::swap(ctx->descriptor_set_bindings_destroy_count, rec_entry->descriptor_destroy_count); + ctx->recording_replay = false; + ctx->active_replay_cmd_pool = nullptr; + + if (!ctx->replay_abort && !ctx->compute_ctx.expired()) { + compute_ctx = ctx->compute_ctx.lock(); + ggml_vk_ctx_end(compute_ctx); + + const bool single_seq = compute_ctx->seqs.size() == 1 && + compute_ctx->seqs[0].size() == 1 && + compute_ctx->seqs[0][0].wait_semaphores.empty() && + compute_ctx->seqs[0][0].signal_semaphores.empty(); + const vk::CommandBuffer recorded_cmd_buf = single_seq ? compute_ctx->seqs[0][0].buffer->buf : vk::CommandBuffer{}; + const bool no_staging = compute_ctx->in_memcpys.empty() && compute_ctx->out_memcpys.empty(); + const bool pre_unchanged = + rec_entry->pre_x_handle == pre_buf_h(ctx->prealloc_x) && rec_entry->pre_x_size == pre_buf_sz(ctx->prealloc_x) && + rec_entry->pre_y_handle == pre_buf_h(ctx->prealloc_y) && rec_entry->pre_y_size == pre_buf_sz(ctx->prealloc_y) && + rec_entry->pre_sk_handle == pre_buf_h(ctx->prealloc_split_k) && rec_entry->pre_sk_size == pre_buf_sz(ctx->prealloc_split_k) && + rec_entry->pre_arp_handle == pre_buf_h(ctx->prealloc_add_rms_partials) && rec_entry->pre_arp_size == pre_buf_sz(ctx->prealloc_add_rms_partials) && + rec_entry->pre_mo_handle == pre_buf_h(ctx->prealloc_mask_opt) && rec_entry->pre_mo_size == pre_buf_sz(ctx->prealloc_mask_opt) && + rec_entry->destroy_count == ctx->device->buffer_destroy_count.load(std::memory_order_acquire); + + for (auto & cpy : compute_ctx->in_memcpys) { + memcpy(cpy.dst, cpy.src, cpy.n); + } + for (auto & mset : compute_ctx->memsets) { + memset(mset.dst, mset.val, mset.n); + } + + ggml_vk_ensure_fence_free(ctx); + const bool need_transfer_sem = ctx->device->async_use_transfer_queue && + ctx->transfer_semaphore_last_submitted < ctx->transfer_semaphore.value; + if (!need_transfer_sem && no_staging && !compute_ctx->seqs.empty()) { + ggml_vk_submit(compute_ctx, ctx->fence); + ctx->fence_submitted = true; + } else { + ggml_vk_submit(compute_ctx, {}); + } + ctx->submit_pending = true; + submit_count = 1; + + if (single_seq && no_staging && pre_unchanged) { + rec_entry->cmd_buf = recorded_cmd_buf; + rec_entry->memsets = compute_ctx->memsets; + rec_entry->total_flops = total_flops; + rec_entry->stat_dispatches = ctx->stat_dispatches; + rec_entry->stat_syncs = ctx->stat_syncs; + rec_entry->ready = true; + if (ctx->fence_submitted) { + ctx->active_replay_entry = rec_entry; + ctx->replay_submit_us = ggml_time_us(); + } + } + for (auto & cpy : compute_ctx->out_memcpys) { + memcpy(cpy.dst, cpy.src, cpy.n); + } + compute_ctx->in_memcpys.clear(); + compute_ctx->out_memcpys.clear(); + compute_ctx->memsets.clear(); + ctx->compute_ctx.reset(); + } else if (!ctx->compute_ctx.expired()) { + compute_ctx = ctx->compute_ctx.lock(); + ggml_vk_ctx_end(compute_ctx); + for (auto & cpy : compute_ctx->in_memcpys) { + memcpy(cpy.dst, cpy.src, cpy.n); + } + for (auto & mset : compute_ctx->memsets) { + memset(mset.dst, mset.val, mset.n); + } + ggml_vk_ensure_fence_free(ctx); + ggml_vk_submit(compute_ctx, {}); + ctx->submit_pending = true; + ctx->compute_ctx.reset(); + } + } + ctx->last_total_flops = total_flops; + if (gen_env.stats) { + fprintf(stderr, "ggml_vk_graph_stats: nodes=%d dispatches=%llu syncs=%llu submits=%u dups=%u flops_per_submit=%llu cpu_record_us=%lld\n", + cgraph->n_nodes, (unsigned long long) ctx->stat_dispatches, (unsigned long long) ctx->stat_syncs, + submit_count, gen_dups, (unsigned long long) flops_per_submit, (long long) (ggml_time_us() - gen_t_start)); + } + if (vk_perf_logger_enabled) { // End the command buffer and submit/wait GGML_ASSERT(!ctx->compute_ctx.expired()); @@ -15636,6 +21394,22 @@ void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * graph, } }; + auto const &is_gate_up_glu = [&](int s) -> bool { + if (!gen_fuse_enabled("gateup") || s < 0 || s + 2 >= graph->n_nodes) { + return false; + } + if (used[s] || used[s + 1] || used[s + 2]) { + return false; + } + ggml_tensor * a = graph->nodes[s]; + ggml_tensor * b = graph->nodes[s + 1]; + ggml_tensor * g = graph->nodes[s + 2]; + return (a->op == GGML_OP_MUL_MAT || a->op == GGML_OP_MUL_MAT_ID) && + b->op == a->op && a->src[1] == b->src[1] && + g->op == GGML_OP_GLU && + ((g->src[0] == a && g->src[1] == b) || (g->src[0] == b && g->src[1] == a)); + }; + auto const &keep_topk_moe_pattern = [&](const std::initializer_list &pattern) -> bool { if (!match_pattern(pattern, first_unused)) { return false; @@ -15643,13 +21417,55 @@ void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * graph, int last_node = first_unused + (int) pattern.size() - 1; // Some TOPK_MOE variants fuse a trailing scale. - if (last_node + 1 < graph->n_nodes && graph->nodes[last_node + 1]->op == GGML_OP_SCALE) { + const bool has_trailing_scale = last_node + 1 < graph->n_nodes && graph->nodes[last_node + 1]->op == GGML_OP_SCALE; + if (has_trailing_scale) { last_node++; } + ggml_tensor * topk_src0 = graph->nodes[first_unused]->src[0]; add_pattern_alloc_deps(pattern, last_node); - return keep_pattern(pattern); + keep_pattern(pattern); + + // gen (s8): overlap 1-workgroup TOPK_MOE with ready shared-expert gate+up+glu + if (gen_fuse_enabled("s8") && !has_trailing_scale) { + for (int k = first_unused; k + 2 < std::min(first_unused + 32, graph->n_nodes); ++k) { + if (!is_gate_up_glu(k) || graph->nodes[k]->op != GGML_OP_MUL_MAT) { + continue; + } + ggml_tensor * g0 = graph->nodes[k]; + ggml_tensor * g1 = graph->nodes[k + 1]; + ggml_tensor * glu = graph->nodes[k + 2]; + if (g0->src[0]->op != GGML_OP_NONE || g1->src[0]->op != GGML_OP_NONE || + !(g0->src[1]->op == GGML_OP_NONE || used_node_set.find(g0->src[1]) != used_node_set.end())) { + continue; + } + bool dep_ok = true; + for (int c = first_unused; c < k; ++c) { + if (!used[c] && (is_src_of(g0, graph->nodes[c]) || is_src_of(g1, graph->nodes[c]) || is_src_of(glu, graph->nodes[c]))) { + dep_ok = false; + break; + } + } + if (dep_ok) { + if (params && params->add_alloc_dep && topk_src0) { + params->add_alloc_dep(params->user_data, topk_src0, g0); + params->add_alloc_dep(params->user_data, topk_src0, g1); + params->add_alloc_dep(params->user_data, topk_src0, glu); + } + for (int t = 0; t < 3; ++t) { + new_order.push_back(graph->nodes[k + t]); + used_node_set.insert(graph->nodes[k + t]); + used[k + t] = true; + } + while (first_unused < graph->n_nodes && used[first_unused]) { + first_unused++; + } + break; + } + } + } + return true; }; if (keep_topk_moe_pattern(topk_moe_early_softmax_norm)) { @@ -15661,43 +21477,646 @@ void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * graph, if (keep_topk_moe_pattern(topk_moe_sqrt_softplus_norm_bias)) { continue; } - if (keep_topk_moe_pattern(topk_moe_early_softmax)) { - continue; + if (keep_topk_moe_pattern(topk_moe_early_softmax)) { + continue; + } + if (keep_topk_moe_pattern(topk_moe_late_softmax)) { + continue; + } + if (keep_pattern(snake_pattern)) { + continue; + } + if (keep_pattern(topk_qsa_pattern)) { + continue; + } + + if (keep_pattern(rms_norm_mul_add_mul_pattern)) { + continue; + } + if (keep_pattern(rms_norm_mul_add_pattern)) { + continue; + } + if (keep_pattern(rms_norm_mul_rope_view_set_rows_pattern)) { + continue; + } + if (keep_pattern(rms_norm_view_set_rows_pattern)) { + continue; + } + if (keep_pattern(rope_view_set_rows_pattern)) { + continue; + } + if (match_pattern(hc_post_gate_pattern, first_unused)) { + add_pattern_alloc_deps(hc_post_gate_pattern, first_unused + (int) hc_post_gate_pattern.size() - 1); + keep_pattern(hc_post_gate_pattern); + continue; + } + + // Fused dynamic LoRA reordering: hoist [la_*] ahead of [mm_*, lb_*, ls_*, add_*] and extend alloc lifetimes + if (!gen_lora_fuse_disabled() && ggml_vk_is_apple_agx_coopmat(ctx->device) && !ggml_vk_exact_s8_enabled()) { + auto match_lora5 = [&](int s) -> bool { + if (s < 0 || s + 4 >= graph->n_nodes) { + return false; + } + for (int t = 0; t < 5; ++t) { + if (used[s + t]) { return false; } + } + ggml_tensor * mm = graph->nodes[s + 0]; + ggml_tensor * la = graph->nodes[s + 1]; + ggml_tensor * lb = graph->nodes[s + 2]; + ggml_tensor * ls = graph->nodes[s + 3]; + ggml_tensor * add = graph->nodes[s + 4]; + if (mm->op != GGML_OP_MUL_MAT || la->op != GGML_OP_MUL_MAT || lb->op != GGML_OP_MUL_MAT || + ls->op != GGML_OP_SCALE || add->op != GGML_OP_ADD) { + return false; + } + if (!mm->src[0] || !mm->src[1] || mm->src[0]->op != GGML_OP_NONE || + !gen_supports_pair_type(mm->src[0]->type) || (mm->src[0]->ne[0] & 3) != 0 || + mm->ne[2] != 1 || mm->ne[3] != 1 || mm->src[1]->ne[2] * mm->src[1]->ne[3] != 1 || + mm->type != GGML_TYPE_F32 || + !gen_uses_dmmv_noswap(ctx, mm)) { + return false; + } + if (!(mm->src[1]->op == GGML_OP_NONE || used_node_set.find(mm->src[1]) != used_node_set.end())) { + return false; + } + if (!la->src[0] || la->src[0]->op != GGML_OP_NONE || la->src[0]->type != GGML_TYPE_F32 || + la->src[1] != mm->src[1] || la->type != GGML_TYPE_F32 || + la->ne[0] <= 0 || la->ne[0] > 64) { + return false; + } + if (!lb->src[0] || lb->src[0]->op != GGML_OP_NONE || lb->src[0]->type != GGML_TYPE_F32 || + lb->src[1] != la || lb->src[0]->ne[0] != la->ne[0] || + lb->ne[0] != mm->ne[0] || lb->ne[1] != mm->ne[1] || lb->type != GGML_TYPE_F32) { + return false; + } + if (ls->src[0] != lb || ls->type != GGML_TYPE_F32 || ggml_get_op_params_f32(ls, 1) != 0.0f) { + return false; + } + return ((add->src[0] == mm && add->src[1] == ls) || (add->src[0] == ls && add->src[1] == mm)) && + add->type == GGML_TYPE_F32; + }; + auto emit_node = [&](int idx) { + new_order.push_back(graph->nodes[idx]); + used_node_set.insert(graph->nodes[idx]); + used[idx] = true; + }; + auto emit_tail4 = [&](int s) { + emit_node(s + 0); + emit_node(s + 2); + emit_node(s + 3); + emit_node(s + 4); + }; + auto dep = [&](ggml_tensor * src, ggml_tensor * dst_node) { + if (params && params->add_alloc_dep && src && dst_node) { + params->add_alloc_dep(params->user_data, src, dst_node); + } + }; + + if (match_lora5(first_unused)) { + const int s0 = first_unused; + ggml_tensor * cur = graph->nodes[s0]->src[1]; + const int64_t r0 = graph->nodes[s0 + 1]->ne[0]; + + // Case A: 3-way QKV [Q, K, V] (15 nodes) where Q > K == V + if (match_lora5(s0 + 5) && match_lora5(s0 + 10) && + graph->nodes[s0 + 5]->src[1] == cur && graph->nodes[s0 + 10]->src[1] == cur && + graph->nodes[s0 + 6]->ne[0] == r0 && graph->nodes[s0 + 11]->ne[0] == r0 && + graph->nodes[s0]->src[0]->type == graph->nodes[s0 + 5]->src[0]->type && + graph->nodes[s0 + 5]->src[0]->type == graph->nodes[s0 + 10]->src[0]->type && + graph->nodes[s0]->ne[0] > graph->nodes[s0 + 5]->ne[0] && + graph->nodes[s0 + 5]->ne[0] == graph->nodes[s0 + 10]->ne[0]) { + ggml_tensor * la_q = graph->nodes[s0 + 1], * add_q = graph->nodes[s0 + 4]; + ggml_tensor * la_k = graph->nodes[s0 + 6], * add_k = graph->nodes[s0 + 9]; + ggml_tensor * la_v = graph->nodes[s0 + 11], * add_v = graph->nodes[s0 + 14]; + dep(cur, add_v); + dep(la_q, add_q); + dep(la_k, add_v); + dep(la_v, add_v); + dep(add_k, add_v); + emit_node(s0 + 1); + emit_node(s0 + 6); + emit_node(s0 + 11); + emit_tail4(s0); + emit_tail4(s0 + 5); + emit_tail4(s0 + 10); + while (first_unused < graph->n_nodes && used[first_unused]) { first_unused++; } + continue; + } + + // Case B & C: 2-way paired [0, 1] + optional SWIGLU (11 or 10 nodes) + if (match_lora5(s0 + 5) && + graph->nodes[s0 + 5]->src[1] == cur && + graph->nodes[s0 + 6]->ne[0] == r0 && + graph->nodes[s0]->src[0]->type == graph->nodes[s0 + 5]->src[0]->type && + graph->nodes[s0]->src[0]->ne[0] == graph->nodes[s0 + 5]->src[0]->ne[0]) { + ggml_tensor * la_0 = graph->nodes[s0 + 1], * add_0 = graph->nodes[s0 + 4]; + ggml_tensor * la_1 = graph->nodes[s0 + 6], * add_1 = graph->nodes[s0 + 9]; + const bool has_glu = (s0 + 10 < graph->n_nodes) && !used[s0 + 10] && + graph->nodes[s0 + 10]->op == GGML_OP_GLU && + ggml_get_glu_op(graph->nodes[s0 + 10]) == GGML_GLU_OP_SWIGLU && + ((graph->nodes[s0 + 10]->src[0] == add_0 && graph->nodes[s0 + 10]->src[1] == add_1) || + (graph->nodes[s0 + 10]->src[0] == add_1 && graph->nodes[s0 + 10]->src[1] == add_0)); + if (has_glu) { + ggml_tensor * glu = graph->nodes[s0 + 10]; + dep(cur, glu); + dep(la_0, glu); + dep(la_1, glu); + emit_node(s0 + 1); + emit_node(s0 + 6); + emit_tail4(s0); + emit_tail4(s0 + 5); + emit_node(s0 + 10); + while (first_unused < graph->n_nodes && used[first_unused]) { first_unused++; } + continue; + } + const bool t1_feeds_add = (s0 + 10 < graph->n_nodes) && !used[s0 + 10] && + graph->nodes[s0 + 10]->op == GGML_OP_ADD && + (graph->nodes[s0 + 10]->src[0] == add_1 || graph->nodes[s0 + 10]->src[1] == add_1); + if (!t1_feeds_add && (graph->nodes[s0]->ne[0] & 7) == 0) { + dep(cur, add_1); + dep(la_0, add_1); + dep(la_1, add_1); + dep(add_0, add_1); + emit_node(s0 + 1); + emit_node(s0 + 6); + emit_tail4(s0); + emit_tail4(s0 + 5); + while (first_unused < graph->n_nodes && used[first_unused]) { first_unused++; } + continue; + } + } + + // Case D: Single LoRA + optional residual ADD (6 or 5 nodes) + ggml_tensor * la_0 = graph->nodes[s0 + 1]; + ggml_tensor * add_0 = graph->nodes[s0 + 4]; + ggml_tensor * radd = nullptr; + if (s0 + 5 < graph->n_nodes && !used[s0 + 5] && + graph->nodes[s0 + 5]->op == GGML_OP_ADD && + (graph->nodes[s0 + 5]->src[0] == add_0 || graph->nodes[s0 + 5]->src[1] == add_0)) { + ggml_tensor * bias = (graph->nodes[s0 + 5]->src[0] == add_0) + ? graph->nodes[s0 + 5]->src[1] : graph->nodes[s0 + 5]->src[0]; + if (bias && bias != add_0 && + (bias->op == GGML_OP_NONE || used_node_set.find(bias) != used_node_set.end())) { + radd = graph->nodes[s0 + 5]; + dep(bias, radd); + } + } + ggml_tensor * end_t = radd ? radd : add_0; + dep(cur, end_t); + dep(la_0, end_t); + emit_node(s0 + 1); + emit_tail4(s0); + if (radd) { + emit_node(s0 + 5); + } + while (first_unused < graph->n_nodes && used[first_unused]) { first_unused++; } + continue; + } + } + auto const &mm_softplus_gate_len = [&](int s) -> int { + if (!gen_fuse_enabled("mmgate") || s < 0 || s + 3 >= graph->n_nodes || used[s]) { + return 0; + } + ggml_tensor * mm = graph->nodes[s]; + if (mm->op != GGML_OP_MUL_MAT) { + return 0; + } + int off = 1; + ggml_tensor * mv = graph->nodes[s + 1]; + if ((mv->op == GGML_OP_RESHAPE || mv->op == GGML_OP_VIEW) && mv->view_src == mm) { + off = 2; + } + if (s + off + 2 >= graph->n_nodes) { + return 0; + } + for (int t = 1; t <= off + 2; ++t) { + if (used[s + t]) { + return 0; + } + } + ggml_tensor * v_or_mm = off == 2 ? mv : mm; + ggml_tensor * add = graph->nodes[s + off]; + ggml_tensor * sp = graph->nodes[s + off + 1]; + ggml_tensor * mul = graph->nodes[s + off + 2]; + if (add->op != GGML_OP_ADD || (add->src[0] != v_or_mm && add->src[1] != v_or_mm)) { + return 0; + } + ggml_tensor * dt = add->src[0] == v_or_mm ? add->src[1] : add->src[0]; + if (!dt || dt->op != GGML_OP_NONE) { + return 0; + } + if (sp->op != GGML_OP_UNARY || ggml_get_unary_op(sp) != GGML_UNARY_OP_SOFTPLUS || sp->src[0] != add) { + return 0; + } + if (mul->op != GGML_OP_MUL || (mul->src[0] != sp && mul->src[1] != sp)) { + return 0; + } + ggml_tensor * aw = mul->src[0] == sp ? mul->src[1] : mul->src[0]; + return (aw && aw->op == GGML_OP_NONE) ? (off + 3) : 0; + }; + auto const &mm_sigmoid_len = [&](int s) -> int { + if (!gen_fuse_enabled("mmgate") || s < 0 || s + 1 >= graph->n_nodes || used[s]) { + return 0; + } + ggml_tensor * mm = graph->nodes[s]; + if (mm->op != GGML_OP_MUL_MAT) { + return 0; + } + if (!used[s + 1] && graph->nodes[s + 1]->op == GGML_OP_UNARY && + ggml_get_unary_op(graph->nodes[s + 1]) == GGML_UNARY_OP_SIGMOID && + graph->nodes[s + 1]->src[0] == mm) { + return 2; + } + if (s + 2 < graph->n_nodes && !used[s + 1] && !used[s + 2] && + (graph->nodes[s + 1]->op == GGML_OP_RESHAPE || graph->nodes[s + 1]->op == GGML_OP_VIEW) && + graph->nodes[s + 1]->view_src == mm && + graph->nodes[s + 2]->op == GGML_OP_UNARY && + ggml_get_unary_op(graph->nodes[s + 2]) == GGML_UNARY_OP_SIGMOID && + graph->nodes[s + 2]->src[0] == graph->nodes[s + 1]) { + return 3; + } + return 0; + }; + auto const &in_mm_epilogue_group = [&](int idx) -> bool { + for (int o = 0; o < 5; ++o) { + int len = mm_softplus_gate_len(idx - o); + if (len > o) { + return true; + } + } + for (int o = 0; o < 3; ++o) { + int len = mm_sigmoid_len(idx - o); + if (len > o) { + return true; + } + } + return false; + }; + auto const &match_rms_mul_silu = [&](int s, int out_idxs[5]) -> int { + if (!gen_fuse_enabled("s8") || !gen_fuse_enabled("gate") || s < 0 || s + 3 >= graph->n_nodes || used[s] || used[s + 1]) { + return 0; + } + ggml_tensor * rms = graph->nodes[s]; + ggml_tensor * mul = graph->nodes[s + 1]; + if (rms->op != GGML_OP_RMS_NORM || mul->op != GGML_OP_MUL || (mul->src[0] != rms && mul->src[1] != rms)) { + return 0; + } + int p = s + 2; + while (p < graph->n_nodes && p <= s + 5 && (used[p] || graph->nodes[p]->op == GGML_OP_MUL_MAT)) { + p++; + } + if (p + 1 >= graph->n_nodes) { + return 0; + } + int n = 0; + out_idxs[n++] = s; + out_idxs[n++] = s + 1; + ggml_tensor * maybe_view = nullptr; + if ((graph->nodes[p]->op == GGML_OP_RESHAPE || graph->nodes[p]->op == GGML_OP_VIEW) && !used[p] && p + 2 < graph->n_nodes) { + maybe_view = graph->nodes[p]; + out_idxs[n++] = p; + p++; + } + if (p + 1 >= graph->n_nodes || used[p] || used[p + 1]) { + return 0; + } + ggml_tensor * silu = graph->nodes[p]; + ggml_tensor * out = graph->nodes[p + 1]; + if (silu->op != GGML_OP_UNARY || ggml_get_unary_op(silu) != GGML_UNARY_OP_SILU || out->op != GGML_OP_MUL) { + return 0; + } + if (maybe_view && silu->src[0] != maybe_view) { + return 0; + } + if (!((out->src[0] == mul && out->src[1] == silu) || (out->src[0] == silu && out->src[1] == mul))) { + return 0; + } + out_idxs[n++] = p; + out_idxs[n++] = p + 1; + return n; + }; + auto const &cont_unary_mul_len = [&](int s) -> int { + if (!gen_fuse_enabled("s8") || s < 0 || s + 2 >= graph->n_nodes || used[s]) { + return 0; + } + ggml_tensor * cont = graph->nodes[s]; + if (cont->op != GGML_OP_CONT || !cont->src[0]) { + return 0; + } + int u_off = 1; + while (u_off <= 2 && s + u_off + 1 < graph->n_nodes && is_empty(graph->nodes[s + u_off]) && + graph->nodes[s + u_off]->op != GGML_OP_NONE && !used[s + u_off]) { + u_off++; + } + if (s + u_off + 1 >= graph->n_nodes || used[s + u_off] || used[s + u_off + 1]) { + return 0; + } + ggml_tensor * unary = graph->nodes[s + u_off]; + ggml_tensor * mul = graph->nodes[s + u_off + 1]; + if (unary->op != GGML_OP_UNARY || mul->op != GGML_OP_MUL || + ggml_vk_unary_mul_op_index(ggml_get_unary_op(unary)) < 0) { + return 0; + } + const bool unary_reads_cont = unary->src[0] == cont || + ((unary->src[0]->op == GGML_OP_RESHAPE || unary->src[0]->op == GGML_OP_VIEW) && unary->src[0]->view_src == cont); + if (!unary_reads_cont || (mul->src[0] != unary && mul->src[1] != unary)) { + return 0; + } + return u_off + 2; + }; + auto const &in_s8_protected_group = [&](int idx) -> bool { + int r_idxs[5]; + for (int o = 0; o <= 6; ++o) { + int nr = match_rms_mul_silu(idx - o, r_idxs); + for (int k = 0; k < nr; ++k) { + if (r_idxs[k] == idx) { + return true; + } + } + } + for (int o = 0; o < 5; ++o) { + if (cont_unary_mul_len(idx - o) > o) { + return true; + } + } + return false; + }; + auto const &idxs_srcs_ready = [&](const int * idxs, int len) -> bool { + for (int t = 0; t < len; ++t) { + ggml_tensor * n = graph->nodes[idxs[t]]; + for (uint32_t k = 0; k < GGML_MAX_SRC; ++k) { + ggml_tensor * src = n->src[k]; + if (!src || src->op == GGML_OP_NONE || used_node_set.find(src) != used_node_set.end()) { + continue; + } + bool in_group = false; + for (int p = 0; p < t; ++p) { + if (graph->nodes[idxs[p]] == src) { + in_group = true; + break; + } + } + if (!in_group) { + return false; + } + } + } + return true; + }; + auto const &group_srcs_ready = [&](int s, int len) -> bool { + int seq[8]; + for (int t = 0; t < len; ++t) { + seq[t] = s + t; + } + return idxs_srcs_ready(seq, len); + }; + if (is_gate_up_glu(first_unused)) { + keep_pattern({ graph->nodes[first_unused]->op, graph->nodes[first_unused + 1]->op, GGML_OP_GLU }); + continue; + } + { + int rms_idxs[5]; + int rms_len = match_rms_mul_silu(first_unused, rms_idxs); + if (rms_len > 0 && idxs_srcs_ready(rms_idxs, rms_len)) { + if (rms_len == 5) { + // Emit z_2d view before RMS_NORM so [RMS_NORM, MUL, UNARY, MUL] has no external-source view inside subgraph + new_order.push_back(graph->nodes[rms_idxs[2]]); + used_node_set.insert(graph->nodes[rms_idxs[2]]); + used[rms_idxs[2]] = true; + } + for (int t = 0; t < rms_len; ++t) { + if (rms_len == 5 && t == 2) { + continue; + } + new_order.push_back(graph->nodes[rms_idxs[t]]); + used_node_set.insert(graph->nodes[rms_idxs[t]]); + used[rms_idxs[t]] = true; + } + while (first_unused < graph->n_nodes && used[first_unused]) { + first_unused++; + } + continue; + } } - if (keep_topk_moe_pattern(topk_moe_late_softmax)) { + if (int cum_len = cont_unary_mul_len(first_unused); cum_len > 0 && group_srcs_ready(first_unused, cum_len)) { + ggml_tensor * cont_node = graph->nodes[first_unused]; + for (int t = 1; t < cum_len - 2; ++t) { + ggml_tensor * ev = graph->nodes[first_unused + t]; + if (ev->view_src != cont_node) { + new_order.push_back(ev); + used_node_set.insert(ev); + used[first_unused + t] = true; + } + } + for (int t = 0; t < cum_len; ++t) { + if (!used[first_unused + t]) { + new_order.push_back(graph->nodes[first_unused + t]); + used_node_set.insert(graph->nodes[first_unused + t]); + used[first_unused + t] = true; + } + } + while (first_unused < graph->n_nodes && used[first_unused]) { + first_unused++; + } continue; } - if (keep_pattern(snake_pattern)) { + if (int sp_len = mm_softplus_gate_len(first_unused)) { + for (int t = 0; t < sp_len; ++t) { + new_order.push_back(graph->nodes[first_unused + t]); + used_node_set.insert(graph->nodes[first_unused + t]); + used[first_unused + t] = true; + } + while (first_unused < graph->n_nodes && used[first_unused]) { + first_unused++; + } continue; } - if (keep_pattern(topk_qsa_pattern)) { + if (int sg_len = mm_sigmoid_len(first_unused)) { + for (int t = 0; t < sg_len; ++t) { + new_order.push_back(graph->nodes[first_unused + t]); + used_node_set.insert(graph->nodes[first_unused + t]); + used[first_unused + t] = true; + } + while (first_unused < graph->n_nodes && used[first_unused]) { + first_unused++; + } continue; } - if (keep_pattern(rms_norm_mul_add_mul_pattern)) { - continue; - } - if (keep_pattern(rms_norm_mul_add_pattern)) { - continue; - } - if (keep_pattern(rms_norm_mul_rope_view_set_rows_pattern)) { - continue; - } - if (keep_pattern(rms_norm_view_set_rows_pattern)) { - continue; - } - if (keep_pattern(rope_view_set_rows_pattern)) { - continue; - } - if (match_pattern(hc_post_gate_pattern, first_unused)) { - add_pattern_alloc_deps(hc_post_gate_pattern, first_unused + (int) hc_post_gate_pattern.size() - 1); - keep_pattern(hc_post_gate_pattern); + if (gen_fuse_enabled("s8") && is_empty(graph->nodes[first_unused]) && !in_s8_protected_group(first_unused) && + group_srcs_ready(first_unused, 1)) { + while (first_unused < graph->n_nodes) { + if (used[first_unused]) { + first_unused++; + continue; + } + if (!is_empty(graph->nodes[first_unused]) || in_s8_protected_group(first_unused) || !group_srcs_ready(first_unused, 1)) { + break; + } + new_order.push_back(graph->nodes[first_unused]); + used_node_set.insert(graph->nodes[first_unused]); + used[first_unused] = true; + first_unused++; + } continue; } // First, grab the next unused node. current_set.push_back(first_unused); + // gen (pairmm): pull [ffn_gate_inp_shexp, shared_gate_sigmoid] or sibling MUL_MAT (e.g. wqkv_gate) adjacent to mm0 + if (gen_fuse_enabled("pairmm") && graph->nodes[first_unused]->op == GGML_OP_MUL_MAT && + (graph->nodes[first_unused]->ne[0] & 7) == 0 && + gen_supports_pair_type(graph->nodes[first_unused]->src[0]->type)) { + ggml_tensor * mm0 = graph->nodes[first_unused]; + bool paired = false; + for (int k = first_unused + 1; k + 1 < std::min(first_unused + 48, graph->n_nodes); ++k) { + if (used[k] || used[k + 1]) { + continue; + } + ggml_tensor * mm1 = graph->nodes[k]; + ggml_tensor * sig1 = graph->nodes[k + 1]; + if (mm1->op == GGML_OP_MUL_MAT && mm1->src[1] == mm0->src[1] && + mm1->src[0]->type == mm0->src[0]->type && mm1->ne[0] == 1 && + sig1->op == GGML_OP_UNARY && ggml_get_unary_op(sig1) == GGML_UNARY_OP_SIGMOID && + sig1->src[0] == mm1 && + (mm1->src[0]->op == GGML_OP_NONE || used_node_set.find(mm1->src[0]) != used_node_set.end())) { + current_set.push_back(k); + current_set.push_back(k + 1); + paired = true; + if (gen_fuse_enabled("gateup")) { + for (int g = first_unused + 1; g + 2 < k; ++g) { + if (!used[g] && !used[g + 1] && !used[g + 2] && + is_gate_up_glu(g) && + graph->nodes[g]->op == GGML_OP_MUL_MAT && + graph->nodes[g]->src[1] == mm0->src[1] && + graph->nodes[g]->src[0]->op == GGML_OP_NONE && + graph->nodes[g + 1]->src[0]->op == GGML_OP_NONE) { + current_set.push_back(g); + current_set.push_back(g + 1); + current_set.push_back(g + 2); + break; + } + } + } + break; + } + } + auto const &mm_has_own_fusion = [&](int idx) -> bool { + if (is_gate_up_glu(idx) || is_gate_up_glu(idx - 1) || in_mm_epilogue_group(idx)) { + return true; + } + ggml_tensor * m = graph->nodes[idx]; + for (int u = idx + 1; u < std::min(idx + 12, graph->n_nodes); ++u) { + if (used[u]) { + continue; + } + ggml_tensor * n = graph->nodes[u]; + if ((n->op == GGML_OP_GLU || n->op == GGML_OP_ADD) && (n->src[0] == m || n->src[1] == m)) { + return true; + } + if (gen_fuse_enabled("shexp") && n->op == GGML_OP_MUL && (n->src[0] == m || n->src[1] == m)) { + ggml_tensor * g = n->src[0] == m ? n->src[1] : n->src[0]; + if (g && g->ne[0] == 1 && g->ne[1] == m->ne[1]) { + return true; + } + } + } + return false; + }; + if (!paired && mm0->src[0]->op == GGML_OP_NONE && !mm_has_own_fusion(first_unused)) { + std::vector cands; + for (int k = first_unused + 1; k < std::min(first_unused + 64, graph->n_nodes); ++k) { + if (used[k]) { + continue; + } + ggml_tensor * mm1 = graph->nodes[k]; + if (mm1->op == GGML_OP_MUL_MAT && mm1->src[1] == mm0->src[1] && + mm1->src[0]->op == GGML_OP_NONE && mm1->src[0]->type == mm0->src[0]->type && + mm1->ne[1] == mm0->ne[1] && mm1->src[0]->ne[0] == mm0->src[0]->ne[0] && + !mm_has_own_fusion(k)) { + cands.push_back(k); + if (cands.size() >= 2) { + break; + } + } + } + // In [Q, K, V] with matching types (Q larger than K == V), emit [Q, K, V] together in one wave: + // Q runs standalone and [K, V] fuse into GEN_PAIR_MM in the same barrier-free wave. + const bool is_qkv_head = cands.size() >= 2 && + mm0->ne[0] > graph->nodes[cands[0]]->ne[0] && + graph->nodes[cands[0]]->ne[0] == graph->nodes[cands[1]]->ne[0]; + if (!cands.empty() && !is_qkv_head) { + current_set.push_back(cands[0]); + paired = true; + if (gen_fuse_enabled("s8")) { + for (int k = first_unused + 1; k < std::min(first_unused + 48, graph->n_nodes); ++k) { + if (used[k] || std::find(current_set.begin(), current_set.end(), k) != current_set.end()) { + continue; + } + int ep_len = mm_softplus_gate_len(k); + if (!ep_len) { + ep_len = mm_sigmoid_len(k); + } + if (ep_len > 0 && graph->nodes[k]->src[0]->op == GGML_OP_NONE && graph->nodes[k]->src[1] == mm0->src[1]) { + for (int t = 0; t < ep_len; ++t) { + current_set.push_back(k + t); + } + k += ep_len - 1; + } + } + } + } else if (is_qkv_head) { + current_set.push_back(cands[0]); + current_set.push_back(cands[1]); + paired = true; + } else if (gen_fuse_enabled("s8")) { + // Mixed-quant GDN layer (wqkv and z differ in quant type): still group z + ssm_alpha + ssm_beta with wqkv + bool has_gdn_ep = false; + for (int k = first_unused + 1; k < std::min(first_unused + 48, graph->n_nodes); ++k) { + if (!used[k] && (mm_softplus_gate_len(k) > 0 || mm_sigmoid_len(k) > 0) && + graph->nodes[k]->src[0]->op == GGML_OP_NONE && graph->nodes[k]->src[1] == mm0->src[1]) { + has_gdn_ep = true; + break; + } + } + if (has_gdn_ep) { + for (int k = first_unused + 1; k < std::min(first_unused + 48, graph->n_nodes); ++k) { + if (used[k]) { + continue; + } + int ep_len = mm_softplus_gate_len(k); + if (!ep_len) { + ep_len = mm_sigmoid_len(k); + } + if (ep_len > 0 && graph->nodes[k]->src[0]->op == GGML_OP_NONE && graph->nodes[k]->src[1] == mm0->src[1]) { + for (int t = 0; t < ep_len; ++t) { + current_set.push_back(k + t); + } + k += ep_len - 1; + paired = true; + } else if (graph->nodes[k]->op == GGML_OP_MUL_MAT && graph->nodes[k]->src[0]->op == GGML_OP_NONE && + graph->nodes[k]->src[1] == mm0->src[1] && !mm_has_own_fusion(k)) { + current_set.push_back(k); + paired = true; + } + } + } + } + } + if (paired) { + for (auto c : current_set) { + new_order.push_back(graph->nodes[c]); + used_node_set.insert(graph->nodes[c]); + used[c] = true; + } + while (first_unused < graph->n_nodes && used[first_unused]) { + first_unused++; + } + continue; + } + } + // Loop through the next N nodes. Grab any that don't depend on other nodes that // haven't already been run. Nodes that have already been run have used[i] set // to true. Allow nodes that depend on the previous node if it's a fusion pattern @@ -15705,13 +22124,77 @@ void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * graph, // This first pass only grabs "real" (non-view nodes). Second pass grabs view nodes. // The goal is to not interleave real and view nodes in a way that breaks fusion. const int NUM_TO_CHECK = 20; - for (int j = first_unused+1; j < std::min(first_unused + NUM_TO_CHECK, graph->n_nodes); ++j) { + const auto pass1_limit = [&]() { + int lim = first_unused + NUM_TO_CHECK; + if (gen_fuse_enabled("s8") && !current_set.empty() && graph->nodes[first_unused]->op == GGML_OP_ADD) { + lim = std::max(lim, current_set.back() + NUM_TO_CHECK); + } + return std::min(lim, graph->n_nodes); + }; + for (int j = first_unused+1; j < pass1_limit(); ++j) { if (used[j]) { continue; } if (is_empty(graph->nodes[j])) { continue; } + if (is_gate_up_glu(j - 1) || is_gate_up_glu(j - 2) || in_mm_epilogue_group(j) || in_s8_protected_group(j)) { + continue; + } + const bool is_zero_scale = graph->nodes[j]->op == GGML_OP_SCALE && + ggml_get_op_params_f32(graph->nodes[j], 0) == 0.0f; + if (gen_fuse_enabled("s8") && + ((graph->nodes[first_unused]->op == GGML_OP_MUL_MAT_ID && graph->nodes[j]->op == GGML_OP_GET_ROWS) || + (graph->nodes[first_unused]->op != GGML_OP_ADD && is_zero_scale))) { + continue; + } + if (gen_fuse_enabled("s8") && gen_fuse_enabled("pairmm") && + graph->nodes[first_unused]->op != GGML_OP_MUL_MAT && + graph->nodes[j]->op == GGML_OP_MUL_MAT && graph->nodes[j]->src[0]->op == GGML_OP_NONE && + (graph->nodes[j]->ne[0] & 7) == 0 && gen_supports_pair_type(graph->nodes[j]->src[0]->type)) { + continue; + } + if (gen_fuse_enabled("s8") && graph->nodes[current_set.back()]->op == GGML_OP_ADD && graph->nodes[j]->op != GGML_OP_ADD) { + ggml_tensor * last_add = graph->nodes[current_set.back()]; + bool more_add_ahead = false; + for (int a = j + 1; a < pass1_limit(); ++a) { + if (used[a] || graph->nodes[a]->op != GGML_OP_ADD) { + continue; + } + ggml_tensor * na = graph->nodes[a]; + if (na->src[0] != last_add && na->src[1] != last_add) { + continue; + } + ggml_tensor * other = (na->src[0] == last_add) ? na->src[1] : na->src[0]; + if (other && (other->op == GGML_OP_NONE || used_node_set.find(other) != used_node_set.end() || + (other->view_src && (other->view_src->op == GGML_OP_NONE || used_node_set.find(other->view_src) != used_node_set.end())))) { + more_add_ahead = true; + break; + } + } + if (more_add_ahead) { + continue; + } + } + if (is_gate_up_glu(j)) { + bool pair_ok = true; + for (int c = first_unused; c < j; ++c) { + if (!used[c] && std::find(current_set.begin(), current_set.end(), c) == current_set.end() && + (is_src_of(graph->nodes[j], graph->nodes[c]) || is_src_of(graph->nodes[j + 1], graph->nodes[c]))) { + pair_ok = false; + break; + } + } + if (pair_ok) { + current_set.push_back(j); + current_set.push_back(j + 1); + current_set.push_back(j + 2); + used[j] = true; + used[j + 1] = true; + used[j + 2] = true; + } + continue; + } // Protect every interior QSA node (not just the start): the mask branch is // independent, so it gets pulled out and breaks keep_pattern otherwise. auto const &in_qsa_pattern = [&](int n) -> bool { @@ -15738,22 +22221,54 @@ void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * graph, continue; } bool ok = true; + std::vector pending_empty_deps; + const bool allow_empty_chain = gen_fuse_enabled("s8") && + graph->nodes[current_set.back()]->op == GGML_OP_ADD && + (is_zero_scale || graph->nodes[j]->op == GGML_OP_GET_ROWS); for (int c = first_unused; c < j; ++c) { if (!used[c] && - is_src_of(graph->nodes[j], graph->nodes[c]) && + is_src_of(graph->nodes[j], graph->nodes[c])) { + if (allow_empty_chain && is_empty(graph->nodes[c]) && !in_s8_protected_group(c)) { + bool emp_ok = true; + for (int e = first_unused; e < c; ++e) { + if (!used[e] && !is_empty(graph->nodes[e]) && + std::find(current_set.begin(), current_set.end(), e) == current_set.end() && + is_src_of(graph->nodes[c], graph->nodes[e])) { + emp_ok = false; + break; + } + } + if (emp_ok) { + pending_empty_deps.push_back(c); + continue; + } + } + if ( !(c == current_set.back() && graph->nodes[c]->op == GGML_OP_RMS_NORM && graph->nodes[j]->op == GGML_OP_MUL && empty_or_scheduled_between(c+1, j)) && + // gen: keep mat-vec + SWIGLU adjacent for the GGML_GEN_FUSE=glu / gateup fusion + !((gen_fuse_enabled("glu") || gen_fuse_enabled("gateup")) && graph->nodes[j]->op == GGML_OP_GLU && + std::find(current_set.begin(), current_set.end(), c) != current_set.end() && + (graph->nodes[current_set.back()]->op == GGML_OP_MUL_MAT || graph->nodes[current_set.back()]->op == GGML_OP_MUL_MAT_ID) && + (graph->nodes[j]->src[0] == graph->nodes[current_set.back()] || graph->nodes[j]->src[1] == graph->nodes[current_set.back()])) && !(c == current_set.back() && graph->nodes[c]->op == GGML_OP_UNARY && graph->nodes[j]->op == GGML_OP_MUL && empty_or_scheduled_between(c+1, j)) && !(j == c+1 && c == current_set.back() && graph->nodes[c]->op == GGML_OP_MUL_MAT && graph->nodes[j]->op == GGML_OP_ADD) && !(j == c+1 && c == current_set.back() && graph->nodes[c]->op == GGML_OP_MUL_MAT_ID && graph->nodes[j]->op == GGML_OP_ADD_ID) && !(j == c+1 && c == current_set.back() && graph->nodes[c]->op == GGML_OP_MUL_MAT_ID && graph->nodes[j]->op == GGML_OP_MUL) && - !(j == c+1 && c == current_set.back() && graph->nodes[c]->op == GGML_OP_ADD && graph->nodes[j]->op == GGML_OP_ADD) && + !((j == c+1 || (gen_fuse_enabled("s8") && empty_or_scheduled_between(c+1, j))) && c == current_set.back() && graph->nodes[c]->op == GGML_OP_ADD && graph->nodes[j]->op == GGML_OP_ADD) && !(j == c+1 && c == current_set.back() && graph->nodes[c]->op == GGML_OP_SSM_CONV && graph->nodes[j]->op == GGML_OP_ADD) && !(j == c+1 && c == current_set.back() && graph->nodes[c]->op == GGML_OP_SSM_CONV && graph->nodes[j]->op == GGML_OP_UNARY)) { - ok = false; - break; + ok = false; + break; + } } } if (ok) { + for (int ec : pending_empty_deps) { + if (!used[ec] && std::find(current_set.begin(), current_set.end(), ec) == current_set.end()) { + current_set.push_back(ec); + used[ec] = true; + } + } current_set.push_back(j); int rope_idx = j; @@ -15827,6 +22342,80 @@ void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * graph, } } } + // gen (s8): overlap ffn_moe_down + ffn_moe_weighted with ready ffn_shexp + ffn_shexp_gated so all 9 ADDs fuse + if (gen_fuse_enabled("s8") && gen_fuse_enabled("shexp") && + current_set.size() >= 2 && graph->nodes[current_set.back()]->op == GGML_OP_MUL) { + const int prev_idx = current_set[current_set.size() - 2]; + const int mmid_idx = graph->nodes[prev_idx]->op == GGML_OP_MUL_MAT_ID ? prev_idx : + (current_set.size() >= 3 && graph->nodes[prev_idx]->op == GGML_OP_ADD_ID && + graph->nodes[current_set[current_set.size() - 3]]->op == GGML_OP_MUL_MAT_ID ? current_set[current_set.size() - 3] : -1); + if (mmid_idx >= 0) { + ggml_tensor * mmid_node = graph->nodes[mmid_idx]; + ggml_tensor * mmid_mul = graph->nodes[current_set.back()]; + const int scan_hi = std::min(current_set.back() + 28, graph->n_nodes); + for (int k = current_set.back() + 1; k < scan_hi; ++k) { + if (used[k] || graph->nodes[k]->op != GGML_OP_MUL_MAT) { + continue; + } + ggml_tensor * mm = graph->nodes[k]; + if (mm->src[0]->op != GGML_OP_NONE || !mm->src[1] || + !(mm->src[1]->op == GGML_OP_NONE || used_node_set.find(mm->src[1]) != used_node_set.end())) { + continue; + } + int found_mul = -1; + for (int m = k + 1; m < std::min(k + 6, graph->n_nodes); ++m) { + if (used[m] || !empty_or_scheduled_between(k + 1, m)) { + continue; + } + ggml_tensor * mul = graph->nodes[m]; + if (mul->op != GGML_OP_MUL || (mul->src[0] != mm && mul->src[1] != mm)) { + continue; + } + ggml_tensor * gate = (mul->src[0] == mm) ? mul->src[1] : mul->src[0]; + if (gate && gate->ne[0] == 1 && gate->ne[1] == mm->ne[1] && + (gate->op == GGML_OP_NONE || used_node_set.find(gate) != used_node_set.end())) { + found_mul = m; + } + break; + } + if (found_mul < 0) { + continue; + } + bool dep_ok = true; + for (int c = first_unused; c < found_mul; ++c) { + if (!used[c] && c != k && std::find(current_set.begin(), current_set.end(), c) == current_set.end() && + (is_src_of(mm, graph->nodes[c]) || is_src_of(graph->nodes[found_mul], graph->nodes[c]))) { + dep_ok = false; + break; + } + } + if (dep_ok) { + if (params && params->add_alloc_dep) { + ggml_tensor * shexp_outs[2] = { mm, graph->nodes[found_mul] }; + for (ggml_tensor * out_t : shexp_outs) { + if (mmid_node->src[1]) { + params->add_alloc_dep(params->user_data, mmid_node->src[1], out_t); + } + if (mmid_node->src[2]) { + params->add_alloc_dep(params->user_data, mmid_node->src[2], out_t); + } + if (mmid_mul->src[0]) { + params->add_alloc_dep(params->user_data, mmid_mul->src[0], out_t); + } + if (mmid_mul->src[1]) { + params->add_alloc_dep(params->user_data, mmid_mul->src[1], out_t); + } + } + } + current_set.push_back(k); + used[k] = true; + current_set.push_back(found_mul); + used[found_mul] = true; + break; + } + } + } + } // Look for MUL_MAT + ADD + ADD if (j > 0 && graph->nodes[j]->op == GGML_OP_ADD && @@ -15885,7 +22474,7 @@ void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * graph, if (used[j]) { continue; } - if (!is_empty(graph->nodes[j])) { + if (!is_empty(graph->nodes[j]) || in_s8_protected_group(j)) { continue; } bool ok = true; @@ -15919,6 +22508,336 @@ void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * graph, for (int i = 0; i < graph->n_nodes; ++i) { graph->nodes[i] = new_order[i]; } + if (params && params->add_alloc_dep) { + if (gen_fuse_enabled("gateup")) { + for (int i = 0; i + 2 < graph->n_nodes; ++i) { + ggml_tensor * n0 = graph->nodes[i]; + ggml_tensor * n1 = graph->nodes[i + 1]; + ggml_tensor * glu = graph->nodes[i + 2]; + if ((n0->op == GGML_OP_MUL_MAT || n0->op == GGML_OP_MUL_MAT_ID) && + n1->op == n0->op && n0->src[1] == n1->src[1] && + glu->op == GGML_OP_GLU && + ((glu->src[0] == n0 && glu->src[1] == n1) || (glu->src[0] == n1 && glu->src[1] == n0))) { + if (n0->src[1]) { + params->add_alloc_dep(params->user_data, n0->src[1], glu); + } + if (n0->src[2]) { + params->add_alloc_dep(params->user_data, n0->src[2], glu); + } + } + } + } + auto base_tensor_opt = [](ggml_tensor * t) -> ggml_tensor * { + while (t && t->view_src) { + t = t->view_src; + } + return t; + }; + auto add_dep_safe = [&](ggml_tensor * a, ggml_tensor * b) { + if (a && b && a != b) { + params->add_alloc_dep(params->user_data, a, b); + } + }; + if (gen_fuse_enabled("pairmm")) { + for (int i = 0; i + 1 < graph->n_nodes; ++i) { + ggml_tensor * n0 = graph->nodes[i]; + if (n0->op != GGML_OP_MUL_MAT) { + continue; + } + if (!s10_pair_gdn_disabled() && gen_fuse_enabled("mmgate") && i + 5 < graph->n_nodes) { + int pos = i + 1; + if (pos < graph->n_nodes && is_empty(graph->nodes[pos]) && graph->nodes[pos]->view_src == n0) { + pos++; + } + if (pos + 4 < graph->n_nodes && + graph->nodes[pos]->op == GGML_OP_ADD && + graph->nodes[pos + 1]->op == GGML_OP_UNARY && + ggml_get_unary_op(graph->nodes[pos + 1]) == GGML_UNARY_OP_SOFTPLUS && + graph->nodes[pos + 2]->op == GGML_OP_MUL && + graph->nodes[pos + 3]->op == GGML_OP_MUL_MAT && + graph->nodes[pos + 3]->src[1] == n0->src[1]) { + ggml_tensor * mul0 = graph->nodes[pos + 2]; + ggml_tensor * mm1 = graph->nodes[pos + 3]; + int pos1 = pos + 4; + if (pos1 < graph->n_nodes && is_empty(graph->nodes[pos1]) && graph->nodes[pos1]->view_src == mm1) { + pos1++; + } + if (pos1 < graph->n_nodes && graph->nodes[pos1]->op == GGML_OP_UNARY && + ggml_get_unary_op(graph->nodes[pos1]) == GGML_UNARY_OP_SIGMOID) { + ggml_tensor * sig1 = graph->nodes[pos1]; + add_dep_safe(base_tensor_opt(n0->src[1]), sig1); + add_dep_safe(mul0, sig1); + continue; + } + } + } + ggml_tensor * n1 = graph->nodes[i + 1]; + if (n1->op == GGML_OP_MUL_MAT && n0 != n1 && n0->src[1] == n1->src[1] && n0->src[0]->type == n1->src[0]->type) { + ggml_tensor * last_out = n1; + int s_pos = i + 2; + if (s_pos + 1 < graph->n_nodes && is_empty(graph->nodes[s_pos]) && graph->nodes[s_pos]->view_src == n1) { + s_pos++; + } + if (s_pos < graph->n_nodes && graph->nodes[s_pos]->op == GGML_OP_UNARY && + ggml_get_unary_op(graph->nodes[s_pos]) == GGML_UNARY_OP_SIGMOID && + base_tensor_opt(graph->nodes[s_pos]->src[0]) == n1) { + last_out = graph->nodes[s_pos]; + } + add_dep_safe(base_tensor_opt(n0->src[1]), last_out); + add_dep_safe(n0, last_out); + } + } + } + if (!s10_ssm_conv_disabled()) { + for (int i = 0; i < graph->n_nodes; ++i) { + ggml_tensor * n0 = graph->nodes[i]; + if (n0->op == GGML_OP_CONCAT && i + 3 < graph->n_nodes && + graph->nodes[i + 1]->op == GGML_OP_VIEW && + graph->nodes[i + 2]->op == GGML_OP_CPY && + graph->nodes[i + 3]->op == GGML_OP_SSM_CONV && + graph->nodes[i + 3]->src[0] == n0) { + ggml_tensor * cpy = graph->nodes[i + 2]; + ggml_tensor * dst = graph->nodes[i + 3]; + if (i + 4 < graph->n_nodes && graph->nodes[i + 4]->op == GGML_OP_UNARY && + ggml_get_unary_op(graph->nodes[i + 4]) == GGML_UNARY_OP_SILU && + graph->nodes[i + 4]->src[0] == dst) { + dst = graph->nodes[i + 4]; + } + add_dep_safe(base_tensor_opt(n0->src[0]), dst); + add_dep_safe(base_tensor_opt(n0->src[1]), dst); + add_dep_safe(base_tensor_opt(n0->src[1]), cpy); + add_dep_safe(base_tensor_opt(cpy), dst); + } else if (n0->op == GGML_OP_CPY && i + 1 < graph->n_nodes && + graph->nodes[i + 1]->op == GGML_OP_SSM_CONV && + n0->src[0] && n0->src[0]->view_src == graph->nodes[i + 1]->src[0]) { + ggml_tensor * dst = graph->nodes[i + 1]; + if (i + 2 < graph->n_nodes && graph->nodes[i + 2]->op == GGML_OP_UNARY && + ggml_get_unary_op(graph->nodes[i + 2]) == GGML_UNARY_OP_SILU && + graph->nodes[i + 2]->src[0] == dst) { + dst = graph->nodes[i + 2]; + } + add_dep_safe(base_tensor_opt(graph->nodes[i + 1]->src[0]), dst); + add_dep_safe(base_tensor_opt(n0), dst); + } else if (n0->op == GGML_OP_SSM_CONV && n0->src[0] && n0->src[0]->op == GGML_OP_CONCAT) { + ggml_tensor * concat = n0->src[0]; + ggml_tensor * dst = n0; + if (i + 1 < graph->n_nodes && graph->nodes[i + 1]->op == GGML_OP_UNARY && + ggml_get_unary_op(graph->nodes[i + 1]) == GGML_UNARY_OP_SILU && + graph->nodes[i + 1]->src[0] == dst) { + dst = graph->nodes[i + 1]; + } + add_dep_safe(base_tensor_opt(concat->src[0]), dst); + add_dep_safe(base_tensor_opt(concat->src[1]), dst); + add_dep_safe(base_tensor_opt(concat), dst); + for (int k = std::max(0, i - 24); k < i; ++k) { + ggml_tensor * sr = graph->nodes[k]; + if (sr->op == GGML_OP_SET_ROWS && sr->src[0] && sr->src[0]->view_src == concat) { + add_dep_safe(base_tensor_opt(sr), dst); + break; + } + } + } + } + } + if (gen_fuse_enabled("l2")) { + for (int i = 0; i < graph->n_nodes; ++i) { + ggml_tensor * rms0 = graph->nodes[i]; + if (rms0->op != GGML_OP_RMS_NORM || !rms0->src[0]) { + continue; + } + ggml_tensor * sc0 = nullptr; + int sc0_idx = -1; + for (int j = i + 1; j < std::min(graph->n_nodes, i + 24); ++j) { + if (graph->nodes[j]->op == GGML_OP_SCALE && graph->nodes[j]->src[0] == rms0) { + sc0 = graph->nodes[j]; + sc0_idx = j; + break; + } + } + if (!sc0) { + continue; + } + add_dep_safe(base_tensor_opt(rms0->src[0]), sc0); + for (int j = i + 1; j <= std::min(graph->n_nodes - 1, sc0_idx + 6); ++j) { + if (j == sc0_idx || graph->nodes[j]->op != GGML_OP_RMS_NORM || !graph->nodes[j]->src[0]) { + continue; + } + ggml_tensor * rms1 = graph->nodes[j]; + for (int k = j + 1; k <= std::min(graph->n_nodes - 1, sc0_idx + 8); ++k) { + if (graph->nodes[k]->op == GGML_OP_SCALE && graph->nodes[k]->src[0] == rms1 && k > sc0_idx) { + ggml_tensor * sc1 = graph->nodes[k]; + add_dep_safe(base_tensor_opt(rms0->src[0]), sc1); + add_dep_safe(base_tensor_opt(rms1->src[0]), sc1); + add_dep_safe(base_tensor_opt(sc0), sc1); + break; + } + } + break; + } + } + } + if (!s10_add_rms_disabled()) { + for (int i = 0; i < graph->n_nodes; ++i) { + if (graph->nodes[i]->op != GGML_OP_ADD) { + continue; + } + int end = i; + while (end + 1 < graph->n_nodes && graph->nodes[end + 1]->op == GGML_OP_ADD && end - i + 1 < MAX_FUSED_ADDS) { + end++; + } + if (end == i && i > 0) { + const ggml_tensor * prev_node = graph->nodes[i - 1]; + if ((prev_node->op == GGML_OP_MUL_MAT || prev_node->op == GGML_OP_MUL_MAT_ID) && + (graph->nodes[i]->src[0] == prev_node || graph->nodes[i]->src[1] == prev_node)) { + continue; + } + } + ggml_tensor * last_add = graph->nodes[end]; + int rms_idx = -1; + for (int j = end + 1; j < std::min(graph->n_nodes, end + 17); ++j) { + if (graph->nodes[j]->op == GGML_OP_RMS_NORM && graph->nodes[j]->src[0] == last_add) { + rms_idx = j; + break; + } + } + if (rms_idx >= 0) { + ggml_tensor * rms = graph->nodes[rms_idx]; + ggml_tensor * final_dst = rms; + int final_idx = rms_idx; + if (rms_idx + 1 < graph->n_nodes && graph->nodes[rms_idx + 1]->op == GGML_OP_MUL && + (graph->nodes[rms_idx + 1]->src[0] == rms || graph->nodes[rms_idx + 1]->src[1] == rms)) { + final_dst = graph->nodes[rms_idx + 1]; + final_idx = rms_idx + 1; + } + for (int a = i; a <= end; ++a) { + for (int s = 0; s < 2; ++s) { + add_dep_safe(base_tensor_opt(graph->nodes[a]->src[s]), final_dst); + } + } + add_dep_safe(base_tensor_opt(last_add), final_dst); + for (int k = end + 1; k < final_idx; ++k) { + if (k == rms_idx || is_empty(graph->nodes[k])) { + continue; + } + add_dep_safe(base_tensor_opt(graph->nodes[k]), final_dst); + for (int s = 0; s < GGML_MAX_SRC; ++s) { + add_dep_safe(base_tensor_opt(graph->nodes[k]->src[s]), final_dst); + } + } + i = end; + } + } + } + if (gen_fuse_enabled("mmgate")) { + for (int i = 0; i < graph->n_nodes; ++i) { + ggml_tensor * mm = graph->nodes[i]; + if (mm->op != GGML_OP_MUL_MAT || !mm->src[1]) { + continue; + } + auto same_mm = [&](const ggml_tensor * t) { + return t == mm || ((t->op == GGML_OP_RESHAPE || t->op == GGML_OP_VIEW) && t->view_src == mm); + }; + for (int j = i + 1; j < std::min(i + 32, graph->n_nodes); ++j) { + ggml_tensor * n = graph->nodes[j]; + if (n->op == GGML_OP_UNARY && ggml_get_unary_op(n) == GGML_UNARY_OP_SIGMOID && same_mm(n->src[0])) { + params->add_alloc_dep(params->user_data, mm->src[1], n); + break; + } + if (n->op == GGML_OP_ADD && j + 2 < graph->n_nodes && (same_mm(n->src[0]) || same_mm(n->src[1]))) { + ggml_tensor * sp = graph->nodes[j + 1]; + ggml_tensor * mul = graph->nodes[j + 2]; + if (sp->op == GGML_OP_UNARY && ggml_get_unary_op(sp) == GGML_UNARY_OP_SOFTPLUS && sp->src[0] == n && + mul->op == GGML_OP_MUL && (mul->src[0] == sp || mul->src[1] == sp)) { + params->add_alloc_dep(params->user_data, mm->src[1], mul); + } + break; + } + } + } + } + if (gen_fuse_enabled("shexp")) { + for (int i = 0; i < graph->n_nodes; ++i) { + ggml_tensor * mm = graph->nodes[i]; + if (mm->op != GGML_OP_MUL_MAT || !mm->src[1]) { + continue; + } + for (int j = i + 1; j < std::min(i + 16, graph->n_nodes); ++j) { + ggml_tensor * mul = graph->nodes[j]; + if (mul->op == GGML_OP_MUL && (mul->src[0] == mm || mul->src[1] == mm)) { + ggml_tensor * gate = (mul->src[0] == mm) ? mul->src[1] : mul->src[0]; + if (gate && gate->ne[0] == 1 && gate->ne[1] == mm->ne[1]) { + params->add_alloc_dep(params->user_data, mm->src[1], mul); + params->add_alloc_dep(params->user_data, gate, mul); + } + break; + } + } + } + } + if (gen_fuse_enabled("s8")) { + auto base_tensor = [](ggml_tensor * t) -> ggml_tensor * { + while (t && t->view_src) { + t = t->view_src; + } + return t; + }; + for (int i = 0; i < graph->n_nodes; ++i) { + ggml_tensor * n0 = graph->nodes[i]; + if (n0->op == GGML_OP_RMS_NORM && i + 3 < graph->n_nodes && graph->nodes[i + 1]->op == GGML_OP_MUL) { + int silu_off = 2; + if ((graph->nodes[i + 2]->op == GGML_OP_RESHAPE || graph->nodes[i + 2]->op == GGML_OP_VIEW) && i + 4 < graph->n_nodes) { + silu_off = 3; + } + ggml_tensor * silu = graph->nodes[i + silu_off]; + ggml_tensor * out = graph->nodes[i + silu_off + 1]; + if (silu->op == GGML_OP_UNARY && ggml_get_unary_op(silu) == GGML_UNARY_OP_SILU && + out->op == GGML_OP_MUL && (out->src[0] == silu || out->src[1] == silu)) { + if (ggml_tensor * b0 = base_tensor(n0->src[0])) { + params->add_alloc_dep(params->user_data, b0, out); + } + if (ggml_tensor * bz = base_tensor(silu->src[0])) { + params->add_alloc_dep(params->user_data, bz, out); + } + } + } + if (n0->op == GGML_OP_CONT && n0->src[0] && i + 2 < graph->n_nodes) { + int u_off = 1; + while (u_off <= 2 && i + u_off + 1 < graph->n_nodes && is_empty(graph->nodes[i + u_off]) && + graph->nodes[i + u_off]->op != GGML_OP_NONE) { + u_off++; + } + if (i + u_off + 1 < graph->n_nodes) { + ggml_tensor * unary = graph->nodes[i + u_off]; + ggml_tensor * mul = graph->nodes[i + u_off + 1]; + if (unary->op == GGML_OP_UNARY && mul->op == GGML_OP_MUL && + (mul->src[0] == unary || mul->src[1] == unary)) { + if (ggml_tensor * bg = base_tensor(n0->src[0])) { + params->add_alloc_dep(params->user_data, bg, mul); + } + } + } + } + if (n0->op == GGML_OP_ADD) { + int end = i; + while (end + 1 < graph->n_nodes && graph->nodes[end + 1]->op == GGML_OP_ADD && end - i + 1 < MAX_FUSED_ADDS) { + end++; + } + if (end - i + 1 >= 3) { + ggml_tensor * last_add = graph->nodes[end]; + for (int a = i; a <= end; ++a) { + for (int s = 0; s < 2; ++s) { + ggml_tensor * src = graph->nodes[a]->src[s]; + if (src && src->view_src) { + params->add_alloc_dep(params->user_data, src->view_src, last_add); + } + } + } + i = end; + } + } + } + } + } } static void ggml_backend_vk_event_record(ggml_backend_t backend, ggml_backend_event_t event) { @@ -16280,7 +23199,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm { ggml_type src0_type = op->src[0]->type; if (op->op == GGML_OP_MUL_MAT_ID) { - if (!ggml_vk_use_mul_mat_vec_id(op)) { + if (!ggml_vk_use_mul_mat_vec_id(op, device->mmv_id_max_tokens)) { const ggml_tensor * ids = op->src[2]; // Shaders store row IDs as uint16_t slot and token indices. if (ids->ne[0] > int64_t(UINT16_MAX) + 1 || ids->ne[1] > int64_t(UINT16_MAX) + 1) { diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/argmax.comp b/ggml/src/ggml-vulkan/vulkan-shaders/argmax.comp index 7c128776710e..aea76f5f61f2 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/argmax.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/argmax.comp @@ -9,52 +9,63 @@ layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in; -layout (binding = 0) readonly buffer A {A_TYPE data_a[];}; -layout (binding = 1) writeonly buffer D {D_TYPE data_d[];}; +layout (binding = 0) readonly buffer A { A_TYPE data_a[]; }; +layout (binding = 0) readonly buffer AV4 { vec4 data_a_v4[]; }; +layout (binding = 1) writeonly buffer D { D_TYPE data_d[]; }; -layout (constant_id = 0) const uint BLOCK_SIZE = 32; +layout (constant_id = 0) const uint BLOCK_SIZE = 512; -shared FLOAT_TYPE tmpmax[BLOCK_SIZE]; -shared uint tmp[BLOCK_SIZE]; +shared FLOAT_TYPE tmpmax[512]; +shared uint tmp[512]; + +void update_best(inout FLOAT_TYPE amax, inout uint acol, FLOAT_TYPE val, uint idx) { + if (val > amax || (val == amax && idx < acol)) { + amax = val; + acol = idx; + } +} void main() { const uint row = gl_WorkGroupID.z * 262144 + gl_WorkGroupID.y * 512 + gl_WorkGroupID.x; - const uint col = gl_LocalInvocationID.x; + const uint tid = gl_LocalInvocationID.x; if (row >= p.KY) { return; } - A_TYPE amax = -FLT_MAX; - uint acol = col; - - if (col < p.KX) { - amax = data_a[row*p.KX + col]; - } + FLOAT_TYPE amax = FLOAT_TYPE(uintBitsToFloat(0xFF800000u)); + uint acol = 0xFFFFFFFFu; + const uint row_base = row * p.KX; - for (uint i = col + BLOCK_SIZE; i < p.KX; i += BLOCK_SIZE) { - A_TYPE val = data_a[row*p.KX + i]; - if (val > amax) { - amax = val; - acol = i; + if ((p.KX & 3u) == 0u) { + const uint kx4 = p.KX >> 2u; + const uint base4 = row_base >> 2u; + for (uint v4_i = tid; v4_i < kx4; v4_i += BLOCK_SIZE) { + const vec4 v = data_a_v4[base4 + v4_i]; + const uint c0 = v4_i << 2u; + update_best(amax, acol, FLOAT_TYPE(v.x), c0 + 0u); + update_best(amax, acol, FLOAT_TYPE(v.y), c0 + 1u); + update_best(amax, acol, FLOAT_TYPE(v.z), c0 + 2u); + update_best(amax, acol, FLOAT_TYPE(v.w), c0 + 3u); + } + } else { + for (uint i = tid; i < p.KX; i += BLOCK_SIZE) { + update_best(amax, acol, FLOAT_TYPE(data_a[row_base + i]), i); } } - tmp[col] = acol; - tmpmax[col] = amax; - + tmp[tid] = acol; + tmpmax[tid] = amax; barrier(); - [[unroll]] for (int s = int(BLOCK_SIZE) / 2; s > 0; s >>= 1) { - if (col < s && col + s < p.KX) { - if (tmpmax[col] < tmpmax[col + s]) { - tmpmax[col] = tmpmax[col + s]; - tmp[col] = tmp[col + s]; - } + + [[unroll]] for (uint s = BLOCK_SIZE >> 1u; s > 0u; s >>= 1u) { + if (tid < s) { + update_best(tmpmax[tid], tmp[tid], tmpmax[tid + s], tmp[tid + s]); } barrier(); } - if (col == 0) { + if (tid == 0u) { data_d[row] = D_TYPE(tmp[0]); } } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/count_experts.comp b/ggml/src/ggml-vulkan/vulkan-shaders/count_experts.comp index c7c0ac3416e5..2998302f73db 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/count_experts.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/count_experts.comp @@ -91,7 +91,6 @@ void zero_skipped_rows() { void main() { const uint num_elements = p.ne00 * p.ne01; const uint tid = gl_LocalInvocationID.x; - const uint tile_shift = findLSB(p.row_tile_size); uint total_tiles = 0; uint local_skipped = 0; if (tid == 0) { @@ -127,7 +126,7 @@ void main() { for (uint e = gl_SubgroupInvocationID; e < n_experts_padded; e += gl_SubgroupSize) { const bool in_range = e < n_experts; const uint count = in_range ? vals[e] : 0; - const uint n_tiles = (count + p.row_tile_size - 1) >> tile_shift; + const uint n_tiles = (count + p.row_tile_size - 1) / p.row_tile_size; const uint tile_begin = base + subgroupExclusiveAdd(n_tiles); if (in_range) { offsets[e] = tile_begin * p.row_tile_size; @@ -144,7 +143,7 @@ void main() { if (tid == 0) { for (uint e = 0; e < n_experts; ++e) { const uint count = vals[e]; - const uint n_tiles = (count + p.row_tile_size - 1) >> tile_shift; + const uint n_tiles = (count + p.row_tile_size - 1) / p.row_tile_size; offsets[e] = total_tiles * p.row_tile_size; cursors[e] = 0; write_tiles(expert_base + e, count, total_tiles, n_tiles); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl index 911ceac22cf6..b62bcbcd4a75 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl @@ -121,9 +121,9 @@ vec2 dequantize(uint ib, uint iqs, uint a_offset) { return vec2(int(data_a[a_offset + ib].qs[iqs]), int(data_a[a_offset + ib].qs[iqs + 1])); } vec4 dequantize4(uint ib, uint iqs, uint a_offset) { - const i8vec2 v0 = unpack8(int32_t(data_a_packed16[a_offset + ib].qs[iqs/2])).xy; // vec4 used due to #12147 - const i8vec2 v1 = unpack8(int32_t(data_a_packed16[a_offset + ib].qs[iqs/2 + 1])).xy; - return vec4(v0.x, v0.y, v1.x, v1.y); + return vec4(unpack8(pack32(i16vec2( + data_a_packed16[a_offset + ib].qs[iqs/2], + data_a_packed16[a_offset + ib].qs[iqs/2 + 1])))); } #endif diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl index 072edfa68cc7..e39a3a2e9092 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl @@ -30,6 +30,7 @@ const bool Q_IN_REGS = (Flags & 32) != 0; // coopmat1: subgroups own Q rows, O in coopmat accumulators. K/V staged in LDS, or loaded from global memory with FLIP_DIRECT. const bool FLIP = (Flags & 64) != 0; const bool FLIP_DIRECT = (Flags & 128) != 0; +const bool USE_LPT_GQA_SWIZZLE = (Flags & 256) != 0; // Round up head sizes to a multiple of 16, for coopmat1/coopmat2 paths const uint32_t HSK_pad = (HSK + 15) & ~15; @@ -103,7 +104,7 @@ layout (binding = 7) readonly buffer SP {int32_t data_sparse[];}; #include "fa_types.glsl" -#if defined(BFLOAT16) +#if defined(BFLOAT16) || defined(COOPMAT_8X8) #define O_TYPE float #define O_TYPEV4 vec4 #else @@ -222,6 +223,22 @@ void init_indices() // and breaking the alignment detection. m_stride = (p.gqa_ratio > 1) ? (p.gqa_ratio >> 16) : KV; + if (USE_LPT_GQA_SWIZZLE) { + if (!USE_SPARSE) { + if (p.k_num == 1u && p.gqa_ratio == 1u && p.nek2 == p.nev2 && rk2 > 1u && p.neq2 == p.nek2 * rk2) { + const uint32_t wg_lin = gl_WorkGroupID.y * Tr + gl_WorkGroupID.x; + const uint32_t h_sub = wg_lin % rk2; + const uint32_t tmp = wg_lin / rk2; + const uint32_t i_rev = tmp % Tr; + const uint32_t ik_lin = tmp / Tr; + i = (Tr - 1u) - i_rev; + iq2 = ik_lin * rk2 + h_sub; + ik2 = ik_lin; + iv2 = ik_lin; + } + } + } + // Sparse: the tile shares one mask row (gqa heads, or Br==1). split_k // partitions the n_kv_max blocks. if (USE_SPARSE) { diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp index bb9755ab8ac9..6e53229089c9 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp @@ -18,7 +18,7 @@ #include "types.glsl" #include "flash_attn_base.glsl" -#if !defined(BFLOAT16) +#if !defined(BFLOAT16) && !defined(COOPMAT_8X8) #include "flash_attn_dequant.glsl" #endif @@ -39,6 +39,504 @@ layout (binding = 2) readonly buffer V {FLOAT_TYPE data_v[];}; layout (binding = 2) readonly buffer VV4 {FLOAT_TYPEV4 data_vv4[];}; layout (binding = 3) readonly buffer M {float16_t data_m[];}; +#if defined(COOPMAT_8X8) && !defined(BFLOAT16) +// Series 13: 15,776-byte <= 16 KB COOPMAT_8X8 layout for 2-WG/core occupancy on Apple AGX G17P +// Supports both WG=128 (4 subgroups -> 256 threads/core) and WG=256 (8 subgroups -> 512 threads/core, +// halving per-thread OfMat/SfMat GPRs from 80 -> 40 half-GPRs for zero register spills!): +// - 8x8 block-packed transposed Qf_T (RowMajor f16vec2 loads, 0 bank conflicts) +// - RowMajor [Br][Bc+8] Psh_s (f16vec2 loads, 36-dword stride -> 0 bank conflicts) +// - Direct OfMat[num_hsv_tiles][2][2] register accumulation (eliminates pvsh + 9 barriers/iter) +// - Compact 1 KB kvsh[128] used only on partial tail blocks +const uint32_t num_sg = WorkGroupSize / SubGroupSize; +const uint32_t k_tiles_per_sg = Bc / (num_sg * 8u); +const uint32_t sm_rows_per_th = Br / 4u; +const uint32_t sm_cols_per_th = Bc / 32u; + +shared float tmpsh[num_sg]; +shared float sh_eMf[Br]; +shared ACC_TYPE slope[Br]; +shared FLOAT_TYPE Qf_T[HSK_pad * Br]; +const uint32_t psh_s_stride = Bc + 8u; +shared FLOAT_TYPE Psh_s[Br * psh_s_stride]; +const uint32_t sfshstride = (HSK <= 128u) ? (Br / 4u + 1u) : (Br / 4u); +const uint32_t osh_stride = 16u; +const uint32_t sfsh_elems = (Bc * sfshstride >= Br * osh_stride) ? (Bc * sfshstride) : (Br * osh_stride); +shared ACC_TYPEV4 sfsh[sfsh_elems]; +shared FLOAT_TYPEV4 kvsh[128]; + +void main() { + init_indices(); + + const uint32_t tid = gl_LocalInvocationIndex; + const uint32_t row_tid = tid / 32u; + const uint32_t col_tid = tid & 31u; + +#define tile_row(r) (row_tid * sm_rows_per_th + (r)) + + const uint32_t q_offset = gqa_iq1 * p.nb01 + (iq2 * p.nb02 + iq3 * p.nb03) / 4u; + + [[unroll]] for (uint32_t idx = 0u; idx < Br * HSK_pad / 4u; idx += WorkGroupSize) { + const uint32_t idx0 = idx + tid; + if (idx0 < Br * HSK_pad / 4u) { + const uint32_t d4 = idx0 % (HSK_pad / 4u); + const uint32_t r = idx0 / (HSK_pad / 4u); + vec4 qv = vec4(0.0); + if (i * Br + r < N && d4 * 4u < HSK) { + const uint32_t q_elem = q_offset + (i * Br + r) * q_stride + d4 * 4u; + if (!KV_bounds_check || (q_elem & 3u) == 0u) { + qv = data_qv4[q_elem >> 2u] * p.scale; + } else { + qv = vec4(data_q[q_elem + 0u], data_q[q_elem + 1u], + data_q[q_elem + 2u], data_q[q_elem + 3u]) * p.scale; + } + } + const uint32_t d8 = d4 >> 1u; + const uint32_t k_base = (d4 & 1u) * 4u; + const uint32_t n_blk = r >> 3u; + const uint32_t c_in = r & 7u; + const uint32_t blk = d8 * (Br / 8u) + n_blk; + [[unroll]] for (uint32_t t = 0u; t < 4u; ++t) { + Qf_T[((blk * 8u) + k_base + t) * 8u + c_in] = FLOAT_TYPE(qv[t]); + } + } + } + + if (tid < Br) { + slope[tid] = (p.max_bias > 0.0f) ? perElemOpComputeSlope(tid, col_tid, ACC_TYPE(0), iq2) : ACC_TYPE(1.0); + } + barrier(); + + const uint32_t num_hsv_tiles = (HSV + MatBc * num_sg - 1u) / (MatBc * num_sg); + coopmat OfMat[num_hsv_tiles][2][2]; + [[unroll]] for (uint32_t ht = 0u; ht < num_hsv_tiles; ++ht) { + [[unroll]] for (uint32_t m = 0u; m < 2u; ++m) { + [[unroll]] for (uint32_t n = 0u; n < 2u; ++n) { + OfMat[ht][m][n] = coopmat(0.0); + } + } + } + + float Lf[sm_rows_per_th], Mf[sm_rows_per_th]; + [[unroll]] for (uint32_t r = 0u; r < sm_rows_per_th; ++r) { + Lf[r] = 0.0f; + Mf[r] = NEG_FLT_MAX_OVER_2; + } + + const uint32_t mo_stride = CEIL_DIV(KV, 16u * Bc); + uint32_t mo_offset = mo_stride * i; + const uint32_t k_offset = (ik2 * p.nb12 + ik3 * p.nb13) / FaBlockBytesK; + const uint32_t v_offset = (iv2 * p.nb22 + iv3 * p.nb23) / FaBlockBytesV; + uint32_t m_offset = gqa_iq1 * KV; + if (p.nem2 != 1u || p.nem3 != 1u) { + m_offset += ((iq3 % p.nem3) * p.nem2 + (iq2 % p.nem2)) * p.nem1 * KV; + mo_offset += ((iq3 % p.nem3) * p.nem2 + (iq2 % p.nem2)) * CEIL_DIV(p.nem1, Br) * mo_stride; + } + + uint32_t mask_opt = 0u; + uint32_t mask_opt_idx = ~0u; + + [[dont_unroll]] + for (uint32_t j = start_j; j < end_j; ++j) { + uint32_t mask_opt_bits = 0u; + if (MASK_ENABLE) { + if (USE_MASK_OPT) { + if (mask_opt_idx != j / 16u) { + mask_opt_idx = j / 16u; + mask_opt = data_mask_opt[mo_offset + mask_opt_idx]; + } + const uint32_t rem_shift = (j & 15u) * 2u; + const uint32_t chunk_end = min(end_j, (j & ~15u) + 16u); + const uint32_t n_rem = chunk_end - j; + const uint32_t rem_mask = (n_rem >= 16u) ? 0xFFFFFFFFu : ((1u << (2u * n_rem)) - 1u); + if (((mask_opt >> rem_shift) & rem_mask) == (0x55555555u & rem_mask)) { + j = chunk_end - 1u; + continue; + } + mask_opt_bits = (mask_opt >> rem_shift) & 0x3u; + if (mask_opt_bits == MASK_OPT_ALL_NEG_INF) { + continue; + } + } else { + const bool nem1_bounds_check = !(p.gqa_ratio > 1u) && (p.nem1 % Br) != 0u; + float max_mask = NEG_FLT_MAX_OVER_2; + [[unroll]] for (uint32_t idx = 0u; idx < Bc * Br / 4u; idx += WorkGroupSize) { + const uint32_t idx0 = idx + tid; + if (idx0 < Bc * Br / 4u) { + const uint32_t c = idx0 / (Br / 4u); + const uint32_t r = idx0 % (Br / 4u); + uint32_t kcol; + if (fa_kv_index(j * Bc + c, kcol)) { + if (USE_SPARSE) { + max_mask = max(max_mask, float(data_m[m_offset + kcol])); + } else { + [[unroll]] for (uint32_t rr = 0u; rr < 4u; ++rr) { + if (!nem1_bounds_check || i * Br + r * 4u + rr < p.nem1) { + max_mask = max(max_mask, float(data_m[m_offset + (i * Br + r * 4u + rr) * m_stride + kcol])); + } + } + } + } + } + } + const bool all_less = subgroupAll(max_mask <= NEG_FLT_MAX_OVER_2); + barrier(); + if (gl_SubgroupInvocationID == 0u) { + tmpsh[gl_SubgroupID] = all_less ? NEG_FLT_MAX_OVER_2 : 0.0f; + } + barrier(); + [[unroll]] for (uint32_t s = 0u; s < num_sg; ++s) { + max_mask = max(max_mask, tmpsh[s]); + } + if (max_mask <= NEG_FLT_MAX_OVER_2) { + continue; + } + } + } + + coopmat SfMat[k_tiles_per_sg][2]; + [[unroll]] for (uint32_t m = 0u; m < k_tiles_per_sg; ++m) { + [[unroll]] for (uint32_t n = 0u; n < 2u; ++n) { + SfMat[m][n] = coopmat(0.0); + } + } + coopmat KMatK[k_tiles_per_sg]; + coopmat KMatP[2]; + coopmat QMat[2]; + + const bool stage_k_block = (SHMEM_STAGING != 0u) || USE_SPARSE || + (KV_bounds_check && (j * Bc + Bc > KV || ((k_offset | k_stride) & 3u) != 0u)); + + [[unroll]] for (uint32_t d = 0u; d < HSK_pad / 16u; ++d) { + [[unroll]] for (uint32_t dk = 0u; dk < 2u; ++dk) { + const bool stage_k_now = stage_k_block || (d * 16u + dk * 8u + 8u > HSK); + if (stage_k_now) { + barrier(); + if (tid < 128u) { + const uint32_t row = tid >> 1u; + const uint32_t col_vec = tid & 1u; + FLOAT_TYPEV4 K_Tf = FLOAT_TYPEV4(0.0); + uint32_t kcol; + if (fa_kv_index(j * Bc + row, kcol) && (d * 16u + dk * 8u + col_vec * 4u < HSK)) { + const uint32_t elem_idx = k_offset + kcol * k_stride + d * 16u + dk * 8u + col_vec * 4u; + if ((elem_idx & 3u) == 0u) { + K_Tf = data_kv4[elem_idx >> 2u]; + } else { + K_Tf = FLOAT_TYPEV4(data_k[elem_idx + 0u], data_k[elem_idx + 1u], + data_k[elem_idx + 2u], data_k[elem_idx + 3u]); + } + } + kvsh[tid] = K_Tf; + } + barrier(); + } + [[unroll]] for (uint32_t m = 0u; m < k_tiles_per_sg; ++m) { + const uint32_t kv_local_row = (gl_SubgroupID * k_tiles_per_sg + m) * 8u; + if (!stage_k_now) { + const uint32_t coord = k_offset / 4u + (j * Bc + kv_local_row) * (k_stride / 4u) + d * 4u + dk * 2u; + coopMatLoad(KMatK[m], data_kv4, coord, k_stride / 4u, gl_CooperativeMatrixLayoutRowMajor); + } else { + const uint32_t coord = kv_local_row * 2u; + coopMatLoad(KMatK[m], kvsh, coord, 2u, gl_CooperativeMatrixLayoutRowMajor); + } + } + [[unroll]] for (uint32_t n = 0u; n < 2u; ++n) { + const uint32_t q_coord = ((d * 2u + dk) * (Br / 8u) + n) * 64u; + coopMatLoad(QMat[n], Qf_T, q_coord, 8u, gl_CooperativeMatrixLayoutRowMajor); + } + [[unroll]] for (uint32_t m = 0u; m < k_tiles_per_sg; ++m) { + [[unroll]] for (uint32_t n = 0u; n < 2u; ++n) { + SfMat[m][n] = coopMatMulAdd(KMatK[m], QMat[n], SfMat[m][n]); + } + } + } + } + + [[unroll]] for (uint32_t m = 0u; m < k_tiles_per_sg; ++m) { + const uint32_t kv_local_row = (gl_SubgroupID * k_tiles_per_sg + m) * 8u; + [[unroll]] for (uint32_t n = 0u; n < 2u; ++n) { + const uint32_t coord = kv_local_row * sfshstride + n * 2u; + coopMatStore(SfMat[m][n], sfsh, coord, sfshstride, gl_CooperativeMatrixLayoutRowMajor); + } + } + barrier(); // Barrier #1: sfsh ready; prior j's Psh_s/sh_eMf reads finished + + if (LOGIT_SOFTCAP) { + [[unroll]] for (uint32_t idx = 0u; idx < Bc * Br / 4u; idx += WorkGroupSize) { + const uint32_t idx0 = idx + tid; + if (idx0 < Bc * Br / 4u) { + const uint32_t c = idx0 / (Br / 4u); + const uint32_t r = idx0 % (Br / 4u); + sfsh[c * sfshstride + r] = ACC_TYPEV4(p.logit_softcap * tanh(sfsh[c * sfshstride + r])); + } + } + barrier(); + } + + if (MASK_ENABLE && mask_opt_bits != MASK_OPT_ALL_ZERO) { + const bool nem1_bounds_check = !(p.gqa_ratio > 1u) && (p.nem1 % Br) != 0u; + [[unroll]] for (uint32_t idx = 0u; idx < Bc * Br / 4u; idx += WorkGroupSize) { + const uint32_t idx0 = idx + tid; + if (idx0 < Bc * Br / 4u) { + const uint32_t c = idx0 / (Br / 4u); + const uint32_t r = idx0 % (Br / 4u); + uint32_t kcol; + const bool kv_active = fa_kv_index(j * Bc + c, kcol); + if (!KV_bounds_check || j * Bc + c < KV) { + ACC_TYPEV4 masks = ACC_TYPEV4(0.0); + if (kv_active) { + if (USE_SPARSE) { + masks = ACC_TYPEV4(float(data_m[m_offset + kcol])); + } else { + [[unroll]] for (uint32_t rr = 0u; rr < 4u; ++rr) { + if (!nem1_bounds_check || i * Br + r * 4u + rr < p.nem1) { + masks[rr] = ACC_TYPE(data_m[m_offset + (i * Br + r * 4u + rr) * m_stride + kcol]); + } + } + } + } else if (USE_SPARSE) { + masks = ACC_TYPEV4(NEG_FLT_MAX_OVER_2); + } + const ACC_TYPEV4 slopes = ACC_TYPEV4(slope[r * 4u], slope[r * 4u + 1u], slope[r * 4u + 2u], slope[r * 4u + 3u]); + sfsh[c * sfshstride + r] += slopes * masks; + } + } + } + barrier(); + } + + // Single-pass sfsh read -> online softmax -> Psh_s & sh_eMf write (first 4 subgroups = 128 threads) + if (gl_SubgroupID < 4u) { + [[unroll]] for (uint32_t r4 = 0u; r4 < sm_rows_per_th; r4 += 4u) { + const uint32_t r_vec = tile_row(r4) / 4u; + vec4 sf_reg[sm_cols_per_th]; + vec4 rowmax4 = vec4(NEG_FLT_MAX_OVER_2); + [[unroll]] for (uint32_t c = 0u; c < sm_cols_per_th; ++c) { + const uint32_t col = c * 32u + col_tid; + if (!KV_bounds_check || j * Bc + col < KV) { + sf_reg[c] = vec4(sfsh[col * sfshstride + r_vec]); + rowmax4 = max(rowmax4, sf_reg[c]); + } else { + sf_reg[c] = vec4(0.0); + } + } + rowmax4 += vec4(FATTN_KQ_MAX_OFFSET); + [[unroll]] for (uint32_t v = 0u; v < 4u; ++v) { + const uint32_t r = r4 + v; + const float Moldf = Mf[r]; + const float rmax = subgroupMax(rowmax4[v]); + Mf[r] = max(rmax, Moldf); + const float eM = exp(Moldf - Mf[r]); + Lf[r] = eM * Lf[r]; + if (col_tid == 0u) { + sh_eMf[tile_row(r)] = eM; + } + } + const vec4 mfvec = vec4(Mf[r4], Mf[r4 + 1u], Mf[r4 + 2u], Mf[r4 + 3u]); + const uint32_t base_row = tile_row(r4); + [[unroll]] for (uint32_t c = 0u; c < sm_cols_per_th; ++c) { + const uint32_t col = c * 32u + col_tid; + FLOAT_TYPEV4 Pf = FLOAT_TYPEV4(0.0); + if (!KV_bounds_check || j * Bc + col < KV) { + Pf = FLOAT_TYPEV4(exp(sf_reg[c] - mfvec)); + [[unroll]] for (uint32_t v = 0u; v < 4u; ++v) { + Lf[r4 + v] += float(Pf[v]); + } + } + [[unroll]] for (uint32_t v = 0u; v < 4u; ++v) { + Psh_s[(base_row + v) * psh_s_stride + col] = Pf[v]; + } + } + } + } + barrier(); // Barrier #2: Psh_s & sh_eMf ready; sfsh reads finished for next j + + // Rescale OfMat directly in registers (both elements in lane belong to row cm_r) + const uint32_t lane = gl_SubgroupInvocationID; + const uint32_t cm_r = ((lane >> 1u) & 3u) | ((lane >> 2u) & 4u); + const float eM0 = sh_eMf[cm_r]; + const float eM1 = sh_eMf[8u + cm_r]; + [[unroll]] for (uint32_t ht = 0u; ht < num_hsv_tiles; ++ht) { + [[unroll]] for (uint32_t n = 0u; n < 2u; ++n) { + OfMat[ht][0][n][0] *= eM0; + OfMat[ht][0][n][1] *= eM0; + OfMat[ht][1][n][0] *= eM1; + OfMat[ht][1][n][1] *= eM1; + } + } + + const bool stage_v_block = (SHMEM_STAGING != 0u) || USE_SPARSE || + (KV_bounds_check && (j * Bc + Bc > KV || ((v_offset | v_stride) & 3u) != 0u)); + const uint32_t n_sub_passes = (num_sg + 3u) / 4u; + + [[unroll]] for (uint32_t bc_chunk = 0u; bc_chunk < Bc / MatBc; ++bc_chunk) { + [[unroll]] for (uint32_t dk = 0u; dk < 2u; ++dk) { + [[unroll]] for (uint32_t m = 0u; m < 2u; ++m) { + coopMatLoad(KMatP[m], Psh_s, (m * 8u) * psh_s_stride + bc_chunk * MatBc + dk * 8u, psh_s_stride, gl_CooperativeMatrixLayoutRowMajor); + } + [[unroll]] for (uint32_t hsv_tile = 0u; hsv_tile < num_hsv_tiles; ++hsv_tile) { + const uint32_t hsv_offset = (hsv_tile * num_sg + gl_SubgroupID) * 16u; + const bool hsv_tile_partial = (hsv_tile * MatBc * num_sg + MatBc * num_sg) > HSV; + const bool stage_v_now = stage_v_block || hsv_tile_partial; + if (!stage_v_now) { + if (hsv_offset < HSV_pad) { + const uint32_t v_tile_row = j * Bc + bc_chunk * MatBc + dk * 8u; + [[unroll]] for (uint32_t n = 0u; n < 2u; ++n) { + const uint32_t v_tile_offset = v_offset / 4u + v_tile_row * (v_stride / 4u) + hsv_offset / 4u + n * 2u; + coopMatLoad(QMat[n], data_vv4, v_tile_offset, v_stride / 4u, gl_CooperativeMatrixLayoutRowMajor); + } + [[unroll]] for (uint32_t m = 0u; m < 2u; ++m) { + [[unroll]] for (uint32_t n = 0u; n < 2u; ++n) { + OfMat[hsv_tile][m][n] = coopMatMulAdd(KMatP[m], QMat[n], OfMat[hsv_tile][m][n]); + } + } + } + } else { + [[unroll]] for (uint32_t sp = 0u; sp < n_sub_passes; ++sp) { + barrier(); + if (tid < 128u) { + const uint32_t r8 = tid >> 4u; + const uint32_t c16 = tid & 15u; + const uint32_t v_col = (hsv_tile * num_sg + sp * 4u) * 16u + c16 * 4u; + FLOAT_TYPEV4 V_Tf = FLOAT_TYPEV4(0.0); + uint32_t v_row; + if (v_col < HSV && fa_kv_index(j * Bc + bc_chunk * MatBc + dk * 8u + r8, v_row)) { + const uint32_t elem_idx = v_offset + v_row * v_stride + v_col; + if ((elem_idx & 3u) == 0u) { + V_Tf = data_vv4[elem_idx >> 2u]; + } else { + V_Tf = FLOAT_TYPEV4(data_v[elem_idx + 0u], data_v[elem_idx + 1u], + data_v[elem_idx + 2u], data_v[elem_idx + 3u]); + } + } + kvsh[tid] = V_Tf; + } + barrier(); + if ((gl_SubgroupID / 4u) == sp && hsv_offset < HSV_pad) { + [[unroll]] for (uint32_t n = 0u; n < 2u; ++n) { + const uint32_t v_tile_offset = (gl_SubgroupID & 3u) * 4u + n * 2u; + coopMatLoad(QMat[n], kvsh, v_tile_offset, 16u, gl_CooperativeMatrixLayoutRowMajor); + } + [[unroll]] for (uint32_t m = 0u; m < 2u; ++m) { + [[unroll]] for (uint32_t n = 0u; n < 2u; ++n) { + OfMat[hsv_tile][m][n] = coopMatMulAdd(KMatP[m], QMat[n], OfMat[hsv_tile][m][n]); + } + } + } + } + } + } + } + } + } + + if (gl_SubgroupID < 4u) { + [[unroll]] for (uint32_t r = 0u; r < sm_rows_per_th; ++r) { + Lf[r] = subgroupAdd(Lf[r]); + } + + if (p.k_num > 1u) { + if (col_tid == 0u) { + [[unroll]] for (uint32_t r = 0u; r < sm_rows_per_th; ++r) { + sh_eMf[tile_row(r)] = 1.0f; + } + } + if (p.gqa_ratio > 1u) { + const uint32_t lm_offset = HSV * p.ne1 * p.k_num * p.ne2 * p.ne3 + p.ne1 * 2u * (split_k_index + p.k_num * (gqa_iq1 + p.ne2 * iq3)); + [[unroll]] for (uint32_t r = 0u; r < sm_rows_per_th; ++r) { + if (tile_row(r) < N) { + perElemOpStoreCol0(tile_row(r), 0u, ACC_TYPE(Lf[r]), lm_offset, iq2, N); + perElemOpStoreCol0(tile_row(r), 0u, ACC_TYPE(Mf[r]), lm_offset + p.ne1, iq2, N); + } + } + } else { + [[unroll]] for (uint32_t r = 0u; r < sm_rows_per_th; ++r) { + const uint32_t global_row = i * Br + tile_row(r); + if (global_row < N && col_tid == 0u) { + const uint32_t lm_offset = HSV * p.ne1 * p.k_num * p.ne2 * p.ne3 + p.ne1 * 2u * (split_k_index + p.k_num * (global_row + p.ne2 * iq3)); + data_o[lm_offset + iq2] = D_TYPE(Lf[r]); + data_o[lm_offset + p.ne1 + iq2] = D_TYPE(Mf[r]); + } + } + } + } else { + [[unroll]] for (uint32_t r = 0u; r < sm_rows_per_th; ++r) { + float ms = 1.0f; + if ((p.mask_n_head_log2 & SINK_ENABLE_BIT) != 0u) { + const float sink = perElemOpGetSink(tile_row(r), 0u, ACC_TYPE(0), iq2); + float vs = 1.0f; + if (sink > Mf[r]) { + ms = exp(Mf[r] - sink); + } else { + vs = exp(sink - Mf[r]); + } + Lf[r] = Lf[r] * ms + vs; + } + if (col_tid == 0u) { + sh_eMf[tile_row(r)] = (Lf[r] == 0.0f) ? 0.0f : (ms / Lf[r]); + } + } + } + } + + const uint32_t o_base_dense = (gqa_iq1 * p.ne1 * HSV + iq3 * p.ne2 * p.ne1 * HSV) / 4u; + const uint32_t o_base_sk_gqa = HSV * p.ne1 * (split_k_index + p.k_num * (gqa_iq1 + p.ne2 * iq3)) / 4u; + const uint32_t n_sub_passes = (num_sg + 3u) / 4u; + + [[unroll]] for (uint32_t hsv_tile = 0u; hsv_tile < num_hsv_tiles; ++hsv_tile) { + const uint32_t hsv_offset = (hsv_tile * num_sg + gl_SubgroupID) * 16u; + [[unroll]] for (uint32_t sp = 0u; sp < n_sub_passes; ++sp) { + barrier(); + if ((gl_SubgroupID / 4u) == sp && hsv_offset < HSV_pad) { + const uint32_t o_offset_sh = (gl_SubgroupID & 3u) * 4u; + [[unroll]] for (uint32_t m = 0u; m < 2u; ++m) { + [[unroll]] for (uint32_t n = 0u; n < 2u; ++n) { + coopMatStore(OfMat[hsv_tile][m][n], sfsh, o_offset_sh + m * 8u * osh_stride + n * 2u, osh_stride, gl_CooperativeMatrixLayoutRowMajor); + } + } + } + barrier(); + if (gl_SubgroupID < 4u && col_tid < 16u) { + const uint32_t d = (hsv_tile * num_sg + sp * 4u) * 4u + col_tid; + if (d < HSV / 4u) { + [[unroll]] for (uint32_t r = 0u; r < sm_rows_per_th; ++r) { + const uint32_t row = tile_row(r); + O_TYPEV4 out_v = sfsh[row * osh_stride + col_tid] * O_TYPE(sh_eMf[row]); +#if defined(FLOAT_TYPE_MAX) + if (p.k_num <= 1u) { + out_v = clamp(out_v, -O_TYPE(FLOAT_TYPE_MAX), O_TYPE(FLOAT_TYPE_MAX)); + } +#endif + if (p.k_num > 1u) { + if (p.gqa_ratio > 1u) { + if (row < N) { + gqaStore(row, d, out_v, o_base_sk_gqa, iq2, N); + } + } else { + const uint32_t global_row = i * Br + row; + if (global_row < N) { + const uint32_t o_off = HSV * p.ne1 * (split_k_index + p.k_num * (global_row + p.ne2 * iq3)) / 4u; + data_ov4[o_off + iq2 * (HSV / 4u) + d] = D_TYPEV4(out_v); + } + } + } else if (p.gqa_ratio > 1u) { + if (row < N) { + gqaStore(row, d, out_v, o_base_dense, iq2, N); + } + } else { + const uint32_t global_row = i * Br + row; + if (global_row < N) { + data_ov4[o_base_dense + (iq2 * HSV + global_row * p.ne1 * HSV) / 4u + d] = D_TYPEV4(out_v); + } + } + } + } + } + } + } +} + +#else // !(defined(COOPMAT_8X8) && !defined(BFLOAT16)) + shared float tmpsh[row_split]; const uint32_t qstride = HSK_pad / 4 + 2; @@ -117,6 +615,16 @@ void main() { #define tile_row(r) (row_tid * rows_per_thread + (r)) + // Zero-initialize shared memory for Q/K when HSK is not a multiple of 16 (HSK_pad > HSK) or N is partial. + if (!Q_IN_REGS && ((HSK % 16) != 0 || i * Br + Br > N)) { + [[unroll]] for (uint idx0 = 0; idx0 < Br * qstride; idx0 += gl_WorkGroupSize.x) { + if (idx0 + tid < Br * qstride) { + Qf[idx0 + tid] = FLOAT_TYPEV4(0); + } + } + barrier(); + } + const uint32_t q_tiles = Br / MatBr; const uint32_t d_tiles = HSK_pad / 16; @@ -329,22 +837,33 @@ void main() { // K * Q^T -> S^T: Bc x HSK_pad * HSK_pad x Br -> Bc x Br // Bc split across workgroup (four subgroups), loop over HSK in chunks of 16: 16 x 16 * 16 x 16 -> 16 x 16 // This is written transposed in order to allow for N being 8 if implementations need it +#ifdef COOPMAT_8X8 + coopmat SfMat[2][2]; + [[unroll]] for (uint32_t m = 0; m < 2; ++m) { + [[unroll]] for (uint32_t n = 0; n < 2; ++n) { + SfMat[m][n] = coopmat(0); + } + } + coopmat KMat[2]; + coopmat QMat[2]; +#else coopmat SfMat[q_tiles]; [[unroll]] for (uint32_t q = 0; q < q_tiles; ++q) { SfMat[q] = coopmat(0); } coopmat KMat; coopmat QMat; +#endif [[unroll]] for (uint32_t d = 0; d < d_tiles; ++d) { // If SHMEM_STAGING is set, a Bc * HSK_pad size tile of K is loaded to shmem // If not, K is loaded directly from global memory if aligned, otherwise // staged through a Bc * MatBr size staging buffer. // If K is a quant type, then it is always staged for dequantization. + const bool stage_k = USE_DECODE_K || KV_bounds_check || USE_SPARSE || d * 16 + 16 > HSK; if (SHMEM_STAGING == 0) { // For quants we always need to dequant into kvsh; for f16/bf16 we can load // directly from global memory when alignment / bounds allow it. - const bool stage_k = USE_DECODE_K || KV_bounds_check || USE_SPARSE || d * 16 + 16 > HSK; if (stage_k) { barrier(); [[unroll]] for (uint32_t idx = 0; idx < Bc * MatBr / 4; idx += gl_WorkGroupSize.x) { @@ -373,7 +892,33 @@ void main() { } barrier(); } - +#ifdef COOPMAT_8X8 + } + [[unroll]] for (uint32_t dk = 0; dk < 2; ++dk) { + [[unroll]] for (uint32_t m = 0; m < 2; ++m) { + if (SHMEM_STAGING == 0) { + if (stage_k) { + uint coord = (gl_SubgroupID * MatBc + m * 8) * kvsh_stride + dk * 2; + coopMatLoad(KMat[m], kvsh, coord, kvsh_stride, gl_CooperativeMatrixLayoutRowMajor); + } else { + const uint coord = k_offset / 4 + (j * Bc + gl_SubgroupID * MatBc + m * 8) * (k_stride / 4) + d * 4 + dk * 2; + coopMatLoad(KMat[m], data_kv4, coord, k_stride / 4, gl_CooperativeMatrixLayoutRowMajor); + } + } else { + uint coord = (gl_SubgroupID * MatBc + m * 8) * kvsh_stride + d * 4 + dk * 2; + coopMatLoad(KMat[m], kvsh, coord, kvsh_stride, gl_CooperativeMatrixLayoutRowMajor); + } + } + [[unroll]] for (uint32_t n = 0; n < 2; ++n) { + coopMatLoad(QMat[n], Qf, d * 4 + n * 8 * qstride + dk * 2, qstride, gl_CooperativeMatrixLayoutColumnMajor); + } + [[unroll]] for (uint32_t m = 0; m < 2; ++m) { + [[unroll]] for (uint32_t n = 0; n < 2; ++n) { + SfMat[m][n] = coopMatMulAdd(KMat[m], QMat[n], SfMat[m][n]); + } + } + } +#else if (stage_k) { uint coord = (gl_SubgroupID * MatBc) * kvsh_stride; coopMatLoad(KMat, kvsh, coord, kvsh_stride, gl_CooperativeMatrixLayoutRowMajor); @@ -394,12 +939,22 @@ void main() { SfMat[q] = coopMatMulAdd(KMat, QMat, SfMat[q]); } } +#endif } +#ifdef COOPMAT_8X8 + [[unroll]] for (uint32_t m = 0; m < 2; ++m) { + [[unroll]] for (uint32_t n = 0; n < 2; ++n) { + uint coord = (gl_SubgroupID * MatBc + m * 8) * sfshstride + n * 2; + coopMatStore(SfMat[m][n], sfsh, coord, sfshstride, gl_CooperativeMatrixLayoutRowMajor); + } + } +#else [[unroll]] for (uint32_t q = 0; q < q_tiles; ++q) { uint coord = gl_SubgroupID * MatBc * sfshstride + q * (MatBr / 4); coopMatStore(SfMat[q], sfsh, coord, sfshstride, gl_CooperativeMatrixLayoutRowMajor); } +#endif barrier(); if (LOGIT_SOFTCAP) { @@ -516,10 +1071,19 @@ void main() { [[unroll]] for (uint32_t hsv_tile = 0; hsv_tile < num_hsv_tiles; ++hsv_tile) { const uint hsv_offset = (hsv_tile * row_split + gl_SubgroupID) * 16; +#ifdef COOPMAT_8X8 + coopmat PVMat[2][2]; + [[unroll]] for (uint32_t m = 0; m < 2; ++m) { + [[unroll]] for (uint32_t n = 0; n < 2; ++n) { + PVMat[m][n] = coopmat(0); + } + } +#else coopmat PVMat[q_tiles]; [[unroll]] for (uint32_t q = 0; q < q_tiles; ++q) { PVMat[q] = coopmat(0); } +#endif // Preload V tiles for [Bc, 16 * num subgroups] const uint v_rows = Bc; @@ -530,10 +1094,11 @@ void main() { // If not, V is loaded directly from global memory if aligned, otherwise // staged through a Bc * MatBr size staging buffer. // If V is a quant type, then it is always staged for dequantization. + const bool hsv_partial = (hsv_tile * MatBc * row_split + MatBc * row_split) > HSV; if (SHMEM_STAGING == 0) { // For quants we always preload via kvsh. For f16/bf16 we only preload when // alignment / bounds force it (otherwise we coopMatLoad direct from data_vv4). - const bool stage_v = USE_DECODE_V || KV_bounds_check || USE_SPARSE; + const bool stage_v = USE_DECODE_V || KV_bounds_check || USE_SPARSE || hsv_partial; if (stage_v) { [[unroll]] for (uint32_t i = 0; i < v_loads_per_thread; ++i) { const uint idx = i * gl_WorkGroupSize.x + tid; @@ -548,7 +1113,7 @@ void main() { const uint ib = coord / BLOCK_SIZE_V; const uint iqs = coord % BLOCK_SIZE_V; - if (USE_SPARSE ? (kv_active && v_col < HSV) : (!KV_bounds_check || (v_row < KV && v_col < HSV))) { + if (USE_SPARSE ? (kv_active && v_col < HSV) : (v_col < HSV && (!KV_bounds_check || v_row < KV))) { #if !defined(BFLOAT16) if (USE_DECODE_V) { kvsh[row * vsh_stride + col] = dequantize4(ib, iqs, v_offset, BINDING_IDX_V); @@ -568,9 +1133,43 @@ void main() { const uint o_offset = gl_SubgroupID * MatBr / 4; if (hsv_offset < HSV_pad) { +#ifdef COOPMAT_8X8 + [[unroll]] for (uint32_t bc_chunk = 0; bc_chunk < Bc / MatBc; ++bc_chunk) { + [[unroll]] for (uint32_t dk = 0; dk < 2; ++dk) { + [[unroll]] for (uint32_t m = 0; m < 2; ++m) { + coopMatLoad(KMat[m], Psh, (bc_chunk * MatBc + dk * 8) * psh_stride + m * 2, psh_stride, gl_CooperativeMatrixLayoutColumnMajor); + } + [[unroll]] for (uint32_t n = 0; n < 2; ++n) { + if (SHMEM_STAGING == 0) { + if (!USE_DECODE_V && !KV_bounds_check && !USE_SPARSE && !hsv_partial) { + const uint v_tile_row = j * Bc + bc_chunk * MatBc + dk * 8; + const uint v_tile_offset = v_offset / 4 + v_tile_row * (v_stride / 4) + hsv_offset / 4 + n * 2; + coopMatLoad(QMat[n], data_vv4, v_tile_offset, v_stride / 4, gl_CooperativeMatrixLayoutRowMajor); + } else { + const uint v_tile_offset = (bc_chunk * MatBr + dk * 8) * v_cols + gl_SubgroupID * (MatBc / 4) + n * 2; + coopMatLoad(QMat[n], kvsh, v_tile_offset, vsh_stride, gl_CooperativeMatrixLayoutRowMajor); + } + } else { + const uint v_tile_offset = (bc_chunk * MatBc + dk * 8) * kvsh_stride + (hsv_tile * row_split + gl_SubgroupID) * (MatBc / 4) + n * 2; + coopMatLoad(QMat[n], kvsh, v_tile_offset, kvsh_stride, gl_CooperativeMatrixLayoutRowMajor); + } + } + [[unroll]] for (uint32_t m = 0; m < 2; ++m) { + [[unroll]] for (uint32_t n = 0; n < 2; ++n) { + PVMat[m][n] = coopMatMulAdd(KMat[m], QMat[n], PVMat[m][n]); + } + } + } + } + [[unroll]] for (uint32_t m = 0; m < 2; ++m) { + [[unroll]] for (uint32_t n = 0; n < 2; ++n) { + coopMatStore(PVMat[m][n], pvsh, o_offset + m * 8 * osh_stride + n * 2, osh_stride, gl_CooperativeMatrixLayoutRowMajor); + } + } +#else [[unroll]] for (uint32_t bc_chunk = 0; bc_chunk < Bc / MatBc; ++bc_chunk) { if (SHMEM_STAGING == 0) { - if (!USE_DECODE_V && !KV_bounds_check && !USE_SPARSE) { + if (!USE_DECODE_V && !KV_bounds_check && !USE_SPARSE && !hsv_partial) { // F16/BF16 values can be loaded directly from global memory const uint v_tile_row = j * Bc + bc_chunk * MatBc; const uint v_tile_offset = v_offset / 4 + v_tile_row * v_stride / 4 + hsv_offset / 4; @@ -597,6 +1196,7 @@ void main() { coopMatStore(PVMat[q], pvsh, o_offset + q * MatBc * osh_stride, osh_stride, gl_CooperativeMatrixLayoutRowMajor); } } +#endif } barrier(); @@ -713,7 +1313,7 @@ void main() { [[unroll]] for (uint32_t r = 0; r < rows_per_thread; ++r) { Of[r][d_local] *= O_TYPE(Lfrcp[r]); #if defined(FLOAT_TYPE_MAX) - Of[r][d_local] = clamp(Of[r][d_local], -FLOAT_TYPE_MAX, FLOAT_TYPE_MAX); + Of[r][d_local] = clamp(Of[r][d_local], -O_TYPE(FLOAT_TYPE_MAX), O_TYPE(FLOAT_TYPE_MAX)); #endif } } @@ -744,3 +1344,4 @@ void main() { } } } +#endif // defined(COOPMAT_8X8) && !defined(BFLOAT16) diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/gated_delta_net.comp b/ggml/src/ggml-vulkan/vulkan-shaders/gated_delta_net.comp index 0e384330b9b9..c59a40541ead 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/gated_delta_net.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/gated_delta_net.comp @@ -15,11 +15,15 @@ layout(constant_id = 0) const uint S_V = 128; layout(constant_id = 1) const uint KDA = 0; layout(constant_id = 2) const uint SUBGROUP_SIZE = 32; layout(constant_id = 3) const uint LANES_PER_COLUMN = 32; +layout(constant_id = 4) const uint COLS_PER_SUB = 1; +layout(constant_id = 5) const uint SUBGROUPS_PER_WG = 1; +layout(constant_id = 6) const uint CONTIG_VEC4 = 0; -const uint COLS_PER_WG = SUBGROUP_SIZE / LANES_PER_COLUMN; +const uint COLS_PER_SUB_STEP = SUBGROUP_SIZE / LANES_PER_COLUMN; +const uint COLS_PER_WG = COLS_PER_SUB_STEP * SUBGROUPS_PER_WG; const uint ROWS_PER_LANE = S_V / LANES_PER_COLUMN; -layout(local_size_x_id = 2, local_size_y = 1, local_size_z = 1) in; +layout(local_size_x_id = 2, local_size_y_id = 5, local_size_z = 1) in; layout(push_constant) uniform Parameters { uint H; @@ -32,15 +36,26 @@ layout(push_constant) uniform Parameters { uint neq1, rq3; float scale; uint K; + uint sout_nb2; }; layout(binding = 0) readonly buffer QBuf { FLOAT_TYPE data_q[]; }; +layout(binding = 0) readonly buffer QBufV4 { vec4 data_q_v4[]; }; layout(binding = 1) readonly buffer KBuf { FLOAT_TYPE data_k[]; }; +layout(binding = 1) readonly buffer KBufV4 { vec4 data_k_v4[]; }; layout(binding = 2) readonly buffer VBuf { FLOAT_TYPE data_v[]; }; -layout(binding = 3) readonly buffer GBuf { FLOAT_TYPE data_g[]; }; -layout(binding = 4) readonly buffer BetaBuf { FLOAT_TYPE data_beta[]; }; -layout(binding = 5) readonly buffer StateBuf { FLOAT_TYPE data_state[]; }; -layout(binding = 6) buffer DstBuf { FLOAT_TYPE data_dst[]; }; +layout(binding = 2) readonly buffer VBufV4 { vec4 data_v_v4[]; }; +layout(binding = 3) readonly restrict buffer GBuf { FLOAT_TYPE data_g[]; }; +layout(binding = 4) readonly restrict buffer BetaBuf { FLOAT_TYPE data_beta[]; }; +layout(binding = 5) readonly buffer StateBuf { FLOAT_TYPE data_state[]; }; +layout(binding = 5) readonly buffer StateBufV4 { vec4 data_state_v4[]; }; +layout(binding = 6) writeonly buffer DstBuf { FLOAT_TYPE data_dst[]; }; +layout(binding = 6) writeonly buffer DstBufV4 { vec4 data_dst_v4[]; }; +#if STATE_OUT +// gen: the final state goes straight to its destination (the recurrent cache row), not into dst +layout(binding = 7) buffer StateOutBuf { FLOAT_TYPE data_state_out[]; }; +layout(binding = 7) buffer StateOutBufV4 { vec4 data_state_out_v4[]; }; +#endif #if !USE_SUBGROUP_ADD && !USE_SUBGROUP_CLUSTERED shared FLOAT_TYPE temp[SUBGROUP_SIZE]; @@ -96,7 +111,7 @@ void main() { const uint head_id = gl_WorkGroupID.x; const uint seq_id = gl_WorkGroupID.y; const uint lane = gl_SubgroupInvocationID % LANES_PER_COLUMN; - const uint col = gl_WorkGroupID.z * COLS_PER_WG + (gl_SubgroupInvocationID / LANES_PER_COLUMN); + const uint col_base = (gl_WorkGroupID.z * COLS_PER_SUB) * COLS_PER_WG + gl_LocalInvocationID.y * COLS_PER_SUB_STEP + (gl_SubgroupInvocationID / LANES_PER_COLUMN); const uint iq1 = head_id % neq1; const uint iq3 = seq_id / rq3; @@ -107,10 +122,195 @@ void main() { // output state layout per slot: same per-(seq,head) offset as the single-slot case. const uint state_out_base = (seq_id * H + head_id) * state_size; const uint state_size_per_snap = state_size * H * n_seqs; + const bool zero_state = (K & 0x80000000u) != 0u; + const uint eff_K = K & 0x7FFFFFFFu; + +#if USE_SUBGROUP_ADD + if (CONTIG_VEC4 != 0u && KDA == 0u && ROWS_PER_LANE == 4u && LANES_PER_COLUMN == 32u && (COLS_PER_SUB == 4u || COLS_PER_SUB == 2u)) { + const bool qk_v4_ok = ((sq1 & 3u) == 0u) && ((sq2 & 3u) == 0u) && ((sq3 & 3u) == 0u); + const bool v_v4_ok = (COLS_PER_SUB == 4u) && (COLS_PER_WG == 1u) && ((sv1 & 3u) == 0u) && ((sv2 & 3u) == 0u) && ((sv3 & 3u) == 0u) && ((col_base & 3u) == 0u); + const bool st_v4_ok = ((sout_nb2 & 3u) == 0u) && ((s_off & 3u) == 0u); + vec4 s0 = vec4(0.0f), s1 = vec4(0.0f), s2 = vec4(0.0f), s3 = vec4(0.0f); + if (!zero_state) { + s0 = data_state_v4[((state_in_base + (col_base + 0u * COLS_PER_WG) * S_V) >> 2u) + lane]; + s1 = data_state_v4[((state_in_base + (col_base + 1u * COLS_PER_WG) * S_V) >> 2u) + lane]; + if (COLS_PER_SUB == 4u) { + s2 = data_state_v4[((state_in_base + (col_base + 2u * COLS_PER_WG) * S_V) >> 2u) + lane]; + s3 = data_state_v4[((state_in_base + (col_base + 3u * COLS_PER_WG) * S_V) >> 2u) + lane]; + } + } + + uint attn_off = (seq_id * n_tokens * H + head_id) * S_V; + uint q_off = iq3 * sq3 + iq1 * sq1; + uint v_off = seq_id * sv3 + head_id * sv1; + uint gb_off = seq_id * sb3 + head_id * sb1; + for (uint t = 0u; t < n_tokens; ++t) { + const float beta_val = data_beta[gb_off]; + const float g_val = exp(data_g[gb_off]); + + const vec4 k_vec = qk_v4_ok + ? data_k_v4[(q_off >> 2u) + lane] + : vec4(data_k[q_off + 4u * lane + 0u], data_k[q_off + 4u * lane + 1u], data_k[q_off + 4u * lane + 2u], data_k[q_off + 4u * lane + 3u]); + const vec4 q_vec = qk_v4_ok + ? data_q_v4[(q_off >> 2u) + lane] + : vec4(data_q[q_off + 4u * lane + 0u], data_q[q_off + 4u * lane + 1u], data_q[q_off + 4u * lane + 2u], data_q[q_off + 4u * lane + 3u]); + + if (COLS_PER_SUB == 4u) { + vec4 attn_cols; + if (CONTIG_VEC4 >= 2u) { + const vec4 kv_sh = vec4( + dot(s0, k_vec), + dot(s1, k_vec), + dot(s2, k_vec), + dot(s3, k_vec) + ) * g_val; + const vec4 sq_sh = vec4( + dot(s0, q_vec), + dot(s1, q_vec), + dot(s2, q_vec), + dot(s3, q_vec) + ); + const float kq_sh = dot(k_vec, q_vec); + const vec4 kv_cols = subgroupAdd(kv_sh); + const vec4 sq_cols = subgroupAdd(sq_sh); + const float kq_dot = subgroupAdd(kq_sh); + const vec4 v_cols = v_v4_ok + ? data_v_v4[(v_off + col_base) >> 2u] + : vec4(data_v[v_off + col_base], data_v[v_off + col_base + COLS_PER_WG], data_v[v_off + col_base + 2u * COLS_PER_WG], data_v[v_off + col_base + 3u * COLS_PER_WG]); + const vec4 delta = (v_cols - kv_cols) * beta_val; + attn_cols = fma(vec4(g_val), sq_cols, delta * kq_dot) * scale; + s0 = fma(vec4(g_val), s0, k_vec * delta.x); + s1 = fma(vec4(g_val), s1, k_vec * delta.y); + s2 = fma(vec4(g_val), s2, k_vec * delta.z); + s3 = fma(vec4(g_val), s3, k_vec * delta.w); + } else { + vec4 kv_sh = vec4( + dot(s0, k_vec), + dot(s1, k_vec), + dot(s2, k_vec), + dot(s3, k_vec) + ) * g_val; + const vec4 kv_cols = subgroupAdd(kv_sh); + const vec4 v_cols = v_v4_ok + ? data_v_v4[(v_off + col_base) >> 2u] + : vec4(data_v[v_off + col_base], data_v[v_off + col_base + COLS_PER_WG], data_v[v_off + col_base + 2u * COLS_PER_WG], data_v[v_off + col_base + 3u * COLS_PER_WG]); + const vec4 delta = (v_cols - kv_cols) * beta_val; + s0 = fma(vec4(g_val), s0, k_vec * delta.x); + s1 = fma(vec4(g_val), s1, k_vec * delta.y); + s2 = fma(vec4(g_val), s2, k_vec * delta.z); + s3 = fma(vec4(g_val), s3, k_vec * delta.w); + + attn_cols = subgroupAdd(vec4( + dot(s0, q_vec), + dot(s1, q_vec), + dot(s2, q_vec), + dot(s3, q_vec) + )) * scale; + } + if (COLS_PER_WG == 1u && ((col_base & 3u) == 0u)) { + if (lane == 0u) { + data_dst_v4[(attn_off + col_base) >> 2u] = attn_cols; + } + } else if (lane < 4u) { + data_dst[attn_off + col_base + lane * COLS_PER_WG] = attn_cols[lane]; + } + } else { + vec2 attn_cols; + if (CONTIG_VEC4 >= 2u) { + const vec2 kv_sh = vec2(dot(s0, k_vec), dot(s1, k_vec)) * g_val; + const vec2 sq_sh = vec2(dot(s0, q_vec), dot(s1, q_vec)); + const float kq_sh = dot(k_vec, q_vec); + const vec2 kv_cols = subgroupAdd(kv_sh); + const vec2 sq_cols = subgroupAdd(sq_sh); + const float kq_dot = subgroupAdd(kq_sh); + const vec2 v_cols = vec2(data_v[v_off + col_base], data_v[v_off + col_base + COLS_PER_WG]); + const vec2 delta = (v_cols - kv_cols) * beta_val; + attn_cols = fma(vec2(g_val), sq_cols, delta * kq_dot) * scale; + s0 = fma(vec4(g_val), s0, k_vec * delta.x); + s1 = fma(vec4(g_val), s1, k_vec * delta.y); + } else { + vec2 kv_sh = vec2(dot(s0, k_vec), dot(s1, k_vec)) * g_val; + const vec2 kv_cols = subgroupAdd(kv_sh); + const vec2 v_cols = vec2(data_v[v_off + col_base], data_v[v_off + col_base + COLS_PER_WG]); + const vec2 delta = (v_cols - kv_cols) * beta_val; + s0 = fma(vec4(g_val), s0, k_vec * delta.x); + s1 = fma(vec4(g_val), s1, k_vec * delta.y); + + attn_cols = subgroupAdd(vec2(dot(s0, q_vec), dot(s1, q_vec))) * scale; + } + if (lane < 2u) { + data_dst[attn_off + col_base + lane * COLS_PER_WG] = attn_cols[lane]; + } + } + + attn_off += S_V * H; + q_off += sq2; + v_off += sv2; + gb_off += sb2; + + if (eff_K > 1u) { + const int target_slot = int(n_tokens) - 1 - int(t); + if (target_slot >= 0 && target_slot < int(eff_K)) { + for (uint c = 0u; c < COLS_PER_SUB; ++c) { + const vec4 sv = (c == 0u) ? s0 : ((c == 1u) ? s1 : ((c == 2u) ? s2 : s3)); + const uint col = col_base + c * COLS_PER_WG; +#if STATE_OUT + const uint slot_base = uint(target_slot) * sout_nb2 + state_out_base; + if (st_v4_ok) { + data_state_out_v4[((slot_base + col * S_V) >> 2u) + lane] = sv; + } else { + const uint b = slot_base + col * S_V + 4u * lane; + data_state_out[b + 0u] = sv.x; data_state_out[b + 1u] = sv.y; + data_state_out[b + 2u] = sv.z; data_state_out[b + 3u] = sv.w; + } +#else + const uint slot_base = s_off + uint(target_slot) * state_size_per_snap + state_out_base; + if (st_v4_ok) { + data_dst_v4[((slot_base + col * S_V) >> 2u) + lane] = sv; + } else { + const uint b = slot_base + col * S_V + 4u * lane; + data_dst[b + 0u] = sv.x; data_dst[b + 1u] = sv.y; + data_dst[b + 2u] = sv.z; data_dst[b + 3u] = sv.w; + } +#endif + } + } + } + } + + if (eff_K == 1u) { + for (uint c = 0u; c < COLS_PER_SUB; ++c) { + const vec4 sv = (c == 0u) ? s0 : ((c == 1u) ? s1 : ((c == 2u) ? s2 : s3)); + const uint col = col_base + c * COLS_PER_WG; +#if STATE_OUT + data_state_out_v4[((state_out_base + col * S_V) >> 2u) + lane] = sv; +#else + if (st_v4_ok) { + data_dst_v4[((s_off + state_out_base + col * S_V) >> 2u) + lane] = sv; + } else { + const uint b = s_off + state_out_base + col * S_V + 4u * lane; + data_dst[b + 0u] = sv.x; data_dst[b + 1u] = sv.y; + data_dst[b + 2u] = sv.z; data_dst[b + 3u] = sv.w; + } +#endif + } + } + return; + } +#endif - FLOAT_TYPE s_shard[ROWS_PER_LANE]; - [[unroll]] for (uint r = 0; r < ROWS_PER_LANE; r++) { - s_shard[r] = FLOAT_TYPE(data_state[state_in_base + col * S_V + r * LANES_PER_COLUMN + lane]); + FLOAT_TYPE s_shard[COLS_PER_SUB * ROWS_PER_LANE]; + if (zero_state) { + [[unroll]] for (uint idx_s = 0; idx_s < COLS_PER_SUB * ROWS_PER_LANE; idx_s++) { + s_shard[idx_s] = FLOAT_TYPE(0.0); + } + } else { + [[unroll]] for (uint c = 0; c < COLS_PER_SUB; c++) { + const uint col = col_base + c * COLS_PER_WG; + [[unroll]] for (uint r = 0; r < ROWS_PER_LANE; r++) { + s_shard[c * ROWS_PER_LANE + r] = FLOAT_TYPE(data_state[state_in_base + col * S_V + r * LANES_PER_COLUMN + lane]); + } + } } // snapshot slot mapping: slot 0 = most recent state, slot s = s tokens back. @@ -126,64 +326,138 @@ void main() { const FLOAT_TYPE beta_val = FLOAT_TYPE(data_beta[gb_off]); FLOAT_TYPE k_reg[ROWS_PER_LANE]; - FLOAT_TYPE q_reg[ROWS_PER_LANE]; [[unroll]] for (uint r = 0; r < ROWS_PER_LANE; r++) { const uint i = r * LANES_PER_COLUMN + lane; k_reg[r] = FLOAT_TYPE(data_k[k_off + i]); - q_reg[r] = FLOAT_TYPE(data_q[q_off + i]); } - FLOAT_TYPE g_exp[ROWS_PER_LANE]; if (KDA == 0) { const FLOAT_TYPE g_val = exp(FLOAT_TYPE(data_g[gb_off])); - [[unroll]] for (uint r = 0; r < ROWS_PER_LANE; r++) { - g_exp[r] = g_val; + FLOAT_TYPE kv_shards[COLS_PER_SUB]; + [[unroll]] for (uint c = 0; c < COLS_PER_SUB; c++) { + const uint sb = c * ROWS_PER_LANE; + FLOAT_TYPE kv_shard = 0.0; + if (COLS_PER_SUB == 1u) { + [[unroll]] for (uint r = 0; r < ROWS_PER_LANE; r++) { + kv_shard += g_val * s_shard[sb + r] * k_reg[r]; + } + kv_shards[c] = kv_shard; + } else { + [[unroll]] for (uint r = 0; r < ROWS_PER_LANE; r++) { + kv_shard += s_shard[sb + r] * k_reg[r]; + } + kv_shards[c] = kv_shard * g_val; + } + } + FLOAT_TYPE kv_cols[COLS_PER_SUB]; + [[unroll]] for (uint c = 0; c < COLS_PER_SUB; c++) { + kv_cols[c] = reduce_partial(kv_shards[c]); + } + [[unroll]] for (uint c = 0; c < COLS_PER_SUB; c++) { + const uint col = col_base + c * COLS_PER_WG; + const uint sb = c * ROWS_PER_LANE; + const FLOAT_TYPE delta_col = (FLOAT_TYPE(data_v[v_off + col]) - kv_cols[c]) * beta_val; + [[unroll]] for (uint r = 0; r < ROWS_PER_LANE; r++) { + s_shard[sb + r] = g_val * s_shard[sb + r] + k_reg[r] * delta_col; + } } } else { + FLOAT_TYPE g_exp[ROWS_PER_LANE]; const uint g_base = gb_off * S_V; [[unroll]] for (uint r = 0; r < ROWS_PER_LANE; r++) { const uint i = r * LANES_PER_COLUMN + lane; g_exp[r] = exp(FLOAT_TYPE(data_g[g_base + i])); } - } + [[unroll]] for (uint c = 0; c < COLS_PER_SUB; c++) { + const uint col = col_base + c * COLS_PER_WG; + const uint sb = c * ROWS_PER_LANE; + const FLOAT_TYPE v_val = FLOAT_TYPE(data_v[v_off + col]); - const FLOAT_TYPE v_val = FLOAT_TYPE(data_v[v_off + col]); + FLOAT_TYPE kv_shard = 0.0; + [[unroll]] for (uint r = 0; r < ROWS_PER_LANE; r++) { + kv_shard += g_exp[r] * s_shard[sb + r] * k_reg[r]; + } + FLOAT_TYPE kv_col = reduce_partial(kv_shard); + FLOAT_TYPE delta_col = (v_val - kv_col) * beta_val; - FLOAT_TYPE kv_shard = 0.0; - [[unroll]] for (uint r = 0; r < ROWS_PER_LANE; r++) { - kv_shard += g_exp[r] * s_shard[r] * k_reg[r]; + [[unroll]] for (uint r = 0; r < ROWS_PER_LANE; r++) { + s_shard[sb + r] = g_exp[r] * s_shard[sb + r] + k_reg[r] * delta_col; + } + } } - FLOAT_TYPE kv_col = reduce_partial(kv_shard); - FLOAT_TYPE delta_col = (v_val - kv_col) * beta_val; + if (COLS_PER_SUB == 1u) { + FLOAT_TYPE attn_partial = 0.0; + [[unroll]] for (uint r = 0; r < ROWS_PER_LANE; r++) { + const uint i = r * LANES_PER_COLUMN + lane; + attn_partial += s_shard[r] * FLOAT_TYPE(data_q[q_off + i]); + } + FLOAT_TYPE attn_col = reduce_partial(attn_partial); - FLOAT_TYPE attn_partial = 0.0; - [[unroll]] for (uint r = 0; r < ROWS_PER_LANE; r++) { - s_shard[r] = g_exp[r] * s_shard[r] + k_reg[r] * delta_col; - attn_partial += s_shard[r] * q_reg[r]; - } - FLOAT_TYPE attn_col = reduce_partial(attn_partial); + if (lane == 0) { + data_dst[attn_off + col_base] = attn_col * scale; + } + } else { + FLOAT_TYPE q_reg[ROWS_PER_LANE]; + [[unroll]] for (uint r = 0; r < ROWS_PER_LANE; r++) { + const uint i = r * LANES_PER_COLUMN + lane; + q_reg[r] = FLOAT_TYPE(data_q[q_off + i]); + } - if (lane == 0) { - data_dst[attn_off + col] = attn_col * scale; + FLOAT_TYPE attn_partials[COLS_PER_SUB]; + [[unroll]] for (uint c = 0; c < COLS_PER_SUB; c++) { + const uint sb = c * ROWS_PER_LANE; + FLOAT_TYPE attn_partial = 0.0; + [[unroll]] for (uint r = 0; r < ROWS_PER_LANE; r++) { + attn_partial += s_shard[sb + r] * q_reg[r]; + } + attn_partials[c] = attn_partial; + } + [[unroll]] for (uint c = 0; c < COLS_PER_SUB; c++) { + const FLOAT_TYPE attn_col = reduce_partial(attn_partials[c]); + if (lane == 0) { + const uint col = col_base + c * COLS_PER_WG; + data_dst[attn_off + col] = attn_col * scale; + } + } } attn_off += S_V * H; - if (K > 1u) { + if (eff_K > 1u) { const int target_slot = int(n_tokens) - 1 - int(t); - if (target_slot >= 0 && target_slot < int(K)) { + if (target_slot >= 0 && target_slot < int(eff_K)) { +#if STATE_OUT + const uint slot_base = uint(target_slot) * sout_nb2 + state_out_base; + [[unroll]] for (uint c = 0; c < COLS_PER_SUB; c++) { + const uint col = col_base + c * COLS_PER_WG; + [[unroll]] for (uint r = 0; r < ROWS_PER_LANE; r++) { + data_state_out[slot_base + col * S_V + r * LANES_PER_COLUMN + lane] = s_shard[c * ROWS_PER_LANE + r]; + } + } +#else const uint slot_base = s_off + uint(target_slot) * state_size_per_snap + state_out_base; - [[unroll]] for (uint r = 0; r < ROWS_PER_LANE; r++) { - data_dst[slot_base + col * S_V + r * LANES_PER_COLUMN + lane] = s_shard[r]; + [[unroll]] for (uint c = 0; c < COLS_PER_SUB; c++) { + const uint col = col_base + c * COLS_PER_WG; + [[unroll]] for (uint r = 0; r < ROWS_PER_LANE; r++) { + data_dst[slot_base + col * S_V + r * LANES_PER_COLUMN + lane] = s_shard[c * ROWS_PER_LANE + r]; + } } +#endif } } } - if (K == 1u) { - [[unroll]] for (uint r = 0; r < ROWS_PER_LANE; r++) { - data_dst[s_off + state_out_base + col * S_V + r * LANES_PER_COLUMN + lane] = s_shard[r]; + if (eff_K == 1u) { + [[unroll]] for (uint c = 0; c < COLS_PER_SUB; c++) { + const uint col = col_base + c * COLS_PER_WG; + [[unroll]] for (uint r = 0; r < ROWS_PER_LANE; r++) { +#if STATE_OUT + data_state_out[state_out_base + col * S_V + r * LANES_PER_COLUMN + lane] = s_shard[c * ROWS_PER_LANE + r]; +#else + data_dst[s_off + state_out_base + col * S_V + r * LANES_PER_COLUMN + lane] = s_shard[c * ROWS_PER_LANE + r]; +#endif + } } } } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp index 34be1f72ba7b..efc48175e800 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp @@ -5,7 +5,11 @@ #include "mul_mat_vec_base.glsl" #include "dequant_funcs.glsl" +#if defined(USE_SUBGROUP_ADD_NO_SHMEM) && !defined(NEEDS_INIT_IQ_SHMEM) +layout(local_size_x_id = 0, local_size_y_id = 4, local_size_z = 1) in; +#else layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in; +#endif #if !defined(DATA_A_F32) && !defined(DATA_A_F16) && !defined(DATA_A_BF16) #define K_PER_ITER 8 @@ -36,9 +40,11 @@ vec4 load_b(const uint j, const uint iybs, const uint iqs, const bool lastiter, return vec4(FLOAT_TYPE(data_b[j*p.batch_stride_b + b_offset + iybs + iqs]), FLOAT_TYPE(data_b[j*p.batch_stride_b + b_offset + iybs + iqs + y_offset]), 0, 0); - } else { + } else if (!lastiter || iybs + iqs < p.ncols) { return vec4(FLOAT_TYPE(data_b[j*p.batch_stride_b + b_offset + iybs + iqs]), 0, 0, 0); + } else { + return vec4(0); } } @@ -69,14 +75,15 @@ void iter(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const uint first_row, const const vec4 b = load_b(j, iybs, iqs, lastiter, OOB_y, OOB_z, OOB_w); #endif uint ibi = first_row*p.ncols; - [[unroll]] for (uint n = 0; n < num_rows; ++n) { - const uint ib = (ibi + col)/QUANT_K; // block index - ibi += p.ncols; + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + const uint rn = (n == 0 || n < num_rows) ? n : 0; + const uint ib = (ibi + rn*p.ncols + col)/QUANT_K; // block index #if K_PER_ITER == 8 vec4 v = dequantize4(ib, iqs, a_offset); vec4 v2 = dequantize4(ib, iqs+(4/QUANT_R), a_offset); +#if defined(DATA_A_Q4_1) || defined(DATA_A_Q5_1) const vec2 dm = get_dm(ib, a_offset); if (dm.y != 0) { // quant has min component v = v * dm.x + dm.y; @@ -89,6 +96,14 @@ void iter(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const uint first_row, const if (dm.y == 0) rowtmp *= dm.x; +#elif defined(DATA_A_NVFP4) + FLOAT_TYPE rowtmp = dot(bv0, v); + rowtmp += dot(bv1, v2); +#else + FLOAT_TYPE rowtmp = dot(bv0, v); + rowtmp += dot(bv1, v2); + rowtmp *= get_dm(ib, a_offset).x; +#endif temp[j][n] += rowtmp; #else @@ -115,33 +130,486 @@ void iter(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const uint first_row, const } #if defined(DATA_A_F32) || defined(DATA_A_F16) || defined(DATA_A_BF16) -void iter_aligned_nonquant(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const uint first_row, const uint num_rows, const uint tid, const uint i) +vec4 dequantize4_2aligned_a2(uint ib, uint a_offset) { +#if defined(DATA_A_F32) + const vec2 a = data_a2_packed64[(a_offset + ib)/2]; + const vec2 b = data_a2_packed64[(a_offset + ib)/2 + 1]; + return vec4(a, b); +#elif defined(DATA_A_F16) + const vec2 a = data_a2_packed32[(a_offset + ib)/2]; + const vec2 b = data_a2_packed32[(a_offset + ib)/2 + 1]; + return vec4(a, b); +#else + const uint a = data_a2_packed32[(a_offset + ib)/2]; + const uint b = data_a2_packed32[(a_offset + ib)/2 + 1]; + return vec4(uintBitsToFloat((a & 0x0000ffff) << 16), + uintBitsToFloat( a & 0xffff0000), + uintBitsToFloat((b & 0x0000ffff) << 16), + uintBitsToFloat( b & 0xffff0000)); +#endif +} + +void iter_aligned_nonquant_a(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], inout FLOAT_TYPE temp2[NUM_COLS][NUM_ROWS], const uint first_row, const uint num_rows, const uint tid, const uint i) { - [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { - const uint col = i*BLOCK_SIZE + K_PER_ITER*tid; - const uint iqs = 0; // quant index - const uint iybs = col; // y block start index + const uint col = i*BLOCK_SIZE + K_PER_ITER*tid; + uint ibi = first_row*p.ncols; + if (NUM_COLS <= 4) { + vec4 b_arr[4]; + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + b_arr[j] = data_b_v4[(j*p.batch_stride_b + b_offset + col) / 4]; + } + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + const uint rn = (n == 0 || n < num_rows) ? n : 0; + const uint ib = (ibi + rn*p.ncols + col)/QUANT_K; + const vec4 v = dequantize4_2aligned(ib, 0, a_offset); + vec4 u = vec4(0.0); + if (GEN_GLU == 3) { + u = dequantize4_2aligned_a2(ib, a_offset); + } + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + temp[j][n] += dot(v, b_arr[j]); + if (GEN_GLU == 3) { + temp2[j][n] += dot(u, b_arr[j]); + } + } + } + } else { + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + const uint rn = (n == 0 || n < num_rows) ? n : 0; + const uint ib = (ibi + rn*p.ncols + col)/QUANT_K; + const vec4 v = dequantize4_2aligned(ib, 0, a_offset); + vec4 u = vec4(0.0); + if (GEN_GLU == 3) { + u = dequantize4_2aligned_a2(ib, a_offset); + } + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + const vec4 b = data_b_v4[(j*p.batch_stride_b + b_offset + col) / 4]; + temp[j][n] += dot(v, b); + if (GEN_GLU == 3) { + temp2[j][n] += dot(u, b); + } + } + } + } +} + +void iter_aligned_nonquant_a2(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const uint first_row, const uint num_rows, const uint tid, const uint i) +{ + const uint col = i*BLOCK_SIZE + K_PER_ITER*tid; + uint ibi = first_row*p.ncols; + if (NUM_COLS <= 4) { + vec4 b_arr[4]; + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + b_arr[j] = data_b_v4[(j*p.batch_stride_b + b_offset + col) / 4]; + } + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + const uint rn = (n == 0 || n < num_rows) ? n : 0; + const uint ib = (ibi + rn*p.ncols + col)/QUANT_K; + const vec4 v = dequantize4_2aligned_a2(ib, a_offset); + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + temp[j][n] += dot(v, b_arr[j]); + } + } + } else { + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + const uint rn = (n == 0 || n < num_rows) ? n : 0; + const uint ib = (ibi + rn*p.ncols + col)/QUANT_K; + const vec4 v = dequantize4_2aligned_a2(ib, a_offset); + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + const vec4 b = data_b_v4[(j*p.batch_stride_b + b_offset + col) / 4]; + temp[j][n] += dot(v, b); + } + } + } +} +#endif - const vec4 b = data_b_v4[(j*p.batch_stride_b + b_offset + iybs + iqs) / 4]; +#if defined(DATA_A_Q8_0) +void iter_aligned_q8_0_a(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], inout FLOAT_TYPE temp2[NUM_COLS][NUM_ROWS], const uint row_base, const uint num_rows, const uint tid, const uint iter_idx) +{ + const uint b_col4 = iter_idx * (2 * BLOCK_SIZE) + 2 * tid; + const uint iqs_h = 4 * (tid & 3); + const uint ib_col = iter_idx * (BLOCK_SIZE >> 2) + (tid >> 2); + const uint blocks_per_row = p.ncols >> 5; + + if (NUM_COLS <= 4) { + vec4 bv0_arr[NUM_COLS]; + vec4 bv1_arr[NUM_COLS]; + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + const uint b4 = (j * p.batch_stride_b + b_offset) >> 2; + bv0_arr[j] = vec4(data_b_v4[b4 + b_col4]); + bv1_arr[j] = vec4(data_b_v4[b4 + b_col4 + 1]); + } + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + const uint rn = (n == 0 || n < num_rows) ? n : 0; + const uint ib = a_offset + (row_base + rn) * blocks_per_row + ib_col; + const vec4 v = vec4(unpack8(pack32(i16vec2(data_a_packed16[ib].qs[iqs_h ], data_a_packed16[ib].qs[iqs_h + 1])))); + const vec4 v2 = vec4(unpack8(pack32(i16vec2(data_a_packed16[ib].qs[iqs_h + 2], data_a_packed16[ib].qs[iqs_h + 3])))); + const FLOAT_TYPE da = float(data_a_packed16[ib].d); + vec4 u = vec4(0.0), u2 = vec4(0.0); + FLOAT_TYPE da2 = FLOAT_TYPE(0); + if (GEN_GLU == 3) { + u = vec4(unpack8(pack32(i16vec2(data_a2_packed16[ib].qs[iqs_h ], data_a2_packed16[ib].qs[iqs_h + 1])))); + u2 = vec4(unpack8(pack32(i16vec2(data_a2_packed16[ib].qs[iqs_h + 2], data_a2_packed16[ib].qs[iqs_h + 3])))); + da2 = float(data_a2_packed16[ib].d); + } + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + FLOAT_TYPE rowtmp = dot(bv0_arr[j], v); + rowtmp += dot(bv1_arr[j], v2); + rowtmp *= da; + temp[j][n] += rowtmp; + if (GEN_GLU == 3) { + FLOAT_TYPE rowtmp2 = dot(bv0_arr[j], u); + rowtmp2 += dot(bv1_arr[j], u2); + rowtmp2 *= da2; + temp2[j][n] += rowtmp2; + } + } + } + } else { + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + const uint rn = (n == 0 || n < num_rows) ? n : 0; + const uint ib = a_offset + (row_base + rn) * blocks_per_row + ib_col; + const vec4 v = vec4(unpack8(pack32(i16vec2(data_a_packed16[ib].qs[iqs_h ], data_a_packed16[ib].qs[iqs_h + 1])))); + const vec4 v2 = vec4(unpack8(pack32(i16vec2(data_a_packed16[ib].qs[iqs_h + 2], data_a_packed16[ib].qs[iqs_h + 3])))); + const FLOAT_TYPE da = float(data_a_packed16[ib].d); + vec4 u = vec4(0.0), u2 = vec4(0.0); + FLOAT_TYPE da2 = FLOAT_TYPE(0); + if (GEN_GLU == 3) { + u = vec4(unpack8(pack32(i16vec2(data_a2_packed16[ib].qs[iqs_h ], data_a2_packed16[ib].qs[iqs_h + 1])))); + u2 = vec4(unpack8(pack32(i16vec2(data_a2_packed16[ib].qs[iqs_h + 2], data_a2_packed16[ib].qs[iqs_h + 3])))); + da2 = float(data_a2_packed16[ib].d); + } + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + const uint b4 = (j * p.batch_stride_b + b_offset) >> 2; + const vec4 bv0 = vec4(data_b_v4[b4 + b_col4]); + const vec4 bv1 = vec4(data_b_v4[b4 + b_col4 + 1]); + FLOAT_TYPE rowtmp = dot(bv0, v); + rowtmp += dot(bv1, v2); + rowtmp *= da; + temp[j][n] += rowtmp; + if (GEN_GLU == 3) { + FLOAT_TYPE rowtmp2 = dot(bv0, u); + rowtmp2 += dot(bv1, u2); + rowtmp2 *= da2; + temp2[j][n] += rowtmp2; + } + } + } + } +} - uint ibi = first_row*p.ncols; - [[unroll]] for (uint n = 0; n < num_rows; ++n) { - const uint ib = (ibi + col)/QUANT_K; // block index - ibi += p.ncols; +void iter_aligned_q8_0_a2(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const uint row_base, const uint num_rows, const uint tid, const uint iter_idx) +{ + const uint b_col4 = iter_idx * (2 * BLOCK_SIZE) + 2 * tid; + const uint iqs_h = 4 * (tid & 3); + const uint ib_col = iter_idx * (BLOCK_SIZE >> 2) + (tid >> 2); + const uint blocks_per_row = p.ncols >> 5; + + if (NUM_COLS <= 4) { + vec4 bv0_arr[NUM_COLS]; + vec4 bv1_arr[NUM_COLS]; + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + const uint b4 = (j * p.batch_stride_b + b_offset) >> 2; + bv0_arr[j] = vec4(data_b_v4[b4 + b_col4]); + bv1_arr[j] = vec4(data_b_v4[b4 + b_col4 + 1]); + } + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + const uint rn = (n == 0 || n < num_rows) ? n : 0; + const uint ib = a_offset + (row_base + rn) * blocks_per_row + ib_col; + const vec4 v = vec4(unpack8(pack32(i16vec2(data_a2_packed16[ib].qs[iqs_h ], data_a2_packed16[ib].qs[iqs_h + 1])))); + const vec4 v2 = vec4(unpack8(pack32(i16vec2(data_a2_packed16[ib].qs[iqs_h + 2], data_a2_packed16[ib].qs[iqs_h + 3])))); + const FLOAT_TYPE da = float(data_a2_packed16[ib].d); + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + FLOAT_TYPE rowtmp = dot(bv0_arr[j], v); + rowtmp += dot(bv1_arr[j], v2); + rowtmp *= da; + temp[j][n] += rowtmp; + } + } + } else { + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + const uint rn = (n == 0 || n < num_rows) ? n : 0; + const uint ib = a_offset + (row_base + rn) * blocks_per_row + ib_col; + const vec4 v = vec4(unpack8(pack32(i16vec2(data_a2_packed16[ib].qs[iqs_h ], data_a2_packed16[ib].qs[iqs_h + 1])))); + const vec4 v2 = vec4(unpack8(pack32(i16vec2(data_a2_packed16[ib].qs[iqs_h + 2], data_a2_packed16[ib].qs[iqs_h + 3])))); + const FLOAT_TYPE da = float(data_a2_packed16[ib].d); + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + const uint b4 = (j * p.batch_stride_b + b_offset) >> 2; + const vec4 bv0 = vec4(data_b_v4[b4 + b_col4]); + const vec4 bv1 = vec4(data_b_v4[b4 + b_col4 + 1]); + FLOAT_TYPE rowtmp = dot(bv0, v); + rowtmp += dot(bv1, v2); + rowtmp *= da; + temp[j][n] += rowtmp; + } + } + } +} + +#ifndef MUL_MAT_ID +void iter2_aligned_q8_0_a(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], inout FLOAT_TYPE temp2[NUM_COLS][NUM_ROWS], const uint row_base, const uint num_rows, const uint tid, const uint iter_idx) +{ + const uint b_col4_0 = iter_idx * (2 * BLOCK_SIZE) + 2 * tid; + const uint b_col4_1 = b_col4_0 + (2 * BLOCK_SIZE); + const uint iqs_h = 4 * (tid & 3); + const uint ib_col_0 = iter_idx * (BLOCK_SIZE >> 2) + (tid >> 2); + const uint ib_col_1 = ib_col_0 + (BLOCK_SIZE >> 2); + const uint blocks_per_row = p.ncols >> 5; + + if (NUM_COLS <= 4) { + vec4 bv0_0[NUM_COLS], bv1_0[NUM_COLS]; + vec4 bv0_1[NUM_COLS], bv1_1[NUM_COLS]; + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + const uint b4 = (j * p.batch_stride_b + b_offset) >> 2; + bv0_0[j] = vec4(data_b_v4[b4 + b_col4_0]); + bv1_0[j] = vec4(data_b_v4[b4 + b_col4_0 + 1]); + bv0_1[j] = vec4(data_b_v4[b4 + b_col4_1]); + bv1_1[j] = vec4(data_b_v4[b4 + b_col4_1 + 1]); + } + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + const uint rn = (n == 0 || n < num_rows) ? n : 0; + const uint row_off = a_offset + (row_base + rn) * blocks_per_row; + const uint ib0 = row_off + ib_col_0; + const uint ib1 = row_off + ib_col_1; + const i16vec2 q0_lo = i16vec2(data_a_packed16[ib0].qs[iqs_h ], data_a_packed16[ib0].qs[iqs_h + 1]); + const i16vec2 q0_hi = i16vec2(data_a_packed16[ib0].qs[iqs_h + 2], data_a_packed16[ib0].qs[iqs_h + 3]); + const FLOAT_TYPE da0 = float(data_a_packed16[ib0].d); + const i16vec2 q1_lo = i16vec2(data_a_packed16[ib1].qs[iqs_h ], data_a_packed16[ib1].qs[iqs_h + 1]); + const i16vec2 q1_hi = i16vec2(data_a_packed16[ib1].qs[iqs_h + 2], data_a_packed16[ib1].qs[iqs_h + 3]); + const FLOAT_TYPE da1 = float(data_a_packed16[ib1].d); + + i16vec2 u0_lo = i16vec2(0), u0_hi = i16vec2(0); + i16vec2 u1_lo = i16vec2(0), u1_hi = i16vec2(0); + FLOAT_TYPE du0 = FLOAT_TYPE(0), du1 = FLOAT_TYPE(0); + if (GEN_GLU == 3) { + u0_lo = i16vec2(data_a2_packed16[ib0].qs[iqs_h ], data_a2_packed16[ib0].qs[iqs_h + 1]); + u0_hi = i16vec2(data_a2_packed16[ib0].qs[iqs_h + 2], data_a2_packed16[ib0].qs[iqs_h + 3]); + du0 = float(data_a2_packed16[ib0].d); + u1_lo = i16vec2(data_a2_packed16[ib1].qs[iqs_h ], data_a2_packed16[ib1].qs[iqs_h + 1]); + u1_hi = i16vec2(data_a2_packed16[ib1].qs[iqs_h + 2], data_a2_packed16[ib1].qs[iqs_h + 3]); + du1 = float(data_a2_packed16[ib1].d); + } - const vec4 v = dequantize4_2aligned(ib, iqs, a_offset); + const vec4 v0_0 = vec4(unpack8(pack32(q0_lo))); + const vec4 v1_0 = vec4(unpack8(pack32(q0_hi))); + const vec4 v0_1 = vec4(unpack8(pack32(q1_lo))); + const vec4 v1_1 = vec4(unpack8(pack32(q1_hi))); + + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + temp[j][n] += (dot(bv0_0[j], v0_0) + dot(bv1_0[j], v1_0)) * da0 + + (dot(bv0_1[j], v0_1) + dot(bv1_1[j], v1_1)) * da1; + if (GEN_GLU == 3) { + const vec4 w0_0 = vec4(unpack8(pack32(u0_lo))); + const vec4 w1_0 = vec4(unpack8(pack32(u0_hi))); + const vec4 w0_1 = vec4(unpack8(pack32(u1_lo))); + const vec4 w1_1 = vec4(unpack8(pack32(u1_hi))); + temp2[j][n] += (dot(bv0_0[j], w0_0) + dot(bv1_0[j], w1_0)) * du0 + + (dot(bv0_1[j], w0_1) + dot(bv1_1[j], w1_1)) * du1; + } + } + } + } else { + iter_aligned_q8_0_a(temp, temp2, row_base, num_rows, tid, iter_idx); + iter_aligned_q8_0_a(temp, temp2, row_base, num_rows, tid, iter_idx + 1); + } +} - // matrix multiplication - temp[j][n] += dot(v, b); +void iter2_aligned_q8_0_a2(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const uint row_base, const uint num_rows, const uint tid, const uint iter_idx) +{ + const uint b_col4_0 = iter_idx * (2 * BLOCK_SIZE) + 2 * tid; + const uint b_col4_1 = b_col4_0 + (2 * BLOCK_SIZE); + const uint iqs_h = 4 * (tid & 3); + const uint ib_col_0 = iter_idx * (BLOCK_SIZE >> 2) + (tid >> 2); + const uint ib_col_1 = ib_col_0 + (BLOCK_SIZE >> 2); + const uint blocks_per_row = p.ncols >> 5; + + if (NUM_COLS <= 4) { + vec4 bv0_0[NUM_COLS], bv1_0[NUM_COLS]; + vec4 bv0_1[NUM_COLS], bv1_1[NUM_COLS]; + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + const uint b4 = (j * p.batch_stride_b + b_offset) >> 2; + bv0_0[j] = vec4(data_b_v4[b4 + b_col4_0]); + bv1_0[j] = vec4(data_b_v4[b4 + b_col4_0 + 1]); + bv0_1[j] = vec4(data_b_v4[b4 + b_col4_1]); + bv1_1[j] = vec4(data_b_v4[b4 + b_col4_1 + 1]); + } + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + const uint rn = (n == 0 || n < num_rows) ? n : 0; + const uint row_off = a_offset + (row_base + rn) * blocks_per_row; + const uint ib0 = row_off + ib_col_0; + const uint ib1 = row_off + ib_col_1; + const i16vec2 q0_lo = i16vec2(data_a2_packed16[ib0].qs[iqs_h ], data_a2_packed16[ib0].qs[iqs_h + 1]); + const i16vec2 q0_hi = i16vec2(data_a2_packed16[ib0].qs[iqs_h + 2], data_a2_packed16[ib0].qs[iqs_h + 3]); + const FLOAT_TYPE da0 = float(data_a2_packed16[ib0].d); + const i16vec2 q1_lo = i16vec2(data_a2_packed16[ib1].qs[iqs_h ], data_a2_packed16[ib1].qs[iqs_h + 1]); + const i16vec2 q1_hi = i16vec2(data_a2_packed16[ib1].qs[iqs_h + 2], data_a2_packed16[ib1].qs[iqs_h + 3]); + const FLOAT_TYPE da1 = float(data_a2_packed16[ib1].d); + + const vec4 v0_0 = vec4(unpack8(pack32(q0_lo))); + const vec4 v1_0 = vec4(unpack8(pack32(q0_hi))); + const vec4 v0_1 = vec4(unpack8(pack32(q1_lo))); + const vec4 v1_1 = vec4(unpack8(pack32(q1_hi))); + + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + temp[j][n] += (dot(bv0_0[j], v0_0) + dot(bv1_0[j], v1_0)) * da0 + + (dot(bv0_1[j], v0_1) + dot(bv1_1[j], v1_1)) * da1; + } + } + } else { + iter_aligned_q8_0_a2(temp, row_base, num_rows, tid, iter_idx); + iter_aligned_q8_0_a2(temp, row_base, num_rows, tid, iter_idx + 1); + } +} + +void iter4_aligned_q8_0_a(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], inout FLOAT_TYPE temp2[NUM_COLS][NUM_ROWS], const uint row_base, const uint num_rows, const uint tid, const uint iter_idx) +{ + const uint b4 = b_offset >> 2; + const uint b_col4_0 = iter_idx * (2 * BLOCK_SIZE) + 2 * tid; + const uint b_col4_1 = b_col4_0 + (2 * BLOCK_SIZE); + const uint b_col4_2 = b_col4_0 + (4 * BLOCK_SIZE); + const uint b_col4_3 = b_col4_0 + (6 * BLOCK_SIZE); + const vec4 bv0_0 = vec4(data_b_v4[b4 + b_col4_0]); + const vec4 bv1_0 = vec4(data_b_v4[b4 + b_col4_0 + 1]); + const vec4 bv0_1 = vec4(data_b_v4[b4 + b_col4_1]); + const vec4 bv1_1 = vec4(data_b_v4[b4 + b_col4_1 + 1]); + const vec4 bv0_2 = vec4(data_b_v4[b4 + b_col4_2]); + const vec4 bv1_2 = vec4(data_b_v4[b4 + b_col4_2 + 1]); + const vec4 bv0_3 = vec4(data_b_v4[b4 + b_col4_3]); + const vec4 bv1_3 = vec4(data_b_v4[b4 + b_col4_3 + 1]); + + const uint iqs_h = 4 * (tid & 3); + const uint ib_col_0 = iter_idx * (BLOCK_SIZE >> 2) + (tid >> 2); + const uint ib_col_1 = ib_col_0 + (BLOCK_SIZE >> 2); + const uint ib_col_2 = ib_col_0 + 2 * (BLOCK_SIZE >> 2); + const uint ib_col_3 = ib_col_0 + 3 * (BLOCK_SIZE >> 2); + const uint blocks_per_row = p.ncols >> 5; + + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + const uint rn = (n == 0 || n < num_rows) ? n : 0; + const uint row_off = a_offset + (row_base + rn) * blocks_per_row; + const uint ib0 = row_off + ib_col_0; + const uint ib1 = row_off + ib_col_1; + const uint ib2 = row_off + ib_col_2; + const uint ib3 = row_off + ib_col_3; + + const i16vec2 q0_lo = i16vec2(data_a_packed16[ib0].qs[iqs_h ], data_a_packed16[ib0].qs[iqs_h + 1]); + const i16vec2 q0_hi = i16vec2(data_a_packed16[ib0].qs[iqs_h + 2], data_a_packed16[ib0].qs[iqs_h + 3]); + const FLOAT_TYPE da0 = float(data_a_packed16[ib0].d); + const i16vec2 q1_lo = i16vec2(data_a_packed16[ib1].qs[iqs_h ], data_a_packed16[ib1].qs[iqs_h + 1]); + const i16vec2 q1_hi = i16vec2(data_a_packed16[ib1].qs[iqs_h + 2], data_a_packed16[ib1].qs[iqs_h + 3]); + const FLOAT_TYPE da1 = float(data_a_packed16[ib1].d); + const i16vec2 q2_lo = i16vec2(data_a_packed16[ib2].qs[iqs_h ], data_a_packed16[ib2].qs[iqs_h + 1]); + const i16vec2 q2_hi = i16vec2(data_a_packed16[ib2].qs[iqs_h + 2], data_a_packed16[ib2].qs[iqs_h + 3]); + const FLOAT_TYPE da2 = float(data_a_packed16[ib2].d); + const i16vec2 q3_lo = i16vec2(data_a_packed16[ib3].qs[iqs_h ], data_a_packed16[ib3].qs[iqs_h + 1]); + const i16vec2 q3_hi = i16vec2(data_a_packed16[ib3].qs[iqs_h + 2], data_a_packed16[ib3].qs[iqs_h + 3]); + const FLOAT_TYPE da3 = float(data_a_packed16[ib3].d); + + i16vec2 u0_lo = i16vec2(0), u0_hi = i16vec2(0); + i16vec2 u1_lo = i16vec2(0), u1_hi = i16vec2(0); + i16vec2 u2_lo = i16vec2(0), u2_hi = i16vec2(0); + i16vec2 u3_lo = i16vec2(0), u3_hi = i16vec2(0); + FLOAT_TYPE du0 = FLOAT_TYPE(0), du1 = FLOAT_TYPE(0), du2 = FLOAT_TYPE(0), du3 = FLOAT_TYPE(0); + if (GEN_GLU == 3) { + u0_lo = i16vec2(data_a2_packed16[ib0].qs[iqs_h ], data_a2_packed16[ib0].qs[iqs_h + 1]); + u0_hi = i16vec2(data_a2_packed16[ib0].qs[iqs_h + 2], data_a2_packed16[ib0].qs[iqs_h + 3]); + du0 = float(data_a2_packed16[ib0].d); + u1_lo = i16vec2(data_a2_packed16[ib1].qs[iqs_h ], data_a2_packed16[ib1].qs[iqs_h + 1]); + u1_hi = i16vec2(data_a2_packed16[ib1].qs[iqs_h + 2], data_a2_packed16[ib1].qs[iqs_h + 3]); + du1 = float(data_a2_packed16[ib1].d); + u2_lo = i16vec2(data_a2_packed16[ib2].qs[iqs_h ], data_a2_packed16[ib2].qs[iqs_h + 1]); + u2_hi = i16vec2(data_a2_packed16[ib2].qs[iqs_h + 2], data_a2_packed16[ib2].qs[iqs_h + 3]); + du2 = float(data_a2_packed16[ib2].d); + u3_lo = i16vec2(data_a2_packed16[ib3].qs[iqs_h ], data_a2_packed16[ib3].qs[iqs_h + 1]); + u3_hi = i16vec2(data_a2_packed16[ib3].qs[iqs_h + 2], data_a2_packed16[ib3].qs[iqs_h + 3]); + du3 = float(data_a2_packed16[ib3].d); + } + + temp[0][n] += (dot(bv0_0, vec4(unpack8(pack32(q0_lo)))) + dot(bv1_0, vec4(unpack8(pack32(q0_hi))))) * da0 + + (dot(bv0_1, vec4(unpack8(pack32(q1_lo)))) + dot(bv1_1, vec4(unpack8(pack32(q1_hi))))) * da1 + + (dot(bv0_2, vec4(unpack8(pack32(q2_lo)))) + dot(bv1_2, vec4(unpack8(pack32(q2_hi))))) * da2 + + (dot(bv0_3, vec4(unpack8(pack32(q3_lo)))) + dot(bv1_3, vec4(unpack8(pack32(q3_hi))))) * da3; + if (GEN_GLU == 3) { + temp2[0][n] += (dot(bv0_0, vec4(unpack8(pack32(u0_lo)))) + dot(bv1_0, vec4(unpack8(pack32(u0_hi))))) * du0 + + (dot(bv0_1, vec4(unpack8(pack32(u1_lo)))) + dot(bv1_1, vec4(unpack8(pack32(u1_hi))))) * du1 + + (dot(bv0_2, vec4(unpack8(pack32(u2_lo)))) + dot(bv1_2, vec4(unpack8(pack32(u2_hi))))) * du2 + + (dot(bv0_3, vec4(unpack8(pack32(u3_lo)))) + dot(bv1_3, vec4(unpack8(pack32(u3_hi))))) * du3; } } } + +void iter4_aligned_q8_0_a2(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const uint row_base, const uint num_rows, const uint tid, const uint iter_idx) +{ + const uint b4 = b_offset >> 2; + const uint b_col4_0 = iter_idx * (2 * BLOCK_SIZE) + 2 * tid; + const uint b_col4_1 = b_col4_0 + (2 * BLOCK_SIZE); + const uint b_col4_2 = b_col4_0 + (4 * BLOCK_SIZE); + const uint b_col4_3 = b_col4_0 + (6 * BLOCK_SIZE); + const vec4 bv0_0 = vec4(data_b_v4[b4 + b_col4_0]); + const vec4 bv1_0 = vec4(data_b_v4[b4 + b_col4_0 + 1]); + const vec4 bv0_1 = vec4(data_b_v4[b4 + b_col4_1]); + const vec4 bv1_1 = vec4(data_b_v4[b4 + b_col4_1 + 1]); + const vec4 bv0_2 = vec4(data_b_v4[b4 + b_col4_2]); + const vec4 bv1_2 = vec4(data_b_v4[b4 + b_col4_2 + 1]); + const vec4 bv0_3 = vec4(data_b_v4[b4 + b_col4_3]); + const vec4 bv1_3 = vec4(data_b_v4[b4 + b_col4_3 + 1]); + + const uint iqs_h = 4 * (tid & 3); + const uint ib_col_0 = iter_idx * (BLOCK_SIZE >> 2) + (tid >> 2); + const uint ib_col_1 = ib_col_0 + (BLOCK_SIZE >> 2); + const uint ib_col_2 = ib_col_0 + 2 * (BLOCK_SIZE >> 2); + const uint ib_col_3 = ib_col_0 + 3 * (BLOCK_SIZE >> 2); + const uint blocks_per_row = p.ncols >> 5; + + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + const uint rn = (n == 0 || n < num_rows) ? n : 0; + const uint row_off = a_offset + (row_base + rn) * blocks_per_row; + const uint ib0 = row_off + ib_col_0; + const uint ib1 = row_off + ib_col_1; + const uint ib2 = row_off + ib_col_2; + const uint ib3 = row_off + ib_col_3; + + const i16vec2 q0_lo = i16vec2(data_a2_packed16[ib0].qs[iqs_h ], data_a2_packed16[ib0].qs[iqs_h + 1]); + const i16vec2 q0_hi = i16vec2(data_a2_packed16[ib0].qs[iqs_h + 2], data_a2_packed16[ib0].qs[iqs_h + 3]); + const FLOAT_TYPE da0 = float(data_a2_packed16[ib0].d); + const i16vec2 q1_lo = i16vec2(data_a2_packed16[ib1].qs[iqs_h ], data_a2_packed16[ib1].qs[iqs_h + 1]); + const i16vec2 q1_hi = i16vec2(data_a2_packed16[ib1].qs[iqs_h + 2], data_a2_packed16[ib1].qs[iqs_h + 3]); + const FLOAT_TYPE da1 = float(data_a2_packed16[ib1].d); + const i16vec2 q2_lo = i16vec2(data_a2_packed16[ib2].qs[iqs_h ], data_a2_packed16[ib2].qs[iqs_h + 1]); + const i16vec2 q2_hi = i16vec2(data_a2_packed16[ib2].qs[iqs_h + 2], data_a2_packed16[ib2].qs[iqs_h + 3]); + const FLOAT_TYPE da2 = float(data_a2_packed16[ib2].d); + const i16vec2 q3_lo = i16vec2(data_a2_packed16[ib3].qs[iqs_h ], data_a2_packed16[ib3].qs[iqs_h + 1]); + const i16vec2 q3_hi = i16vec2(data_a2_packed16[ib3].qs[iqs_h + 2], data_a2_packed16[ib3].qs[iqs_h + 3]); + const FLOAT_TYPE da3 = float(data_a2_packed16[ib3].d); + + temp[0][n] += (dot(bv0_0, vec4(unpack8(pack32(q0_lo)))) + dot(bv1_0, vec4(unpack8(pack32(q0_hi))))) * da0 + + (dot(bv0_1, vec4(unpack8(pack32(q1_lo)))) + dot(bv1_1, vec4(unpack8(pack32(q1_hi))))) * da1 + + (dot(bv0_2, vec4(unpack8(pack32(q2_lo)))) + dot(bv1_2, vec4(unpack8(pack32(q2_hi))))) * da2 + + (dot(bv0_3, vec4(unpack8(pack32(q3_lo)))) + dot(bv1_3, vec4(unpack8(pack32(q3_hi))))) * da3; + } +} +#endif #endif void compute_outputs(const uint32_t first_row, const uint32_t num_rows) { +#if defined(USE_SUBGROUP_ADD_NO_SHMEM) && !defined(NEEDS_INIT_IQ_SHMEM) + const uint tid = gl_SubgroupInvocationID; +#else const uint tid = gl_LocalInvocationID.x; +#endif get_offsets(a_offset, b_offset, d_offset); +#ifndef MUL_MAT_ID + const bool use_a2 = (GEN_GLU == 2) && (first_row >= p.pair_m0); + const uint row_base = use_a2 ? (first_row - p.pair_m0) : first_row; + const uint eff_rows = (!use_a2 && GEN_GLU == 2) ? min(num_rows, p.pair_m0 - first_row) : num_rows; +#else + const bool use_a2 = false; + const uint row_base = first_row; + const uint eff_rows = num_rows; +#endif + const bool is_aligned_nonquant = p.batch_stride_b % 4 == 0 && p.ncols % 4 == 0 && a_offset % 4 == 0 && b_offset % 4 == 0 && @@ -150,115 +618,154 @@ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) { y_offset = QUANT_R == 1 ? 1 : QUANT_K/2; FLOAT_TYPE temp[NUM_COLS][NUM_ROWS]; + FLOAT_TYPE temp2[NUM_COLS][NUM_ROWS]; [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { [[unroll]] for (uint i = 0; i < NUM_ROWS; ++i) { temp[j][i] = FLOAT_TYPE(0); + if (GEN_GLU == 3) { + temp2[j][i] = FLOAT_TYPE(0); + } } } - uint num_iters = p.ncols / (K_PER_ITER * BLOCK_SIZE); - if (num_iters * K_PER_ITER * BLOCK_SIZE + K_PER_ITER*tid < p.ncols) { - num_iters++; - } - int unroll_count = 4; - uint unrolled_iters = num_iters & ~(unroll_count - 1); - - uint i = 0; - -#if K_PER_ITER == 4 - // If the K dimension is odd, we need lastiter==true on the last iteration - // so OOB is computed correctly. Skip some unrolling to make that happen. - if ((p.ncols & 3) != 0 && - unrolled_iters == num_iters && - unrolled_iters > 0) { - unrolled_iters -= unroll_count; - } - if (is_aligned_nonquant) { - while (i < unrolled_iters) { - // Manually partially unroll the loop - [[unroll]] for (uint k = 0; k < unroll_count; ++k) { - iter_aligned_nonquant(temp, first_row, num_rows, tid, i*K_PER_ITER); - i++; +#if defined(DATA_A_Q8_0) + if ((p.ncols & 31) == 0 && (p.batch_stride_b & 3) == 0 && (b_offset & 3) == 0 && (BLOCK_SIZE & 3) == 0) { + const uint full_iters = p.ncols / (8 * BLOCK_SIZE); + const bool has_tail = (full_iters * (8 * BLOCK_SIZE) + 8 * tid < p.ncols); + if (GEN_GLU == 2 && use_a2) { + uint it = 0; +#ifndef MUL_MAT_ID + if (NUM_ROWS == 1) { + if (NUM_COLS == 1) { + for (; it + 3 < full_iters; it += 4) { + iter4_aligned_q8_0_a2(temp, row_base, eff_rows, tid, it); + } + } + if (NUM_COLS <= 2) { + for (; it + 1 < full_iters; it += 2) { + iter2_aligned_q8_0_a2(temp, row_base, eff_rows, tid, it); + } + } + } +#endif + for (; it < full_iters; ++it) { + iter_aligned_q8_0_a2(temp, row_base, eff_rows, tid, it); + } + if (has_tail) { + iter_aligned_q8_0_a2(temp, row_base, eff_rows, tid, full_iters); + } + } else { + uint it = 0; +#ifndef MUL_MAT_ID + if (NUM_ROWS == 1) { + if (NUM_COLS == 1) { + for (; it + 3 < full_iters; it += 4) { + iter4_aligned_q8_0_a(temp, temp2, row_base, eff_rows, tid, it); + } + } + if (NUM_COLS <= 2) { + for (; it + 1 < full_iters; it += 2) { + iter2_aligned_q8_0_a(temp, temp2, row_base, eff_rows, tid, it); + } + } + } +#endif + for (; it < full_iters; ++it) { + iter_aligned_q8_0_a(temp, temp2, row_base, eff_rows, tid, it); + } + if (has_tail) { + iter_aligned_q8_0_a(temp, temp2, row_base, eff_rows, tid, full_iters); } } - } else { + if (GEN_GLU == 3) { + reduce_pair_glu(temp, temp2, d_offset, first_row, eff_rows, tid); + } +#ifndef MUL_MAT_ID + else if (GEN_GLU == 2 && use_a2) { + reduce_result_d2(temp, row_base, eff_rows, tid); + } #endif - while (i < unrolled_iters) { - // Manually partially unroll the loop - [[unroll]] for (uint k = 0; k < unroll_count; ++k) { - iter(temp, first_row, num_rows, tid, i*K_PER_ITER, false); - i++; + else { + reduce_result(temp, d_offset, first_row, eff_rows, tid); } - } -#if K_PER_ITER == 4 + return; } #endif - unroll_count = 2; - unrolled_iters = num_iters & ~(unroll_count - 1); - -#if K_PER_ITER == 4 - if ((p.ncols & 3) != 0 && - unrolled_iters == num_iters && - unrolled_iters > 0) { - unrolled_iters -= unroll_count; + uint num_iters = p.ncols / (K_PER_ITER * BLOCK_SIZE); + if (num_iters * K_PER_ITER * BLOCK_SIZE + K_PER_ITER*tid < p.ncols) { + num_iters++; } +#if K_PER_ITER == 4 && (defined(DATA_A_F32) || defined(DATA_A_F16) || defined(DATA_A_BF16)) if (is_aligned_nonquant) { - while (i < unrolled_iters && is_aligned_nonquant) { - // Manually partially unroll the loop - [[unroll]] for (uint k = 0; k < unroll_count; ++k) { - iter_aligned_nonquant(temp, first_row, num_rows, tid, i*K_PER_ITER); - i++; + if (GEN_GLU == 2 && use_a2) { + for (uint it = 0; it < num_iters; ++it) { + iter_aligned_nonquant_a2(temp, row_base, eff_rows, tid, it*K_PER_ITER); + } + } else { + for (uint it = 0; it < num_iters; ++it) { + iter_aligned_nonquant_a(temp, temp2, row_base, eff_rows, tid, it*K_PER_ITER); } } - } else { + if (GEN_GLU == 3) { + reduce_pair_glu(temp, temp2, d_offset, first_row, eff_rows, tid); + } +#ifndef MUL_MAT_ID + else if (GEN_GLU == 2 && use_a2) { + reduce_result_d2(temp, row_base, eff_rows, tid); + } #endif - while (i < unrolled_iters) { - // Manually partially unroll the loop - [[unroll]] for (uint k = 0; k < unroll_count; ++k) { - iter(temp, first_row, num_rows, tid, i*K_PER_ITER, false); - i++; + else { + reduce_result(temp, d_offset, first_row, eff_rows, tid); } - } -#if K_PER_ITER == 4 + return; } #endif + uint unrolled_iters = num_iters; #if K_PER_ITER == 4 - if (is_aligned_nonquant) { - while (i < num_iters) { - iter_aligned_nonquant(temp, first_row, num_rows, tid, i*K_PER_ITER); - i++; - } - } else { + if ((p.ncols & 3) != 0 && unrolled_iters > 0) { + unrolled_iters--; + } #endif + uint i = 0; + while (i < unrolled_iters) { + iter(temp, row_base, eff_rows, tid, i*K_PER_ITER, false); + i++; + } while (i < num_iters) { - iter(temp, first_row, num_rows, tid, i*K_PER_ITER, true); + iter(temp, row_base, eff_rows, tid, i*K_PER_ITER, true); i++; } -#if K_PER_ITER == 4 + + if (GEN_GLU == 3) { + reduce_pair_glu(temp, temp2, d_offset, first_row, eff_rows, tid); + } +#ifndef MUL_MAT_ID + else if (GEN_GLU == 2 && use_a2) { + reduce_result_d2(temp, row_base, eff_rows, tid); } #endif - - reduce_result(temp, d_offset, first_row, num_rows, tid); + else { + reduce_result(temp, d_offset, first_row, eff_rows, tid); + } } void main() { +#if defined(USE_SUBGROUP_ADD_NO_SHMEM) && !defined(NEEDS_INIT_IQ_SHMEM) + const uint first_row = NUM_ROWS * (NUM_SUBGROUPS * (gl_WorkGroupID.x + gl_NumWorkGroups.x * gl_WorkGroupID.z) + gl_SubgroupID); +#else const uint first_row = NUM_ROWS * (gl_WorkGroupID.x + gl_NumWorkGroups.x * gl_WorkGroupID.z); +#endif #ifdef NEEDS_INIT_IQ_SHMEM init_iq_shmem(gl_WorkGroupSize); #endif - // do NUM_ROWS at a time, unless there aren't enough remaining rows - if (first_row + NUM_ROWS <= p.stride_d) { - compute_outputs(first_row, NUM_ROWS); - } else { - if (first_row >= p.stride_d) { - return; - } - compute_outputs(first_row, min(NUM_ROWS, p.stride_d - first_row)); + if (first_row >= p.stride_d) { + return; } + compute_outputs(first_row, min(NUM_ROWS, p.stride_d - first_row)); } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.glsl index 12d9cbf6b7cd..72d1411585fc 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.glsl @@ -13,6 +13,38 @@ #include "mul_mat_vec_iface.glsl" +#define MAT_VEC_FUSION_FLAGS_PAIR_GLU 0x400 +#define MAT_VEC_FUSION_FLAGS_PAIR_MM 0x800 +#define MAT_VEC_FUSION_FLAGS_PAIR_MM_SIGMOID1 0x1000 +#define MAT_VEC_FUSION_FLAGS_SCALE0_TOK 0x2000 +#define MAT_VEC_FUSION_FLAGS_SCALE1_TOK 0x4000 +#define MAT_VEC_FUSION_FLAGS_COL_SPLIT 0x8000 +#define MAT_VEC_FUSION_FLAGS_LORA0 0x80000 +#define MAT_VEC_FUSION_FLAGS_LORA1 0x100000 + +layout (binding = 3) readonly buffer A2 {A_TYPE data_a2[];}; +#if defined(A_TYPEV4) +layout (binding = 3) readonly buffer A2V4 {A_TYPEV4 data_a2_v4[];}; +#endif +#if defined(A_TYPE_PACKED16) +layout (binding = 3) readonly buffer A2_PACKED16 {A_TYPE_PACKED16 data_a2_packed16[];}; +#endif +#if defined(A_TYPE_PACKED32) +layout (binding = 3) readonly buffer A2_PACKED32 {A_TYPE_PACKED32 data_a2_packed32[];}; +#endif +#if defined(A_TYPE_PACKED64) +layout (binding = 3) readonly buffer A2_PACKED64 {A_TYPE_PACKED64 data_a2_packed64[];}; +#endif + +#ifndef MUL_MAT_ID +layout (binding = 5) writeonly buffer D2 {D_TYPE data_d2[];}; +layout (binding = 6) readonly buffer Fuse2 {D_TYPE data_fuse2[];}; +layout (binding = 7) readonly buffer LoRAB0 {float data_lora_b0[];}; +layout (binding = 8) readonly buffer LoRAH0 {float data_lora_h0[];}; +layout (binding = 9) readonly buffer LoRAB1 {float data_lora_b1[];}; +layout (binding = 10) readonly buffer LoRAH1 {float data_lora_h1[];}; +#endif + layout (push_constant) uniform parameter { uint ncols; @@ -37,24 +69,52 @@ layout (push_constant) uniform parameter uint ne12; uint broadcast2; uint broadcast3; + uint pair_m0; + uint stride_d2; + uint batch_stride_d2; + uint lora_r; + float lora_scale0; + float lora_scale1; #endif } p; #ifdef MUL_MAT_ID uint expert_id; bool expert_skip; // an id of -1 skips the slot, the result row is zeroed + +// gen: the expert slot and the token of this workgroup (one dispatch per token, or all tokens in y with GEN_TOKY) +uint gen_expert_i0() { return (p.fusion_flags & MAT_VEC_FUSION_FLAGS_GEN_TOKY) != 0 ? gl_WorkGroupID.y % p.nei0 : gl_WorkGroupID.y; } +uint gen_expert_i1() { return (p.fusion_flags & MAT_VEC_FUSION_FLAGS_GEN_TOKY) != 0 ? gl_WorkGroupID.y / p.nei0 : p.expert_i1; } #endif +layout (constant_id = 0) const uint BLOCK_SIZE = 32; +layout (constant_id = 1) const uint NUM_ROWS = 1; +layout (constant_id = 2) const uint NUM_COLS = 1; +// gen: 0 = plain, 1 = single-op epilogue (glu/mmgate/shexp), 2 = PAIR_MM, 3 = PAIR_GLU +layout (constant_id = 3) const uint GEN_GLU = 0; +// gen: independent subgroups per workgroup in SHADER_REDUCTION_MODE_SUBGROUP (row-split) +layout (constant_id = 4) const uint NUM_SUBGROUPS = 1; +// gen: K-quant fast reduction mode (0 = strict 0-ULP upstream layout, 1 = 16-lane vec4/packed32, 2 = 8-lane/superblock) +layout (constant_id = 5) const uint GEN_FAST_KQ = 0; +layout (constant_id = 6) const uint GEN_LORA = 0; + +#define MMV_HAS_LORA0 (GEN_LORA != 0 && (p.fusion_flags & MAT_VEC_FUSION_FLAGS_LORA0) != 0) +#define MMV_HAS_LORA1 (GEN_LORA != 0 && (p.fusion_flags & MAT_VEC_FUSION_FLAGS_LORA1) != 0) +#define MMV_HAS_ANY_LORA (GEN_LORA != 0 && (p.fusion_flags & (MAT_VEC_FUSION_FLAGS_LORA0 | MAT_VEC_FUSION_FLAGS_LORA1)) != 0) + void get_offsets(out uint a_offset, out uint b_offset, out uint d_offset) { #ifdef MUL_MAT_ID - const uint expert_i0 = gl_WorkGroupID.y; + const uint expert_i0 = gen_expert_i0(); + const uint expert_i1 = gen_expert_i1(); #else - const uint batch_idx = gl_WorkGroupID.y + p.base_work_group_y; + const bool col_split = (p.fusion_flags & MAT_VEC_FUSION_FLAGS_COL_SPLIT) != 0; + const uint batch_idx = col_split ? (gl_WorkGroupID.y * NUM_COLS + p.base_work_group_y) + : (gl_WorkGroupID.y + p.base_work_group_y); #endif #ifndef MUL_MAT_ID uint batch_idx_a = 0; - if (batch_idx != 0) { + if (!col_split && batch_idx != 0) { const uint i13 = batch_idx / p.ne12; const uint i12 = batch_idx % p.ne12; @@ -64,7 +124,7 @@ void get_offsets(out uint a_offset, out uint b_offset, out uint d_offset) { batch_idx_a = i03 * p.ne02 + i02; } #else - const int expert = data_ids[expert_i0 + p.expert_i1 * p.nbi1]; + const int expert = data_ids[expert_i0 + expert_i1 * p.nbi1]; expert_skip = expert == -1; expert_id = expert_skip ? 0 : uint(expert); // clamp, so that the offsets below stay in range #endif @@ -77,44 +137,144 @@ void get_offsets(out uint a_offset, out uint b_offset, out uint d_offset) { #endif b_offset = #ifdef MUL_MAT_ID - (expert_i0 % p.ne11) * p.stride_b + p.expert_i1 * p.batch_stride_b; + (expert_i0 % p.ne11) * p.stride_b + expert_i1 * p.batch_stride_b; #else batch_idx * p.batch_stride_b; #endif d_offset = #ifdef MUL_MAT_ID - expert_i0 * p.stride_d + p.expert_i1 * p.batch_stride_d; + expert_i0 * p.stride_d + expert_i1 * p.batch_stride_d; #else batch_idx * p.batch_stride_d; #endif } -layout (constant_id = 0) const uint BLOCK_SIZE = 32; -layout (constant_id = 1) const uint NUM_ROWS = 1; -layout (constant_id = 2) const uint NUM_COLS = 1; +#ifndef MUL_MAT_ID +FLOAT_TYPE gen_apply_mm_epilogue(FLOAT_TYPE val, const uint j, const uint first_row, const uint n) { + if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_SIGMOID) != 0) { + precise float gen_x = float(D_TYPE(val)); + precise float gen_y = 1.0f / (1.0f + exp(-gen_x)); + return FLOAT_TYPE(gen_y); + } + if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_SOFTPLUS_GATE) != 0) { + precise float gen_x = float(D_TYPE(val)) + float(data_fuse0[first_row + n]); + precise float gen_sp = (gen_x > 20.0f) ? gen_x : log(1.0f + exp(gen_x)); + precise float gen_y = gen_sp * float(data_fuse1[first_row + n]); + return FLOAT_TYPE(gen_y); + } + if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_ROW_SCALE0) != 0) { + const uint col_j = (p.fusion_flags & MAT_VEC_FUSION_FLAGS_COL_SPLIT) != 0 + ? (gl_WorkGroupID.y * NUM_COLS + p.base_work_group_y + j) : j; + precise float gen_x = float(D_TYPE(val)); + precise float gen_g = float(data_fuse0[col_j]); + precise float gen_y = gen_x * gen_g; + return FLOAT_TYPE(gen_y); + } + return val; +} + +FLOAT_TYPE gen_apply_pair_mm0_epilogue(FLOAT_TYPE val, const uint first_row, const uint n) { + if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_SOFTPLUS_GATE) != 0) { + precise float gen_x = float(D_TYPE(val)) + float(data_fuse1[first_row + n]); + precise float gen_sp = (gen_x > 20.0f) ? gen_x : log(1.0f + exp(gen_x)); + precise float gen_y = gen_sp * float(data_fuse2[first_row + n]); + return FLOAT_TYPE(gen_y); + } + return val; +} + +uint mmv_tok_j(const uint j) { + return (((p.fusion_flags & MAT_VEC_FUSION_FLAGS_COL_SPLIT) != 0) + ? (gl_WorkGroupID.y * NUM_COLS + p.base_work_group_y) + : (gl_WorkGroupID.y + p.base_work_group_y)) + j; +} + +float mmv_eval_lora0_scalar(const uint row, const uint col_j) { + if (GEN_LORA == 0) { return 0.0f; } + float acc = 0.0f; + const uint b_off = row * p.lora_r; + const uint h_off = col_j * p.lora_r; + for (uint k = 0; k < p.lora_r; ++k) { + acc = fma(data_lora_b0[b_off + k], data_lora_h0[h_off + k], acc); + } + return acc * p.lora_scale0; +} + +float mmv_eval_lora1_scalar(const uint row, const uint col_j) { + if (GEN_LORA == 0) { return 0.0f; } + float acc = 0.0f; + const uint b_off = row * p.lora_r; + const uint h_off = col_j * p.lora_r; + for (uint k = 0; k < p.lora_r; ++k) { + acc = fma(data_lora_b1[b_off + k], data_lora_h1[h_off + k], acc); + } + return acc * p.lora_scale1; +} + +#if USE_SUBGROUP_ADD || USE_SUBGROUP_ADD_NO_SHMEM +float mmv_eval_lora0_sg(const uint row, const uint col_j) { + if (GEN_LORA == 0) { return 0.0f; } + float acc = 0.0f; + const uint b_off = row * p.lora_r; + const uint h_off = col_j * p.lora_r; + for (uint k = gl_SubgroupInvocationID; k < p.lora_r; k += gl_SubgroupSize) { + acc = fma(data_lora_b0[b_off + k], data_lora_h0[h_off + k], acc); + } + return subgroupAdd(acc) * p.lora_scale0; +} + +float mmv_eval_lora1_sg(const uint row, const uint col_j) { + if (GEN_LORA == 0) { return 0.0f; } + float acc = 0.0f; + const uint b_off = row * p.lora_r; + const uint h_off = col_j * p.lora_r; + for (uint k = gl_SubgroupInvocationID; k < p.lora_r; k += gl_SubgroupSize) { + acc = fma(data_lora_b1[b_off + k], data_lora_h1[h_off + k], acc); + } + return subgroupAdd(acc) * p.lora_scale1; +} +#endif +#endif #ifdef USE_SUBGROUP_ADD_NO_SHMEM void reduce_result(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const in uint32_t d_offset, const in uint32_t first_row, const in uint32_t num_rows, const in uint32_t tid) { [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { - [[unroll]] for (uint n = 0; n < num_rows; ++n) { + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { temp[j][n] = subgroupAdd(temp[j][n]); } } +#ifndef MUL_MAT_ID + if (GEN_GLU != 0 && MMV_HAS_LORA0 && num_rows > 0) { + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + const uint tok_j = mmv_tok_j(j); + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + const uint rn = (n == 0 || n < num_rows) ? n : 0; + const float lv = mmv_eval_lora0_sg(first_row + rn, tok_j); + if (n < num_rows) { + temp[j][n] += FLOAT_TYPE(lv); + } + } + } + } +#endif if (tid == 0) { [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { - [[unroll]] for (uint n = 0; n < num_rows; ++n) { + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + if (n >= num_rows) { + continue; + } #ifdef MUL_MAT_ID if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS0) != 0) { temp[j][n] += FLOAT_TYPE(data_fuse0[expert_id*p.stride_d + first_row + n]); } if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_SCALE0) != 0) { - const uint expert_i0 = gl_GlobalInvocationID.y; - temp[j][n] *= FLOAT_TYPE(data_fuse0[expert_i0]); + const uint s0_idx = gen_expert_i0() + ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_SCALE0_TOK) != 0 ? gen_expert_i1() * p.nei0 : 0u); + temp[j][n] *= FLOAT_TYPE(data_fuse0[s0_idx]); } if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_SCALE1) != 0) { - const uint expert_i0 = gl_GlobalInvocationID.y; - temp[j][n] *= FLOAT_TYPE(data_fuse1[expert_i0]); + const uint s1_idx = gen_expert_i0() + ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_SCALE1_TOK) != 0 ? gen_expert_i1() * p.nei0 : 0u); + temp[j][n] *= FLOAT_TYPE(data_fuse1[s1_idx]); } if (expert_skip) { temp[j][n] = FLOAT_TYPE(0); @@ -126,14 +286,115 @@ void reduce_result(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const in uint32_t if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS1) != 0) { temp[j][n] += FLOAT_TYPE(data_fuse1[j*p.batch_stride_d + d_offset + first_row + n]); } + if (GEN_GLU == 1 && (p.fusion_flags & (MAT_VEC_FUSION_FLAGS_SIGMOID | MAT_VEC_FUSION_FLAGS_SOFTPLUS_GATE | MAT_VEC_FUSION_FLAGS_ROW_SCALE0)) != 0) { + temp[j][n] = gen_apply_mm_epilogue(temp[j][n], j, first_row, n); + } + if (GEN_GLU == 2 && (p.fusion_flags & MAT_VEC_FUSION_FLAGS_SOFTPLUS_GATE) != 0) { + temp[j][n] = gen_apply_pair_mm0_epilogue(temp[j][n], first_row, n); + } #endif + if (GEN_GLU == 1 && (p.fusion_flags & (MAT_VEC_FUSION_FLAGS_GLU_A | MAT_VEC_FUSION_FLAGS_GLU_B)) != 0) { + // gen: same expression as swiglu.comp op(a, b) = a / (1 + exp(-a)) * b + const float glu_o = float(data_fuse0[j*p.batch_stride_d + d_offset + first_row + n]); + const float glu_r = float(temp[j][n]); + temp[j][n] = FLOAT_TYPE((p.fusion_flags & MAT_VEC_FUSION_FLAGS_GLU_A) != 0 ? + glu_r / (1.0f + exp(-glu_r)) * glu_o : glu_o / (1.0f + exp(-glu_o)) * glu_r); + } data_d[j*p.batch_stride_d + d_offset + first_row + n] = D_TYPE(temp[j][n]); } } } } + +void reduce_pair_glu(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], inout FLOAT_TYPE temp2[NUM_COLS][NUM_ROWS], const in uint32_t d_offset, const in uint32_t first_row, const in uint32_t num_rows, const in uint32_t tid) { + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + temp[j][n] = subgroupAdd(temp[j][n]); + temp2[j][n] = subgroupAdd(temp2[j][n]); + } + } +#ifndef MUL_MAT_ID + if (MMV_HAS_ANY_LORA && num_rows > 0) { + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + const uint tok_j = mmv_tok_j(j); + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + const uint rn = (n == 0 || n < num_rows) ? n : 0; + if (MMV_HAS_LORA0) { + const float lv0 = mmv_eval_lora0_sg(first_row + rn, tok_j); + if (n < num_rows) { temp[j][n] += FLOAT_TYPE(lv0); } + } + if (MMV_HAS_LORA1) { + const float lv1 = mmv_eval_lora1_sg(first_row + rn, tok_j); + if (n < num_rows) { temp2[j][n] += FLOAT_TYPE(lv1); } + } + } + } + } +#endif + if (tid == 0) { + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + if (n >= num_rows) { + continue; + } +#ifdef MUL_MAT_ID + if (expert_skip) { + data_d[j*p.batch_stride_d + d_offset + first_row + n] = D_TYPE(0); + continue; + } +#endif + const float glu_g = float(D_TYPE(temp[j][n])); + const float glu_u = float(D_TYPE(temp2[j][n])); + data_d[j*p.batch_stride_d + d_offset + first_row + n] = D_TYPE(FLOAT_TYPE(glu_g / (1.0f + exp(-glu_g)) * glu_u)); + } + } + } +} + +#ifndef MUL_MAT_ID +void reduce_result_d2(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const in uint32_t first_row, const in uint32_t num_rows, const in uint32_t tid) { + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + temp[j][n] = subgroupAdd(temp[j][n]); + } + } + if (MMV_HAS_LORA1 && num_rows > 0) { + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + const uint tok_j = mmv_tok_j(j); + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + const uint rn = (n == 0 || n < num_rows) ? n : 0; + const float lv1 = mmv_eval_lora1_sg(first_row + rn, tok_j); + if (n < num_rows) { temp[j][n] += FLOAT_TYPE(lv1); } + } + } + } + if (tid == 0) { + const uint d2_offset = (((p.fusion_flags & MAT_VEC_FUSION_FLAGS_COL_SPLIT) != 0) + ? (gl_WorkGroupID.y * NUM_COLS + p.base_work_group_y) + : (gl_WorkGroupID.y + p.base_work_group_y)) * p.batch_stride_d2; + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + if (n >= num_rows) { + continue; + } + FLOAT_TYPE val = temp[j][n]; + if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_PAIR_MM_SIGMOID1) != 0) { + precise float gen_x = float(D_TYPE(val)); + precise float gen_y = 1.0f / (1.0f + exp(-gen_x)); + val = FLOAT_TYPE(gen_y); + } + data_d2[j*p.batch_stride_d2 + d2_offset + first_row + n] = D_TYPE(val); + } + } + } +} +#endif +#else +#if USE_SUBGROUP_ADD +shared FLOAT_TYPE tmpsh[NUM_COLS][NUM_ROWS][2 * (BLOCK_SIZE / 8) + 2]; #else shared FLOAT_TYPE tmpsh[NUM_COLS][NUM_ROWS][BLOCK_SIZE]; +#endif void reduce_result(FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const in uint32_t d_offset, const in uint32_t first_row, const in uint32_t num_rows, const in uint32_t tid) { // subgroupAdd is probably faster on devices that support it, @@ -141,7 +402,7 @@ void reduce_result(FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const in uint32_t d_offs #if USE_SUBGROUP_ADD // sum up partial sums within a subgroup [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { - [[unroll]] for (uint n = 0; n < num_rows; ++n) { + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { temp[j][n] = subgroupAdd(temp[j][n]); } } @@ -149,15 +410,30 @@ void reduce_result(FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const in uint32_t d_offs // Go through shared memory to sum partials across subgroups if (gl_SubgroupInvocationID == 0) { [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { - [[unroll]] for (uint n = 0; n < num_rows; ++n) { + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { tmpsh[j][n][gl_SubgroupID] = temp[j][n]; } } } barrier(); +#ifndef MUL_MAT_ID + FLOAT_TYPE lora0_acc[NUM_COLS][NUM_ROWS]; + if (GEN_GLU != 0 && MMV_HAS_LORA0 && gl_SubgroupID == 0 && num_rows > 0) { + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + const uint tok_j = mmv_tok_j(j); + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + const uint rn = (n == 0 || n < num_rows) ? n : 0; + lora0_acc[j][n] = FLOAT_TYPE(mmv_eval_lora0_sg(first_row + rn, tok_j)); + } + } + } +#endif if (tid == 0) { [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { - [[unroll]] for (uint n = 0; n < num_rows; ++n) { + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + if (n >= num_rows) { + continue; + } temp[j][n] = FLOAT_TYPE(0); [[unroll]] for (uint s = 0; s < gl_NumSubgroups; ++s) { temp[j][n] += tmpsh[j][n][s]; @@ -167,24 +443,40 @@ void reduce_result(FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const in uint32_t d_offs temp[j][n] += FLOAT_TYPE(data_fuse0[expert_id*p.stride_d + first_row + n]); } if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_SCALE0) != 0) { - const uint expert_i0 = gl_GlobalInvocationID.y; - temp[j][n] *= FLOAT_TYPE(data_fuse0[expert_i0]); + const uint s0_idx = gen_expert_i0() + ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_SCALE0_TOK) != 0 ? gen_expert_i1() * p.nei0 : 0u); + temp[j][n] *= FLOAT_TYPE(data_fuse0[s0_idx]); } if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_SCALE1) != 0) { - const uint expert_i0 = gl_GlobalInvocationID.y; - temp[j][n] *= FLOAT_TYPE(data_fuse1[expert_i0]); + const uint s1_idx = gen_expert_i0() + ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_SCALE1_TOK) != 0 ? gen_expert_i1() * p.nei0 : 0u); + temp[j][n] *= FLOAT_TYPE(data_fuse1[s1_idx]); } if (expert_skip) { temp[j][n] = FLOAT_TYPE(0); } #else + if (GEN_GLU != 0 && MMV_HAS_LORA0) { + temp[j][n] += lora0_acc[j][n]; + } if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS0) != 0) { temp[j][n] += FLOAT_TYPE(data_fuse0[j*p.batch_stride_d + d_offset + first_row + n]); } if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS1) != 0) { temp[j][n] += FLOAT_TYPE(data_fuse1[j*p.batch_stride_d + d_offset + first_row + n]); } + if (GEN_GLU == 1 && (p.fusion_flags & (MAT_VEC_FUSION_FLAGS_SIGMOID | MAT_VEC_FUSION_FLAGS_SOFTPLUS_GATE | MAT_VEC_FUSION_FLAGS_ROW_SCALE0)) != 0) { + temp[j][n] = gen_apply_mm_epilogue(temp[j][n], j, first_row, n); + } + if (GEN_GLU == 2 && (p.fusion_flags & MAT_VEC_FUSION_FLAGS_SOFTPLUS_GATE) != 0) { + temp[j][n] = gen_apply_pair_mm0_epilogue(temp[j][n], first_row, n); + } #endif + if (GEN_GLU == 1 && (p.fusion_flags & (MAT_VEC_FUSION_FLAGS_GLU_A | MAT_VEC_FUSION_FLAGS_GLU_B)) != 0) { + // gen: same expression as swiglu.comp op(a, b) = a / (1 + exp(-a)) * b + const float glu_o = float(data_fuse0[j*p.batch_stride_d + d_offset + first_row + n]); + const float glu_r = float(temp[j][n]); + temp[j][n] = FLOAT_TYPE((p.fusion_flags & MAT_VEC_FUSION_FLAGS_GLU_A) != 0 ? + glu_r / (1.0f + exp(-glu_r)) * glu_o : glu_o / (1.0f + exp(-glu_o)) * glu_r); + } data_d[j*p.batch_stride_d + d_offset + first_row + n] = D_TYPE(temp[j][n]); } } @@ -192,7 +484,7 @@ void reduce_result(FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const in uint32_t d_offs #else // sum up partial sums and write back result [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { - [[unroll]] for (uint n = 0; n < num_rows; ++n) { + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { tmpsh[j][n][tid] = temp[j][n]; } } @@ -200,7 +492,7 @@ void reduce_result(FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const in uint32_t d_offs [[unroll]] for (uint s = BLOCK_SIZE/2; s > 0; s >>= 1) { if (tid < s) { [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { - [[unroll]] for (uint n = 0; n < num_rows; ++n) { + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { tmpsh[j][n][tid] += tmpsh[j][n][tid + s]; } } @@ -209,34 +501,281 @@ void reduce_result(FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const in uint32_t d_offs } if (tid == 0) { [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { - [[unroll]] for (uint n = 0; n < num_rows; ++n) { + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + if (n >= num_rows) { + continue; + } #ifdef MUL_MAT_ID if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS0) != 0) { tmpsh[j][n][0] += FLOAT_TYPE(data_fuse0[expert_id*p.stride_d + first_row + n]); } if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_SCALE0) != 0) { - const uint expert_i0 = gl_GlobalInvocationID.y; - tmpsh[j][n][0] *= FLOAT_TYPE(data_fuse0[expert_i0]); + const uint s0_idx = gen_expert_i0() + ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_SCALE0_TOK) != 0 ? gen_expert_i1() * p.nei0 : 0u); + tmpsh[j][n][0] *= FLOAT_TYPE(data_fuse0[s0_idx]); } if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_SCALE1) != 0) { - const uint expert_i0 = gl_GlobalInvocationID.y; - tmpsh[j][n][0] *= FLOAT_TYPE(data_fuse1[expert_i0]); + const uint s1_idx = gen_expert_i0() + ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_SCALE1_TOK) != 0 ? gen_expert_i1() * p.nei0 : 0u); + tmpsh[j][n][0] *= FLOAT_TYPE(data_fuse1[s1_idx]); } if (expert_skip) { tmpsh[j][n][0] = FLOAT_TYPE(0); } #else + if (GEN_GLU != 0 && MMV_HAS_LORA0) { + tmpsh[j][n][0] += FLOAT_TYPE(mmv_eval_lora0_scalar(first_row + n, mmv_tok_j(j))); + } if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS0) != 0) { tmpsh[j][n][0] += FLOAT_TYPE(data_fuse0[j*p.batch_stride_d + d_offset + first_row + n]); } if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS1) != 0) { tmpsh[j][n][0] += FLOAT_TYPE(data_fuse1[j*p.batch_stride_d + d_offset + first_row + n]); } + if (GEN_GLU == 1 && (p.fusion_flags & (MAT_VEC_FUSION_FLAGS_SIGMOID | MAT_VEC_FUSION_FLAGS_SOFTPLUS_GATE | MAT_VEC_FUSION_FLAGS_ROW_SCALE0)) != 0) { + tmpsh[j][n][0] = gen_apply_mm_epilogue(tmpsh[j][n][0], j, first_row, n); + } + if (GEN_GLU == 2 && (p.fusion_flags & MAT_VEC_FUSION_FLAGS_SOFTPLUS_GATE) != 0) { + tmpsh[j][n][0] = gen_apply_pair_mm0_epilogue(tmpsh[j][n][0], first_row, n); + } #endif + if (GEN_GLU == 1 && (p.fusion_flags & (MAT_VEC_FUSION_FLAGS_GLU_A | MAT_VEC_FUSION_FLAGS_GLU_B)) != 0) { + // gen: same expression as swiglu.comp op(a, b) = a / (1 + exp(-a)) * b + const float glu_o = float(data_fuse0[j*p.batch_stride_d + d_offset + first_row + n]); + const float glu_r = float(tmpsh[j][n][0]); + tmpsh[j][n][0] = FLOAT_TYPE((p.fusion_flags & MAT_VEC_FUSION_FLAGS_GLU_A) != 0 ? + glu_r / (1.0f + exp(-glu_r)) * glu_o : glu_o / (1.0f + exp(-glu_o)) * glu_r); + } data_d[j*p.batch_stride_d + d_offset + first_row + n] = D_TYPE(tmpsh[j][n][0]); } } } #endif } + +void reduce_pair_glu(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], inout FLOAT_TYPE temp2[NUM_COLS][NUM_ROWS], const in uint32_t d_offset, const in uint32_t first_row, const in uint32_t num_rows, const in uint32_t tid) { +#if USE_SUBGROUP_ADD + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + temp[j][n] = subgroupAdd(temp[j][n]); + temp2[j][n] = subgroupAdd(temp2[j][n]); + } + } + if (gl_SubgroupInvocationID == 0) { + const uint sg_u = gl_NumSubgroups + gl_SubgroupID; + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + tmpsh[j][n][gl_SubgroupID] = temp[j][n]; + tmpsh[j][n][sg_u] = temp2[j][n]; + } + } + } + barrier(); +#ifndef MUL_MAT_ID + FLOAT_TYPE lora0_acc[NUM_COLS][NUM_ROWS]; + FLOAT_TYPE lora1_acc[NUM_COLS][NUM_ROWS]; + if (MMV_HAS_ANY_LORA && gl_SubgroupID == 0 && num_rows > 0) { + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + const uint tok_j = mmv_tok_j(j); + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + const uint rn = (n == 0 || n < num_rows) ? n : 0; + if (MMV_HAS_LORA0) { + lora0_acc[j][n] = FLOAT_TYPE(mmv_eval_lora0_sg(first_row + rn, tok_j)); + } + if (MMV_HAS_LORA1) { + lora1_acc[j][n] = FLOAT_TYPE(mmv_eval_lora1_sg(first_row + rn, tok_j)); + } + } + } + } +#endif + if (tid == 0) { + const uint sg_base_u = gl_NumSubgroups; + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + if (n >= num_rows) { + continue; + } + FLOAT_TYPE acc = FLOAT_TYPE(0); + FLOAT_TYPE acc2 = FLOAT_TYPE(0); + [[unroll]] for (uint s = 0; s < gl_NumSubgroups; ++s) { + acc += tmpsh[j][n][s]; + acc2 += tmpsh[j][n][sg_base_u + s]; + } +#ifdef MUL_MAT_ID + if (expert_skip) { + data_d[j*p.batch_stride_d + d_offset + first_row + n] = D_TYPE(0); + continue; + } +#else + if (MMV_HAS_LORA0) { + acc += lora0_acc[j][n]; + } + if (MMV_HAS_LORA1) { + acc2 += lora1_acc[j][n]; + } +#endif + const float glu_g = float(D_TYPE(acc)); + const float glu_u = float(D_TYPE(acc2)); + data_d[j*p.batch_stride_d + d_offset + first_row + n] = D_TYPE(FLOAT_TYPE(glu_g / (1.0f + exp(-glu_g)) * glu_u)); + } + } + } +#else + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + tmpsh[j][n][tid] = temp[j][n]; + } + } + barrier(); + [[unroll]] for (uint s = BLOCK_SIZE/2; s > 0; s >>= 1) { + if (tid < s) { + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + tmpsh[j][n][tid] += tmpsh[j][n][tid + s]; + } + } + } + barrier(); + } + if (tid == 0) { + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + temp[j][n] = tmpsh[j][n][0]; + } + } + } + barrier(); + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + tmpsh[j][n][tid] = temp2[j][n]; + } + } + barrier(); + [[unroll]] for (uint s = BLOCK_SIZE/2; s > 0; s >>= 1) { + if (tid < s) { + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + tmpsh[j][n][tid] += tmpsh[j][n][tid + s]; + } + } + } + barrier(); + } + if (tid == 0) { + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + if (n >= num_rows) { + continue; + } +#ifdef MUL_MAT_ID + if (expert_skip) { + data_d[j*p.batch_stride_d + d_offset + first_row + n] = D_TYPE(0); + continue; + } +#else + if (MMV_HAS_LORA0) { + temp[j][n] += FLOAT_TYPE(mmv_eval_lora0_scalar(first_row + n, mmv_tok_j(j))); + } + if (MMV_HAS_LORA1) { + tmpsh[j][n][0] += FLOAT_TYPE(mmv_eval_lora1_scalar(first_row + n, mmv_tok_j(j))); + } +#endif + const float glu_g = float(D_TYPE(temp[j][n])); + const float glu_u = float(D_TYPE(tmpsh[j][n][0])); + data_d[j*p.batch_stride_d + d_offset + first_row + n] = D_TYPE(FLOAT_TYPE(glu_g / (1.0f + exp(-glu_g)) * glu_u)); + } + } + } +#endif +} + +#ifndef MUL_MAT_ID +void reduce_result_d2(FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const in uint32_t first_row, const in uint32_t num_rows, const in uint32_t tid) { + const uint d2_offset = (((p.fusion_flags & MAT_VEC_FUSION_FLAGS_COL_SPLIT) != 0) + ? (gl_WorkGroupID.y * NUM_COLS + p.base_work_group_y) + : (gl_WorkGroupID.y + p.base_work_group_y)) * p.batch_stride_d2; +#if USE_SUBGROUP_ADD + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + temp[j][n] = subgroupAdd(temp[j][n]); + } + } + if (gl_SubgroupInvocationID == 0) { + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + tmpsh[j][n][gl_SubgroupID] = temp[j][n]; + } + } + } + barrier(); + FLOAT_TYPE lora1_acc[NUM_COLS][NUM_ROWS]; + if (MMV_HAS_LORA1 && gl_SubgroupID == 0 && num_rows > 0) { + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + const uint tok_j = mmv_tok_j(j); + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + const uint rn = (n == 0 || n < num_rows) ? n : 0; + lora1_acc[j][n] = FLOAT_TYPE(mmv_eval_lora1_sg(first_row + rn, tok_j)); + } + } + } + if (tid == 0) { + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + if (n >= num_rows) { + continue; + } + FLOAT_TYPE val = FLOAT_TYPE(0); + [[unroll]] for (uint s = 0; s < gl_NumSubgroups; ++s) { + val += tmpsh[j][n][s]; + } + if (MMV_HAS_LORA1) { + val += lora1_acc[j][n]; + } + if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_PAIR_MM_SIGMOID1) != 0) { + precise float gen_x = float(D_TYPE(val)); + precise float gen_y = 1.0f / (1.0f + exp(-gen_x)); + val = FLOAT_TYPE(gen_y); + } + data_d2[j*p.batch_stride_d2 + d2_offset + first_row + n] = D_TYPE(val); + } + } + } +#else + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + tmpsh[j][n][tid] = temp[j][n]; + } + } + barrier(); + [[unroll]] for (uint s = BLOCK_SIZE/2; s > 0; s >>= 1) { + if (tid < s) { + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + tmpsh[j][n][tid] += tmpsh[j][n][tid + s]; + } + } + } + barrier(); + } + if (tid == 0) { + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + if (n >= num_rows) { + continue; + } + FLOAT_TYPE val = tmpsh[j][n][0]; + if (MMV_HAS_LORA1) { + val += FLOAT_TYPE(mmv_eval_lora1_scalar(first_row + n, mmv_tok_j(j))); + } + if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_PAIR_MM_SIGMOID1) != 0) { + precise float gen_x = float(D_TYPE(val)); + precise float gen_y = 1.0f / (1.0f + exp(-gen_x)); + val = FLOAT_TYPE(gen_y); + } + data_d2[j*p.batch_stride_d2 + d2_offset + first_row + n] = D_TYPE(val); + } + } + } +#endif +} +#endif #endif diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iface.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iface.glsl index a2b7cef64906..4c531a665c8c 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iface.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iface.glsl @@ -4,6 +4,15 @@ #define MAT_VEC_FUSION_FLAGS_BIAS1 0x2 #define MAT_VEC_FUSION_FLAGS_SCALE0 0x4 #define MAT_VEC_FUSION_FLAGS_SCALE1 0x8 +// gen: SWIGLU epilogue, this result is the gate (A) or the up operand (B); the other comes from fuse0 +#define MAT_VEC_FUSION_FLAGS_GLU_A 0x10 +#define MAT_VEC_FUSION_FLAGS_GLU_B 0x20 +// gen: MUL_MAT_ID with all tokens in one dispatch: workgroup y = expert slot + nei0 * token +#define MAT_VEC_FUSION_FLAGS_GEN_TOKY 0x40 +// gen: dense mat-vec epilogues (GGML_GEN_FUSE=mmgate,shexp), compiled into the GEN_GLU=1 twin +#define MAT_VEC_FUSION_FLAGS_SIGMOID 0x80 +#define MAT_VEC_FUSION_FLAGS_SOFTPLUS_GATE 0x100 +#define MAT_VEC_FUSION_FLAGS_ROW_SCALE0 0x200 layout (binding = 0) readonly buffer A {A_TYPE data_a[];}; #if defined(A_TYPEV4) diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq4_xs.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq4_xs.comp index a2b99d9ab16c..bee9bd102180 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq4_xs.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq4_xs.comp @@ -14,8 +14,9 @@ FLOAT_TYPE temp[NUM_COLS][NUM_ROWS]; void calc_superblock(const uint a_offset, const uint b_offset, const uint ib32, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows) { const uint y_idx = i * QUANT_K + 32 * ib32; - uint ibi = a_offset + first_row * num_blocks_per_row + i; - [[unroll]] for (uint n = 0; n < num_rows; ++n) { + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + const uint rn = (n == 0 || n < num_rows) ? n : 0; + const uint ibi = a_offset + (first_row + rn) * num_blocks_per_row + i; const float d = float(data_a[ibi].d); const uint sl = (data_a[ibi].scales_l[ib32/2] >> (4 * (ib32 & 1))) & 0xF; const uint sh = (data_a[ibi].scales_h >> (2 * ib32)) & 3; @@ -50,8 +51,6 @@ void calc_superblock(const uint a_offset, const uint b_offset, const uint ib32, [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { temp[j][n] = fma(dscale, sum[j], temp[j][n]); } - - ibi += num_blocks_per_row; } } @@ -74,7 +73,7 @@ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) { } } - [[unroll]] for (uint i = ix; i < num_blocks_per_row; i += blocks_per_wg) + for (uint i = ix; i < num_blocks_per_row; i += blocks_per_wg) calc_superblock(a_offset, b_offset, itid, i, num_blocks_per_row, first_row, num_rows); reduce_result(temp, d_offset, first_row, num_rows, tid); @@ -85,13 +84,8 @@ void main() { init_iq_shmem(gl_WorkGroupSize); - // do NUM_ROWS at a time, unless there aren't enough remaining rows - if (first_row + NUM_ROWS <= p.stride_d) { - compute_outputs(first_row, NUM_ROWS); - } else { - if (first_row >= p.stride_d) { - return; - } - compute_outputs(first_row, p.stride_d - first_row); + if (first_row >= p.stride_d) { + return; } + compute_outputs(first_row, min(NUM_ROWS, p.stride_d - first_row)); } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp index 975cec8013f2..2ce5a1e58935 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp @@ -14,8 +14,9 @@ uint csel = 0; void calc_superblock(const uint a_offset, const uint b_offset, const uint itid, const uint v_im, const uint ix, const uint q_offset, const uint y_offset, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows, const bool all_threads) { const uint y_idx = i * QUANT_K + y_offset; - [[unroll]] for (uint n = 0; n < num_rows; ++n) { - const uint ib0 = a_offset + (first_row+n)*num_blocks_per_row; + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + const uint rn = (n == 0 || n < num_rows) ? n : 0; + const uint ib0 = a_offset + (first_row+rn)*num_blocks_per_row; csel ^= 1; if (!all_threads) { // when we don't have enough blocks to use all threads @@ -106,7 +107,7 @@ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) { const uint nbr_par_th = num_blocks_per_row%it_size; const uint nbr_all_th = num_blocks_per_row - nbr_par_th; uint i0 = 0; - [[unroll]] for (; i0 < nbr_all_th; i0 += it_size) + for (; i0 < nbr_all_th; i0 += it_size) calc_superblock(a_offset, b_offset, itid, v_im, ix, q_offset, y_offset, i0 + ix, num_blocks_per_row, first_row, num_rows, true); calc_superblock(a_offset, b_offset, itid, v_im, ix, q_offset, y_offset, i0 + ix, num_blocks_per_row, first_row, num_rows, false); @@ -116,13 +117,8 @@ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) { void main() { const uint first_row = NUM_ROWS * (gl_WorkGroupID.x + gl_NumWorkGroups.x * gl_WorkGroupID.z); - // do NUM_ROWS at a time, unless there aren't enough remaining rows - if (first_row + NUM_ROWS <= p.stride_d) { - compute_outputs(first_row, NUM_ROWS); - } else { - if (first_row >= p.stride_d) { - return; - } - compute_outputs(first_row, p.stride_d - first_row); + if (first_row >= p.stride_d) { + return; } + compute_outputs(first_row, min(NUM_ROWS, p.stride_d - first_row)); } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp index 93e48b790122..2fb61538384e 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp @@ -13,8 +13,9 @@ uint csel = 0; void calc_superblock(const uint a_offset, const uint b_offset, const uint ix, const uint itid8, const uint v_im, const uint v_im4, const uint v_in, const uint32_t hm_m[4], const uint q_offset, const uint y_offset, const uint s_shift, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows, const bool all_threads) { const uint y_idx = i * QUANT_K + y_offset; - [[unroll]] for (uint n = 0; n < num_rows; ++n) { - const uint ib0 = a_offset + (first_row+n)*num_blocks_per_row; + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + const uint rn = (n == 0 || n < num_rows) ? n : 0; + const uint ib0 = a_offset + (first_row+rn)*num_blocks_per_row; csel ^= 1; if (!all_threads) { // when we don't have enough blocks to use all threads @@ -110,7 +111,7 @@ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) { const uint nbr_par_th = num_blocks_per_row%it_size; const uint nbr_all_th = num_blocks_per_row - nbr_par_th; uint i0 = 0; - [[unroll]] for (; i0 < nbr_all_th; i0 += it_size) + for (; i0 < nbr_all_th; i0 += it_size) calc_superblock(a_offset, b_offset, ix, itid8, v_im, v_im4, v_in, hm_m, q_offset, y_offset, s_shift, i0 + ix, num_blocks_per_row, first_row, num_rows, true); calc_superblock(a_offset, b_offset, ix, itid8, v_im, v_im4, v_in, hm_m, q_offset, y_offset, s_shift, i0 + ix, num_blocks_per_row, first_row, num_rows, false); @@ -120,13 +121,8 @@ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) { void main() { const uint first_row = NUM_ROWS * (gl_WorkGroupID.x + gl_NumWorkGroups.x * gl_WorkGroupID.z); - // do NUM_ROWS at a time, unless there aren't enough remaining rows - if (first_row + NUM_ROWS <= p.stride_d) { - compute_outputs(first_row, NUM_ROWS); - } else { - if (first_row >= p.stride_d) { - return; - } - compute_outputs(first_row, p.stride_d - first_row); + if (first_row >= p.stride_d) { + return; } + compute_outputs(first_row, min(NUM_ROWS, p.stride_d - first_row)); } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp index 93fbacc6282c..859383127698 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp @@ -4,82 +4,326 @@ #include "mul_mat_vec_base.glsl" +#ifdef USE_SUBGROUP_ADD_NO_SHMEM +layout(local_size_x_id = 0, local_size_y_id = 4, local_size_z = 1) in; +#else layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in; +#endif -FLOAT_TYPE temp[NUM_COLS][NUM_ROWS]; +#define LOAD_Q4_K_16(PFX, ARR_A, ARR_P16, ARR_P32) \ + const FLOAT_TYPEV2 PFX##_dm = FLOAT_TYPEV2(ARR_A[ib0 + i].dm); \ + const uint32_t PFX##_scale0_u32 = ARR_P16[ib0 + i].scales[v_im ]; \ + const uint32_t PFX##_scale4_u32 = ARR_P16[ib0 + i].scales[v_im + 2]; \ + const uint32_t PFX##_scale8_u32 = ARR_P16[ib0 + i].scales[v_im + 4]; \ + const uint32_t PFX##_scale_0_4_l = (PFX##_scale4_u32 << 16) | PFX##_scale0_u32; \ + const uint32_t PFX##_scale_0_4_h = (PFX##_scale_0_4_l & 0xC0C0C0C0u) >> 2; \ + const vec4 PFX##_sc03 = vec4(unpack8(PFX##_scale_0_4_l & 0x3F3F3F3Fu)); \ + const vec4 PFX##_sc47 = vec4(unpack8((((PFX##_scale8_u32 << 12) | PFX##_scale8_u32) & 0x0F0F0F0Fu) | PFX##_scale_0_4_h)); \ + const uint32_t PFX##_qs0_u32 = ARR_P32[ib0 + i].qs[q_offset / 4]; \ + const uint32_t PFX##_qs64_u32 = ARR_P32[ib0 + i].qs[q_offset / 4 + 16]; \ + const vec4 PFX##_qs0_lo4 = vec4(unpack8(PFX##_qs0_u32 & 0x0F0F0F0Fu)); \ + const vec4 PFX##_qs64_lo4 = vec4(unpack8(PFX##_qs64_u32 & 0x0F0F0F0Fu)); \ + const vec4 PFX##_qs0_hi4 = vec4(unpack8((PFX##_qs0_u32 >> 4) & 0x0F0F0F0Fu)); \ + const vec4 PFX##_qs64_hi4 = vec4(unpack8((PFX##_qs64_u32 >> 4) & 0x0F0F0F0Fu)) -void calc_superblock(const uint a_offset, const uint b_offset, const uint v_im, const uint q_offset, const uint y_offset, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows) { +#define DOT_Q4_K_16_WITH_BY(PFX, DST_TEMP, BY10, BY132, BY20, BY232, SY10, SY132, SY20, SY232) do { \ + const FLOAT_TYPE sc0 = PFX##_sc03.x, sc1 = PFX##_sc03.y, sc2 = PFX##_sc03.z, sc3 = PFX##_sc03.w; \ + const FLOAT_TYPE sc4 = PFX##_sc47.x, sc5 = PFX##_sc47.y, sc6 = PFX##_sc47.z, sc7 = PFX##_sc47.w; \ + const FLOAT_TYPE sx = fma(FLOAT_TYPE((BY10).x), PFX##_qs0_lo4.x, fma(FLOAT_TYPE((BY10).y), PFX##_qs0_lo4.y, fma(FLOAT_TYPE((BY10).z), PFX##_qs0_lo4.z, FLOAT_TYPE((BY10).w) * PFX##_qs0_lo4.w))); \ + const FLOAT_TYPE sy = fma(FLOAT_TYPE((BY132).x), PFX##_qs0_hi4.x, fma(FLOAT_TYPE((BY132).y), PFX##_qs0_hi4.y, fma(FLOAT_TYPE((BY132).z), PFX##_qs0_hi4.z, FLOAT_TYPE((BY132).w) * PFX##_qs0_hi4.w))); \ + const FLOAT_TYPE sz = fma(FLOAT_TYPE((BY20).x), PFX##_qs64_lo4.x, fma(FLOAT_TYPE((BY20).y), PFX##_qs64_lo4.y, fma(FLOAT_TYPE((BY20).z), PFX##_qs64_lo4.z, FLOAT_TYPE((BY20).w) * PFX##_qs64_lo4.w))); \ + const FLOAT_TYPE sw = fma(FLOAT_TYPE((BY232).x), PFX##_qs64_hi4.x, fma(FLOAT_TYPE((BY232).y), PFX##_qs64_hi4.y, fma(FLOAT_TYPE((BY232).z), PFX##_qs64_hi4.z, FLOAT_TYPE((BY232).w) * PFX##_qs64_hi4.w))); \ + FLOAT_TYPE smin; \ + if (GEN_FAST_KQ == 0) { \ + smin = \ + fma(FLOAT_TYPE((BY10).x), sc2, fma(FLOAT_TYPE((BY132).x), sc3, fma(FLOAT_TYPE((BY20).x), sc6, fma(FLOAT_TYPE((BY232).x), sc7, \ + fma(FLOAT_TYPE((BY10).y), sc2, fma(FLOAT_TYPE((BY132).y), sc3, fma(FLOAT_TYPE((BY20).y), sc6, fma(FLOAT_TYPE((BY232).y), sc7, \ + fma(FLOAT_TYPE((BY10).z), sc2, fma(FLOAT_TYPE((BY132).z), sc3, fma(FLOAT_TYPE((BY20).z), sc6, fma(FLOAT_TYPE((BY232).z), sc7, \ + fma(FLOAT_TYPE((BY10).w), sc2, fma(FLOAT_TYPE((BY132).w), sc3, fma(FLOAT_TYPE((BY20).w), sc6, FLOAT_TYPE((BY232).w) * sc7))))))))))))))); \ + } else { \ + smin = fma(SY10, sc2, fma(SY132, sc3, fma(SY20, sc6, (SY232) * sc7))); \ + } \ + DST_TEMP[j][n] = fma(PFX##_dm.x, fma(sx, sc0, fma(sy, sc1, fma(sz, sc4, sw * sc5))), fma(-PFX##_dm.y, smin, DST_TEMP[j][n])); \ +} while (false) + +void calc_superblock_a(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], inout FLOAT_TYPE temp2[NUM_COLS][NUM_ROWS], const uint a_offset, const uint b_offset, const uint v_im, const uint q_offset, const uint y_offset, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows) { + const uint y1_idx = i * QUANT_K + y_offset; + const uint y2_idx = y1_idx + 128; + + if (NUM_COLS <= 4) { + vec4 by10_a[NUM_COLS], by132_a[NUM_COLS], by20_a[NUM_COLS], by232_a[NUM_COLS]; + FLOAT_TYPE sy10_a[NUM_COLS], sy132_a[NUM_COLS], sy20_a[NUM_COLS], sy232_a[NUM_COLS]; + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + const uint b4_1 = (j * p.batch_stride_b + b_offset + y1_idx) / 4; + const uint b4_2 = (j * p.batch_stride_b + b_offset + y2_idx) / 4; + by10_a[j] = vec4(data_b_v4[b4_1 ]); + by132_a[j] = vec4(data_b_v4[b4_1 + 8]); + by20_a[j] = vec4(data_b_v4[b4_2 ]); + by232_a[j] = vec4(data_b_v4[b4_2 + 8]); + sy10_a[j] = FLOAT_TYPE(0); sy132_a[j] = FLOAT_TYPE(0); sy20_a[j] = FLOAT_TYPE(0); sy232_a[j] = FLOAT_TYPE(0); + if (GEN_FAST_KQ != 0) { + sy10_a[j] = (FLOAT_TYPE(by10_a[j].x) + FLOAT_TYPE(by10_a[j].y)) + (FLOAT_TYPE(by10_a[j].z) + FLOAT_TYPE(by10_a[j].w)); + sy132_a[j] = (FLOAT_TYPE(by132_a[j].x) + FLOAT_TYPE(by132_a[j].y)) + (FLOAT_TYPE(by132_a[j].z) + FLOAT_TYPE(by132_a[j].w)); + sy20_a[j] = (FLOAT_TYPE(by20_a[j].x) + FLOAT_TYPE(by20_a[j].y)) + (FLOAT_TYPE(by20_a[j].z) + FLOAT_TYPE(by20_a[j].w)); + sy232_a[j] = (FLOAT_TYPE(by232_a[j].x) + FLOAT_TYPE(by232_a[j].y)) + (FLOAT_TYPE(by232_a[j].z) + FLOAT_TYPE(by232_a[j].w)); + } + } + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + const uint rn = (n == 0 || n < num_rows) ? n : 0; + const uint ib0 = a_offset + (first_row + rn) * num_blocks_per_row; + LOAD_Q4_K_16(a1, data_a, data_a_packed16, data_a_packed32); + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + DOT_Q4_K_16_WITH_BY(a1, temp, by10_a[j], by132_a[j], by20_a[j], by232_a[j], sy10_a[j], sy132_a[j], sy20_a[j], sy232_a[j]); + } + if (GEN_GLU == 3) { + LOAD_Q4_K_16(a2, data_a2, data_a2_packed16, data_a2_packed32); + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + DOT_Q4_K_16_WITH_BY(a2, temp2, by10_a[j], by132_a[j], by20_a[j], by232_a[j], sy10_a[j], sy132_a[j], sy20_a[j], sy232_a[j]); + } + } + } + } else { + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + const uint rn = (n == 0 || n < num_rows) ? n : 0; + const uint ib0 = a_offset + (first_row + rn) * num_blocks_per_row; + LOAD_Q4_K_16(a1, data_a, data_a_packed16, data_a_packed32); + if (GEN_GLU == 3) { + LOAD_Q4_K_16(a2, data_a2, data_a2_packed16, data_a2_packed32); + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + const uint b4_1 = (j * p.batch_stride_b + b_offset + y1_idx) / 4; + const uint b4_2 = (j * p.batch_stride_b + b_offset + y2_idx) / 4; + const vec4 by10 = vec4(data_b_v4[b4_1 ]); + const vec4 by132 = vec4(data_b_v4[b4_1 + 8]); + const vec4 by20 = vec4(data_b_v4[b4_2 ]); + const vec4 by232 = vec4(data_b_v4[b4_2 + 8]); + FLOAT_TYPE sumy10 = FLOAT_TYPE(0), sumy132 = FLOAT_TYPE(0), sumy20 = FLOAT_TYPE(0), sumy232 = FLOAT_TYPE(0); + if (GEN_FAST_KQ != 0) { + sumy10 = (FLOAT_TYPE(by10.x) + FLOAT_TYPE(by10.y)) + (FLOAT_TYPE(by10.z) + FLOAT_TYPE(by10.w)); + sumy132 = (FLOAT_TYPE(by132.x) + FLOAT_TYPE(by132.y)) + (FLOAT_TYPE(by132.z) + FLOAT_TYPE(by132.w)); + sumy20 = (FLOAT_TYPE(by20.x) + FLOAT_TYPE(by20.y)) + (FLOAT_TYPE(by20.z) + FLOAT_TYPE(by20.w)); + sumy232 = (FLOAT_TYPE(by232.x) + FLOAT_TYPE(by232.y)) + (FLOAT_TYPE(by232.z) + FLOAT_TYPE(by232.w)); + } + DOT_Q4_K_16_WITH_BY(a1, temp, by10, by132, by20, by232, sumy10, sumy132, sumy20, sumy232); + DOT_Q4_K_16_WITH_BY(a2, temp2, by10, by132, by20, by232, sumy10, sumy132, sumy20, sumy232); + } + } else { + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + const uint b4_1 = (j * p.batch_stride_b + b_offset + y1_idx) / 4; + const uint b4_2 = (j * p.batch_stride_b + b_offset + y2_idx) / 4; + const vec4 by10 = vec4(data_b_v4[b4_1 ]); + const vec4 by132 = vec4(data_b_v4[b4_1 + 8]); + const vec4 by20 = vec4(data_b_v4[b4_2 ]); + const vec4 by232 = vec4(data_b_v4[b4_2 + 8]); + FLOAT_TYPE sumy10 = FLOAT_TYPE(0), sumy132 = FLOAT_TYPE(0), sumy20 = FLOAT_TYPE(0), sumy232 = FLOAT_TYPE(0); + if (GEN_FAST_KQ != 0) { + sumy10 = (FLOAT_TYPE(by10.x) + FLOAT_TYPE(by10.y)) + (FLOAT_TYPE(by10.z) + FLOAT_TYPE(by10.w)); + sumy132 = (FLOAT_TYPE(by132.x) + FLOAT_TYPE(by132.y)) + (FLOAT_TYPE(by132.z) + FLOAT_TYPE(by132.w)); + sumy20 = (FLOAT_TYPE(by20.x) + FLOAT_TYPE(by20.y)) + (FLOAT_TYPE(by20.z) + FLOAT_TYPE(by20.w)); + sumy232 = (FLOAT_TYPE(by232.x) + FLOAT_TYPE(by232.y)) + (FLOAT_TYPE(by232.z) + FLOAT_TYPE(by232.w)); + } + DOT_Q4_K_16_WITH_BY(a1, temp, by10, by132, by20, by232, sumy10, sumy132, sumy20, sumy232); + } + } + } + } +} + +void calc_superblock_a2(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const uint a_offset, const uint b_offset, const uint v_im, const uint q_offset, const uint y_offset, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows) { const uint y1_idx = i * QUANT_K + y_offset; const uint y2_idx = y1_idx + 128; - [[unroll]] for (uint n = 0; n < num_rows; ++n) { - const uint ib0 = a_offset + (first_row+n)*num_blocks_per_row; - const FLOAT_TYPEV2 dm = FLOAT_TYPEV2(data_a[ib0 + i].dm); - - const uint32_t scale0_u32 = data_a_packed16[ib0 + i].scales[v_im ]; - const uint32_t scale4_u32 = data_a_packed16[ib0 + i].scales[v_im + 2]; - const uint32_t scale8_u32 = data_a_packed16[ib0 + i].scales[v_im + 4]; - - const uint32_t scale_0_4_l = (scale4_u32 << 16) | scale0_u32; - const uint32_t scale_0_4_h = (scale_0_4_l & 0xC0C0C0C0) >> 2; - const vec4 scale_0_4_l_f = vec4(unpack8(scale_0_4_l & 0x3F3F3F3F)); - const vec4 scale8_f = vec4(unpack8((((scale8_u32 << 12) | scale8_u32) & 0x0F0F0F0F) | scale_0_4_h)); - - const FLOAT_TYPE sc0 = scale_0_4_l_f.x; - const FLOAT_TYPE sc1 = scale_0_4_l_f.y; - const FLOAT_TYPE sc2 = scale_0_4_l_f.z; - const FLOAT_TYPE sc3 = scale_0_4_l_f.w; - const FLOAT_TYPE sc4 = scale8_f.x; - const FLOAT_TYPE sc5 = scale8_f.y; - const FLOAT_TYPE sc6 = scale8_f.z; - const FLOAT_TYPE sc7 = scale8_f.w; - - const uint32_t qs0_u32 = data_a_packed32[ib0 + i].qs[q_offset / 4]; - const uint32_t qs64_u32 = data_a_packed32[ib0 + i].qs[q_offset / 4 + 16]; - - const uint32_t qs0_u32_lo4 = qs0_u32 & 0x0F0F0F0F; - const uint32_t qs0_u32_hi4 = (qs0_u32 >> 4) & 0x0F0F0F0F; - const uint32_t qs64_u32_lo4 = qs64_u32 & 0x0F0F0F0F; - const uint32_t qs64_u32_hi4 = (qs64_u32 >> 4) & 0x0F0F0F0F; - - const vec4 qs0_lo4 = vec4(unpack8(qs0_u32_lo4)); - const vec4 qs64_lo4 = vec4(unpack8(qs64_u32_lo4)); - const vec4 qs0_hi4 = vec4(unpack8(qs0_u32_hi4)); - const vec4 qs64_hi4 = vec4(unpack8(qs64_u32_hi4)); - - const FLOAT_TYPE q4_0 = qs0_lo4.x; - const FLOAT_TYPE q4_1 = qs0_lo4.y; - const FLOAT_TYPE q4_2 = qs0_lo4.z; - const FLOAT_TYPE q4_3 = qs0_lo4.w; - const FLOAT_TYPE q4_4 = qs0_hi4.x; - const FLOAT_TYPE q4_5 = qs0_hi4.y; - const FLOAT_TYPE q4_6 = qs0_hi4.z; - const FLOAT_TYPE q4_7 = qs0_hi4.w; - const FLOAT_TYPE q4_8 = qs64_lo4.x; - const FLOAT_TYPE q4_9 = qs64_lo4.y; - const FLOAT_TYPE q4_10 = qs64_lo4.z; - const FLOAT_TYPE q4_11 = qs64_lo4.w; - const FLOAT_TYPE q4_12 = qs64_hi4.x; - const FLOAT_TYPE q4_13 = qs64_hi4.y; - const FLOAT_TYPE q4_14 = qs64_hi4.z; - const FLOAT_TYPE q4_15 = qs64_hi4.w; + if (NUM_COLS <= 4) { + vec4 by10_a[NUM_COLS], by132_a[NUM_COLS], by20_a[NUM_COLS], by232_a[NUM_COLS]; + FLOAT_TYPE sy10_a[NUM_COLS], sy132_a[NUM_COLS], sy20_a[NUM_COLS], sy232_a[NUM_COLS]; + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + const uint b4_1 = (j * p.batch_stride_b + b_offset + y1_idx) / 4; + const uint b4_2 = (j * p.batch_stride_b + b_offset + y2_idx) / 4; + by10_a[j] = vec4(data_b_v4[b4_1 ]); + by132_a[j] = vec4(data_b_v4[b4_1 + 8]); + by20_a[j] = vec4(data_b_v4[b4_2 ]); + by232_a[j] = vec4(data_b_v4[b4_2 + 8]); + sy10_a[j] = FLOAT_TYPE(0); sy132_a[j] = FLOAT_TYPE(0); sy20_a[j] = FLOAT_TYPE(0); sy232_a[j] = FLOAT_TYPE(0); + if (GEN_FAST_KQ != 0) { + sy10_a[j] = (FLOAT_TYPE(by10_a[j].x) + FLOAT_TYPE(by10_a[j].y)) + (FLOAT_TYPE(by10_a[j].z) + FLOAT_TYPE(by10_a[j].w)); + sy132_a[j] = (FLOAT_TYPE(by132_a[j].x) + FLOAT_TYPE(by132_a[j].y)) + (FLOAT_TYPE(by132_a[j].z) + FLOAT_TYPE(by132_a[j].w)); + sy20_a[j] = (FLOAT_TYPE(by20_a[j].x) + FLOAT_TYPE(by20_a[j].y)) + (FLOAT_TYPE(by20_a[j].z) + FLOAT_TYPE(by20_a[j].w)); + sy232_a[j] = (FLOAT_TYPE(by232_a[j].x) + FLOAT_TYPE(by232_a[j].y)) + (FLOAT_TYPE(by232_a[j].z) + FLOAT_TYPE(by232_a[j].w)); + } + } + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + const uint rn = (n == 0 || n < num_rows) ? n : 0; + const uint ib0 = a_offset + (first_row + rn) * num_blocks_per_row; + LOAD_Q4_K_16(a2, data_a2, data_a2_packed16, data_a2_packed32); + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + DOT_Q4_K_16_WITH_BY(a2, temp, by10_a[j], by132_a[j], by20_a[j], by232_a[j], sy10_a[j], sy132_a[j], sy20_a[j], sy232_a[j]); + } + } + } else { + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + const uint rn = (n == 0 || n < num_rows) ? n : 0; + const uint ib0 = a_offset + (first_row + rn) * num_blocks_per_row; + LOAD_Q4_K_16(a2, data_a2, data_a2_packed16, data_a2_packed32); + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + const uint b4_1 = (j * p.batch_stride_b + b_offset + y1_idx) / 4; + const uint b4_2 = (j * p.batch_stride_b + b_offset + y2_idx) / 4; + const vec4 by10 = vec4(data_b_v4[b4_1 ]); + const vec4 by132 = vec4(data_b_v4[b4_1 + 8]); + const vec4 by20 = vec4(data_b_v4[b4_2 ]); + const vec4 by232 = vec4(data_b_v4[b4_2 + 8]); + FLOAT_TYPE sumy10 = FLOAT_TYPE(0), sumy132 = FLOAT_TYPE(0), sumy20 = FLOAT_TYPE(0), sumy232 = FLOAT_TYPE(0); + if (GEN_FAST_KQ != 0) { + sumy10 = (FLOAT_TYPE(by10.x) + FLOAT_TYPE(by10.y)) + (FLOAT_TYPE(by10.z) + FLOAT_TYPE(by10.w)); + sumy132 = (FLOAT_TYPE(by132.x) + FLOAT_TYPE(by132.y)) + (FLOAT_TYPE(by132.z) + FLOAT_TYPE(by132.w)); + sumy20 = (FLOAT_TYPE(by20.x) + FLOAT_TYPE(by20.y)) + (FLOAT_TYPE(by20.z) + FLOAT_TYPE(by20.w)); + sumy232 = (FLOAT_TYPE(by232.x) + FLOAT_TYPE(by232.y)) + (FLOAT_TYPE(by232.z) + FLOAT_TYPE(by232.w)); + } + DOT_Q4_K_16_WITH_BY(a2, temp, by10, by132, by20, by232, sumy10, sumy132, sumy20, sumy232); + } + } + } +} + +#define LOAD_Q4_K_8(PFX, ARR_P32) \ + const FLOAT_TYPEV2 PFX##_dm = FLOAT_TYPEV2(ARR_P32[ib].dm); \ + const uint32_t PFX##_s0 = ARR_P32[ib].scales[0]; \ + const uint32_t PFX##_s1 = ARR_P32[ib].scales[1]; \ + const uint32_t PFX##_s2 = ARR_P32[ib].scales[2]; \ + const vec4 PFX##_sc03 = vec4(unpack8(PFX##_s0 & 0x3F3F3F3Fu)); \ + const vec4 PFX##_m03 = vec4(unpack8(PFX##_s1 & 0x3F3F3F3Fu)); \ + const vec4 PFX##_sc47 = vec4(unpack8((PFX##_s2 & 0x0F0F0F0Fu) | ((PFX##_s0 >> 2) & 0x30303030u))); \ + const vec4 PFX##_m47 = vec4(unpack8(((PFX##_s2 >> 4) & 0x0F0F0F0Fu) | ((PFX##_s1 >> 2) & 0x30303030u))); \ + const uint32_t PFX##_q0 = ARR_P32[ib].qs[itid8 ]; \ + const uint32_t PFX##_q1 = ARR_P32[ib].qs[itid8 + 8]; \ + const uint32_t PFX##_q2 = ARR_P32[ib].qs[itid8 + 16]; \ + const uint32_t PFX##_q3 = ARR_P32[ib].qs[itid8 + 24]; \ + const vec4 PFX##_q0_lo = vec4(unpack8(PFX##_q0 & 0x0F0F0F0Fu)); \ + const vec4 PFX##_q0_hi = vec4(unpack8((PFX##_q0 >> 4) & 0x0F0F0F0Fu)); \ + const vec4 PFX##_q1_lo = vec4(unpack8(PFX##_q1 & 0x0F0F0F0Fu)); \ + const vec4 PFX##_q1_hi = vec4(unpack8((PFX##_q1 >> 4) & 0x0F0F0F0Fu)); \ + const vec4 PFX##_q2_lo = vec4(unpack8(PFX##_q2 & 0x0F0F0F0Fu)); \ + const vec4 PFX##_q2_hi = vec4(unpack8((PFX##_q2 >> 4) & 0x0F0F0F0Fu)); \ + const vec4 PFX##_q3_lo = vec4(unpack8(PFX##_q3 & 0x0F0F0F0Fu)); \ + const vec4 PFX##_q3_hi = vec4(unpack8((PFX##_q3 >> 4) & 0x0F0F0F0Fu)) + +#define DOT_Q4_K_8_BY(PFX, DST_TEMP, BY0, BY1, BY2, BY3, BY4, BY5, BY6, BY7, SY0, SY1, SY2, SY3, SY4, SY5, SY6, SY7) do { \ + const FLOAT_TYPE d0 = dot(BY0, PFX##_q0_lo) * PFX##_sc03.x + dot(BY1, PFX##_q0_hi) * PFX##_sc03.y; \ + const FLOAT_TYPE d1 = dot(BY2, PFX##_q1_lo) * PFX##_sc03.z + dot(BY3, PFX##_q1_hi) * PFX##_sc03.w; \ + const FLOAT_TYPE d2 = dot(BY4, PFX##_q2_lo) * PFX##_sc47.x + dot(BY5, PFX##_q2_hi) * PFX##_sc47.y; \ + const FLOAT_TYPE d3 = dot(BY6, PFX##_q3_lo) * PFX##_sc47.z + dot(BY7, PFX##_q3_hi) * PFX##_sc47.w; \ + const FLOAT_TYPE sm = (SY0 * PFX##_m03.x + SY1 * PFX##_m03.y) + (SY2 * PFX##_m03.z + SY3 * PFX##_m03.w) + \ + (SY4 * PFX##_m47.x + SY5 * PFX##_m47.y) + (SY6 * PFX##_m47.z + SY7 * PFX##_m47.w); \ + DST_TEMP[j][n] = fma(PFX##_dm.x, (d0 + d1) + (d2 + d3), fma(-PFX##_dm.y, sm, DST_TEMP[j][n])); \ +} while (false) + +#define DOT_Q4_K_8(PFX, DST_TEMP) do { \ + const uint b4 = (j * p.batch_stride_b + b_offset + y_idx) / 4; \ + const vec4 by0 = vec4(data_b_v4[b4 ]), by1 = vec4(data_b_v4[b4 + 8]); \ + const vec4 by2 = vec4(data_b_v4[b4 + 16]), by3 = vec4(data_b_v4[b4 + 24]); \ + const vec4 by4 = vec4(data_b_v4[b4 + 32]), by5 = vec4(data_b_v4[b4 + 40]); \ + const vec4 by6 = vec4(data_b_v4[b4 + 48]), by7 = vec4(data_b_v4[b4 + 56]); \ + const FLOAT_TYPE sy0 = (FLOAT_TYPE(by0.x) + FLOAT_TYPE(by0.y)) + (FLOAT_TYPE(by0.z) + FLOAT_TYPE(by0.w)); \ + const FLOAT_TYPE sy1 = (FLOAT_TYPE(by1.x) + FLOAT_TYPE(by1.y)) + (FLOAT_TYPE(by1.z) + FLOAT_TYPE(by1.w)); \ + const FLOAT_TYPE sy2 = (FLOAT_TYPE(by2.x) + FLOAT_TYPE(by2.y)) + (FLOAT_TYPE(by2.z) + FLOAT_TYPE(by2.w)); \ + const FLOAT_TYPE sy3 = (FLOAT_TYPE(by3.x) + FLOAT_TYPE(by3.y)) + (FLOAT_TYPE(by3.z) + FLOAT_TYPE(by3.w)); \ + const FLOAT_TYPE sy4 = (FLOAT_TYPE(by4.x) + FLOAT_TYPE(by4.y)) + (FLOAT_TYPE(by4.z) + FLOAT_TYPE(by4.w)); \ + const FLOAT_TYPE sy5 = (FLOAT_TYPE(by5.x) + FLOAT_TYPE(by5.y)) + (FLOAT_TYPE(by5.z) + FLOAT_TYPE(by5.w)); \ + const FLOAT_TYPE sy6 = (FLOAT_TYPE(by6.x) + FLOAT_TYPE(by6.y)) + (FLOAT_TYPE(by6.z) + FLOAT_TYPE(by6.w)); \ + const FLOAT_TYPE sy7 = (FLOAT_TYPE(by7.x) + FLOAT_TYPE(by7.y)) + (FLOAT_TYPE(by7.z) + FLOAT_TYPE(by7.w)); \ + DOT_Q4_K_8_BY(PFX, DST_TEMP, by0, by1, by2, by3, by4, by5, by6, by7, sy0, sy1, sy2, sy3, sy4, sy5, sy6, sy7); \ +} while (false) + +void calc_superblock_8_a(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], inout FLOAT_TYPE temp2[NUM_COLS][NUM_ROWS], const uint a_offset, const uint b_offset, const uint itid8, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows) { + const uint y_idx = i * QUANT_K + 4 * itid8; + if (NUM_COLS <= 4 && NUM_ROWS > 1) { + vec4 by0_a[NUM_COLS], by1_a[NUM_COLS], by2_a[NUM_COLS], by3_a[NUM_COLS]; + vec4 by4_a[NUM_COLS], by5_a[NUM_COLS], by6_a[NUM_COLS], by7_a[NUM_COLS]; + FLOAT_TYPE sy0_a[NUM_COLS], sy1_a[NUM_COLS], sy2_a[NUM_COLS], sy3_a[NUM_COLS]; + FLOAT_TYPE sy4_a[NUM_COLS], sy5_a[NUM_COLS], sy6_a[NUM_COLS], sy7_a[NUM_COLS]; + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + const uint b4 = (j * p.batch_stride_b + b_offset + y_idx) / 4; + by0_a[j] = vec4(data_b_v4[b4 ]); by1_a[j] = vec4(data_b_v4[b4 + 8]); + by2_a[j] = vec4(data_b_v4[b4 + 16]); by3_a[j] = vec4(data_b_v4[b4 + 24]); + by4_a[j] = vec4(data_b_v4[b4 + 32]); by5_a[j] = vec4(data_b_v4[b4 + 40]); + by6_a[j] = vec4(data_b_v4[b4 + 48]); by7_a[j] = vec4(data_b_v4[b4 + 56]); + sy0_a[j] = (FLOAT_TYPE(by0_a[j].x) + FLOAT_TYPE(by0_a[j].y)) + (FLOAT_TYPE(by0_a[j].z) + FLOAT_TYPE(by0_a[j].w)); + sy1_a[j] = (FLOAT_TYPE(by1_a[j].x) + FLOAT_TYPE(by1_a[j].y)) + (FLOAT_TYPE(by1_a[j].z) + FLOAT_TYPE(by1_a[j].w)); + sy2_a[j] = (FLOAT_TYPE(by2_a[j].x) + FLOAT_TYPE(by2_a[j].y)) + (FLOAT_TYPE(by2_a[j].z) + FLOAT_TYPE(by2_a[j].w)); + sy3_a[j] = (FLOAT_TYPE(by3_a[j].x) + FLOAT_TYPE(by3_a[j].y)) + (FLOAT_TYPE(by3_a[j].z) + FLOAT_TYPE(by3_a[j].w)); + sy4_a[j] = (FLOAT_TYPE(by4_a[j].x) + FLOAT_TYPE(by4_a[j].y)) + (FLOAT_TYPE(by4_a[j].z) + FLOAT_TYPE(by4_a[j].w)); + sy5_a[j] = (FLOAT_TYPE(by5_a[j].x) + FLOAT_TYPE(by5_a[j].y)) + (FLOAT_TYPE(by5_a[j].z) + FLOAT_TYPE(by5_a[j].w)); + sy6_a[j] = (FLOAT_TYPE(by6_a[j].x) + FLOAT_TYPE(by6_a[j].y)) + (FLOAT_TYPE(by6_a[j].z) + FLOAT_TYPE(by6_a[j].w)); + sy7_a[j] = (FLOAT_TYPE(by7_a[j].x) + FLOAT_TYPE(by7_a[j].y)) + (FLOAT_TYPE(by7_a[j].z) + FLOAT_TYPE(by7_a[j].w)); + } + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + const uint rn = (n == 0 || n < num_rows) ? n : 0; + const uint ib = a_offset + (first_row + rn) * num_blocks_per_row + i; + LOAD_Q4_K_8(a1, data_a_packed32); + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + DOT_Q4_K_8_BY(a1, temp, by0_a[j], by1_a[j], by2_a[j], by3_a[j], by4_a[j], by5_a[j], by6_a[j], by7_a[j], sy0_a[j], sy1_a[j], sy2_a[j], sy3_a[j], sy4_a[j], sy5_a[j], sy6_a[j], sy7_a[j]); + } + if (GEN_GLU == 3) { + LOAD_Q4_K_8(a2, data_a2_packed32); + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + DOT_Q4_K_8_BY(a2, temp2, by0_a[j], by1_a[j], by2_a[j], by3_a[j], by4_a[j], by5_a[j], by6_a[j], by7_a[j], sy0_a[j], sy1_a[j], sy2_a[j], sy3_a[j], sy4_a[j], sy5_a[j], sy6_a[j], sy7_a[j]); + } + } + } + } else { + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + const uint rn = (n == 0 || n < num_rows) ? n : 0; + const uint ib = a_offset + (first_row + rn) * num_blocks_per_row + i; + LOAD_Q4_K_8(a1, data_a_packed32); + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + DOT_Q4_K_8(a1, temp); + } + if (GEN_GLU == 3) { + LOAD_Q4_K_8(a2, data_a2_packed32); + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + DOT_Q4_K_8(a2, temp2); + } + } + } + } +} +void calc_superblock_8_a2(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const uint a_offset, const uint b_offset, const uint itid8, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows) { + const uint y_idx = i * QUANT_K + 4 * itid8; + if (NUM_COLS <= 4 && NUM_ROWS > 1) { + vec4 by0_a[NUM_COLS], by1_a[NUM_COLS], by2_a[NUM_COLS], by3_a[NUM_COLS]; + vec4 by4_a[NUM_COLS], by5_a[NUM_COLS], by6_a[NUM_COLS], by7_a[NUM_COLS]; + FLOAT_TYPE sy0_a[NUM_COLS], sy1_a[NUM_COLS], sy2_a[NUM_COLS], sy3_a[NUM_COLS]; + FLOAT_TYPE sy4_a[NUM_COLS], sy5_a[NUM_COLS], sy6_a[NUM_COLS], sy7_a[NUM_COLS]; [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { - vec4 by10 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y1_idx) / 4 ]); - vec4 by132 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y1_idx) / 4 + 8]); - vec4 by20 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y2_idx) / 4 ]); - vec4 by232 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y2_idx) / 4 + 8]); - - const FLOAT_TYPE sx = fma(FLOAT_TYPE(by10.x), q4_0, fma(FLOAT_TYPE(by10.y), q4_1, fma(FLOAT_TYPE(by10.z), q4_2, FLOAT_TYPE(by10.w) * q4_3))); - const FLOAT_TYPE sy = fma(FLOAT_TYPE(by132.x), q4_4, fma(FLOAT_TYPE(by132.y), q4_5, fma(FLOAT_TYPE(by132.z), q4_6, FLOAT_TYPE(by132.w) * q4_7))); - const FLOAT_TYPE sz = fma(FLOAT_TYPE(by20.x), q4_8, fma(FLOAT_TYPE(by20.y), q4_9, fma(FLOAT_TYPE(by20.z), q4_10, FLOAT_TYPE(by20.w) * q4_11))); - const FLOAT_TYPE sw = fma(FLOAT_TYPE(by232.x), q4_12, fma(FLOAT_TYPE(by232.y), q4_13, fma(FLOAT_TYPE(by232.z), q4_14, FLOAT_TYPE(by232.w) * q4_15))); - const FLOAT_TYPE smin = - fma(FLOAT_TYPE(by10.x), sc2, fma(FLOAT_TYPE(by132.x), sc3, fma(FLOAT_TYPE(by20.x), sc6, fma(FLOAT_TYPE(by232.x), sc7, - fma(FLOAT_TYPE(by10.y), sc2, fma(FLOAT_TYPE(by132.y), sc3, fma(FLOAT_TYPE(by20.y), sc6, fma(FLOAT_TYPE(by232.y), sc7, - fma(FLOAT_TYPE(by10.z), sc2, fma(FLOAT_TYPE(by132.z), sc3, fma(FLOAT_TYPE(by20.z), sc6, fma(FLOAT_TYPE(by232.z), sc7, - fma(FLOAT_TYPE(by10.w), sc2, fma(FLOAT_TYPE(by132.w), sc3, fma(FLOAT_TYPE(by20.w), sc6, FLOAT_TYPE(by232.w) * sc7))))))))))))))); - temp[j][n] = fma(dm.x, fma(sx, sc0, fma(sy, sc1, fma(sz, sc4, sw * sc5))), fma(-dm.y, smin, temp[j][n])); + const uint b4 = (j * p.batch_stride_b + b_offset + y_idx) / 4; + by0_a[j] = vec4(data_b_v4[b4 ]); by1_a[j] = vec4(data_b_v4[b4 + 8]); + by2_a[j] = vec4(data_b_v4[b4 + 16]); by3_a[j] = vec4(data_b_v4[b4 + 24]); + by4_a[j] = vec4(data_b_v4[b4 + 32]); by5_a[j] = vec4(data_b_v4[b4 + 40]); + by6_a[j] = vec4(data_b_v4[b4 + 48]); by7_a[j] = vec4(data_b_v4[b4 + 56]); + sy0_a[j] = (FLOAT_TYPE(by0_a[j].x) + FLOAT_TYPE(by0_a[j].y)) + (FLOAT_TYPE(by0_a[j].z) + FLOAT_TYPE(by0_a[j].w)); + sy1_a[j] = (FLOAT_TYPE(by1_a[j].x) + FLOAT_TYPE(by1_a[j].y)) + (FLOAT_TYPE(by1_a[j].z) + FLOAT_TYPE(by1_a[j].w)); + sy2_a[j] = (FLOAT_TYPE(by2_a[j].x) + FLOAT_TYPE(by2_a[j].y)) + (FLOAT_TYPE(by2_a[j].z) + FLOAT_TYPE(by2_a[j].w)); + sy3_a[j] = (FLOAT_TYPE(by3_a[j].x) + FLOAT_TYPE(by3_a[j].y)) + (FLOAT_TYPE(by3_a[j].z) + FLOAT_TYPE(by3_a[j].w)); + sy4_a[j] = (FLOAT_TYPE(by4_a[j].x) + FLOAT_TYPE(by4_a[j].y)) + (FLOAT_TYPE(by4_a[j].z) + FLOAT_TYPE(by4_a[j].w)); + sy5_a[j] = (FLOAT_TYPE(by5_a[j].x) + FLOAT_TYPE(by5_a[j].y)) + (FLOAT_TYPE(by5_a[j].z) + FLOAT_TYPE(by5_a[j].w)); + sy6_a[j] = (FLOAT_TYPE(by6_a[j].x) + FLOAT_TYPE(by6_a[j].y)) + (FLOAT_TYPE(by6_a[j].z) + FLOAT_TYPE(by6_a[j].w)); + sy7_a[j] = (FLOAT_TYPE(by7_a[j].x) + FLOAT_TYPE(by7_a[j].y)) + (FLOAT_TYPE(by7_a[j].z) + FLOAT_TYPE(by7_a[j].w)); + } + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + const uint rn = (n == 0 || n < num_rows) ? n : 0; + const uint ib = a_offset + (first_row + rn) * num_blocks_per_row + i; + LOAD_Q4_K_8(a2, data_a2_packed32); + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + DOT_Q4_K_8_BY(a2, temp, by0_a[j], by1_a[j], by2_a[j], by3_a[j], by4_a[j], by5_a[j], by6_a[j], by7_a[j], sy0_a[j], sy1_a[j], sy2_a[j], sy3_a[j], sy4_a[j], sy5_a[j], sy6_a[j], sy7_a[j]); + } + } + } else { + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + const uint rn = (n == 0 || n < num_rows) ? n : 0; + const uint ib = a_offset + (first_row + rn) * num_blocks_per_row + i; + LOAD_Q4_K_8(a2, data_a2_packed32); + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + DOT_Q4_K_8(a2, temp); + } } } } @@ -87,48 +331,99 @@ void calc_superblock(const uint a_offset, const uint b_offset, const uint v_im, void compute_outputs(const uint32_t first_row, const uint32_t num_rows) { uint a_offset, b_offset, d_offset; get_offsets(a_offset, b_offset, d_offset); +#ifndef MUL_MAT_ID + const bool use_a2 = (GEN_GLU == 2) && (first_row >= p.pair_m0); + const uint row_base = use_a2 ? (first_row - p.pair_m0) : first_row; + const uint eff_rows = (!use_a2 && GEN_GLU == 2) ? min(num_rows, p.pair_m0 - first_row) : num_rows; +#else + const bool use_a2 = false; + const uint row_base = first_row; + const uint eff_rows = num_rows; +#endif const uint num_blocks_per_row = p.ncols / QUANT_K; - // 16 threads are used to process each block - const uint it_size = gl_WorkGroupSize.x/16; +#ifdef USE_SUBGROUP_ADD_NO_SHMEM + const uint tid = gl_SubgroupInvocationID; +#else const uint tid = gl_LocalInvocationID.x; - const uint itid = tid%16; // 0...15 - const uint ix = tid/16; +#endif - const uint il = itid/4; // 0...3 - const uint ir = itid - 4*il; // 0...3 - const uint n = 4; - - const uint v_im = il / 2; // 0 or 1. 0 computes 0,32 + 128,160, 1 computes 64,96 + 192,224 - const uint v_in = il % 2; - - const uint l0 = n * (2 * ir + v_in); // 0...15 - const uint q_offset = 32*v_im + l0; - const uint y_offset = 64*v_im + l0; + FLOAT_TYPE temp[NUM_COLS][NUM_ROWS]; + FLOAT_TYPE temp2[NUM_COLS][NUM_ROWS]; [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { [[unroll]] for (uint i = 0; i < NUM_ROWS; ++i) { temp[j][i] = FLOAT_TYPE(0); + if (GEN_GLU == 3) { + temp2[j][i] = FLOAT_TYPE(0); + } } } - [[unroll]] for (uint i = ix; i < num_blocks_per_row; i += it_size) - calc_superblock(a_offset, b_offset, v_im, q_offset, y_offset, i, num_blocks_per_row, first_row, num_rows); + if (GEN_FAST_KQ >= 2) { + const uint it_size8 = gl_WorkGroupSize.x / 8; + const uint itid8 = tid & 7; + const uint ix8 = tid >> 3; + if (GEN_GLU == 2 && use_a2) { + for (uint i = ix8; i < num_blocks_per_row; i += it_size8) { + calc_superblock_8_a2(temp, a_offset, b_offset, itid8, i, num_blocks_per_row, row_base, eff_rows); + } + } else { + for (uint i = ix8; i < num_blocks_per_row; i += it_size8) { + calc_superblock_8_a(temp, temp2, a_offset, b_offset, itid8, i, num_blocks_per_row, row_base, eff_rows); + } + } + } else { + // 16 threads are used to process each block + const uint it_size = gl_WorkGroupSize.x/16; + const uint itid = tid%16; // 0...15 + const uint ix = tid/16; + + const uint il = itid/4; // 0...3 + const uint ir = itid - 4*il; // 0...3 + const uint n = 4; + + const uint v_im = il / 2; // 0 or 1. 0 computes 0,32 + 128,160, 1 computes 64,96 + 192,224 + const uint v_in = il % 2; + + const uint l0 = n * (2 * ir + v_in); // 0...15 + const uint q_offset = 32*v_im + l0; + const uint y_offset = 64*v_im + l0; + + if (GEN_GLU == 2 && use_a2) { + for (uint i = ix; i < num_blocks_per_row; i += it_size) { + calc_superblock_a2(temp, a_offset, b_offset, v_im, q_offset, y_offset, i, num_blocks_per_row, row_base, eff_rows); + } + } else { + for (uint i = ix; i < num_blocks_per_row; i += it_size) { + calc_superblock_a(temp, temp2, a_offset, b_offset, v_im, q_offset, y_offset, i, num_blocks_per_row, row_base, eff_rows); + } + } + } - reduce_result(temp, d_offset, first_row, num_rows, tid); + if (GEN_GLU == 3) { + reduce_pair_glu(temp, temp2, d_offset, first_row, eff_rows, tid); + } +#ifndef MUL_MAT_ID + else if (GEN_GLU == 2 && use_a2) { + reduce_result_d2(temp, row_base, eff_rows, tid); + } +#endif + else { + reduce_result(temp, d_offset, first_row, eff_rows, tid); + } } void main() { +#ifdef USE_SUBGROUP_ADD_NO_SHMEM + const uint first_row = NUM_ROWS * (NUM_SUBGROUPS * (gl_WorkGroupID.x + gl_NumWorkGroups.x * gl_WorkGroupID.z) + gl_SubgroupID); +#else const uint first_row = NUM_ROWS * (gl_WorkGroupID.x + gl_NumWorkGroups.x * gl_WorkGroupID.z); +#endif - // do NUM_ROWS at a time, unless there aren't enough remaining rows - if (first_row + NUM_ROWS <= p.stride_d) { - compute_outputs(first_row, NUM_ROWS); - } else { - if (first_row >= p.stride_d) { - return; - } - compute_outputs(first_row, p.stride_d - first_row); + if (first_row >= p.stride_d) { + return; } + compute_outputs(first_row, min(NUM_ROWS, p.stride_d - first_row)); } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp index 54d7e1bcdca6..050d08ea689b 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp @@ -4,114 +4,407 @@ #include "mul_mat_vec_base.glsl" +#ifdef USE_SUBGROUP_ADD_NO_SHMEM +layout(local_size_x_id = 0, local_size_y_id = 4, local_size_z = 1) in; +#else layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in; +#endif -FLOAT_TYPE temp[NUM_COLS][NUM_ROWS]; +#define LOAD_Q5_K_0ULP(PFX, ARR_A, ARR_P16) \ + const FLOAT_TYPEV2 PFX##_dm = FLOAT_TYPEV2(ARR_A[ib0 + i].dm); \ + const uint32_t PFX##_scale0_u32 = ARR_P16[ib0 + i].scales[v_im ]; \ + const uint32_t PFX##_scale4_u32 = ARR_P16[ib0 + i].scales[v_im + 2]; \ + const uint32_t PFX##_scale8_u32 = ARR_P16[ib0 + i].scales[v_im + 4]; \ + const uint32_t PFX##_scale_0_4_l = (PFX##_scale4_u32 << 16) | PFX##_scale0_u32; \ + const uint32_t PFX##_scale_0_4_h = (PFX##_scale_0_4_l & 0xC0C0C0C0u) >> 2; \ + const vec4 PFX##_sc03 = vec4(unpack8(PFX##_scale_0_4_l & 0x3F3F3F3Fu)); \ + const vec4 PFX##_sc47 = vec4(unpack8((((PFX##_scale8_u32 << 12) | PFX##_scale8_u32) & 0x0F0F0F0Fu) | PFX##_scale_0_4_h)); \ + const uint32_t PFX##_qs0_16_u32 = uint32_t(ARR_P16[ib0 + i].qs[q_offset / 2]) | (uint32_t(ARR_P16[ib0 + i].qs[q_offset / 2 + 8]) << 16); \ + const uint32_t PFX##_qs64_80_u32 = uint32_t(ARR_P16[ib0 + i].qs[q_offset / 2 + 32]) | (uint32_t(ARR_P16[ib0 + i].qs[q_offset / 2 + 40]) << 16); \ + const uint32_t PFX##_qh = pack32(u16vec2(ARR_P16[ib0 + i].qh[l0 / 2], ARR_P16[ib0 + i].qh[l0 / 2 + 8])); \ + const vec4 PFX##_qs0_16_lo4 = vec4(unpack8((PFX##_qs0_16_u32 & 0x0F0F0F0Fu) + (((PFX##_qh >> (2*v_im)) & 0x01010101u) << 4))); \ + const vec4 PFX##_qs0_16_hi4 = vec4(unpack8(((PFX##_qs0_16_u32 >> 4) & 0x0F0F0F0Fu) + (((PFX##_qh >> (2*v_im)) & 0x02020202u) << 3))); \ + const vec4 PFX##_qs64_80_lo4 = vec4(unpack8((PFX##_qs64_80_u32 & 0x0F0F0F0Fu) + ((PFX##_qh >> (2*v_im)) & 0x10101010u))); \ + const vec4 PFX##_qs64_80_hi4 = vec4(unpack8(((PFX##_qs64_80_u32 >> 4) & 0x0F0F0F0Fu) + (((PFX##_qh >> (2*v_im)) & 0x20202020u) >> 1))) -void calc_superblock(const uint a_offset, const uint b_offset, const uint v_im, const uint l0, const uint q_offset, const uint y_offset, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows) { +#define DOT_Q5_K_0ULP_WITH_BY(PFX, DST_TEMP, BY10, BY116, BY132, BY148, BY20, BY216, BY232, BY248, SY0, SY1, SY2, SY3) do { \ + const FLOAT_TYPE sx = fma(FLOAT_TYPE((BY10).x), PFX##_qs0_16_lo4.x, fma(FLOAT_TYPE((BY10).y), PFX##_qs0_16_lo4.y, fma(FLOAT_TYPE((BY116).x), PFX##_qs0_16_lo4.z, FLOAT_TYPE((BY116).y) * PFX##_qs0_16_lo4.w))); \ + const FLOAT_TYPE sy = fma(FLOAT_TYPE((BY132).x), PFX##_qs0_16_hi4.x, fma(FLOAT_TYPE((BY132).y), PFX##_qs0_16_hi4.y, fma(FLOAT_TYPE((BY148).x), PFX##_qs0_16_hi4.z, FLOAT_TYPE((BY148).y) * PFX##_qs0_16_hi4.w))); \ + const FLOAT_TYPE sz = fma(FLOAT_TYPE((BY20).x), PFX##_qs64_80_lo4.x, fma(FLOAT_TYPE((BY20).y), PFX##_qs64_80_lo4.y, fma(FLOAT_TYPE((BY216).x), PFX##_qs64_80_lo4.z, FLOAT_TYPE((BY216).y) * PFX##_qs64_80_lo4.w))); \ + const FLOAT_TYPE sw = fma(FLOAT_TYPE((BY232).x), PFX##_qs64_80_hi4.x, fma(FLOAT_TYPE((BY232).y), PFX##_qs64_80_hi4.y, fma(FLOAT_TYPE((BY248).x), PFX##_qs64_80_hi4.z, FLOAT_TYPE((BY248).y) * PFX##_qs64_80_hi4.w))); \ + const FLOAT_TYPE smin = fma(SY0, PFX##_sc03.z, fma(SY1, PFX##_sc03.w, fma(SY2, PFX##_sc47.z, (SY3) * PFX##_sc47.w))); \ + DST_TEMP[j][n] = fma(PFX##_dm.x, fma(sx, PFX##_sc03.x, fma(sy, PFX##_sc03.y, fma(sz, PFX##_sc47.x, sw * PFX##_sc47.y))), fma(-PFX##_dm.y, smin, DST_TEMP[j][n])); \ +} while (false) + +void calc_superblock_a(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], inout FLOAT_TYPE temp2[NUM_COLS][NUM_ROWS], const uint a_offset, const uint b_offset, const uint v_im, const uint l0, const uint q_offset, const uint y_offset, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows) { const uint y1_idx = i * QUANT_K + y_offset; const uint y2_idx = y1_idx + 128; - [[unroll]] for (uint n = 0; n < num_rows; ++n) { - const uint ib0 = a_offset + (first_row+n)*num_blocks_per_row; - const FLOAT_TYPEV2 dm = FLOAT_TYPEV2(data_a[ib0 + i].dm); - - const uint32_t scale0_u32 = data_a_packed16[ib0 + i].scales[v_im ]; - const uint32_t scale4_u32 = data_a_packed16[ib0 + i].scales[v_im + 2]; - const uint32_t scale8_u32 = data_a_packed16[ib0 + i].scales[v_im + 4]; - - const uint32_t scale_0_4_l = (scale4_u32 << 16) | scale0_u32; - const uint32_t scale_0_4_h = (scale_0_4_l & 0xC0C0C0C0) >> 2; - const vec4 scale_0_4_l_f = vec4(unpack8(scale_0_4_l & 0x3F3F3F3F)); - const vec4 scale8_f = vec4(unpack8((((scale8_u32 << 12) | scale8_u32) & 0x0F0F0F0F) | scale_0_4_h)); - - const FLOAT_TYPE sc0 = scale_0_4_l_f.x; - const FLOAT_TYPE sc1 = scale_0_4_l_f.y; - const FLOAT_TYPE sc2 = scale_0_4_l_f.z; - const FLOAT_TYPE sc3 = scale_0_4_l_f.w; - const FLOAT_TYPE sc4 = scale8_f.x; - const FLOAT_TYPE sc5 = scale8_f.y; - const FLOAT_TYPE sc6 = scale8_f.z; - const FLOAT_TYPE sc7 = scale8_f.w; - - const uint32_t qs0_16_u32 = uint32_t(data_a_packed16[ib0 + i].qs[q_offset / 2]) | (uint32_t(data_a_packed16[ib0 + i].qs[q_offset / 2 + 8]) << 16); - const uint32_t qs64_80_u32 = uint32_t(data_a_packed16[ib0 + i].qs[q_offset / 2 + 32]) | (uint32_t(data_a_packed16[ib0 + i].qs[q_offset / 2 + 40]) << 16); - - uint32_t qs0_16_u32_lo4 = qs0_16_u32 & 0x0F0F0F0F; - uint32_t qs0_16_u32_hi4 = (qs0_16_u32 >> 4) & 0x0F0F0F0F; - uint32_t qs64_80_u32_lo4 = qs64_80_u32 & 0x0F0F0F0F; - uint32_t qs64_80_u32_hi4 = (qs64_80_u32 >> 4) & 0x0F0F0F0F; - - const uint32_t qh = pack32(u16vec2(data_a_packed16[ib0 + i].qh[l0 / 2], data_a_packed16[ib0 + i].qh[l0 / 2 + 8])); - - const uint32_t qs0_16_lo4_offset16 = ((qh >> (2*v_im)) & 0x01010101) << 4; - const uint32_t qs0_16_hi4_offset16 = ((qh >> (2*v_im)) & 0x02020202) << 3; - const uint32_t qs64_80_lo4_offset16 = ((qh >> (2*v_im)) & 0x10101010); - const uint32_t qs64_80_hi4_offset16 = ((qh >> (2*v_im)) & 0x20202020) >> 1; - - qs0_16_u32_lo4 += qs0_16_lo4_offset16; - qs0_16_u32_hi4 += qs0_16_hi4_offset16; - qs64_80_u32_lo4 += qs64_80_lo4_offset16; - qs64_80_u32_hi4 += qs64_80_hi4_offset16; - - const vec4 qs0_16_lo4 = vec4(unpack8(qs0_16_u32_lo4)); - const vec4 qs64_80_lo4 = vec4(unpack8(qs64_80_u32_lo4)); - const vec4 qs0_16_hi4 = vec4(unpack8(qs0_16_u32_hi4)); - const vec4 qs64_80_hi4 = vec4(unpack8(qs64_80_u32_hi4)); - - const FLOAT_TYPE q4_0 = qs0_16_lo4.x; - const FLOAT_TYPE q4_1 = qs0_16_lo4.y; - const FLOAT_TYPE q4_2 = qs0_16_lo4.z; - const FLOAT_TYPE q4_3 = qs0_16_lo4.w; - const FLOAT_TYPE q4_4 = qs0_16_hi4.x; - const FLOAT_TYPE q4_5 = qs0_16_hi4.y; - const FLOAT_TYPE q4_6 = qs0_16_hi4.z; - const FLOAT_TYPE q4_7 = qs0_16_hi4.w; - const FLOAT_TYPE q4_8 = qs64_80_lo4.x; - const FLOAT_TYPE q4_9 = qs64_80_lo4.y; - const FLOAT_TYPE q4_10 = qs64_80_lo4.z; - const FLOAT_TYPE q4_11 = qs64_80_lo4.w; - const FLOAT_TYPE q4_12 = qs64_80_hi4.x; - const FLOAT_TYPE q4_13 = qs64_80_hi4.y; - const FLOAT_TYPE q4_14 = qs64_80_hi4.z; - const FLOAT_TYPE q4_15 = qs64_80_hi4.w; + if (NUM_COLS <= 4) { + vec2 by10_a[4], by116_a[4], by132_a[4], by148_a[4]; + vec2 by20_a[4], by216_a[4], by232_a[4], by248_a[4]; + FLOAT_TYPE sy0_a[4], sy1_a[4], sy2_a[4], sy3_a[4]; + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + const uint b2_1 = (j * p.batch_stride_b + b_offset + y1_idx) / 2; + const uint b2_2 = (j * p.batch_stride_b + b_offset + y2_idx) / 2; + by10_a[j] = vec2(data_b_v2[b2_1 ]); by116_a[j] = vec2(data_b_v2[b2_1 + 8]); + by132_a[j] = vec2(data_b_v2[b2_1 + 16]); by148_a[j] = vec2(data_b_v2[b2_1 + 24]); + by20_a[j] = vec2(data_b_v2[b2_2 ]); by216_a[j] = vec2(data_b_v2[b2_2 + 8]); + by232_a[j] = vec2(data_b_v2[b2_2 + 16]); by248_a[j] = vec2(data_b_v2[b2_2 + 24]); + sy0_a[j] = FLOAT_TYPE(by10_a[j].x) + FLOAT_TYPE(by10_a[j].y) + FLOAT_TYPE(by116_a[j].x) + FLOAT_TYPE(by116_a[j].y); + sy1_a[j] = FLOAT_TYPE(by132_a[j].x) + FLOAT_TYPE(by132_a[j].y) + FLOAT_TYPE(by148_a[j].x) + FLOAT_TYPE(by148_a[j].y); + sy2_a[j] = FLOAT_TYPE(by20_a[j].x) + FLOAT_TYPE(by20_a[j].y) + FLOAT_TYPE(by216_a[j].x) + FLOAT_TYPE(by216_a[j].y); + sy3_a[j] = FLOAT_TYPE(by232_a[j].x) + FLOAT_TYPE(by232_a[j].y) + FLOAT_TYPE(by248_a[j].x) + FLOAT_TYPE(by248_a[j].y); + } + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + const uint rn = (n == 0 || n < num_rows) ? n : 0; + const uint ib0 = a_offset + (first_row + rn) * num_blocks_per_row; + LOAD_Q5_K_0ULP(a1, data_a, data_a_packed16); + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + DOT_Q5_K_0ULP_WITH_BY(a1, temp, by10_a[j], by116_a[j], by132_a[j], by148_a[j], by20_a[j], by216_a[j], by232_a[j], by248_a[j], sy0_a[j], sy1_a[j], sy2_a[j], sy3_a[j]); + } + if (GEN_GLU == 3) { + LOAD_Q5_K_0ULP(a2, data_a2, data_a2_packed16); + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + DOT_Q5_K_0ULP_WITH_BY(a2, temp2, by10_a[j], by116_a[j], by132_a[j], by148_a[j], by20_a[j], by216_a[j], by232_a[j], by248_a[j], sy0_a[j], sy1_a[j], sy2_a[j], sy3_a[j]); + } + } + } + } else { + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + const uint rn = (n == 0 || n < num_rows) ? n : 0; + const uint ib0 = a_offset + (first_row + rn) * num_blocks_per_row; + LOAD_Q5_K_0ULP(a1, data_a, data_a_packed16); + if (GEN_GLU == 3) { + LOAD_Q5_K_0ULP(a2, data_a2, data_a2_packed16); + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + const uint b2_1 = (j * p.batch_stride_b + b_offset + y1_idx) / 2; + const uint b2_2 = (j * p.batch_stride_b + b_offset + y2_idx) / 2; + const vec2 by10 = vec2(data_b_v2[b2_1 ]), by116 = vec2(data_b_v2[b2_1 + 8]); + const vec2 by132 = vec2(data_b_v2[b2_1 + 16]), by148 = vec2(data_b_v2[b2_1 + 24]); + const vec2 by20 = vec2(data_b_v2[b2_2 ]), by216 = vec2(data_b_v2[b2_2 + 8]); + const vec2 by232 = vec2(data_b_v2[b2_2 + 16]), by248 = vec2(data_b_v2[b2_2 + 24]); + const FLOAT_TYPE sumy0 = FLOAT_TYPE(by10.x) + FLOAT_TYPE(by10.y) + FLOAT_TYPE(by116.x) + FLOAT_TYPE(by116.y); + const FLOAT_TYPE sumy1 = FLOAT_TYPE(by132.x) + FLOAT_TYPE(by132.y) + FLOAT_TYPE(by148.x) + FLOAT_TYPE(by148.y); + const FLOAT_TYPE sumy2 = FLOAT_TYPE(by20.x) + FLOAT_TYPE(by20.y) + FLOAT_TYPE(by216.x) + FLOAT_TYPE(by216.y); + const FLOAT_TYPE sumy3 = FLOAT_TYPE(by232.x) + FLOAT_TYPE(by232.y) + FLOAT_TYPE(by248.x) + FLOAT_TYPE(by248.y); + DOT_Q5_K_0ULP_WITH_BY(a1, temp, by10, by116, by132, by148, by20, by216, by232, by248, sumy0, sumy1, sumy2, sumy3); + DOT_Q5_K_0ULP_WITH_BY(a2, temp2, by10, by116, by132, by148, by20, by216, by232, by248, sumy0, sumy1, sumy2, sumy3); + } + } else { + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + const uint b2_1 = (j * p.batch_stride_b + b_offset + y1_idx) / 2; + const uint b2_2 = (j * p.batch_stride_b + b_offset + y2_idx) / 2; + const vec2 by10 = vec2(data_b_v2[b2_1 ]), by116 = vec2(data_b_v2[b2_1 + 8]); + const vec2 by132 = vec2(data_b_v2[b2_1 + 16]), by148 = vec2(data_b_v2[b2_1 + 24]); + const vec2 by20 = vec2(data_b_v2[b2_2 ]), by216 = vec2(data_b_v2[b2_2 + 8]); + const vec2 by232 = vec2(data_b_v2[b2_2 + 16]), by248 = vec2(data_b_v2[b2_2 + 24]); + const FLOAT_TYPE sumy0 = FLOAT_TYPE(by10.x) + FLOAT_TYPE(by10.y) + FLOAT_TYPE(by116.x) + FLOAT_TYPE(by116.y); + const FLOAT_TYPE sumy1 = FLOAT_TYPE(by132.x) + FLOAT_TYPE(by132.y) + FLOAT_TYPE(by148.x) + FLOAT_TYPE(by148.y); + const FLOAT_TYPE sumy2 = FLOAT_TYPE(by20.x) + FLOAT_TYPE(by20.y) + FLOAT_TYPE(by216.x) + FLOAT_TYPE(by216.y); + const FLOAT_TYPE sumy3 = FLOAT_TYPE(by232.x) + FLOAT_TYPE(by232.y) + FLOAT_TYPE(by248.x) + FLOAT_TYPE(by248.y); + DOT_Q5_K_0ULP_WITH_BY(a1, temp, by10, by116, by132, by148, by20, by216, by232, by248, sumy0, sumy1, sumy2, sumy3); + } + } + } + } +} +void calc_superblock_a2(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const uint a_offset, const uint b_offset, const uint v_im, const uint l0, const uint q_offset, const uint y_offset, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows) { + const uint y1_idx = i * QUANT_K + y_offset; + const uint y2_idx = y1_idx + 128; + + if (NUM_COLS <= 4) { + vec2 by10_a[4], by116_a[4], by132_a[4], by148_a[4]; + vec2 by20_a[4], by216_a[4], by232_a[4], by248_a[4]; + FLOAT_TYPE sy0_a[4], sy1_a[4], sy2_a[4], sy3_a[4]; [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { - vec2 by10 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y1_idx) / 2 ]); - vec2 by116 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y1_idx) / 2 + 8]); - vec2 by132 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y1_idx) / 2 + 16]); - vec2 by148 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y1_idx) / 2 + 24]); - vec2 by20 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y2_idx) / 2 ]); - vec2 by216 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y2_idx) / 2 + 8]); - vec2 by232 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y2_idx) / 2 + 16]); - vec2 by248 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y2_idx) / 2 + 24]); - - const FLOAT_TYPE sx = - fma(FLOAT_TYPE(by10.x), q4_0, - fma(FLOAT_TYPE(by10.y), q4_1, - fma(FLOAT_TYPE(by116.x), q4_2, - FLOAT_TYPE(by116.y) * q4_3))); - const FLOAT_TYPE sy = - fma(FLOAT_TYPE(by132.x), q4_4, - fma(FLOAT_TYPE(by132.y), q4_5, - fma(FLOAT_TYPE(by148.x), q4_6, - FLOAT_TYPE(by148.y) * q4_7))); - const FLOAT_TYPE sz = - fma(FLOAT_TYPE(by20.x), q4_8, - fma(FLOAT_TYPE(by20.y), q4_9, - fma(FLOAT_TYPE(by216.x), q4_10, - FLOAT_TYPE(by216.y) * q4_11))); - const FLOAT_TYPE sw = - fma(FLOAT_TYPE(by232.x), q4_12, - fma(FLOAT_TYPE(by232.y), q4_13, - fma(FLOAT_TYPE(by248.x), q4_14, - FLOAT_TYPE(by248.y) * q4_15))); - const FLOAT_TYPE smin = - fma(FLOAT_TYPE(by10.x) + FLOAT_TYPE(by10.y) + FLOAT_TYPE(by116.x) + FLOAT_TYPE(by116.y), sc2, - fma(FLOAT_TYPE(by132.x) + FLOAT_TYPE(by132.y) + FLOAT_TYPE(by148.x) + FLOAT_TYPE(by148.y), sc3, - fma(FLOAT_TYPE(by20.x) + FLOAT_TYPE(by20.y) + FLOAT_TYPE(by216.x) + FLOAT_TYPE(by216.y), sc6, - (FLOAT_TYPE(by232.x) + FLOAT_TYPE(by232.y) + FLOAT_TYPE(by248.x) + FLOAT_TYPE(by248.y)) * sc7))); - temp[j][n] = fma(dm.x, fma(sx, sc0, fma(sy, sc1, fma(sz, sc4, sw * sc5))), fma(-dm.y, smin, temp[j][n])); + const uint b2_1 = (j * p.batch_stride_b + b_offset + y1_idx) / 2; + const uint b2_2 = (j * p.batch_stride_b + b_offset + y2_idx) / 2; + by10_a[j] = vec2(data_b_v2[b2_1 ]); by116_a[j] = vec2(data_b_v2[b2_1 + 8]); + by132_a[j] = vec2(data_b_v2[b2_1 + 16]); by148_a[j] = vec2(data_b_v2[b2_1 + 24]); + by20_a[j] = vec2(data_b_v2[b2_2 ]); by216_a[j] = vec2(data_b_v2[b2_2 + 8]); + by232_a[j] = vec2(data_b_v2[b2_2 + 16]); by248_a[j] = vec2(data_b_v2[b2_2 + 24]); + sy0_a[j] = FLOAT_TYPE(by10_a[j].x) + FLOAT_TYPE(by10_a[j].y) + FLOAT_TYPE(by116_a[j].x) + FLOAT_TYPE(by116_a[j].y); + sy1_a[j] = FLOAT_TYPE(by132_a[j].x) + FLOAT_TYPE(by132_a[j].y) + FLOAT_TYPE(by148_a[j].x) + FLOAT_TYPE(by148_a[j].y); + sy2_a[j] = FLOAT_TYPE(by20_a[j].x) + FLOAT_TYPE(by20_a[j].y) + FLOAT_TYPE(by216_a[j].x) + FLOAT_TYPE(by216_a[j].y); + sy3_a[j] = FLOAT_TYPE(by232_a[j].x) + FLOAT_TYPE(by232_a[j].y) + FLOAT_TYPE(by248_a[j].x) + FLOAT_TYPE(by248_a[j].y); + } + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + const uint rn = (n == 0 || n < num_rows) ? n : 0; + const uint ib0 = a_offset + (first_row + rn) * num_blocks_per_row; + LOAD_Q5_K_0ULP(a2, data_a2, data_a2_packed16); + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + DOT_Q5_K_0ULP_WITH_BY(a2, temp, by10_a[j], by116_a[j], by132_a[j], by148_a[j], by20_a[j], by216_a[j], by232_a[j], by248_a[j], sy0_a[j], sy1_a[j], sy2_a[j], sy3_a[j]); + } + } + } else { + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + const uint rn = (n == 0 || n < num_rows) ? n : 0; + const uint ib0 = a_offset + (first_row + rn) * num_blocks_per_row; + LOAD_Q5_K_0ULP(a2, data_a2, data_a2_packed16); + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + const uint b2_1 = (j * p.batch_stride_b + b_offset + y1_idx) / 2; + const uint b2_2 = (j * p.batch_stride_b + b_offset + y2_idx) / 2; + const vec2 by10 = vec2(data_b_v2[b2_1 ]), by116 = vec2(data_b_v2[b2_1 + 8]); + const vec2 by132 = vec2(data_b_v2[b2_1 + 16]), by148 = vec2(data_b_v2[b2_1 + 24]); + const vec2 by20 = vec2(data_b_v2[b2_2 ]), by216 = vec2(data_b_v2[b2_2 + 8]); + const vec2 by232 = vec2(data_b_v2[b2_2 + 16]), by248 = vec2(data_b_v2[b2_2 + 24]); + const FLOAT_TYPE sumy0 = FLOAT_TYPE(by10.x) + FLOAT_TYPE(by10.y) + FLOAT_TYPE(by116.x) + FLOAT_TYPE(by116.y); + const FLOAT_TYPE sumy1 = FLOAT_TYPE(by132.x) + FLOAT_TYPE(by132.y) + FLOAT_TYPE(by148.x) + FLOAT_TYPE(by148.y); + const FLOAT_TYPE sumy2 = FLOAT_TYPE(by20.x) + FLOAT_TYPE(by20.y) + FLOAT_TYPE(by216.x) + FLOAT_TYPE(by216.y); + const FLOAT_TYPE sumy3 = FLOAT_TYPE(by232.x) + FLOAT_TYPE(by232.y) + FLOAT_TYPE(by248.x) + FLOAT_TYPE(by248.y); + DOT_Q5_K_0ULP_WITH_BY(a2, temp, by10, by116, by132, by148, by20, by216, by232, by248, sumy0, sumy1, sumy2, sumy3); + } + } + } +} + +#define LOAD_Q5_K_16_VEC4(PFX, ARR_A, ARR_P16, ARR_P32) \ + const FLOAT_TYPEV2 PFX##_dm = FLOAT_TYPEV2(ARR_A[ib0 + i].dm); \ + const uint32_t PFX##_scale0_u32 = ARR_P16[ib0 + i].scales[v_im ]; \ + const uint32_t PFX##_scale4_u32 = ARR_P16[ib0 + i].scales[v_im + 2]; \ + const uint32_t PFX##_scale8_u32 = ARR_P16[ib0 + i].scales[v_im + 4]; \ + const uint32_t PFX##_scale_0_4_l = (PFX##_scale4_u32 << 16) | PFX##_scale0_u32; \ + const uint32_t PFX##_scale_0_4_h = (PFX##_scale_0_4_l & 0xC0C0C0C0u) >> 2; \ + const vec4 PFX##_sc03 = vec4(unpack8(PFX##_scale_0_4_l & 0x3F3F3F3Fu)); \ + const vec4 PFX##_sc47 = vec4(unpack8((((PFX##_scale8_u32 << 12) | PFX##_scale8_u32) & 0x0F0F0F0Fu) | PFX##_scale_0_4_h)); \ + const uint32_t PFX##_qs0_u32 = ARR_P32[ib0 + i].qs[q_offset / 4]; \ + const uint32_t PFX##_qs64_u32 = ARR_P32[ib0 + i].qs[q_offset / 4 + 16]; \ + const uint32_t PFX##_qh_s = ARR_P32[ib0 + i].qh[l0 / 4] >> (2 * v_im); \ + const vec4 PFX##_q0_lo = vec4(unpack8((PFX##_qs0_u32 & 0x0F0F0F0Fu) | ((PFX##_qh_s & 0x01010101u) << 4))); \ + const vec4 PFX##_q0_hi = vec4(unpack8(((PFX##_qs0_u32 >> 4) & 0x0F0F0F0Fu) | ((PFX##_qh_s & 0x02020202u) << 3))); \ + const vec4 PFX##_q2_lo = vec4(unpack8((PFX##_qs64_u32 & 0x0F0F0F0Fu) | (PFX##_qh_s & 0x10101010u))); \ + const vec4 PFX##_q2_hi = vec4(unpack8(((PFX##_qs64_u32 >> 4) & 0x0F0F0F0Fu) | ((PFX##_qh_s & 0x20202020u) >> 1))) + +#define DOT_Q5_K_16_VEC4(PFX, DST_TEMP) do { \ + const uint b4_1 = (j * p.batch_stride_b + b_offset + y1_idx) / 4; \ + const uint b4_2 = (j * p.batch_stride_b + b_offset + y2_idx) / 4; \ + const vec4 by10 = vec4(data_b_v4[b4_1 ]), by132 = vec4(data_b_v4[b4_1 + 8]); \ + const vec4 by20 = vec4(data_b_v4[b4_2 ]), by232 = vec4(data_b_v4[b4_2 + 8]); \ + const FLOAT_TYPE sumy10 = (FLOAT_TYPE(by10.x) + FLOAT_TYPE(by10.y)) + (FLOAT_TYPE(by10.z) + FLOAT_TYPE(by10.w)); \ + const FLOAT_TYPE sumy132 = (FLOAT_TYPE(by132.x) + FLOAT_TYPE(by132.y)) + (FLOAT_TYPE(by132.z) + FLOAT_TYPE(by132.w)); \ + const FLOAT_TYPE sumy20 = (FLOAT_TYPE(by20.x) + FLOAT_TYPE(by20.y)) + (FLOAT_TYPE(by20.z) + FLOAT_TYPE(by20.w)); \ + const FLOAT_TYPE sumy232 = (FLOAT_TYPE(by232.x) + FLOAT_TYPE(by232.y)) + (FLOAT_TYPE(by232.z) + FLOAT_TYPE(by232.w)); \ + const FLOAT_TYPE sx = dot(by10, PFX##_q0_lo); \ + const FLOAT_TYPE sy = dot(by132, PFX##_q0_hi); \ + const FLOAT_TYPE sz = dot(by20, PFX##_q2_lo); \ + const FLOAT_TYPE sw = dot(by232, PFX##_q2_hi); \ + const FLOAT_TYPE smin = fma(sumy10, PFX##_sc03.z, fma(sumy132, PFX##_sc03.w, fma(sumy20, PFX##_sc47.z, sumy232 * PFX##_sc47.w))); \ + DST_TEMP[j][n] = fma(PFX##_dm.x, fma(sx, PFX##_sc03.x, fma(sy, PFX##_sc03.y, fma(sz, PFX##_sc47.x, sw * PFX##_sc47.y))), fma(-PFX##_dm.y, smin, DST_TEMP[j][n])); \ +} while (false) + +#define DOT_Q5_K_16_VEC4_BY(PFX, DST_TEMP, BY10, BY132, BY20, BY232, SY10, SY132, SY20, SY232) do { \ + const FLOAT_TYPE sx = dot(BY10, PFX##_q0_lo); \ + const FLOAT_TYPE sy = dot(BY132, PFX##_q0_hi); \ + const FLOAT_TYPE sz = dot(BY20, PFX##_q2_lo); \ + const FLOAT_TYPE sw = dot(BY232, PFX##_q2_hi); \ + const FLOAT_TYPE smin = fma(SY10, PFX##_sc03.z, fma(SY132, PFX##_sc03.w, fma(SY20, PFX##_sc47.z, (SY232) * PFX##_sc47.w))); \ + DST_TEMP[j][n] = fma(PFX##_dm.x, fma(sx, PFX##_sc03.x, fma(sy, PFX##_sc03.y, fma(sz, PFX##_sc47.x, sw * PFX##_sc47.y))), fma(-PFX##_dm.y, smin, DST_TEMP[j][n])); \ +} while (false) + +void calc_superblock_vec4_a(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], inout FLOAT_TYPE temp2[NUM_COLS][NUM_ROWS], const uint a_offset, const uint b_offset, const uint v_im, const uint l0, const uint q_offset, const uint y_offset, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows) { + const uint y1_idx = i * QUANT_K + y_offset; + const uint y2_idx = y1_idx + 128; + + if (GEN_FAST_KQ > 0) { + if (NUM_COLS <= 4) { + vec4 by10_a[NUM_COLS], by132_a[NUM_COLS], by20_a[NUM_COLS], by232_a[NUM_COLS]; + FLOAT_TYPE sy10_a[NUM_COLS], sy132_a[NUM_COLS], sy20_a[NUM_COLS], sy232_a[NUM_COLS]; + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + const uint b4_1 = (j * p.batch_stride_b + b_offset + y1_idx) / 4; + const uint b4_2 = (j * p.batch_stride_b + b_offset + y2_idx) / 4; + by10_a[j] = vec4(data_b_v4[b4_1 ]); by132_a[j] = vec4(data_b_v4[b4_1 + 8]); + by20_a[j] = vec4(data_b_v4[b4_2 ]); by232_a[j] = vec4(data_b_v4[b4_2 + 8]); + sy10_a[j] = (FLOAT_TYPE(by10_a[j].x) + FLOAT_TYPE(by10_a[j].y)) + (FLOAT_TYPE(by10_a[j].z) + FLOAT_TYPE(by10_a[j].w)); + sy132_a[j] = (FLOAT_TYPE(by132_a[j].x) + FLOAT_TYPE(by132_a[j].y)) + (FLOAT_TYPE(by132_a[j].z) + FLOAT_TYPE(by132_a[j].w)); + sy20_a[j] = (FLOAT_TYPE(by20_a[j].x) + FLOAT_TYPE(by20_a[j].y)) + (FLOAT_TYPE(by20_a[j].z) + FLOAT_TYPE(by20_a[j].w)); + sy232_a[j] = (FLOAT_TYPE(by232_a[j].x) + FLOAT_TYPE(by232_a[j].y)) + (FLOAT_TYPE(by232_a[j].z) + FLOAT_TYPE(by232_a[j].w)); + } + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + const uint rn = (n == 0 || n < num_rows) ? n : 0; + const uint ib0 = a_offset + (first_row + rn) * num_blocks_per_row; + LOAD_Q5_K_16_VEC4(a1, data_a, data_a_packed16, data_a_packed32); + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + DOT_Q5_K_16_VEC4_BY(a1, temp, by10_a[j], by132_a[j], by20_a[j], by232_a[j], sy10_a[j], sy132_a[j], sy20_a[j], sy232_a[j]); + } + if (GEN_GLU == 3) { + LOAD_Q5_K_16_VEC4(a2, data_a2, data_a2_packed16, data_a2_packed32); + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + DOT_Q5_K_16_VEC4_BY(a2, temp2, by10_a[j], by132_a[j], by20_a[j], by232_a[j], sy10_a[j], sy132_a[j], sy20_a[j], sy232_a[j]); + } + } + } + return; + } + } + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + const uint rn = (n == 0 || n < num_rows) ? n : 0; + const uint ib0 = a_offset + (first_row + rn) * num_blocks_per_row; + LOAD_Q5_K_16_VEC4(a1, data_a, data_a_packed16, data_a_packed32); + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + DOT_Q5_K_16_VEC4(a1, temp); + } + if (GEN_GLU == 3) { + LOAD_Q5_K_16_VEC4(a2, data_a2, data_a2_packed16, data_a2_packed32); + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + DOT_Q5_K_16_VEC4(a2, temp2); + } + } + } +} + +void calc_superblock_vec4_a2(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const uint a_offset, const uint b_offset, const uint v_im, const uint l0, const uint q_offset, const uint y_offset, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows) { + const uint y1_idx = i * QUANT_K + y_offset; + const uint y2_idx = y1_idx + 128; + + if (GEN_FAST_KQ > 0) { + if (NUM_COLS <= 4) { + vec4 by10_a[NUM_COLS], by132_a[NUM_COLS], by20_a[NUM_COLS], by232_a[NUM_COLS]; + FLOAT_TYPE sy10_a[NUM_COLS], sy132_a[NUM_COLS], sy20_a[NUM_COLS], sy232_a[NUM_COLS]; + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + const uint b4_1 = (j * p.batch_stride_b + b_offset + y1_idx) / 4; + const uint b4_2 = (j * p.batch_stride_b + b_offset + y2_idx) / 4; + by10_a[j] = vec4(data_b_v4[b4_1 ]); by132_a[j] = vec4(data_b_v4[b4_1 + 8]); + by20_a[j] = vec4(data_b_v4[b4_2 ]); by232_a[j] = vec4(data_b_v4[b4_2 + 8]); + sy10_a[j] = (FLOAT_TYPE(by10_a[j].x) + FLOAT_TYPE(by10_a[j].y)) + (FLOAT_TYPE(by10_a[j].z) + FLOAT_TYPE(by10_a[j].w)); + sy132_a[j] = (FLOAT_TYPE(by132_a[j].x) + FLOAT_TYPE(by132_a[j].y)) + (FLOAT_TYPE(by132_a[j].z) + FLOAT_TYPE(by132_a[j].w)); + sy20_a[j] = (FLOAT_TYPE(by20_a[j].x) + FLOAT_TYPE(by20_a[j].y)) + (FLOAT_TYPE(by20_a[j].z) + FLOAT_TYPE(by20_a[j].w)); + sy232_a[j] = (FLOAT_TYPE(by232_a[j].x) + FLOAT_TYPE(by232_a[j].y)) + (FLOAT_TYPE(by232_a[j].z) + FLOAT_TYPE(by232_a[j].w)); + } + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + const uint rn = (n == 0 || n < num_rows) ? n : 0; + const uint ib0 = a_offset + (first_row + rn) * num_blocks_per_row; + LOAD_Q5_K_16_VEC4(a2, data_a2, data_a2_packed16, data_a2_packed32); + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + DOT_Q5_K_16_VEC4_BY(a2, temp, by10_a[j], by132_a[j], by20_a[j], by232_a[j], sy10_a[j], sy132_a[j], sy20_a[j], sy232_a[j]); + } + } + return; + } + } + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + const uint rn = (n == 0 || n < num_rows) ? n : 0; + const uint ib0 = a_offset + (first_row + rn) * num_blocks_per_row; + LOAD_Q5_K_16_VEC4(a2, data_a2, data_a2_packed16, data_a2_packed32); + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + DOT_Q5_K_16_VEC4(a2, temp); + } + } +} + +#define LOAD_Q5_K_8(PFX, ARR_P32) \ + const FLOAT_TYPEV2 PFX##_dm = FLOAT_TYPEV2(ARR_P32[ib].dm); \ + const uint32_t PFX##_s0 = ARR_P32[ib].scales[0]; \ + const uint32_t PFX##_s1 = ARR_P32[ib].scales[1]; \ + const uint32_t PFX##_s2 = ARR_P32[ib].scales[2]; \ + const vec4 PFX##_sc03 = vec4(unpack8(PFX##_s0 & 0x3F3F3F3Fu)); \ + const vec4 PFX##_m03 = vec4(unpack8(PFX##_s1 & 0x3F3F3F3Fu)); \ + const vec4 PFX##_sc47 = vec4(unpack8((PFX##_s2 & 0x0F0F0F0Fu) | ((PFX##_s0 >> 2) & 0x30303030u))); \ + const vec4 PFX##_m47 = vec4(unpack8(((PFX##_s2 >> 4) & 0x0F0F0F0Fu) | ((PFX##_s1 >> 2) & 0x30303030u))); \ + const uint32_t PFX##_qh_u32 = ARR_P32[ib].qh[itid8]; \ + const uint32_t PFX##_q0 = ARR_P32[ib].qs[itid8 ]; \ + const uint32_t PFX##_q1 = ARR_P32[ib].qs[itid8 + 8]; \ + const uint32_t PFX##_q2 = ARR_P32[ib].qs[itid8 + 16]; \ + const uint32_t PFX##_q3 = ARR_P32[ib].qs[itid8 + 24]; \ + const vec4 PFX##_q0_lo = vec4(unpack8((PFX##_q0 & 0x0F0F0F0Fu) | ((PFX##_qh_u32 << 4) & 0x10101010u))); \ + const vec4 PFX##_q0_hi = vec4(unpack8(((PFX##_q0 >> 4) & 0x0F0F0F0Fu) | ((PFX##_qh_u32 << 3) & 0x10101010u))); \ + const vec4 PFX##_q1_lo = vec4(unpack8((PFX##_q1 & 0x0F0F0F0Fu) | ((PFX##_qh_u32 << 2) & 0x10101010u))); \ + const vec4 PFX##_q1_hi = vec4(unpack8(((PFX##_q1 >> 4) & 0x0F0F0F0Fu) | ((PFX##_qh_u32 << 1) & 0x10101010u))); \ + const vec4 PFX##_q2_lo = vec4(unpack8((PFX##_q2 & 0x0F0F0F0Fu) | (PFX##_qh_u32 & 0x10101010u))); \ + const vec4 PFX##_q2_hi = vec4(unpack8(((PFX##_q2 >> 4) & 0x0F0F0F0Fu) | ((PFX##_qh_u32 >> 1) & 0x10101010u))); \ + const vec4 PFX##_q3_lo = vec4(unpack8((PFX##_q3 & 0x0F0F0F0Fu) | ((PFX##_qh_u32 >> 2) & 0x10101010u))); \ + const vec4 PFX##_q3_hi = vec4(unpack8(((PFX##_q3 >> 4) & 0x0F0F0F0Fu) | ((PFX##_qh_u32 >> 3) & 0x10101010u))) + +#define DOT_Q5_K_8_BY(PFX, DST_TEMP, BY0, BY1, BY2, BY3, BY4, BY5, BY6, BY7, SY0, SY1, SY2, SY3, SY4, SY5, SY6, SY7) do { \ + const FLOAT_TYPE d0 = dot(BY0, PFX##_q0_lo) * PFX##_sc03.x + dot(BY1, PFX##_q0_hi) * PFX##_sc03.y; \ + const FLOAT_TYPE d1 = dot(BY2, PFX##_q1_lo) * PFX##_sc03.z + dot(BY3, PFX##_q1_hi) * PFX##_sc03.w; \ + const FLOAT_TYPE d2 = dot(BY4, PFX##_q2_lo) * PFX##_sc47.x + dot(BY5, PFX##_q2_hi) * PFX##_sc47.y; \ + const FLOAT_TYPE d3 = dot(BY6, PFX##_q3_lo) * PFX##_sc47.z + dot(BY7, PFX##_q3_hi) * PFX##_sc47.w; \ + const FLOAT_TYPE sm = (SY0 * PFX##_m03.x + SY1 * PFX##_m03.y) + (SY2 * PFX##_m03.z + SY3 * PFX##_m03.w) + \ + (SY4 * PFX##_m47.x + SY5 * PFX##_m47.y) + (SY6 * PFX##_m47.z + SY7 * PFX##_m47.w); \ + DST_TEMP[j][n] = fma(PFX##_dm.x, (d0 + d1) + (d2 + d3), fma(-PFX##_dm.y, sm, DST_TEMP[j][n])); \ +} while (false) + +#define DOT_Q5_K_8(PFX, DST_TEMP) do { \ + const uint b4 = (j * p.batch_stride_b + b_offset + y_idx) / 4; \ + const vec4 by0 = vec4(data_b_v4[b4 ]), by1 = vec4(data_b_v4[b4 + 8]); \ + const vec4 by2 = vec4(data_b_v4[b4 + 16]), by3 = vec4(data_b_v4[b4 + 24]); \ + const vec4 by4 = vec4(data_b_v4[b4 + 32]), by5 = vec4(data_b_v4[b4 + 40]); \ + const vec4 by6 = vec4(data_b_v4[b4 + 48]), by7 = vec4(data_b_v4[b4 + 56]); \ + const FLOAT_TYPE sy0 = (FLOAT_TYPE(by0.x) + FLOAT_TYPE(by0.y)) + (FLOAT_TYPE(by0.z) + FLOAT_TYPE(by0.w)); \ + const FLOAT_TYPE sy1 = (FLOAT_TYPE(by1.x) + FLOAT_TYPE(by1.y)) + (FLOAT_TYPE(by1.z) + FLOAT_TYPE(by1.w)); \ + const FLOAT_TYPE sy2 = (FLOAT_TYPE(by2.x) + FLOAT_TYPE(by2.y)) + (FLOAT_TYPE(by2.z) + FLOAT_TYPE(by2.w)); \ + const FLOAT_TYPE sy3 = (FLOAT_TYPE(by3.x) + FLOAT_TYPE(by3.y)) + (FLOAT_TYPE(by3.z) + FLOAT_TYPE(by3.w)); \ + const FLOAT_TYPE sy4 = (FLOAT_TYPE(by4.x) + FLOAT_TYPE(by4.y)) + (FLOAT_TYPE(by4.z) + FLOAT_TYPE(by4.w)); \ + const FLOAT_TYPE sy5 = (FLOAT_TYPE(by5.x) + FLOAT_TYPE(by5.y)) + (FLOAT_TYPE(by5.z) + FLOAT_TYPE(by5.w)); \ + const FLOAT_TYPE sy6 = (FLOAT_TYPE(by6.x) + FLOAT_TYPE(by6.y)) + (FLOAT_TYPE(by6.z) + FLOAT_TYPE(by6.w)); \ + const FLOAT_TYPE sy7 = (FLOAT_TYPE(by7.x) + FLOAT_TYPE(by7.y)) + (FLOAT_TYPE(by7.z) + FLOAT_TYPE(by7.w)); \ + DOT_Q5_K_8_BY(PFX, DST_TEMP, by0, by1, by2, by3, by4, by5, by6, by7, sy0, sy1, sy2, sy3, sy4, sy5, sy6, sy7); \ +} while (false) + +void calc_superblock_8_a(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], inout FLOAT_TYPE temp2[NUM_COLS][NUM_ROWS], const uint a_offset, const uint b_offset, const uint itid8, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows) { + const uint y_idx = i * QUANT_K + 4 * itid8; + if (NUM_COLS <= 4 && NUM_ROWS > 1) { + vec4 by0_a[NUM_COLS], by1_a[NUM_COLS], by2_a[NUM_COLS], by3_a[NUM_COLS]; + vec4 by4_a[NUM_COLS], by5_a[NUM_COLS], by6_a[NUM_COLS], by7_a[NUM_COLS]; + FLOAT_TYPE sy0_a[NUM_COLS], sy1_a[NUM_COLS], sy2_a[NUM_COLS], sy3_a[NUM_COLS]; + FLOAT_TYPE sy4_a[NUM_COLS], sy5_a[NUM_COLS], sy6_a[NUM_COLS], sy7_a[NUM_COLS]; + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + const uint b4 = (j * p.batch_stride_b + b_offset + y_idx) / 4; + by0_a[j] = vec4(data_b_v4[b4 ]); by1_a[j] = vec4(data_b_v4[b4 + 8]); + by2_a[j] = vec4(data_b_v4[b4 + 16]); by3_a[j] = vec4(data_b_v4[b4 + 24]); + by4_a[j] = vec4(data_b_v4[b4 + 32]); by5_a[j] = vec4(data_b_v4[b4 + 40]); + by6_a[j] = vec4(data_b_v4[b4 + 48]); by7_a[j] = vec4(data_b_v4[b4 + 56]); + sy0_a[j] = (FLOAT_TYPE(by0_a[j].x) + FLOAT_TYPE(by0_a[j].y)) + (FLOAT_TYPE(by0_a[j].z) + FLOAT_TYPE(by0_a[j].w)); + sy1_a[j] = (FLOAT_TYPE(by1_a[j].x) + FLOAT_TYPE(by1_a[j].y)) + (FLOAT_TYPE(by1_a[j].z) + FLOAT_TYPE(by1_a[j].w)); + sy2_a[j] = (FLOAT_TYPE(by2_a[j].x) + FLOAT_TYPE(by2_a[j].y)) + (FLOAT_TYPE(by2_a[j].z) + FLOAT_TYPE(by2_a[j].w)); + sy3_a[j] = (FLOAT_TYPE(by3_a[j].x) + FLOAT_TYPE(by3_a[j].y)) + (FLOAT_TYPE(by3_a[j].z) + FLOAT_TYPE(by3_a[j].w)); + sy4_a[j] = (FLOAT_TYPE(by4_a[j].x) + FLOAT_TYPE(by4_a[j].y)) + (FLOAT_TYPE(by4_a[j].z) + FLOAT_TYPE(by4_a[j].w)); + sy5_a[j] = (FLOAT_TYPE(by5_a[j].x) + FLOAT_TYPE(by5_a[j].y)) + (FLOAT_TYPE(by5_a[j].z) + FLOAT_TYPE(by5_a[j].w)); + sy6_a[j] = (FLOAT_TYPE(by6_a[j].x) + FLOAT_TYPE(by6_a[j].y)) + (FLOAT_TYPE(by6_a[j].z) + FLOAT_TYPE(by6_a[j].w)); + sy7_a[j] = (FLOAT_TYPE(by7_a[j].x) + FLOAT_TYPE(by7_a[j].y)) + (FLOAT_TYPE(by7_a[j].z) + FLOAT_TYPE(by7_a[j].w)); + } + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + const uint rn = (n == 0 || n < num_rows) ? n : 0; + const uint ib = a_offset + (first_row + rn) * num_blocks_per_row + i; + LOAD_Q5_K_8(a1, data_a_packed32); + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + DOT_Q5_K_8_BY(a1, temp, by0_a[j], by1_a[j], by2_a[j], by3_a[j], by4_a[j], by5_a[j], by6_a[j], by7_a[j], sy0_a[j], sy1_a[j], sy2_a[j], sy3_a[j], sy4_a[j], sy5_a[j], sy6_a[j], sy7_a[j]); + } + if (GEN_GLU == 3) { + LOAD_Q5_K_8(a2, data_a2_packed32); + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + DOT_Q5_K_8_BY(a2, temp2, by0_a[j], by1_a[j], by2_a[j], by3_a[j], by4_a[j], by5_a[j], by6_a[j], by7_a[j], sy0_a[j], sy1_a[j], sy2_a[j], sy3_a[j], sy4_a[j], sy5_a[j], sy6_a[j], sy7_a[j]); + } + } + } + } else { + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + const uint rn = (n == 0 || n < num_rows) ? n : 0; + const uint ib = a_offset + (first_row + rn) * num_blocks_per_row + i; + LOAD_Q5_K_8(a1, data_a_packed32); + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + DOT_Q5_K_8(a1, temp); + } + if (GEN_GLU == 3) { + LOAD_Q5_K_8(a2, data_a2_packed32); + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + DOT_Q5_K_8(a2, temp2); + } + } + } + } +} + +void calc_superblock_8_a2(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const uint a_offset, const uint b_offset, const uint itid8, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows) { + const uint y_idx = i * QUANT_K + 4 * itid8; + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + const uint rn = (n == 0 || n < num_rows) ? n : 0; + const uint ib = a_offset + (first_row + rn) * num_blocks_per_row + i; + LOAD_Q5_K_8(a2, data_a2_packed32); + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + DOT_Q5_K_8(a2, temp); } } } @@ -119,47 +412,118 @@ void calc_superblock(const uint a_offset, const uint b_offset, const uint v_im, void compute_outputs(const uint32_t first_row, const uint32_t num_rows) { uint a_offset, b_offset, d_offset; get_offsets(a_offset, b_offset, d_offset); +#ifndef MUL_MAT_ID + const bool use_a2 = (GEN_GLU == 2) && (first_row >= p.pair_m0); + const uint row_base = use_a2 ? (first_row - p.pair_m0) : first_row; + const uint eff_rows = (!use_a2 && GEN_GLU == 2) ? min(num_rows, p.pair_m0 - first_row) : num_rows; +#else + const bool use_a2 = false; + const uint row_base = first_row; + const uint eff_rows = num_rows; +#endif const uint num_blocks_per_row = p.ncols / QUANT_K; - // 16 threads are used to process each block - const uint it_size = gl_WorkGroupSize.x/16; +#ifdef USE_SUBGROUP_ADD_NO_SHMEM + const uint tid = gl_SubgroupInvocationID; +#else const uint tid = gl_LocalInvocationID.x; - const uint itid = tid%16; // 0...15 - const uint ix = tid/16; - - const uint il = itid/4; // 0...3 - const uint ir = itid - 4*il; // 0...3 - - const uint v_im = il / 2; // 0 or 1. 0 computes 0,32 + 128,160, 1 computes 64,96 + 192,224 - const uint v_in = il % 2; +#endif - const uint l0 = 4*ir + 2*v_in; // 0...15 - const uint q_offset = 32*v_im + l0; - const uint y_offset = 64*v_im + l0; + FLOAT_TYPE temp[NUM_COLS][NUM_ROWS]; + FLOAT_TYPE temp2[NUM_COLS][NUM_ROWS]; [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { [[unroll]] for (uint i = 0; i < NUM_ROWS; ++i) { temp[j][i] = FLOAT_TYPE(0); + if (GEN_GLU == 3) { + temp2[j][i] = FLOAT_TYPE(0); + } } } - [[unroll]] for (uint i = ix; i < num_blocks_per_row; i += it_size) - calc_superblock(a_offset, b_offset, v_im, l0, q_offset, y_offset, i, num_blocks_per_row, first_row, num_rows); + if (GEN_FAST_KQ >= 2) { + const uint it_size8 = gl_WorkGroupSize.x / 8; + const uint itid8 = tid & 7; + const uint ix8 = tid >> 3; + if (GEN_GLU == 2 && use_a2) { + for (uint i = ix8; i < num_blocks_per_row; i += it_size8) { + calc_superblock_8_a2(temp, a_offset, b_offset, itid8, i, num_blocks_per_row, row_base, eff_rows); + } + } else { + for (uint i = ix8; i < num_blocks_per_row; i += it_size8) { + calc_superblock_8_a(temp, temp2, a_offset, b_offset, itid8, i, num_blocks_per_row, row_base, eff_rows); + } + } + } else if (GEN_FAST_KQ >= 1) { + const uint it_size = gl_WorkGroupSize.x / 16; + const uint itid = tid % 16; + const uint ix = tid / 16; + const uint il = itid / 4; + const uint ir = itid - 4 * il; + const uint v_im = il / 2; + const uint v_in = il % 2; + const uint l0 = 4 * (2 * ir + v_in); + const uint q_offset = 32 * v_im + l0; + const uint y_offset = 64 * v_im + l0; + if (GEN_GLU == 2 && use_a2) { + for (uint i = ix; i < num_blocks_per_row; i += it_size) { + calc_superblock_vec4_a2(temp, a_offset, b_offset, v_im, l0, q_offset, y_offset, i, num_blocks_per_row, row_base, eff_rows); + } + } else { + for (uint i = ix; i < num_blocks_per_row; i += it_size) { + calc_superblock_vec4_a(temp, temp2, a_offset, b_offset, v_im, l0, q_offset, y_offset, i, num_blocks_per_row, row_base, eff_rows); + } + } + } else { + // 16 threads are used to process each block + const uint it_size = gl_WorkGroupSize.x/16; + const uint itid = tid%16; // 0...15 + const uint ix = tid/16; + + const uint il = itid/4; // 0...3 + const uint ir = itid - 4*il; // 0...3 + + const uint v_im = il / 2; // 0 or 1. 0 computes 0,32 + 128,160, 1 computes 64,96 + 192,224 + const uint v_in = il % 2; + + const uint l0 = 4*ir + 2*v_in; // 0...15 + const uint q_offset = 32*v_im + l0; + const uint y_offset = 64*v_im + l0; - reduce_result(temp, d_offset, first_row, num_rows, tid); + if (GEN_GLU == 2 && use_a2) { + for (uint i = ix; i < num_blocks_per_row; i += it_size) { + calc_superblock_a2(temp, a_offset, b_offset, v_im, l0, q_offset, y_offset, i, num_blocks_per_row, row_base, eff_rows); + } + } else { + for (uint i = ix; i < num_blocks_per_row; i += it_size) { + calc_superblock_a(temp, temp2, a_offset, b_offset, v_im, l0, q_offset, y_offset, i, num_blocks_per_row, row_base, eff_rows); + } + } + } + + if (GEN_GLU == 3) { + reduce_pair_glu(temp, temp2, d_offset, first_row, eff_rows, tid); + } +#ifndef MUL_MAT_ID + else if (GEN_GLU == 2 && use_a2) { + reduce_result_d2(temp, row_base, eff_rows, tid); + } +#endif + else { + reduce_result(temp, d_offset, first_row, eff_rows, tid); + } } void main() { +#ifdef USE_SUBGROUP_ADD_NO_SHMEM + const uint first_row = NUM_ROWS * (NUM_SUBGROUPS * (gl_WorkGroupID.x + gl_NumWorkGroups.x * gl_WorkGroupID.z) + gl_SubgroupID); +#else const uint first_row = NUM_ROWS * (gl_WorkGroupID.x + gl_NumWorkGroups.x * gl_WorkGroupID.z); +#endif - // do NUM_ROWS at a time, unless there aren't enough remaining rows - if (first_row + NUM_ROWS <= p.stride_d) { - compute_outputs(first_row, NUM_ROWS); - } else { - if (first_row >= p.stride_d) { - return; - } - compute_outputs(first_row, p.stride_d - first_row); + if (first_row >= p.stride_d) { + return; } + compute_outputs(first_row, min(NUM_ROWS, p.stride_d - first_row)); } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp index 3e89d91cbb04..c3435e1bba3a 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp @@ -4,127 +4,308 @@ #include "mul_mat_vec_base.glsl" +#ifdef USE_SUBGROUP_ADD_NO_SHMEM +layout(local_size_x_id = 0, local_size_y_id = 4, local_size_z = 1) in; +#else layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in; +#endif -shared FLOAT_TYPE sccache[2][BLOCK_SIZE/16][16]; +#define LOAD_Q6_K_16(PFX, ARR_A, ARR_P16) \ + const uint32_t PFX##_ql0_u32 = uint32_t(ARR_P16[ib0 + i].ql[ql_offset / 2]) | (uint32_t(ARR_P16[ib0 + i].ql[ql_offset / 2 + 1]) << 16); \ + const uint32_t PFX##_ql32_u32 = uint32_t(ARR_P16[ib0 + i].ql[ql_offset / 2 + 16]) | (uint32_t(ARR_P16[ib0 + i].ql[ql_offset / 2 + 17]) << 16); \ + const uint32_t PFX##_qh_u32 = uint32_t(ARR_P16[ib0 + i].qh[qh_offset / 2]) | (uint32_t(ARR_P16[ib0 + i].qh[qh_offset / 2 + 1]) << 16); \ + const vec4 PFX##_q0 = vec4(unpack8((PFX##_ql0_u32 & 0x0F0F0F0Fu) | ((PFX##_qh_u32 & 0x03030303u) << 4))) - 32.0f; \ + const vec4 PFX##_q1 = vec4(unpack8((PFX##_ql32_u32 & 0x0F0F0F0Fu) | ((PFX##_qh_u32 & 0x0C0C0C0Cu) << 2))) - 32.0f; \ + const vec4 PFX##_q2 = vec4(unpack8(((PFX##_ql0_u32 >> 4) & 0x0F0F0F0Fu) | (PFX##_qh_u32 & 0x30303030u))) - 32.0f; \ + const vec4 PFX##_q3 = vec4(unpack8(((PFX##_ql32_u32 >> 4) & 0x0F0F0F0Fu) | ((PFX##_qh_u32 & 0xC0C0C0C0u) >> 2))) - 32.0f; \ + const FLOAT_TYPE PFX##_sc0 = FLOAT_TYPE(ARR_A[ib0 + i].scales[s_offset]); \ + const FLOAT_TYPE PFX##_sc1 = FLOAT_TYPE(ARR_A[ib0 + i].scales[s_offset + 2]); \ + const FLOAT_TYPE PFX##_sc2 = FLOAT_TYPE(ARR_A[ib0 + i].scales[s_offset + 4]); \ + const FLOAT_TYPE PFX##_sc3 = FLOAT_TYPE(ARR_A[ib0 + i].scales[s_offset + 6]); \ + const FLOAT_TYPE PFX##_d = FLOAT_TYPE(ARR_A[ib0 + i].d) -FLOAT_TYPE temp[NUM_COLS][NUM_ROWS]; -uint csel = 0; +#define DOT_Q6_K_16_WITH_BY(PFX, DST_TEMP, BY0, BY32, BY64, BY96) do { \ + FLOAT_TYPE sum[4] = {0, 0, 0, 0}; \ + [[unroll]] for (uint l = 0; l < 4; ++l) { \ + sum[0] = fma(FLOAT_TYPE((BY0)[l]), PFX##_q0[l], sum[0]); \ + sum[1] = fma(FLOAT_TYPE((BY32)[l]), PFX##_q1[l], sum[1]); \ + sum[2] = fma(FLOAT_TYPE((BY64)[l]), PFX##_q2[l], sum[2]); \ + sum[3] = fma(FLOAT_TYPE((BY96)[l]), PFX##_q3[l], sum[3]); \ + } \ + DST_TEMP[j][n] = fma(fma(sum[0], PFX##_sc0, fma(sum[1], PFX##_sc1, fma(sum[2], PFX##_sc2, sum[3] * PFX##_sc3))), PFX##_d, DST_TEMP[j][n]); \ +} while (false) -void calc_superblock(const uint a_offset, const uint b_offset, const uint itid, const uint ix, const uint ql_offset, const uint qh_offset, const uint s_offset, const uint y_offset, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows, const bool all_threads) { +void calc_superblock_a(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], inout FLOAT_TYPE temp2[NUM_COLS][NUM_ROWS], const uint a_offset, const uint b_offset, const uint ql_offset, const uint qh_offset, const uint s_offset, const uint y_offset, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows) { const uint y_idx = i * QUANT_K + y_offset; - [[unroll]] for (uint n = 0; n < num_rows; ++n) { - const uint ib0 = a_offset + (first_row+n)*num_blocks_per_row; - csel ^= 1; + if (NUM_COLS <= 4) { + vec4 by0_a[NUM_COLS], by32_a[NUM_COLS], by64_a[NUM_COLS], by96_a[NUM_COLS]; + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + const uint b4 = (j * p.batch_stride_b + b_offset + y_idx) / 4; + by0_a[j] = vec4(data_b_v4[b4 ]); + by32_a[j] = vec4(data_b_v4[b4 + 8]); + by64_a[j] = vec4(data_b_v4[b4 + 16]); + by96_a[j] = vec4(data_b_v4[b4 + 24]); + } + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + const uint rn = (n == 0 || n < num_rows) ? n : 0; + const uint ib0 = a_offset + (first_row + rn) * num_blocks_per_row; + LOAD_Q6_K_16(a1, data_a, data_a_packed16); + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + DOT_Q6_K_16_WITH_BY(a1, temp, by0_a[j], by32_a[j], by64_a[j], by96_a[j]); + } + if (GEN_GLU == 3) { + LOAD_Q6_K_16(a2, data_a2, data_a2_packed16); + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + DOT_Q6_K_16_WITH_BY(a2, temp2, by0_a[j], by32_a[j], by64_a[j], by96_a[j]); + } + } + } + } else { + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + const uint rn = (n == 0 || n < num_rows) ? n : 0; + const uint ib0 = a_offset + (first_row + rn) * num_blocks_per_row; + LOAD_Q6_K_16(a1, data_a, data_a_packed16); + if (GEN_GLU == 3) { + LOAD_Q6_K_16(a2, data_a2, data_a2_packed16); + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + const uint b4 = (j * p.batch_stride_b + b_offset + y_idx) / 4; + const vec4 by0 = vec4(data_b_v4[b4]), by32 = vec4(data_b_v4[b4 + 8]), by64 = vec4(data_b_v4[b4 + 16]), by96 = vec4(data_b_v4[b4 + 24]); + DOT_Q6_K_16_WITH_BY(a1, temp, by0, by32, by64, by96); + DOT_Q6_K_16_WITH_BY(a2, temp2, by0, by32, by64, by96); + } + } else { + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + const uint b4 = (j * p.batch_stride_b + b_offset + y_idx) / 4; + const vec4 by0 = vec4(data_b_v4[b4]), by32 = vec4(data_b_v4[b4 + 8]), by64 = vec4(data_b_v4[b4 + 16]), by96 = vec4(data_b_v4[b4 + 24]); + DOT_Q6_K_16_WITH_BY(a1, temp, by0, by32, by64, by96); + } + } + } + } +} - if (!all_threads) { // when we don't have enough blocks to use all threads - if (i < num_blocks_per_row) - sccache[csel][ix][itid] = FLOAT_TYPE(data_a[ib0 + i].scales[itid]); - barrier(); +void calc_superblock_a2(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const uint a_offset, const uint b_offset, const uint ql_offset, const uint qh_offset, const uint s_offset, const uint y_offset, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows) { + const uint y_idx = i * QUANT_K + y_offset; - if (i >= num_blocks_per_row) - continue; + if (NUM_COLS <= 4) { + vec4 by0_a[NUM_COLS], by32_a[NUM_COLS], by64_a[NUM_COLS], by96_a[NUM_COLS]; + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + const uint b4 = (j * p.batch_stride_b + b_offset + y_idx) / 4; + by0_a[j] = vec4(data_b_v4[b4 ]); + by32_a[j] = vec4(data_b_v4[b4 + 8]); + by64_a[j] = vec4(data_b_v4[b4 + 16]); + by96_a[j] = vec4(data_b_v4[b4 + 24]); } - - const uint32_t ql0_u32 = uint32_t(data_a_packed16[ib0 + i].ql[ql_offset / 2]) | (uint32_t(data_a_packed16[ib0 + i].ql[ql_offset / 2 + 1]) << 16); - const uint32_t ql32_u32 = uint32_t(data_a_packed16[ib0 + i].ql[ql_offset / 2 + 16]) | (uint32_t(data_a_packed16[ib0 + i].ql[ql_offset / 2 + 17]) << 16); - - const uint32_t ql0_u32_lo4 = ql0_u32 & 0x0F0F0F0F; - const uint32_t ql0_u32_hi4 = (ql0_u32 >> 4) & 0x0F0F0F0F; - const uint32_t ql32_u32_lo4 = ql32_u32 & 0x0F0F0F0F; - const uint32_t ql32_u32_hi4 = (ql32_u32 >> 4) & 0x0F0F0F0F; - - const uint32_t qh_u32 = uint32_t(data_a_packed16[ib0 + i].qh[qh_offset / 2]) | (uint32_t(data_a_packed16[ib0 + i].qh[qh_offset / 2 + 1]) << 16); - const uint32_t qh0_u32 = (qh_u32 & 0x03030303) << 4; - const uint32_t qh2_u32 = (qh_u32 & 0x0C0C0C0C) << 2; - const uint32_t qh4_u32 = (qh_u32 & 0x30303030); - const uint32_t qh6_u32 = (qh_u32 & 0xC0C0C0C0) >> 2; - - const uint32_t q0_u32 = ql0_u32_lo4 | qh0_u32; - const uint32_t q1_u32 = ql32_u32_lo4 | qh2_u32; - const uint32_t q2_u32 = ql0_u32_hi4 | qh4_u32; - const uint32_t q3_u32 = ql32_u32_hi4 | qh6_u32; - - const vec4 q0 = vec4(unpack8(q0_u32)) - 32; - const vec4 q1 = vec4(unpack8(q1_u32)) - 32; - const vec4 q2 = vec4(unpack8(q2_u32)) - 32; - const vec4 q3 = vec4(unpack8(q3_u32)) - 32; - - if (all_threads) { - sccache[csel][ix][itid] = FLOAT_TYPE(data_a[ib0 + i].scales[itid]); - barrier(); + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + const uint rn = (n == 0 || n < num_rows) ? n : 0; + const uint ib0 = a_offset + (first_row + rn) * num_blocks_per_row; + LOAD_Q6_K_16(a2, data_a2, data_a2_packed16); + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + DOT_Q6_K_16_WITH_BY(a2, temp, by0_a[j], by32_a[j], by64_a[j], by96_a[j]); + } + } + } else { + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + const uint rn = (n == 0 || n < num_rows) ? n : 0; + const uint ib0 = a_offset + (first_row + rn) * num_blocks_per_row; + LOAD_Q6_K_16(a2, data_a2, data_a2_packed16); + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + const uint b4 = (j * p.batch_stride_b + b_offset + y_idx) / 4; + const vec4 by0 = vec4(data_b_v4[b4]), by32 = vec4(data_b_v4[b4 + 8]), by64 = vec4(data_b_v4[b4 + 16]), by96 = vec4(data_b_v4[b4 + 24]); + DOT_Q6_K_16_WITH_BY(a2, temp, by0, by32, by64, by96); + } } + } +} - const FLOAT_TYPE d = FLOAT_TYPE(data_a[ib0 + i].d); +#define RUN_Q6_K_8(ARR_A, ARR_P16, DST_TEMP) do { \ + const FLOAT_TYPE d = FLOAT_TYPE(ARR_A[ib].d); \ + [[unroll]] for (uint h = 0; h < 2; ++h) { \ + const uint ql_h = 32 * h + 2 * itid8; \ + const uint qh_h = 16 * h + 2 * itid8; \ + const uint s_h = 8 * h + (itid8 >> 2); \ + const uint32_t ql0_u32 = uint32_t(ARR_P16[ib].ql[ql_h]) | (uint32_t(ARR_P16[ib].ql[ql_h + 1]) << 16); \ + const uint32_t ql32_u32 = uint32_t(ARR_P16[ib].ql[ql_h + 16]) | (uint32_t(ARR_P16[ib].ql[ql_h + 17]) << 16); \ + const uint32_t qh_u32 = uint32_t(ARR_P16[ib].qh[qh_h]) | (uint32_t(ARR_P16[ib].qh[qh_h + 1]) << 16); \ + const vec4 q0 = vec4(unpack8((ql0_u32 & 0x0F0F0F0Fu) | ((qh_u32 & 0x03030303u) << 4))) - 32.0f; \ + const vec4 q1 = vec4(unpack8((ql32_u32 & 0x0F0F0F0Fu) | ((qh_u32 & 0x0C0C0C0Cu) << 2))) - 32.0f; \ + const vec4 q2 = vec4(unpack8(((ql0_u32 >> 4) & 0x0F0F0F0Fu) | (qh_u32 & 0x30303030u))) - 32.0f; \ + const vec4 q3 = vec4(unpack8(((ql32_u32 >> 4) & 0x0F0F0F0Fu) | ((qh_u32 & 0xC0C0C0C0u) >> 2))) - 32.0f; \ + const FLOAT_TYPE sc0 = FLOAT_TYPE(ARR_A[ib].scales[s_h ]) * d; \ + const FLOAT_TYPE sc1 = FLOAT_TYPE(ARR_A[ib].scales[s_h + 2]) * d; \ + const FLOAT_TYPE sc2 = FLOAT_TYPE(ARR_A[ib].scales[s_h + 4]) * d; \ + const FLOAT_TYPE sc3 = FLOAT_TYPE(ARR_A[ib].scales[s_h + 6]) * d; \ + const uint bk = 32 * h; \ + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { \ + const uint b4 = (j * p.batch_stride_b + b_offset + y_idx) / 4 + bk; \ + const vec4 by0 = vec4(data_b_v4[b4 ]), by1 = vec4(data_b_v4[b4 + 8]); \ + const vec4 by2 = vec4(data_b_v4[b4 + 16]), by3 = vec4(data_b_v4[b4 + 24]); \ + DST_TEMP[j][n] += (dot(by0, q0) * sc0 + dot(by1, q1) * sc1) + (dot(by2, q2) * sc2 + dot(by3, q3) * sc3); \ + } \ + } \ +} while (false) - [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { - vec4 by0 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 ]); - vec4 by32 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 8]); - vec4 by64 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 16]); - vec4 by96 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 24]); - - FLOAT_TYPE sum[4] = {0, 0, 0, 0}; - [[unroll]] for (uint l = 0; l < 4; ++l) { - sum[0] = fma(FLOAT_TYPE(by0[l]), q0[l], sum[0]); - sum[1] = fma(FLOAT_TYPE(by32[l]), q1[l], sum[1]); - sum[2] = fma(FLOAT_TYPE(by64[l]), q2[l], sum[2]); - sum[3] = fma(FLOAT_TYPE(by96[l]), q3[l], sum[3]); +#define RUN_Q6_K_8_BY(ARR_A, ARR_P16, DST_TEMP) do { \ + const FLOAT_TYPE d = FLOAT_TYPE(ARR_A[ib].d); \ + [[unroll]] for (uint h = 0; h < 2; ++h) { \ + const uint ql_h = 32 * h + 2 * itid8; \ + const uint qh_h = 16 * h + 2 * itid8; \ + const uint s_h = 8 * h + (itid8 >> 2); \ + const uint32_t ql0_u32 = uint32_t(ARR_P16[ib].ql[ql_h]) | (uint32_t(ARR_P16[ib].ql[ql_h + 1]) << 16); \ + const uint32_t ql32_u32 = uint32_t(ARR_P16[ib].ql[ql_h + 16]) | (uint32_t(ARR_P16[ib].ql[ql_h + 17]) << 16); \ + const uint32_t qh_u32 = uint32_t(ARR_P16[ib].qh[qh_h]) | (uint32_t(ARR_P16[ib].qh[qh_h + 1]) << 16); \ + const vec4 q0 = vec4(unpack8((ql0_u32 & 0x0F0F0F0Fu) | ((qh_u32 & 0x03030303u) << 4))) - 32.0f; \ + const vec4 q1 = vec4(unpack8((ql32_u32 & 0x0F0F0F0Fu) | ((qh_u32 & 0x0C0C0C0Cu) << 2))) - 32.0f; \ + const vec4 q2 = vec4(unpack8(((ql0_u32 >> 4) & 0x0F0F0F0Fu) | (qh_u32 & 0x30303030u))) - 32.0f; \ + const vec4 q3 = vec4(unpack8(((ql32_u32 >> 4) & 0x0F0F0F0Fu) | ((qh_u32 & 0xC0C0C0C0u) >> 2))) - 32.0f; \ + const FLOAT_TYPE sc0 = FLOAT_TYPE(ARR_A[ib].scales[s_h ]) * d; \ + const FLOAT_TYPE sc1 = FLOAT_TYPE(ARR_A[ib].scales[s_h + 2]) * d; \ + const FLOAT_TYPE sc2 = FLOAT_TYPE(ARR_A[ib].scales[s_h + 4]) * d; \ + const FLOAT_TYPE sc3 = FLOAT_TYPE(ARR_A[ib].scales[s_h + 6]) * d; \ + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { \ + DST_TEMP[j][n] += (dot(by_h[h][j][0], q0) * sc0 + dot(by_h[h][j][1], q1) * sc1) + (dot(by_h[h][j][2], q2) * sc2 + dot(by_h[h][j][3], q3) * sc3); \ + } \ + } \ +} while (false) + +void calc_superblock_8_a(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], inout FLOAT_TYPE temp2[NUM_COLS][NUM_ROWS], const uint a_offset, const uint b_offset, const uint itid8, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows) { + const uint y_idx = i * QUANT_K + 4 * itid8; + if (NUM_COLS <= 4 && NUM_ROWS > 1) { + vec4 by_h[2][NUM_COLS][4]; + [[unroll]] for (uint h = 0; h < 2; ++h) { + const uint bk = 32 * h; + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + const uint b4 = (j * p.batch_stride_b + b_offset + y_idx) / 4 + bk; + by_h[h][j][0] = vec4(data_b_v4[b4 ]); by_h[h][j][1] = vec4(data_b_v4[b4 + 8]); + by_h[h][j][2] = vec4(data_b_v4[b4 + 16]); by_h[h][j][3] = vec4(data_b_v4[b4 + 24]); + } + } + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + const uint rn = (n == 0 || n < num_rows) ? n : 0; + const uint ib = a_offset + (first_row + rn) * num_blocks_per_row + i; + RUN_Q6_K_8_BY(data_a, data_a_packed16, temp); + if (GEN_GLU == 3) { + RUN_Q6_K_8_BY(data_a2, data_a2_packed16, temp2); } - temp[j][n] = fma(fma(sum[0], sccache[csel][ix][s_offset], fma(sum[1], sccache[csel][ix][s_offset + 2], fma(sum[2], sccache[csel][ix][s_offset + 4], sum[3] * sccache[csel][ix][s_offset + 6]))), d, temp[j][n]); } + } else { + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + const uint rn = (n == 0 || n < num_rows) ? n : 0; + const uint ib = a_offset + (first_row + rn) * num_blocks_per_row + i; + RUN_Q6_K_8(data_a, data_a_packed16, temp); + if (GEN_GLU == 3) { + RUN_Q6_K_8(data_a2, data_a2_packed16, temp2); + } + } + } +} + +void calc_superblock_8_a2(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const uint a_offset, const uint b_offset, const uint itid8, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows) { + const uint y_idx = i * QUANT_K + 4 * itid8; + [[unroll]] for (uint n = 0; n < NUM_ROWS; ++n) { + const uint rn = (n == 0 || n < num_rows) ? n : 0; + const uint ib = a_offset + (first_row + rn) * num_blocks_per_row + i; + RUN_Q6_K_8(data_a2, data_a2_packed16, temp); } } void compute_outputs(const uint first_row, const uint num_rows) { uint a_offset, b_offset, d_offset; get_offsets(a_offset, b_offset, d_offset); +#ifndef MUL_MAT_ID + const bool use_a2 = (GEN_GLU == 2) && (first_row >= p.pair_m0); + const uint row_base = use_a2 ? (first_row - p.pair_m0) : first_row; + const uint eff_rows = (!use_a2 && GEN_GLU == 2) ? min(num_rows, p.pair_m0 - first_row) : num_rows; +#else + const bool use_a2 = false; + const uint row_base = first_row; + const uint eff_rows = num_rows; +#endif const uint num_blocks_per_row = p.ncols / QUANT_K; - // 16 threads are used to process each block - const uint it_size = gl_WorkGroupSize.x/16; +#ifdef USE_SUBGROUP_ADD_NO_SHMEM + const uint tid = gl_SubgroupInvocationID; +#else const uint tid = gl_LocalInvocationID.x; - const uint itid = tid%16; // 0...15 - const uint ix = tid/16; - - const uint v_im = itid/8; // 0 or 1. 0 computes 0..., 1 computes 128... - const uint v_in = itid - 8*v_im; // 0...7 - - const uint l0 = 4 * v_in; // 0, 4, 8, ..., 28 - const uint is = v_in / 4; +#endif - const uint ql_offset = 64*v_im + l0; - const uint qh_offset = 32*v_im + l0; - const uint s_offset = 8*v_im + is; - const uint y_offset = 128*v_im + l0; + FLOAT_TYPE temp[NUM_COLS][NUM_ROWS]; + FLOAT_TYPE temp2[NUM_COLS][NUM_ROWS]; [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { [[unroll]] for (uint i = 0; i < NUM_ROWS; ++i) { temp[j][i] = FLOAT_TYPE(0); + if (GEN_GLU == 3) { + temp2[j][i] = FLOAT_TYPE(0); + } } } - const uint nbr_par_th = num_blocks_per_row%it_size; - const uint nbr_all_th = num_blocks_per_row - nbr_par_th; - uint i0 = 0; - [[unroll]] for (; i0 < nbr_all_th; i0 += it_size) - calc_superblock(a_offset, b_offset, itid, ix, ql_offset, qh_offset, s_offset, y_offset, i0 + ix, num_blocks_per_row, first_row, num_rows, true); - calc_superblock(a_offset, b_offset, itid, ix, ql_offset, qh_offset, s_offset, y_offset, i0 + ix, num_blocks_per_row, first_row, num_rows, false); + if (GEN_FAST_KQ >= 2) { + const uint it_size8 = gl_WorkGroupSize.x / 8; + const uint itid8 = tid & 7; + const uint ix8 = tid >> 3; + if (GEN_GLU == 2 && use_a2) { + for (uint i = ix8; i < num_blocks_per_row; i += it_size8) { + calc_superblock_8_a2(temp, a_offset, b_offset, itid8, i, num_blocks_per_row, row_base, eff_rows); + } + } else { + for (uint i = ix8; i < num_blocks_per_row; i += it_size8) { + calc_superblock_8_a(temp, temp2, a_offset, b_offset, itid8, i, num_blocks_per_row, row_base, eff_rows); + } + } + } else { + // 16 threads are used to process each block + const uint it_size = gl_WorkGroupSize.x/16; + const uint itid = tid%16; // 0...15 + const uint ix = tid/16; + + const uint v_im = itid/8; // 0 or 1. 0 computes 0..., 1 computes 128... + const uint v_in = itid - 8*v_im; // 0...7 - reduce_result(temp, d_offset, first_row, num_rows, tid); + const uint l0 = 4 * v_in; // 0, 4, 8, ..., 28 + const uint is = v_in / 4; + + const uint ql_offset = 64*v_im + l0; + const uint qh_offset = 32*v_im + l0; + const uint s_offset = 8*v_im + is; + const uint y_offset = 128*v_im + l0; + + if (GEN_GLU == 2 && use_a2) { + for (uint i = ix; i < num_blocks_per_row; i += it_size) { + calc_superblock_a2(temp, a_offset, b_offset, ql_offset, qh_offset, s_offset, y_offset, i, num_blocks_per_row, row_base, eff_rows); + } + } else { + for (uint i = ix; i < num_blocks_per_row; i += it_size) { + calc_superblock_a(temp, temp2, a_offset, b_offset, ql_offset, qh_offset, s_offset, y_offset, i, num_blocks_per_row, row_base, eff_rows); + } + } + } + + if (GEN_GLU == 3) { + reduce_pair_glu(temp, temp2, d_offset, first_row, eff_rows, tid); + } +#ifndef MUL_MAT_ID + else if (GEN_GLU == 2 && use_a2) { + reduce_result_d2(temp, row_base, eff_rows, tid); + } +#endif + else { + reduce_result(temp, d_offset, first_row, eff_rows, tid); + } } void main() { +#ifdef USE_SUBGROUP_ADD_NO_SHMEM + const uint first_row = NUM_ROWS * (NUM_SUBGROUPS * (gl_WorkGroupID.x + gl_NumWorkGroups.x * gl_WorkGroupID.z) + gl_SubgroupID); +#else const uint first_row = NUM_ROWS * (gl_WorkGroupID.x + gl_NumWorkGroups.x * gl_WorkGroupID.z); +#endif - // do NUM_ROWS at a time, unless there aren't enough remaining rows - if (first_row + NUM_ROWS <= p.stride_d) { - compute_outputs(first_row, NUM_ROWS); - } else { - if (first_row >= p.stride_d) { - return; - } - compute_outputs(first_row, p.stride_d - first_row); + if (first_row >= p.stride_d) { + return; } + compute_outputs(first_row, min(NUM_ROWS, p.stride_d - first_row)); } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp index 19d37870bd69..9e9bfcaa07ed 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp @@ -25,6 +25,7 @@ #if defined(COOPMAT) || defined(MUL_MAT_ID_USE_SUBGROUPS) #extension GL_KHR_shader_subgroup_basic : enable #extension GL_KHR_shader_subgroup_ballot : enable +#extension GL_KHR_shader_subgroup_shuffle : enable #endif #ifdef MUL_MAT_ID @@ -50,28 +51,6 @@ layout (constant_id = 12) const uint MmTypeA = 0; layout (constant_id = 11) const uint ALIGNED = 0; -#ifdef MULMAT_QUANT - -uint mm_load_vec_a() { - switch (MmTypeA) { - case GGML_TYPE_Q1_0: - case GGML_TYPE_Q4_0: - case GGML_TYPE_Q4_1: - case GGML_TYPE_Q5_1: - return 8u; - case GGML_TYPE_Q2_0: - case GGML_TYPE_Q5_0: - case GGML_TYPE_Q8_0: - case GGML_TYPE_Q2_K: - case GGML_TYPE_Q4_K: - case GGML_TYPE_Q5_K: - return 4u; - default: - return 2u; - } -} -#endif - #if !defined(TO_FLOAT_TYPE) #define TO_FLOAT_TYPE FLOAT_TYPE #endif @@ -125,6 +104,72 @@ layout (binding = 2) writeonly buffer D {D_TYPE data_d[];}; #ifdef MUL_MAT_ID layout (binding = 3) readonly buffer IDS {int data_ids[];}; layout (binding = 4) readonly buffer RowMap {uint data_row_map[];}; +#define MM_BIND_U 5 +#define MM_BIND_AUX0 6 +#define MM_BIND_AUX1 7 +#else +#define MM_BIND_U 3 +#define MM_BIND_AUX0 4 +#define MM_BIND_AUX1 5 +#endif + +#ifdef MM_FUSED +#ifndef MULMAT_QUANT +layout (binding = MM_BIND_U) readonly buffer U {A_TYPE data_u[];}; +#if defined(DATA_A_F32) +layout (binding = MM_BIND_U) readonly buffer U_SCALAR {float data_u_scalar[];}; +#elif defined(DATA_A_F16) +layout (binding = MM_BIND_U) readonly buffer U_SCALAR {float16_t data_u_scalar[];}; +#elif defined(DATA_A_BF16) +layout (binding = MM_BIND_U) readonly buffer U_SCALAR {uint16_t data_u_scalar[];}; +#endif +#if defined(A_TYPE_PACKED16) +layout (binding = MM_BIND_U) readonly buffer U_PACKED16 {A_TYPE_PACKED16 data_u_packed16[];}; +#endif +#if defined(A_TYPE_PACKED32) +layout (binding = MM_BIND_U) readonly buffer U_PACKED32 {A_TYPE_PACKED32 data_u_packed32[];}; +#endif +#else +layout (binding = MM_BIND_U) readonly buffer UBUF_Q1_0 { block_q1_0 data[]; } u_q1_0; +layout (binding = MM_BIND_U) readonly buffer UBUF_Q2_0 { block_q2_0 data[]; } u_q2_0; +layout (binding = MM_BIND_U) readonly buffer UBUF_Q2_K { block_q2_K data[]; } u_q2_k; +layout (binding = MM_BIND_U) readonly buffer UBUF_Q3_K { block_q3_K data[]; } u_q3_k; +layout (binding = MM_BIND_U) readonly buffer UBUF_Q4_K { block_q4_K data[]; } u_q4_k; +layout (binding = MM_BIND_U) readonly buffer UBUF_Q5_K { block_q5_K data[]; } u_q5_k; +layout (binding = MM_BIND_U) readonly buffer UBUF_Q6_K { block_q6_K data[]; } u_q6_k; +layout (binding = MM_BIND_U) readonly buffer UBUF_TQ1_0 { block_tq1_0 data[]; } u_tq1_0; +layout (binding = MM_BIND_U) readonly buffer UBUF_TQ2_0 { block_tq2_0 data[]; } u_tq2_0; +layout (binding = MM_BIND_U) readonly buffer UBUF_Q4_0_P16 { block_q4_0_packed16 data[]; } u_q4_0_p16; +layout (binding = MM_BIND_U) readonly buffer UBUF_Q5_0_P16 { block_q5_0_packed16 data[]; } u_q5_0_p16; +layout (binding = MM_BIND_U) readonly buffer UBUF_Q8_0_P16 { block_q8_0_packed16 data[]; } u_q8_0_p16; +layout (binding = MM_BIND_U) readonly buffer UBUF_Q3_K_P16 { block_q3_K_packed16 data[]; } u_q3_k_p16; +layout (binding = MM_BIND_U) readonly buffer UBUF_Q6_K_P16 { block_q6_K_packed16 data[]; } u_q6_k_p16; +layout (binding = MM_BIND_U) readonly buffer UBUF_Q4_1_P32 { block_q4_1_packed32 data[]; } u_q4_1_p32; +layout (binding = MM_BIND_U) readonly buffer UBUF_Q5_1_P32 { block_q5_1_packed32 data[]; } u_q5_1_p32; +layout (binding = MM_BIND_U) readonly buffer UBUF_Q2_K_P32 { block_q2_K_packed32 data[]; } u_q2_k_p32; +layout (binding = MM_BIND_U) readonly buffer UBUF_Q4_K_P32 { block_q4_K_packed32 data[]; } u_q4_k_p32; +layout (binding = MM_BIND_U) readonly buffer UBUF_Q5_K_P32 { block_q5_K_packed32 data[]; } u_q5_k_p32; +#endif +layout (binding = MM_BIND_AUX0) readonly buffer AUX0 { float data_aux0[]; }; +layout (binding = MM_BIND_AUX1) readonly buffer AUX1 { float data_aux1[]; }; +#ifndef MUL_MAT_ID +layout (binding = MM_BIND_AUX1) writeonly buffer D2 { D_TYPE data_d2[]; }; +#endif + +#define MAT_VEC_FUSION_FLAGS_BIAS0 0x1u +#define MAT_VEC_FUSION_FLAGS_BIAS1 0x2u +#define MAT_VEC_FUSION_FLAGS_SCALE0 0x4u +#define MAT_VEC_FUSION_FLAGS_SCALE1 0x8u +#define MAT_VEC_FUSION_FLAGS_GLU_A 0x10u +#define MAT_VEC_FUSION_FLAGS_GLU_B 0x20u +#define MAT_VEC_FUSION_FLAGS_SIGMOID 0x80u +#define MAT_VEC_FUSION_FLAGS_SOFTPLUS_GATE 0x100u +#define MAT_VEC_FUSION_FLAGS_ROW_SCALE0 0x200u +#define MAT_VEC_FUSION_FLAGS_PAIR_GLU 0x400u +#define MAT_VEC_FUSION_FLAGS_PAIR_MM 0x800u +#define MAT_VEC_FUSION_FLAGS_PAIR_MM_SIGMOID1 0x1000u +#define MAT_VEC_FUSION_FLAGS_SCALE0_TOK 0x2000u +#define MAT_VEC_FUSION_FLAGS_SCALE1_TOK 0x4000u #endif layout (push_constant) uniform parameter @@ -147,6 +192,9 @@ layout (push_constant) uniform parameter uint ne11; uint n_experts; uint row_ids_offset; +#ifdef MM_FUSED + uint fusion_flags; +#endif #else uint base_work_group_z; uint num_batches; @@ -155,6 +203,13 @@ layout (push_constant) uniform parameter uint ne12; uint broadcast2; uint broadcast3; +#ifdef MM_FUSED + uint padded_N; + uint fusion_flags; + uint pair_m0; + uint stride_d2; + uint batch_stride_d2; +#endif #endif } p; @@ -181,23 +236,80 @@ layout (constant_id = 3) const uint BK = 16; // Assumed to be 32 if working wit #ifdef MULMAT_QUANT layout(constant_id = 13) const uint SHMEM_STRIDE_PAD = 4; layout(constant_id = 14) const bool APPLY_SLM_A_RESHAPE = false; +layout(constant_id = 15) const uint AGX_DIRECT_STORE = 0; +#ifdef MM_FUSED +layout(constant_id = 16) const uint MM_FUSE_MODE = 1; +#else +const uint MM_FUSE_MODE = 0; +#endif #else layout(constant_id = 12) const uint SHMEM_STRIDE_PAD = 4; layout(constant_id = 13) const bool APPLY_SLM_A_RESHAPE = false; +layout(constant_id = 14) const uint AGX_DIRECT_STORE = 0; +#ifdef MM_FUSED +layout(constant_id = 15) const uint MM_FUSE_MODE = 1; +#else +const uint MM_FUSE_MODE = 0; +#endif #endif #else const uint SHMEM_STRIDE_PAD = 1; const bool APPLY_SLM_A_RESHAPE = false; +const uint AGX_DIRECT_STORE = 0; +const uint MM_FUSE_MODE = 0; #endif #define SHMEM_STRIDE (BK / 2 + SHMEM_STRIDE_PAD) +#define SHMEM_A_STRIDE (APPLY_SLM_A_RESHAPE ? (BK / 2) : SHMEM_STRIDE) +#define SHMEM_B_STRIDE SHMEM_STRIDE -shared FLOAT_TYPEV2 buf_a[BM * SHMEM_STRIDE]; -shared FLOAT_TYPEV2 buf_b[BN * SHMEM_STRIDE]; +shared FLOAT_TYPEV2 buf_a[BM * SHMEM_A_STRIDE]; +shared FLOAT_TYPEV2 buf_b[BN * SHMEM_B_STRIDE]; #define NUM_WARPS (BLOCK_SIZE / WARP) #ifdef COOPMAT -shared ACC_TYPE coopmat_stage[TM * TN * NUM_WARPS]; +shared ACC_TYPE coopmat_stage[(AGX_DIRECT_STORE != 0) ? 1 : (TM * TN * NUM_WARPS)]; +#endif + +#ifdef MULMAT_QUANT +uint mm_load_vec_a() { + if (AGX_DIRECT_STORE == 1u) { + switch (MmTypeA) { + case GGML_TYPE_Q1_0: + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_1: + return 8u; + case GGML_TYPE_Q2_0: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q8_0: + case GGML_TYPE_Q2_K: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + return 4u; + default: + return 2u; + } + } + switch (MmTypeA) { + case GGML_TYPE_Q8_0: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: + return 16u; + case GGML_TYPE_Q1_0: + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_1: + return 8u; + case GGML_TYPE_Q2_0: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q2_K: + return 4u; + default: + return 2u; + } +} #endif #include "mul_mm_id_funcs.glsl" @@ -207,6 +319,58 @@ shared ACC_TYPE coopmat_stage[TM * TN * NUM_WARPS]; #include "iq_shmem_init.glsl" #endif +#ifdef MM_FUSED +#ifdef MUL_MAT_ID +float mm_apply_id_epilogue(float val, uint expert_idx, u16vec2 row_idx, uint d_row, uint d_idx) { + if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS0) != 0u) { + val += data_aux0[expert_idx * p.stride_d + d_row]; + } + if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_SCALE0) != 0u) { + const uint s0_idx = uint(row_idx.x) + ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_SCALE0_TOK) != 0u ? uint(row_idx.y) * p.nei0 : 0u); + val *= data_aux0[s0_idx]; + } + if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_SCALE1) != 0u) { + const uint s1_idx = uint(row_idx.x) + ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_SCALE1_TOK) != 0u ? uint(row_idx.y) * p.nei0 : 0u); + val *= data_aux1[s1_idx]; + } + if ((p.fusion_flags & (MAT_VEC_FUSION_FLAGS_GLU_A | MAT_VEC_FUSION_FLAGS_GLU_B)) != 0u) { + const float o = data_aux0[d_idx]; + val = ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_GLU_A) != 0u) + ? (val / (1.0f + exp(-val)) * o) + : (o / (1.0f + exp(-o)) * val); + } + return val; +} +#else +float mm_apply_dense_epilogue(float val, uint d_row, uint d_col, uint d_idx) { + if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS0) != 0u) { + val += data_aux0[d_idx]; + } + if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS1) != 0u) { + val += data_aux1[d_idx]; + } + if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_SIGMOID) != 0u) { + val = 1.0f / (1.0f + exp(-val)); + } + if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_SOFTPLUS_GATE) != 0u) { + const float x = val + data_aux0[d_row]; + const float sp = (x > 20.0f) ? x : log(1.0f + exp(x)); + val = sp * data_aux1[d_row]; + } + if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_ROW_SCALE0) != 0u) { + val *= data_aux0[d_col]; + } + if ((p.fusion_flags & (MAT_VEC_FUSION_FLAGS_GLU_A | MAT_VEC_FUSION_FLAGS_GLU_B)) != 0u) { + const float o = data_aux0[d_idx]; + val = ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_GLU_A) != 0u) + ? (val / (1.0f + exp(-val)) * o) + : (o / (1.0f + exp(-o)) * val); + } + return val; +} +#endif +#endif + void main() { #ifdef MUL_MAT_ID const uint tile_idx = gl_WorkGroupID.y + gl_WorkGroupID.z * gl_NumWorkGroups.y; @@ -236,13 +400,21 @@ void main() { const uint batch_idx_a = i03 * p.ne02 + i02; #endif - const uint blocks_m = (p.M + BM - 1) / BM; + const uint BM_EFF = (MM_FUSE_MODE == 2u) ? (BM / 2u) : BM; + const uint WM_EFF = (MM_FUSE_MODE == 2u) ? (WM / 2u) : WM; + + const uint blocks_m = (p.M + BM_EFF - 1) / BM_EFF; const uint ir = gl_WorkGroupID.x % blocks_m; const uint ik = gl_WorkGroupID.x / blocks_m; - - const uint WNITER = (WM * WN) / (WARP * TM * TN * WMITER); - const uint WSUBM = WM / WMITER; - const uint WSUBN = WN / WNITER; +#if defined(MM_FUSED) && !defined(MUL_MAT_ID) + const bool pair_use_u = (MM_FUSE_MODE == 3u) && (ir * BM_EFF >= p.pair_m0); + const uint ir_mem = pair_use_u ? (ir - (p.pair_m0 / BM_EFF)) : ir; + const uint m_lim = (MM_FUSE_MODE == 3u) ? (pair_use_u ? (p.M - p.pair_m0) : p.pair_m0) : p.M; +#else + const bool pair_use_u = false; + const uint ir_mem = ir; + const uint m_lim = p.M; +#endif #ifdef COOPMAT const uint warp_i = gl_SubgroupID; @@ -256,6 +428,10 @@ void main() { const uint store_r = tiw % TM; const uint store_c = tiw / TM; #else + const uint WNITER = (WM * WN) / (WARP * TM * TN * WMITER); + const uint WSUBM = WM / WMITER; + const uint WSUBN = WN / WNITER; + const uint warp_i = gl_LocalInvocationID.x / WARP; const uint tiw = gl_LocalInvocationID.x % WARP; @@ -285,12 +461,11 @@ void main() { const uint loadr_b = gl_LocalInvocationID.x % (BK / LOAD_VEC_B_EFF / LOAD_VEC_BATCH_B); const uint loadc_b = gl_LocalInvocationID.x / (BK / LOAD_VEC_B_EFF / LOAD_VEC_BATCH_B); - const uint loadstride_a = gl_WorkGroupSize.x * LOAD_VEC_A_EFF * LOAD_VEC_BATCH_A / BK; - const uint loadstride_b = gl_WorkGroupSize.x * LOAD_VEC_B_EFF * LOAD_VEC_BATCH_B / BK; + const uint loadstride_a = BLOCK_SIZE * LOAD_VEC_A_EFF * LOAD_VEC_BATCH_A / BK; + const uint loadstride_b = BLOCK_SIZE * LOAD_VEC_B_EFF * LOAD_VEC_BATCH_B / BK; #ifdef MUL_MAT_ID load_row_ids(row_begin); - uint required_work_items = _ne1 * BK / LOAD_VEC_B_EFF / LOAD_VEC_BATCH_B; uint required_warp_c = (_ne1 + WN - 1) / WN; #endif @@ -308,7 +483,7 @@ void main() { #else batch_idx_a * (p.batch_stride_a / LOAD_VEC_A_EFF) + #endif - (ir * BM * p.stride_a + start_k) / LOAD_VEC_A_EFF; + (ir_mem * BM_EFF * p.stride_a + start_k) / LOAD_VEC_A_EFF; #ifdef MUL_MAT_ID uint pos_b = 0; #else @@ -317,7 +492,7 @@ void main() { #ifdef COOPMAT coopmat cache_a; - coopmat cache_b; + coopmat cache_b_arr[cms_per_col]; coopmat sums[cms_per_row * cms_per_col]; [[unroll]] for (uint i = 0; i < cms_per_row * cms_per_col; i++) { @@ -338,21 +513,325 @@ void main() { } #endif - for (uint block = start_k; block < end_k; block += BK) { - [[unroll]] for (uint l = 0; l < BM; l += loadstride_a) { - load_a_to_shmem(pos_a, loadr_a, loadc_a + l, ir * BM + loadc_a + l, block, end_k); +#ifdef COOPMAT + bool use_sw_pipe = false; +#if (defined(DATA_A_F32) || defined(DATA_A_F16)) && LOAD_VEC_A == 4 + const uint MAX_STAGE_A = 4u; +#else + const uint MAX_STAGE_A = 2u; +#endif + if (AGX_DIRECT_STORE != 1u) { + if (ALIGNED != 0) { + if (mm_can_stage_a()) { + if (LOAD_VEC_B == 8 || LOAD_VEC_B == 4) { + if (BM <= MAX_STAGE_A * loadstride_a && BN <= 2u * loadstride_b) { + use_sw_pipe = true; + } + } + } } - #ifdef MUL_MAT_ID - if (gl_LocalInvocationID.x < required_work_items) { - #endif - [[unroll]] for (uint l = 0; l < BN; l += loadstride_b) { + } + if (use_sw_pipe) { +#ifdef MUL_MAT_ID + const uint active_cols_b = min(BN, ((required_warp_c * WN) + 7u) & ~7u); +#endif + for (uint block = start_k; block < end_k; block += BK) { + FLOAT_TYPEV2 reg_a0[8]; + FLOAT_TYPEV2 reg_a1[8]; +#if (defined(DATA_A_F32) || defined(DATA_A_F16)) && LOAD_VEC_A == 4 + FLOAT_TYPEV2 reg_a2[8]; + FLOAT_TYPEV2 reg_a3[8]; +#endif + FLOAT_TYPEV2 reg_b0[4]; + FLOAT_TYPEV2 reg_b1[4]; +#ifdef MUL_MAT_ID + [[unroll]] for (uint bi = 0u; bi < 4u; ++bi) { + reg_b0[bi] = FLOAT_TYPEV2(0.0f); + reg_b1[bi] = FLOAT_TYPEV2(0.0f); + } +#endif + + if (loadstride_a <= BM || loadc_a < BM) { + const uint c_sh0 = loadc_a; +#ifdef MM_FUSED + if (MM_FUSE_MODE == 2u && c_sh0 >= BM_EFF) { + const uint c_mem0 = c_sh0 - BM_EFF; + stage_u_to_reg(pos_a, loadr_a, c_sh0, c_mem0, ir * BM_EFF + c_mem0, block, end_k, reg_a0); + } else if (pair_use_u) { + stage_u_to_reg(pos_a, loadr_a, c_sh0, c_sh0, ir * BM_EFF + c_sh0, block, end_k, reg_a0); + } else +#endif + { + stage_a_to_reg(pos_a, loadr_a, c_sh0, c_sh0, ir * BM_EFF + c_sh0, block, end_k, reg_a0); + } + } + if (loadstride_a < BM && loadc_a + loadstride_a < BM) { + const uint c_sh1 = loadc_a + loadstride_a; +#ifdef MM_FUSED + if (MM_FUSE_MODE == 2u && c_sh1 >= BM_EFF) { + const uint c_mem1 = c_sh1 - BM_EFF; + stage_u_to_reg(pos_a, loadr_a, c_sh1, c_mem1, ir * BM_EFF + c_mem1, block, end_k, reg_a1); + } else if (pair_use_u) { + stage_u_to_reg(pos_a, loadr_a, c_sh1, c_sh1, ir * BM_EFF + c_sh1, block, end_k, reg_a1); + } else +#endif + { + stage_a_to_reg(pos_a, loadr_a, c_sh1, c_sh1, ir * BM_EFF + c_sh1, block, end_k, reg_a1); + } + } +#if (defined(DATA_A_F32) || defined(DATA_A_F16)) && LOAD_VEC_A == 4 + if (2u * loadstride_a < BM && loadc_a + 2u * loadstride_a < BM) { + const uint c_sh2 = loadc_a + 2u * loadstride_a; +#ifdef MM_FUSED + if (MM_FUSE_MODE == 2u && c_sh2 >= BM_EFF) { + const uint c_mem2 = c_sh2 - BM_EFF; + stage_u_to_reg(pos_a, loadr_a, c_sh2, c_mem2, ir * BM_EFF + c_mem2, block, end_k, reg_a2); + } else if (pair_use_u) { + stage_u_to_reg(pos_a, loadr_a, c_sh2, c_sh2, ir * BM_EFF + c_sh2, block, end_k, reg_a2); + } else +#endif + { + stage_a_to_reg(pos_a, loadr_a, c_sh2, c_sh2, ir * BM_EFF + c_sh2, block, end_k, reg_a2); + } + } + if (3u * loadstride_a < BM && loadc_a + 3u * loadstride_a < BM) { + const uint c_sh3 = loadc_a + 3u * loadstride_a; +#ifdef MM_FUSED + if (MM_FUSE_MODE == 2u && c_sh3 >= BM_EFF) { + const uint c_mem3 = c_sh3 - BM_EFF; + stage_u_to_reg(pos_a, loadr_a, c_sh3, c_mem3, ir * BM_EFF + c_mem3, block, end_k, reg_a3); + } else if (pair_use_u) { + stage_u_to_reg(pos_a, loadr_a, c_sh3, c_sh3, ir * BM_EFF + c_sh3, block, end_k, reg_a3); + } else +#endif + { + stage_a_to_reg(pos_a, loadr_a, c_sh3, c_sh3, ir * BM_EFF + c_sh3, block, end_k, reg_a3); + } + } +#endif + +#ifdef MUL_MAT_ID + if (loadc_b < active_cols_b) { + stage_b_to_reg(pos_b, loadr_b, loadc_b, 0u, reg_b0); + } + if (loadstride_b < BN && loadc_b + loadstride_b < active_cols_b) { + stage_b_to_reg(pos_b, loadr_b, loadc_b + loadstride_b, 0u, reg_b1); + } +#else + if (loadstride_b <= BN || loadc_b < BN) { + stage_b_to_reg(pos_b, loadr_b, loadc_b, ic * BN + loadc_b, reg_b0); + } + if (loadstride_b < BN && loadc_b + loadstride_b < BN) { + const uint c_b1 = loadc_b + loadstride_b; + stage_b_to_reg(pos_b, loadr_b, c_b1, ic * BN + c_b1, reg_b1); + } +#endif + + if (block > start_k) { + barrier(); + } + + if (loadstride_a <= BM || loadc_a < BM) { + commit_stage_a(loadc_a, loadr_a, reg_a0); + } + if (loadstride_a < BM && loadc_a + loadstride_a < BM) { + commit_stage_a(loadc_a + loadstride_a, loadr_a, reg_a1); + } +#if (defined(DATA_A_F32) || defined(DATA_A_F16)) && LOAD_VEC_A == 4 + if (2u * loadstride_a < BM && loadc_a + 2u * loadstride_a < BM) { + commit_stage_a(loadc_a + 2u * loadstride_a, loadr_a, reg_a2); + } + if (3u * loadstride_a < BM && loadc_a + 3u * loadstride_a < BM) { + commit_stage_a(loadc_a + 3u * loadstride_a, loadr_a, reg_a3); + } +#endif +#ifdef MUL_MAT_ID + if (loadc_b < active_cols_b) { + commit_stage_b(loadc_b, loadr_b, reg_b0); + } + if (loadstride_b < BN && loadc_b + loadstride_b < active_cols_b) { + commit_stage_b(loadc_b + loadstride_b, loadr_b, reg_b1); + } +#else + if (loadstride_b <= BN || loadc_b < BN) { + commit_stage_b(loadc_b, loadr_b, reg_b0); + } + if (loadstride_b < BN && loadc_b + loadstride_b < BN) { + commit_stage_b(loadc_b + loadstride_b, loadr_b, reg_b1); + } +#endif + + barrier(); + + pos_a += BK / LOAD_VEC_A_EFF; + pos_b += BK / LOAD_VEC_B_EFF; + +#ifdef MUL_MAT_ID + if (warp_c < required_warp_c) +#endif + { +#define MM_COMPUTE_KTILE(K_OFF) \ + do { \ + const uint _i = (K_OFF); \ + [[unroll]] for (uint cm_col = 0; cm_col < cms_per_col; cm_col++) { \ + MM_LOAD_CACHE_B(_i, cm_col); \ + } \ + [[unroll]] for (uint cm_row = 0; cm_row < cms_per_row; cm_row++) { \ + const uint sh_row_a = (MM_FUSE_MODE == 2u) \ + ? ((cm_row < cms_per_row / 2u) \ + ? (warp_r * WM_EFF + cm_row * TM) \ + : (BM_EFF + warp_r * WM_EFF + (cm_row - cms_per_row / 2u) * TM)) \ + : (warp_r * WM + cm_row * TM); \ + coopMatLoad(cache_a, buf_a, a_shmem_index(sh_row_a, _i / 2), a_shmem_stride(), gl_CooperativeMatrixLayoutRowMajor); \ + [[unroll]] for (uint cm_col = 0; cm_col < cms_per_col; cm_col++) { \ + sums[cm_col * cms_per_row + cm_row] = coopMatMulAdd(cache_a, cache_b_arr[cm_col], sums[cm_col * cms_per_row + cm_row]); \ + } \ + } \ + } while (false) + #if !defined(MUL_MAT_ID) - load_b_to_shmem(pos_b, loadr_b, loadc_b + l, ic * BN + loadc_b + l, block, end_k); +#define MM_LOAD_CACHE_B_COLMAJ(K_I, CM_COL) \ + coopMatLoad(cache_b_arr[CM_COL], buf_b, b_shmem_index(warp_c * WN + (CM_COL) * TN, (K_I) / 2), b_shmem_stride(), gl_CooperativeMatrixLayoutColumnMajor) +#else +#define MM_LOAD_CACHE_B_COLMAJ(K_I, CM_COL) \ + coopMatLoad(cache_b_arr[CM_COL], buf_b, (warp_c * WN + (CM_COL) * TN) * SHMEM_STRIDE + (K_I) / 2, SHMEM_STRIDE, gl_CooperativeMatrixLayoutColumnMajor) +#endif + +#if defined(FLOAT16) && !defined(DATA_A_BF16) && LOAD_VEC_B == 8 +#define MM_LOAD_CACHE_B(K_I, CM_COL) \ + do { \ + if (AGX_DIRECT_STORE != 0) { \ + if (APPLY_SLM_A_RESHAPE) { \ + if (BK == 32) { \ + const uint b_off = ((K_I) >> 3u) * ((BN << 2u) + 8u) + (((warp_c * WN + (CM_COL) * 8u) >> 3u) << 5u); \ + coopMatLoad(cache_b_arr[CM_COL], buf_b, b_off, 4u, gl_CooperativeMatrixLayoutRowMajor); \ + } else { \ + MM_LOAD_CACHE_B_COLMAJ(K_I, CM_COL); \ + } \ + } else { \ + MM_LOAD_CACHE_B_COLMAJ(K_I, CM_COL); \ + } \ + } else { \ + MM_LOAD_CACHE_B_COLMAJ(K_I, CM_COL); \ + } \ + } while (false) #else - load_b_to_shmem(pos_b, loadr_b, loadc_b + l, ic, _ne1, block, end_k); +#define MM_LOAD_CACHE_B(K_I, CM_COL) MM_LOAD_CACHE_B_COLMAJ(K_I, CM_COL) #endif + + if (AGX_DIRECT_STORE == 2u) { + [[dont_unroll]] for (uint i = 0; i < BK; i += 2u * TK) { + MM_COMPUTE_KTILE(i); + MM_COMPUTE_KTILE(i + TK); + } + } else if (AGX_DIRECT_STORE == 3u) { + [[dont_unroll]] for (uint i = 0; i < BK; i += TK) { + MM_COMPUTE_KTILE(i); + } + } else if (AGX_DIRECT_STORE == 4u) { + [[unroll]] for (uint i = 0; i < BK; i += TK) { + controlBarrier(gl_ScopeSubgroup, gl_ScopeSubgroup, gl_StorageSemanticsShared, gl_SemanticsAcquireRelease); + MM_COMPUTE_KTILE(i); + } + } else if (AGX_DIRECT_STORE == 5u) { + [[unroll]] for (uint i = 0; i < BK; i += TK) { + coopmat cache_a_arr[cms_per_row]; + controlBarrier(gl_ScopeSubgroup, gl_ScopeSubgroup, gl_StorageSemanticsShared, gl_SemanticsAcquireRelease); + [[unroll]] for (uint cm_row = 0; cm_row < cms_per_row; cm_row++) { + const uint sh_row_a = (MM_FUSE_MODE == 2u) + ? ((cm_row < cms_per_row / 2u) + ? (warp_r * WM_EFF + cm_row * TM) + : (BM_EFF + warp_r * WM_EFF + (cm_row - cms_per_row / 2u) * TM)) + : (warp_r * WM + cm_row * TM); + coopMatLoad(cache_a_arr[cm_row], buf_a, a_shmem_index(sh_row_a, i / 2), a_shmem_stride(), gl_CooperativeMatrixLayoutRowMajor); + } + controlBarrier(gl_ScopeSubgroup, gl_ScopeSubgroup, gl_StorageSemanticsShared, gl_SemanticsAcquireRelease); + [[unroll]] for (uint cm_col = 0; cm_col < cms_per_col; cm_col++) { + MM_LOAD_CACHE_B(i, cm_col); + } + controlBarrier(gl_ScopeSubgroup, gl_ScopeSubgroup, gl_StorageSemanticsShared, gl_SemanticsAcquireRelease); + [[unroll]] for (uint cm_row = 0; cm_row < cms_per_row; cm_row++) { + [[unroll]] for (uint cm_col = 0; cm_col < cms_per_col; cm_col++) { + sums[cm_col * cms_per_row + cm_row] = coopMatMulAdd(cache_a_arr[cm_row], cache_b_arr[cm_col], sums[cm_col * cms_per_row + cm_row]); + } + } + } + } else { + [[unroll]] for (uint i = 0; i < BK; i += TK) { + MM_COMPUTE_KTILE(i); + } + } + +#undef MM_LOAD_CACHE_B +#undef MM_LOAD_CACHE_B_COLMAJ +#undef MM_COMPUTE_KTILE + } + } + } else +#endif + for (uint block = start_k; block < end_k; block += BK) { + [[unroll]] for (uint l = 0; l < BM; l += loadstride_a) { + if (loadstride_a <= BM || loadc_a + l < BM) { + const uint c_sh = loadc_a + l; +#ifdef MM_FUSED + if (MM_FUSE_MODE == 2u && c_sh >= BM_EFF) { + const uint c_mem = c_sh - BM_EFF; + const uint idx_m = ir * BM_EFF + c_mem; + if (idx_m < p.M) { + load_u_to_shmem(pos_a, loadr_a, c_sh, c_mem, idx_m, block, end_k); + } else { + const uint kp_cnt = (LOAD_VEC_A_EFF * LOAD_VEC_BATCH_A) / 2u; + const uint kp_base = loadr_a * kp_cnt; + [[unroll]] for (uint kp = 0; kp < kp_cnt; ++kp) { + store_a(c_sh, kp_base + kp, FLOAT_TYPEV2(0.0f)); + } + } + } else if (pair_use_u) { + const uint idx_m = ir * BM_EFF + c_sh; + if (idx_m < p.M) { + load_u_to_shmem(pos_a, loadr_a, c_sh, c_sh, idx_m, block, end_k); + } else { + const uint kp_cnt = (LOAD_VEC_A_EFF * LOAD_VEC_BATCH_A) / 2u; + const uint kp_base = loadr_a * kp_cnt; + [[unroll]] for (uint kp = 0; kp < kp_cnt; ++kp) { + store_a(c_sh, kp_base + kp, FLOAT_TYPEV2(0.0f)); + } + } + } else +#endif + { + const uint idx_m = ir * BM_EFF + c_sh; + if (idx_m < m_lim) { + load_a_to_shmem(pos_a, loadr_a, c_sh, c_sh, idx_m, block, end_k); + } else { + const uint kp_cnt = (LOAD_VEC_A_EFF * LOAD_VEC_BATCH_A) / 2u; + const uint kp_base = loadr_a * kp_cnt; + [[unroll]] for (uint kp = 0; kp < kp_cnt; ++kp) { + store_a(c_sh, kp_base + kp, FLOAT_TYPEV2(0.0f)); + } + } + } + } } #ifdef MUL_MAT_ID + const uint active_cols_b = min(BN, required_warp_c * WN); + [[unroll]] for (uint l = 0; l < BN; l += loadstride_b) { + const uint col_b = loadc_b + l; + if (col_b < _ne1) { + load_b_to_shmem(pos_b, loadr_b, col_b, ic, _ne1, block, end_k); + } else if (col_b < active_cols_b) { + const uint kp_cnt = (LOAD_VEC_B_EFF * LOAD_VEC_BATCH_B) / 2u; + const uint buf_idx = col_b * SHMEM_STRIDE + loadr_b * kp_cnt; + [[unroll]] for (uint kp = 0; kp < kp_cnt; ++kp) { + buf_b[buf_idx + kp] = FLOAT_TYPEV2(0.0f); + } + } + } + #else + [[unroll]] for (uint l = 0; l < BN; l += loadstride_b) { + if (loadstride_b <= BN || loadc_b + l < BN) { + load_b_to_shmem(pos_b, loadr_b, loadc_b + l, ic * BN + loadc_b + l, block, end_k); + } } #endif @@ -366,14 +845,24 @@ void main() { if (warp_c < required_warp_c) { #endif [[unroll]] for (uint i = 0; i < BK; i += TK) { + [[unroll]] for (uint cm_col = 0; cm_col < cms_per_col; cm_col++) { +#if !defined(MUL_MAT_ID) + coopMatLoad(cache_b_arr[cm_col], buf_b, b_shmem_index(warp_c * WN + cm_col * TN, i / 2), b_shmem_stride(), gl_CooperativeMatrixLayoutColumnMajor); +#else + coopMatLoad(cache_b_arr[cm_col], buf_b, (warp_c * WN + cm_col * TN) * SHMEM_STRIDE + i / 2, SHMEM_STRIDE, gl_CooperativeMatrixLayoutColumnMajor); +#endif + } [[unroll]] for (uint cm_row = 0; cm_row < cms_per_row; cm_row++) { + const uint sh_row_a = (MM_FUSE_MODE == 2u) + ? ((cm_row < cms_per_row / 2u) + ? (warp_r * WM_EFF + cm_row * TM) + : (BM_EFF + warp_r * WM_EFF + (cm_row - cms_per_row / 2u) * TM)) + : (warp_r * WM + cm_row * TM); // Load from shared into cache - coopMatLoad(cache_a, buf_a, a_shmem_index(warp_r * WM + cm_row * TM, i / 2), a_shmem_stride(), gl_CooperativeMatrixLayoutRowMajor); + coopMatLoad(cache_a, buf_a, a_shmem_index(sh_row_a, i / 2), a_shmem_stride(), gl_CooperativeMatrixLayoutRowMajor); [[unroll]] for (uint cm_col = 0; cm_col < cms_per_col; cm_col++) { - coopMatLoad(cache_b, buf_b, (warp_c * WN + cm_col * TN) * SHMEM_STRIDE + i / 2, SHMEM_STRIDE, gl_CooperativeMatrixLayoutColumnMajor); - - sums[cm_col * cms_per_row + cm_row] = coopMatMulAdd(cache_a, cache_b, sums[cm_col * cms_per_row + cm_row]); + sums[cm_col * cms_per_row + cm_row] = coopMatMulAdd(cache_a, cache_b_arr[cm_col], sums[cm_col * cms_per_row + cm_row]); } } } @@ -434,7 +923,7 @@ void main() { #endif #endif - const uint dr = ir * BM + warp_r * WM; + const uint dr = ir_mem * BM_EFF + warp_r * WM_EFF; const uint dc = ic * BN + warp_c * WN; #ifndef MUL_MAT_ID @@ -442,59 +931,209 @@ void main() { #endif #ifdef COOPMAT + const uint cms_out_rows = (MM_FUSE_MODE == 2u) ? (cms_per_row / 2u) : cms_per_row; + const uint cms_up_off = cms_per_row / 2u; #ifdef MUL_MAT_ID - [[unroll]] for (uint cm_row = 0; cm_row < cms_per_row; cm_row++) { + if (AGX_DIRECT_STORE != 0 && TM == 8 && TN == 8 && WARP == 32) { + const uint agx_m_local = ((tiw >> 1u) & 3u) | ((tiw >> 2u) & 4u); + const uint agx_n_base = ((tiw & 1u) << 1u) | ((tiw >> 1u) & 4u); [[unroll]] for (uint cm_col = 0; cm_col < cms_per_col; cm_col++) { - coopMatStore(sums[cm_col * cms_per_row + cm_row], coopmat_stage, warp_i * TM * TN, TM, gl_CooperativeMatrixLayoutColumnMajor); - - barrier(); - [[unroll]] for (uint col = 0; col < TN; col += storestride) { - const uint row_i = dc + cm_col * TN + col + store_c; - if (row_i >= _ne1) break; - - const u16vec2 row_idx = row_ids[row_i - ic * BN]; - - if (dr + cm_row * TM + store_r < p.M) { - data_d[row_idx.y * p.batch_stride_d + row_idx.x * p.stride_d + dr + cm_row * TM + store_r] = D_TYPE(coopmat_stage[warp_i * TM * TN + (col + store_c) * TM + store_r]); + [[unroll]] for (uint elem = 0; elem < 2; elem++) { + const uint row_i = dc + cm_col * TN + (agx_n_base | elem); + if (row_i < _ne1) { + const u16vec2 row_idx = row_ids[row_i - ic * BN]; + const uint d_col_base = row_idx.y * p.batch_stride_d + row_idx.x * p.stride_d; + float row_scale = 1.0f; +#ifdef MM_FUSED + if (MM_FUSE_MODE == 4u) { + const uint s0_idx = uint(row_idx.x) + ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_SCALE0_TOK) != 0u ? uint(row_idx.y) * p.nei0 : 0u); + row_scale = data_aux0[s0_idx]; + } +#endif + [[unroll]] for (uint cm_row = 0; cm_row < cms_out_rows; cm_row++) { + const uint d_row = dr + cm_row * TM + agx_m_local; + if (d_row < p.M) { + const uint d_idx = d_col_base + d_row; + float val = float(sums[cm_col * cms_per_row + cm_row][elem]); +#ifdef MM_FUSED + if (MM_FUSE_MODE == 2u) { + const float u = float(sums[cm_col * cms_per_row + cms_up_off + cm_row][elem]); + val = (val / (1.0f + exp(-val))) * u; + } else if (MM_FUSE_MODE == 4u) { + val *= row_scale; + } else if (p.fusion_flags != 0u) { + val = mm_apply_id_epilogue(val, expert_idx, row_idx, d_row, d_idx); + } +#endif + data_d[d_idx] = D_TYPE(val); + } + } } } - barrier(); } - } -#else - const bool is_aligned = p.stride_d % 4 == 0; // Assumption: D_TYPE == float - - [[unroll]] for (uint cm_row = 0; cm_row < cms_per_row; cm_row++) { - [[unroll]] for (uint cm_col = 0; cm_col < cms_per_col; cm_col++) { - const bool is_in_bounds = dr + (cm_row + 1) * TM <= p.M && dc + (cm_col + 1) * TN <= p.N; - - if (is_aligned && is_in_bounds) { - // Full coopMat is within bounds and stride_d is aligned with 16B - coopmat cm_dtype = coopmat(sums[cm_col * cms_per_row + cm_row]); - coopMatStore(cm_dtype, data_d, offsets + (dc + cm_col * TN) * p.stride_d + dr + cm_row * TM, p.stride_d, gl_CooperativeMatrixLayoutColumnMajor); - } else if (is_in_bounds) { - // Full coopMat is within bounds, but stride_d is not aligned - coopMatStore(sums[cm_col * cms_per_row + cm_row], coopmat_stage, warp_i * TM * TN, TM, gl_CooperativeMatrixLayoutColumnMajor); - - controlBarrier(gl_ScopeSubgroup, gl_ScopeSubgroup, gl_StorageSemanticsShared, gl_SemanticsAcquireRelease); - [[unroll]] for (uint col = 0; col < TN; col += storestride) { - data_d[offsets + (dc + cm_col * TN + col + store_c) * p.stride_d + dr + cm_row * TM + store_r] = D_TYPE(coopmat_stage[warp_i * TM * TN + (col + store_c) * TM + store_r]); + } else { + [[unroll]] for (uint cm_row = 0; cm_row < cms_out_rows; cm_row++) { + [[unroll]] for (uint cm_col = 0; cm_col < cms_per_col; cm_col++) { + coopmat out_cm = sums[cm_col * cms_per_row + cm_row]; +#ifdef MM_FUSED + if (MM_FUSE_MODE == 2u) { + [[unroll]] for (uint e = 0; e < out_cm.length(); ++e) { + const float g = float(out_cm[e]); + const float u = float(sums[cm_col * cms_per_row + cms_up_off + cm_row][e]); + out_cm[e] = ACC_TYPE((g / (1.0f + exp(-g))) * u); + } } - controlBarrier(gl_ScopeSubgroup, gl_ScopeSubgroup, gl_StorageSemanticsShared, gl_SemanticsAcquireRelease); - } else if (dr + cm_row * TM < p.M && dc + cm_col * TN < p.N) { - // Partial coopMat is within bounds - coopMatStore(sums[cm_col * cms_per_row + cm_row], coopmat_stage, warp_i * TM * TN, TM, gl_CooperativeMatrixLayoutColumnMajor); +#endif + coopMatStore(out_cm, coopmat_stage, warp_i * TM * TN, TM, gl_CooperativeMatrixLayoutColumnMajor); controlBarrier(gl_ScopeSubgroup, gl_ScopeSubgroup, gl_StorageSemanticsShared, gl_SemanticsAcquireRelease); [[unroll]] for (uint col = 0; col < TN; col += storestride) { - if (dr + cm_row * TM + store_r < p.M && dc + cm_col * TN + col + store_c < p.N) { - data_d[offsets + (dc + cm_col * TN + col + store_c) * p.stride_d + dr + cm_row * TM + store_r] = D_TYPE(coopmat_stage[warp_i * TM * TN + (col + store_c) * TM + store_r]); + const uint row_i = dc + cm_col * TN + col + store_c; + if (row_i >= _ne1) break; + + const u16vec2 row_idx = row_ids[row_i - ic * BN]; + const uint d_row = dr + cm_col * 0u + cm_row * TM + store_r; + + if (d_row < p.M) { + const uint d_idx = row_idx.y * p.batch_stride_d + row_idx.x * p.stride_d + d_row; + float val = float(coopmat_stage[warp_i * TM * TN + (col + store_c) * TM + store_r]); +#ifdef MM_FUSED + if (MM_FUSE_MODE != 2u && p.fusion_flags != 0u) { + val = mm_apply_id_epilogue(val, expert_idx, row_idx, d_row, d_idx); + } +#endif + data_d[d_idx] = D_TYPE(val); } } controlBarrier(gl_ScopeSubgroup, gl_ScopeSubgroup, gl_StorageSemanticsShared, gl_SemanticsAcquireRelease); } } } +#else + if (AGX_DIRECT_STORE != 0 && TM == 8 && TN == 8 && WARP == 32) { + const uint agx_m_local = ((tiw >> 1u) & 3u) | ((tiw >> 2u) & 4u); + const uint agx_n_base = ((tiw & 1u) << 1u) | ((tiw >> 1u) & 4u); +#ifdef MM_FUSED + float fast_row_scale[cms_per_col][2]; + if (MM_FUSE_MODE == 4u && (p.fusion_flags & MAT_VEC_FUSION_FLAGS_ROW_SCALE0) != 0u) { + [[unroll]] for (uint cm_col = 0; cm_col < cms_per_col; cm_col++) { + [[unroll]] for (uint elem = 0; elem < 2; elem++) { + const uint d_col = dc + cm_col * TN + (agx_n_base | elem); + fast_row_scale[cm_col][elem] = (d_col < p.N) ? data_aux0[d_col] : 1.0f; + } + } + } +#endif + [[unroll]] for (uint cm_row = 0; cm_row < cms_out_rows; cm_row++) { + const uint d_row = dr + cm_row * TM + agx_m_local; + if (d_row < m_lim) { + [[unroll]] for (uint cm_col = 0; cm_col < cms_per_col; cm_col++) { + [[unroll]] for (uint elem = 0; elem < 2; elem++) { + const uint d_col = dc + cm_col * TN + (agx_n_base | elem); + if (d_col < p.N) { + float val = float(sums[cm_col * cms_per_row + cm_row][elem]); +#ifdef MM_FUSED + if (MM_FUSE_MODE == 3u && pair_use_u) { + if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_PAIR_MM_SIGMOID1) != 0u) { + val = 1.0f / (1.0f + exp(-val)); + } + data_d2[batch_idx * p.batch_stride_d2 + d_col * p.stride_d2 + d_row] = D_TYPE(val); + } else +#endif + { + const uint d_idx = offsets + d_col * p.stride_d + d_row; +#ifdef MM_FUSED + if (MM_FUSE_MODE == 2u) { + const float u = float(sums[cm_col * cms_per_row + cms_up_off + cm_row][elem]); + val = (val / (1.0f + exp(-val))) * u; + } else if (MM_FUSE_MODE == 4u) { + if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS0) != 0u) { + val += data_aux0[d_idx]; + } + if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS1) != 0u) { + val += data_aux1[d_idx]; + } + if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_ROW_SCALE0) != 0u) { + val *= fast_row_scale[cm_col][elem]; + } + } else if (MM_FUSE_MODE == 1u) { + val = mm_apply_dense_epilogue(val, d_row, d_col, d_idx); + } +#endif + data_d[d_idx] = D_TYPE(val); + } + } + } + } + } + } + } else { + const bool is_aligned = p.stride_d % 4 == 0; // Assumption: D_TYPE == float + + [[unroll]] for (uint cm_row = 0; cm_row < cms_out_rows; cm_row++) { + [[unroll]] for (uint cm_col = 0; cm_col < cms_per_col; cm_col++) { + const bool is_in_bounds = dr + (cm_row + 1) * TM <= m_lim && dc + (cm_col + 1) * TN <= p.N; + +#ifndef MM_FUSED + if (is_aligned && is_in_bounds) { + // Full coopMat is within bounds and stride_d is aligned with 16B + coopmat cm_dtype = coopmat(sums[cm_col * cms_per_row + cm_row]); + coopMatStore(cm_dtype, data_d, offsets + (dc + cm_col * TN) * p.stride_d + dr + cm_row * TM, p.stride_d, gl_CooperativeMatrixLayoutColumnMajor); + } else +#endif + if (dr + cm_row * TM < m_lim && dc + cm_col * TN < p.N) { + coopmat out_cm = sums[cm_col * cms_per_row + cm_row]; +#ifdef MM_FUSED + if (MM_FUSE_MODE == 2u) { + [[unroll]] for (uint e = 0; e < out_cm.length(); ++e) { + const float g = float(out_cm[e]); + const float u = float(sums[cm_col * cms_per_row + cms_up_off + cm_row][e]); + out_cm[e] = ACC_TYPE((g / (1.0f + exp(-g))) * u); + } + } +#endif + coopMatStore(out_cm, coopmat_stage, warp_i * TM * TN, TM, gl_CooperativeMatrixLayoutColumnMajor); + + controlBarrier(gl_ScopeSubgroup, gl_ScopeSubgroup, gl_StorageSemanticsShared, gl_SemanticsAcquireRelease); + [[unroll]] for (uint col = 0; col < TN; col += storestride) { + const uint d_row = dr + cm_row * TM + store_r; + const uint d_col = dc + cm_col * TN + col + store_c; + if (is_in_bounds || (d_row < m_lim && d_col < p.N)) { + float val = float(coopmat_stage[warp_i * TM * TN + (col + store_c) * TM + store_r]); +#ifdef MM_FUSED + if (pair_use_u) { + if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_PAIR_MM_SIGMOID1) != 0u) { + val = 1.0f / (1.0f + exp(-val)); + } + data_d2[batch_idx * p.batch_stride_d2 + d_col * p.stride_d2 + d_row] = D_TYPE(val); + } else +#endif + { + const uint d_idx = offsets + d_col * p.stride_d + d_row; +#ifdef MM_FUSED + if (MM_FUSE_MODE == 4u) { + if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS0) != 0u) { + val += data_aux0[d_idx]; + } + if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS1) != 0u) { + val += data_aux1[d_idx]; + } + if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_ROW_SCALE0) != 0u) { + val *= data_aux0[d_col]; + } + } else if (MM_FUSE_MODE == 1u) { + val = mm_apply_dense_epilogue(val, d_row, d_col, d_idx); + } +#endif + data_d[d_idx] = D_TYPE(val); + } + } + } + controlBarrier(gl_ScopeSubgroup, gl_ScopeSubgroup, gl_StorageSemanticsShared, gl_SemanticsAcquireRelease); + } + } + } + } #endif // MUL_MAT_ID #else [[unroll]] for (uint wsic = 0; wsic < WNITER; wsic++) { diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl index 588fb73546a3..00a3d8ef0a93 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl @@ -17,662 +17,328 @@ void store_a(uint m, uint k_pair, FLOAT_TYPEV2 value) { buf_a[a_shmem_index(m, k_pair)] = value; } -void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uint idx_m, const uint block, const uint end_k) { -#if defined(DATA_A_F32) || defined(DATA_A_F16) -#if LOAD_VEC_A == 8 - if (ALIGNED != 0) { - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - const uint k_pair = row * LOAD_VEC_A / 2; - FLOAT_TYPEV8 aa = FLOAT_TYPEV8(data_a[idx]); - store_a(col, k_pair, aa[0].xy); - store_a(col, k_pair + 1, aa[0].zw); - store_a(col, k_pair + 2, aa[1].xy); - store_a(col, k_pair + 3, aa[1].zw); - return; - } -#elif LOAD_VEC_A == 4 - if (ALIGNED != 0) { - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - const uint k_pair = row * LOAD_VEC_A / 2; - FLOAT_TYPEV4 aa = FLOAT_TYPEV4(data_a[idx]); - store_a(col, k_pair, aa.xy); - store_a(col, k_pair + 1, aa.zw); - return; - } -#endif - const uint idx = pos_a + col * p.stride_a + row * 2; - if (idx_m < p.M && block + row * 2 + 1 < end_k) { - store_a(col, row, FLOAT_TYPEV2(data_a_scalar[idx], - data_a_scalar[idx + 1])); - } else if (idx_m < p.M && block + row * 2 < end_k) { - store_a(col, row, FLOAT_TYPEV2(data_a_scalar[idx], 0.0f)); - } else { - store_a(col, row, FLOAT_TYPEV2(0.0f)); - } -#elif defined(DATA_A_BF16) -#if LOAD_VEC_A == 4 - if (ALIGNED != 0) { - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - const uint k_pair = row * LOAD_VEC_A / 2; - FLOAT_TYPEV4 aa = FLOAT_TYPEV4(TO_FLOAT_TYPE(data_a[idx])); - store_a(col, k_pair, aa.xy); - store_a(col, k_pair + 1, aa.zw); - return; - } -#endif - const uint idx = pos_a + col * p.stride_a + row * 2; - if (idx_m < p.M && block + row * 2 + 1 < end_k) { - store_a(col, row, FLOAT_TYPEV2(TO_FLOAT_TYPE(data_a_scalar[idx]), - TO_FLOAT_TYPE(data_a_scalar[idx + 1]))); - } else if (idx_m < p.M && block + row * 2 < end_k) { - store_a(col, row, FLOAT_TYPEV2(TO_FLOAT_TYPE(data_a_scalar[idx]), 0.0f)); - } else { - store_a(col, row, FLOAT_TYPEV2(0.0f)); - } -#elif defined(DATA_A_IQ1_S) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - const uint k_pair = row * LOAD_VEC_A / 2; - - const uint ib = idx / 32; - const uint ib32 = (idx % 32) / 4; - const uint ib8 = idx % 32; - - const float d = float(data_a[ib].d); - const uint qh = data_a[ib].qh[ib32]; - const uint qs = data_a[ib].qs[ib8]; - const float dl = d * (2 * bitfieldExtract(qh, 12, 3) + 1); - const float delta = ((qh & 0x8000) != 0) ? -IQ1S_DELTA : IQ1S_DELTA; - const int16_t grid = int16_t(iq1s_grid[qs | (bitfieldExtract(qh, 3 * int(ib8 & 3), 3) << 8)]); - - [[unroll]] for (int k = 0; k < 4; ++k) { - store_a(col, k_pair + k, FLOAT_TYPEV2(dl * (bitfieldExtract(grid, 4 * k , 2) + delta), - dl * (bitfieldExtract(grid, 4 * k + 2, 2) + delta))); - - } -#elif defined(DATA_A_IQ1_M) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - const uint k_pair = row * LOAD_VEC_A / 2; - - const uint ib = idx / 32; - const uint ib8 = idx % 32; - const uint ib16 = ib8 / 2; - - const uint16_t[4] scales = data_a[ib].scales; - const u16vec4 s = u16vec4(scales[0], scales[1], scales[2], scales[3]) >> 12; - const float d = float(unpackHalf2x16(s.x | (s.y << 4) | (s.z << 8) | (s.w << 12)).x); - const uint sc = scales[ib8 / 8]; - const uint qs = data_a[ib].qs[ib8]; - const uint qh = data_a[ib].qh[ib16] >> (4 * (ib8 & 1)); - const float dl = d * (2 * bitfieldExtract(sc, 3 * int(ib16 & 3), 3) + 1); - const float delta = ((qh & 8) != 0) ? -IQ1M_DELTA : IQ1M_DELTA; - const int16_t grid = int16_t(iq1s_grid[qs | ((qh & 7) << 8)]); - - [[unroll]] for (int k = 0; k < 4; ++k) { - store_a(col, k_pair + k, FLOAT_TYPEV2(dl * (bitfieldExtract(grid, 4 * k , 2) + delta), - dl * (bitfieldExtract(grid, 4 * k + 2, 2) + delta))); - - } -#elif defined(DATA_A_IQ2_XXS) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - const uint k_pair = row * LOAD_VEC_A / 2; - - const uint ib = idx / 32; - const uint ib32 = (idx % 32) / 4; - const uint ib8 = idx % 4; - - const float d = float(data_a[ib].d); - const uint qs = data_a[ib].qs[8 * ib32 + ib8]; - const uint signs = pack32(u8vec4( - data_a[ib].qs[8*ib32 + 4], - data_a[ib].qs[8*ib32 + 5], - data_a[ib].qs[8*ib32 + 6], - data_a[ib].qs[8*ib32 + 7] - )); - const FLOAT_TYPE db = FLOAT_TYPE(d * 0.25 * (0.5 + (signs >> 28))); - const uint32_t sign7 = bitfieldExtract(signs, 7 * int(ib8), 7); - const uint sign = sign7 | (bitCount(sign7) << 7); - const uvec2 grid = iq2xxs_grid[qs]; - const vec4 grid0 = vec4(unpack8(grid.x)); - const vec4 grid1 = vec4(unpack8(grid.y)); - - store_a(col, k_pair, db * FLOAT_TYPEV2((sign & 1) != 0 ? -grid0.x : grid0.x, - (sign & 2) != 0 ? -grid0.y : grid0.y)); - - store_a(col, k_pair + 1, db * FLOAT_TYPEV2((sign & 4) != 0 ? -grid0.z : grid0.z, - (sign & 8) != 0 ? -grid0.w : grid0.w)); - - store_a(col, k_pair + 2, db * FLOAT_TYPEV2((sign & 16) != 0 ? -grid1.x : grid1.x, - (sign & 32) != 0 ? -grid1.y : grid1.y)); - - store_a(col, k_pair + 3, db * FLOAT_TYPEV2((sign & 64) != 0 ? -grid1.z : grid1.z, - (sign & 128) != 0 ? -grid1.w : grid1.w)); - -#elif defined(DATA_A_IQ2_XS) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - const uint k_pair = row * LOAD_VEC_A / 2; - - const uint ib = idx / 32; - const uint ib32 = (idx % 32) / 4; - const uint ib8 = idx % 4; - - const float d = float(data_a[ib].d); - const uint scale = (data_a[ib].scales[ib32] >> (2 * (ib8 & 2))) & 0xf; - const FLOAT_TYPE db = FLOAT_TYPE(d * 0.25 * (0.5 + scale)); - const uint qs = data_a[ib].qs[4 * ib32 + ib8]; - const uint sign7 = qs >> 9; - const uint sign = sign7 | (bitCount(sign7) << 7); - const uvec2 grid = iq2xs_grid[qs & 511]; - const vec4 grid0 = vec4(unpack8(grid.x)); - const vec4 grid1 = vec4(unpack8(grid.y)); - - store_a(col, k_pair, db * FLOAT_TYPEV2((sign & 1) != 0 ? -grid0.x : grid0.x, - (sign & 2) != 0 ? -grid0.y : grid0.y)); - - store_a(col, k_pair + 1, db * FLOAT_TYPEV2((sign & 4) != 0 ? -grid0.z : grid0.z, - (sign & 8) != 0 ? -grid0.w : grid0.w)); - - store_a(col, k_pair + 2, db * FLOAT_TYPEV2((sign & 16) != 0 ? -grid1.x : grid1.x, - (sign & 32) != 0 ? -grid1.y : grid1.y)); - - store_a(col, k_pair + 3, db * FLOAT_TYPEV2((sign & 64) != 0 ? -grid1.z : grid1.z, - (sign & 128) != 0 ? -grid1.w : grid1.w)); - -#elif defined(DATA_A_IQ2_S) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - const uint k_pair = row * LOAD_VEC_A / 2; - - const uint ib = idx / 32; - const uint ib8 = idx % 32; - const uint ib32 = ib8 / 4; - - const uint scale = (data_a[ib].scales[ib32] >> (2 * (ib8 & 2))) & 0xf; - const uint qs = data_a[ib].qs[ib8]; - const uint qh = data_a[ib].qh[ib32]; - const uint qhshift = 2 * (ib8 % 4); - const uint sign = data_a[ib].qs[QUANT_K_IQ2_S / 8 + ib8]; - - const float d = float(data_a[ib].d); - const FLOAT_TYPE db = FLOAT_TYPE(d * 0.25 * (0.5 + scale)); - const uvec2 grid = iq2s_grid[qs | ((qh << (8 - qhshift)) & 0x300)]; - const vec4 grid0 = vec4(unpack8(grid.x)); - const vec4 grid1 = vec4(unpack8(grid.y)); - - store_a(col, k_pair, db * FLOAT_TYPEV2((sign & 1) != 0 ? -grid0.x : grid0.x, - (sign & 2) != 0 ? -grid0.y : grid0.y)); - - store_a(col, k_pair + 1, db * FLOAT_TYPEV2((sign & 4) != 0 ? -grid0.z : grid0.z, - (sign & 8) != 0 ? -grid0.w : grid0.w)); - - store_a(col, k_pair + 2, db * FLOAT_TYPEV2((sign & 16) != 0 ? -grid1.x : grid1.x, - (sign & 32) != 0 ? -grid1.y : grid1.y)); - - store_a(col, k_pair + 3, db * FLOAT_TYPEV2((sign & 64) != 0 ? -grid1.z : grid1.z, - (sign & 128) != 0 ? -grid1.w : grid1.w)); - -#elif defined(DATA_A_IQ3_XXS) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - const uint k_pair = row * LOAD_VEC_A / 2; - - const uint ib = idx / 64; - const uint iqs = idx % 64; - const uint is = QUANT_K_IQ3_XXS / 4 + 4 * (iqs / 8); - - const float d = float(data_a[ib].d); - const uint qs = data_a[ib].qs[iqs]; - const uint signs = pack32(u16vec2( - data_a_packed16[ib].qs[is/2], - data_a_packed16[ib].qs[is/2+1] - )); - const float db = d * 0.5 * (0.5 + (signs >> 28)); - const uint32_t sign7 = bitfieldExtract(signs, 7 * (int(iqs / 2) % 4), 7); - const uint sign = (sign7 | (bitCount(sign7) << 7)) >> (4 * (idx % 2)); - const uint grid = iq3xxs_grid[qs]; - const vec4 v = db * vec4(unpack8(grid)); - - store_a(col, k_pair, FLOAT_TYPEV2((sign & 1) != 0 ? -v.x : v.x, - (sign & 2) != 0 ? -v.y : v.y)); - - store_a(col, k_pair + 1, FLOAT_TYPEV2((sign & 4) != 0 ? -v.z : v.z, - (sign & 8) != 0 ? -v.w : v.w)); - -#elif defined(DATA_A_IQ3_S) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - const uint k_pair = row * LOAD_VEC_A / 2; - - const uint ib = idx / 64; - const uint iqs = idx % 64; - const uint iqh = iqs / 8; - - const float d = float(data_a[ib].d); - const uint qs = data_a[ib].qs[iqs]; - const uint qh = data_a[ib].qh[iqh]; - const int8_t sign = int8_t(data_a[ib].signs[iqs / 2] >> (4 * (idx % 2))); - const uint scale = data_a[ib].scales[iqs / 16]; - const i8vec2 sign01 = i8vec2(1 - (2 & i8vec2(sign << 1, sign))); - const float db = d * (1 + 2 * ((scale >> (4 * (iqh & 1))) & 0xf)); - const uint32_t grid = iq3s_grid[qs | ((qh << (8 - (iqs % 8))) & 256)]; - const vec4 v = db * vec4(unpack8(grid)); +#if !defined(MUL_MAT_ID) +uint b_shmem_index(uint n, uint k_pair) { + return n * SHMEM_STRIDE + k_pair; +} - store_a(col, k_pair, FLOAT_TYPEV2((sign & 1) != 0 ? -v.x : v.x, - (sign & 2) != 0 ? -v.y : v.y)); +uint b_shmem_stride() { + return SHMEM_STRIDE; +} - store_a(col, k_pair + 1, FLOAT_TYPEV2((sign & 4) != 0 ? -v.z : v.z, - (sign & 8) != 0 ? -v.w : v.w)); +void store_b(uint n, uint k_pair, FLOAT_TYPEV2 value) { + buf_b[b_shmem_index(n, k_pair)] = value; +} +#endif +bool mm_can_stage_a() { +#if (defined(DATA_A_F32) || defined(DATA_A_F16)) && (LOAD_VEC_A == 8 || LOAD_VEC_A == 4) + return ALIGNED != 0; #elif defined(DATA_A_IQ4_XS) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - const uint k_pair = row * LOAD_VEC_A / 4; - - const uint ib = idx / 32; - const uint ib32 = (idx % 32) / 4; - const uint iq = 4 * ib32 + (idx % 4); - - const uint sl = (data_a[ib].scales_l[ib32/2] >> (4 * (ib32 & 1))) & 0xF; - const uint sh = ((data_a[ib].scales_h) >> (2 * ib32)) & 3; - const float d = float(data_a[ib].d); - const float dl = d * float(int(sl | (sh << 4)) - 32); - const uint vui = uint(data_a_packed32[ib].qs[iq]); - - const u8vec4 qs0 = unpack8( vui & 0x0F0F0F0F); - const u8vec4 qs1 = unpack8((vui >> 4) & 0x0F0F0F0F); - const vec4 v0 = dl * vec4(kvalues_iq4nl[qs0.x], kvalues_iq4nl[qs0.y], kvalues_iq4nl[qs0.z], kvalues_iq4nl[qs0.w]); - const vec4 v1 = dl * vec4(kvalues_iq4nl[qs1.x], kvalues_iq4nl[qs1.y], kvalues_iq4nl[qs1.z], kvalues_iq4nl[qs1.w]); - - store_a(col, k_pair, FLOAT_TYPEV2(v0.xy)); - store_a(col, k_pair + 1, FLOAT_TYPEV2(v0.zw)); - store_a(col, k_pair + 8, FLOAT_TYPEV2(v1.xy)); - store_a(col, k_pair + 9, FLOAT_TYPEV2(v1.zw)); - -#elif defined(DATA_A_IQ4_NL) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - const uint k_pair = row * LOAD_VEC_A / 4; - - const uint ib = idx / 8; - const uint iqs = idx & 0x07; - - const FLOAT_TYPE d = FLOAT_TYPE(data_a_packed16[ib].d); - const uint vui = uint(data_a_packed16[ib].qs[iqs]); - - store_a(col, k_pair, d * FLOAT_TYPEV2(kvalues_iq4nl[vui & 0xF], - kvalues_iq4nl[bitfieldExtract(vui, 8, 4)])); - - store_a(col, k_pair + 8, d * FLOAT_TYPEV2(kvalues_iq4nl[bitfieldExtract(vui, 4, 4)], - kvalues_iq4nl[vui >> 12])); - -#elif defined(DATA_A_MXFP4) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - const uint k_pair = row * LOAD_VEC_A / 4; - - const uint ib = idx / 8; - const uint iqs = (idx & 0x07) * 2; - - const uint vui = uint(data_a[ib].qs[iqs]); - const uint vui2 = uint(data_a[ib].qs[iqs+1]); - -#ifdef USE_OCP_FP4 - const float d = e8m0_to_fp32(data_a[ib].e); - const u8vec2 packed = u8vec2(vui, vui2); - store_a(col, k_pair, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 0u)) * FLOAT_TYPE(d)); - store_a(col, k_pair + 8, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 4u)) * FLOAT_TYPE(d)); + return true; +#elif defined(MULMAT_QUANT) + return MmTypeA == GGML_TYPE_Q8_0 || MmTypeA == GGML_TYPE_Q4_K || MmTypeA == GGML_TYPE_Q5_K || MmTypeA == GGML_TYPE_Q6_K; #else - const float d = e8m0_to_fp32(data_a[ib].e) * 0.5; - store_a(col, k_pair, FLOAT_TYPEV2(kvalues_mxfp4[vui & 0xF] * d, - kvalues_mxfp4[vui2 & 0xF] * d)); - - store_a(col, k_pair + 8, FLOAT_TYPEV2(kvalues_mxfp4[vui >> 4] * d, - kvalues_mxfp4[vui2 >> 4] * d)); - + return false; #endif -#elif defined(DATA_A_NVFP4) - const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; - const uint eff_row = (row & 3) + (row & ~3) * 2; - - const uint ib = idx / 16u; - const uint sub = (idx & 0xC) >> 2; - const uint iqs = (idx & 0xF) * 2; - const uint vui = uint(data_a[ib].qs[iqs]); - const uint vui2 = uint(data_a[ib].qs[iqs+1]); +} -#ifdef USE_OCP_FP4 - const FLOAT_TYPE d = FLOAT_TYPE(ue4m3_from_bits(data_a[ib].d[sub])); - const u8vec2 packed = u8vec2(vui, vui2); - store_a(col, eff_row, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 0u)) * d); - store_a(col, eff_row + 4, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 4u)) * d); -#else - const float d = ue4m3_to_fp32(data_a[ib].d[sub]) * 0.5; - store_a(col, eff_row, FLOAT_TYPEV2(kvalues_mxfp4[vui & 0xF] * d, - kvalues_mxfp4[vui2 & 0xF] * d)); - store_a(col, eff_row + 4, FLOAT_TYPEV2(kvalues_mxfp4[vui >> 4] * d, - kvalues_mxfp4[vui2 >> 4] * d)); +void commit_stage_a(const uint col_sh, const uint row, const FLOAT_TYPEV2 stage_v[8]) { +#if (defined(DATA_A_F32) || defined(DATA_A_F16)) && LOAD_VEC_A == 8 + const uint kp = row * 4u; + store_a(col_sh, kp + 0u, stage_v[0]); + store_a(col_sh, kp + 1u, stage_v[1]); + store_a(col_sh, kp + 2u, stage_v[2]); + store_a(col_sh, kp + 3u, stage_v[3]); +#elif (defined(DATA_A_F32) || defined(DATA_A_F16)) && LOAD_VEC_A == 4 + const uint kp = row * 2u; + store_a(col_sh, kp + 0u, stage_v[0]); + store_a(col_sh, kp + 1u, stage_v[1]); +#elif defined(DATA_A_IQ4_XS) + const uint kp = row * 2u; + store_a(col_sh, kp + 0u, stage_v[0]); + store_a(col_sh, kp + 1u, stage_v[1]); + store_a(col_sh, kp + 8u, stage_v[2]); + store_a(col_sh, kp + 9u, stage_v[3]); +#elif defined(MULMAT_QUANT) + const uint kp = row * 8u; + store_a(col_sh, kp + 0u, stage_v[0]); + store_a(col_sh, kp + 1u, stage_v[1]); + store_a(col_sh, kp + 2u, stage_v[2]); + store_a(col_sh, kp + 3u, stage_v[3]); + store_a(col_sh, kp + 4u, stage_v[4]); + store_a(col_sh, kp + 5u, stage_v[5]); + store_a(col_sh, kp + 6u, stage_v[6]); + store_a(col_sh, kp + 7u, stage_v[7]); #endif -#else - if (MmTypeA == GGML_TYPE_Q4_0) { - const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; - const uint k_pair = row * mm_load_vec_a() / 4; - - const uint ib = idx / 4; - const uint iqs = idx & 0x03; - - const float d = float(a_q4_0_p16.data[ib].d); - const uint vui = uint(a_q4_0_p16.data[ib].qs[2*iqs]) | (uint(a_q4_0_p16.data[ib].qs[2*iqs + 1]) << 16); - const vec4 v0 = (vec4(unpack8(vui & 0x0F0F0F0F)) - 8.0f) * d; - const vec4 v1 = (vec4(unpack8((vui >> 4) & 0x0F0F0F0F)) - 8.0f) * d; - - store_a(col, k_pair, FLOAT_TYPEV2(v0.xy)); - store_a(col, k_pair + 1, FLOAT_TYPEV2(v0.zw)); - store_a(col, k_pair + 8, FLOAT_TYPEV2(v1.xy)); - store_a(col, k_pair + 9, FLOAT_TYPEV2(v1.zw)); - } else if (MmTypeA == GGML_TYPE_Q4_1) { - const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; - const uint k_pair = row * mm_load_vec_a() / 4; - - const uint ib = idx / 4; - const uint iqs = idx & 0x03; - - const vec2 dm = vec2(a_q4_1_p32.data[ib].dm); - const uint vui = a_q4_1_p32.data[ib].qs[iqs]; - const vec4 v0 = vec4(unpack8(vui & 0x0F0F0F0F)) * dm.x + dm.y; - const vec4 v1 = vec4(unpack8((vui >> 4) & 0x0F0F0F0F)) * dm.x + dm.y; - - store_a(col, k_pair, FLOAT_TYPEV2(v0.xy)); - store_a(col, k_pair + 1, FLOAT_TYPEV2(v0.zw)); - store_a(col, k_pair + 8, FLOAT_TYPEV2(v1.xy)); - store_a(col, k_pair + 9, FLOAT_TYPEV2(v1.zw)); - } else if (MmTypeA == GGML_TYPE_Q5_0) { - const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; - const uint k_pair = row * mm_load_vec_a() / 4; - - const uint ib = idx / 8; - const uint iqs = idx & 0x07; - - const float d = float(a_q5_0_p16.data[ib].d); - const uint uint_qh = uint(a_q5_0_p16.data[ib].qh[1]) << 16 | uint(a_q5_0_p16.data[ib].qh[0]); - const ivec2 qh0 = ivec2(((uint_qh >> 2*iqs) << 4) & 0x10, (uint_qh >> (2*iqs + 12)) & 0x10); - const ivec2 qh1 = ivec2(((uint_qh >> (2*iqs + 1)) << 4) & 0x10, (uint_qh >> (2*iqs + 13)) & 0x10); - - const uint vui = uint(a_q5_0_p16.data[ib].qs[iqs]); - const vec4 v = (vec4((vui & 0xF) | qh0.x, ((vui >> 4) & 0xF) | qh0.y, ((vui >> 8) & 0xF) | qh1.x, (vui >> 12) | qh1.y) - 16.0f) * d; - - store_a(col, k_pair, FLOAT_TYPEV2(v.xz)); - store_a(col, k_pair + 8, FLOAT_TYPEV2(v.yw)); - } else if (MmTypeA == GGML_TYPE_Q5_1) { - const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; - const uint k_pair = row * mm_load_vec_a() / 4; - - const uint ib = idx / 4; - const uint iqs = idx & 0x03; - - const vec2 dm = vec2(a_q5_1_p32.data[ib].dm); - const uint uint_qh = a_q5_1_p32.data[ib].qh; - const uvec2 qh0 = uvec2(((uint_qh >> 4*iqs) << 4) & 0x10, (uint_qh >> (4*iqs + 12)) & 0x10); - const uvec2 qh1 = uvec2(((uint_qh >> (4*iqs + 1)) << 4) & 0x10, (uint_qh >> (4*iqs + 13)) & 0x10); - const uvec2 qh2 = uvec2(((uint_qh >> (4*iqs + 2)) << 4) & 0x10, (uint_qh >> (4*iqs + 14)) & 0x10); - const uvec2 qh3 = uvec2(((uint_qh >> (4*iqs + 3)) << 4) & 0x10, (uint_qh >> (4*iqs + 15)) & 0x10); - - const uint vui = a_q5_1_p32.data[ib].qs[iqs]; - const vec4 v0 = vec4((vui & 0xF) | qh0.x, ((vui >> 4) & 0xF) | qh0.y, ((vui >> 8) & 0xF) | qh1.x, ((vui >> 12) & 0xF) | qh1.y) * dm.x + dm.y; - const vec4 v1 = vec4(((vui >> 16) & 0xF) | qh2.x, ((vui >> 20) & 0xF) | qh2.y, ((vui >> 24) & 0xF) | qh3.x, ((vui >> 28) & 0xF) | qh3.y) * dm.x + dm.y; - - store_a(col, k_pair, FLOAT_TYPEV2(v0.xz)); - store_a(col, k_pair + 1, FLOAT_TYPEV2(v1.xz)); - store_a(col, k_pair + 8, FLOAT_TYPEV2(v0.yw)); - store_a(col, k_pair + 9, FLOAT_TYPEV2(v1.yw)); - } else if (MmTypeA == GGML_TYPE_Q8_0) { - const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; - const uint k_pair = row * mm_load_vec_a() / 2; - - const uint ib = idx / 8; - const uint iqs = idx & 0x07; - - const float d = float(a_q8_0_p16.data[ib].d); - const i8vec2 v0 = unpack8(int32_t(a_q8_0_p16.data[ib].qs[2*iqs])).xy; // vec4 used due to #12147 - const i8vec2 v1 = unpack8(int32_t(a_q8_0_p16.data[ib].qs[2*iqs + 1])).xy; - const vec4 v = vec4(v0.x, v0.y, v1.x, v1.y) * d; - - store_a(col, k_pair, FLOAT_TYPEV2(v.xy)); - store_a(col, k_pair + 1, FLOAT_TYPEV2(v.zw)); - } else if (MmTypeA == GGML_TYPE_Q1_0) { - const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; - const uint k_pair = row * mm_load_vec_a() / 2; - - const uint ib = idx / 16; - const uint iqs = idx & 0xfu; - - const float d = float(a_q1_0.data[ib].d); - const uint bits = uint(a_q1_0.data[ib].qs[iqs]); - - store_a(col, k_pair, FLOAT_TYPEV2((bits & 0x01u) != 0u ? d : -d, (bits & 0x02u) != 0u ? d : -d)); - store_a(col, k_pair + 1, FLOAT_TYPEV2((bits & 0x04u) != 0u ? d : -d, (bits & 0x08u) != 0u ? d : -d)); - store_a(col, k_pair + 2, FLOAT_TYPEV2((bits & 0x10u) != 0u ? d : -d, (bits & 0x20u) != 0u ? d : -d)); - store_a(col, k_pair + 3, FLOAT_TYPEV2((bits & 0x40u) != 0u ? d : -d, (bits & 0x80u) != 0u ? d : -d)); - } else if (MmTypeA == GGML_TYPE_Q2_0) { - const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; - const uint k_pair = row * mm_load_vec_a() / 2; - - const uint ib = idx / 16; - const uint iqs = idx & 0xfu; - - const FLOAT_TYPE d = FLOAT_TYPE(a_q2_0.data[ib].d); - const uint bits = uint(a_q2_0.data[ib].qs[iqs]); - - store_a(col, k_pair, d * (FLOAT_TYPEV2(bits & 3u, (bits >> 2u) & 3u) - FLOAT_TYPEV2(1.0f))); - store_a(col, k_pair + 1, d * (FLOAT_TYPEV2((bits >> 4u) & 3u, bits >> 6u) - FLOAT_TYPEV2(1.0f))); - } else if (MmTypeA == GGML_TYPE_Q2_K) { - const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; - const uint k_pair = row * mm_load_vec_a() / 2; - - const uint ib = idx / 64; // 4 values per idx - const uint iqs = (idx % 64) * 2; // 0,2,4..126 - - const uint qsi = (iqs / 64) * 16 + (iqs % 16); // 0..15 - const uint scalesi = iqs / 8; // 0..15 - const uint qsshift = ((iqs % 64) / 16) * 2; // 0,2,4,6 - - const vec4 qs = vec4(unpack8((a_q2_k_p32.data[ib].qs[qsi / 2] >> qsshift) & 0x03030303)); - const uint scales = a_q2_k.data[ib].scales[scalesi]; - const vec2 dm = vec2(a_q2_k.data[ib].dm); - - const vec4 v = dm.x * float(scales & 0xF) * qs - dm.y * float(scales >> 4); - - store_a(col, k_pair, FLOAT_TYPEV2(v.xy)); - store_a(col, k_pair + 1, FLOAT_TYPEV2(v.zw)); - } else if (MmTypeA == GGML_TYPE_Q3_K) { - const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; - const uint k_pair = row * mm_load_vec_a() / 2; - - const uint ib = idx / 128; // 2 values per idx - const uint iqs = idx % 128; // 0..127 - - const uint n = iqs / 64; // 0,1 - const uint qsi = n * 32 + (iqs % 16) * 2; // 0,2,4..62 - const uint hmi = (iqs % 16) * 2; // 0,2,4..30 - const uint j = (iqs % 64) / 4; // 0..3 - const uint is = iqs / 8; // 0..15 - const uint halfsplit = ((iqs % 64) / 16); // 0,1,2,3 - const uint qsshift = halfsplit * 2; // 0,2,4,6 - - const int8_t us = int8_t(((a_q3_k.data[ib].scales[is % 8] >> (4 * int(is / 8))) & 0xF) - | (((a_q3_k.data[ib].scales[8 + (is % 4)] >> (2 * int(is / 4))) & 3) << 4)); - const float dl = float(a_q3_k.data[ib].d) * float(us - 32); - - const vec2 qs = vec2(unpack8((uint(a_q3_k_p16.data[ib].qs[qsi / 2]) >> qsshift) & 0x0303).xy); - const vec2 hm = vec2(unpack8(((uint(a_q3_k_p16.data[ib].hmask[hmi / 2]) >> (4 * n + halfsplit)) & 0x0101 ^ 0x0101) << 2).xy); - - store_a(col, k_pair, FLOAT_TYPEV2(dl * (qs.x - hm.x), - dl * (qs.y - hm.y))); - - } else if (MmTypeA == GGML_TYPE_Q4_K) { - const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; - const uint k_pair = row * mm_load_vec_a() / 2; - - const uint ib = idx / 64; // 4 values per idx - const uint iqs = (idx % 64) * 2; // 0,2,4..126 - - const uint n = iqs / 32; // 0,1,2,3 - const uint b = (iqs % 32) / 16; // 0,1 - const uint is = 2 * n + b; // 0..7 - const uint qsi = n * 32 + (iqs % 16) * 2; // 0,2,4..126 - - const vec2 loadd = vec2(a_q4_k.data[ib].dm); - - const uvec3 scales = uvec3(a_q4_k_p32.data[ib].scales[0], - a_q4_k_p32.data[ib].scales[1], - a_q4_k_p32.data[ib].scales[2]); - const uint scalesoffs = (is & 3) * 8; - - const uint scidx0 = (is < 4) ? 0 : 2; - const uint scidxshift0 = scalesoffs; - const uint scidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2; - const uint mbidx0 = (is < 4) ? 1 : 2; - const uint mbidxshift0 = (is < 4) ? scalesoffs : scalesoffs + 4; - const uint mbidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2; - - const uint8_t sc = uint8_t(((scales[scidx0] >> scidxshift0) & 0xF) | ((scales[0] >> scidxshift1) & 0x30)); - const uint8_t mbyte = uint8_t(((scales[mbidx0] >> mbidxshift0) & 0xF) | ((scales[1] >> mbidxshift1) & 0x30)); - - const float d = loadd.x * sc; - const float m = -loadd.y * mbyte; - - const vec4 q = vec4(unpack8((a_q4_k_p32.data[ib].qs[qsi / 4] >> (b * 4)) & 0x0F0F0F0F)); - - store_a(col, k_pair, FLOAT_TYPEV2(fma(d, q.x, m), fma(d, q.y, m))); - store_a(col, k_pair + 1, FLOAT_TYPEV2(fma(d, q.z, m), fma(d, q.w, m))); - } else if (MmTypeA == GGML_TYPE_Q5_K) { - const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; - const uint k_pair = row * mm_load_vec_a() / 2; - - const uint ib = idx / 64; // 4 values per idx - const uint iqs = (idx % 64) * 2; // 0,2,4..126 - - const uint n = iqs / 32; // 0,1,2,3 - const uint b = (iqs % 32) / 16; // 0,1 - const uint is = 2 * n + b; // 0..7 - const uint qsi = n * 32 + (iqs % 16) * 2; // 0,2,4..126 - const uint qhi = (iqs % 16) * 2; // 0,2,4..30 - - const vec2 loadd = vec2(a_q5_k.data[ib].dm); - - const uvec3 scales = uvec3(a_q5_k_p32.data[ib].scales[0], - a_q5_k_p32.data[ib].scales[1], - a_q5_k_p32.data[ib].scales[2]); - const uint scalesoffs = (is & 3) * 8; - - const uint scidx0 = (is < 4) ? 0 : 2; - const uint scidxshift0 = scalesoffs; - const uint scidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2; - const uint mbidx0 = (is < 4) ? 1 : 2; - const uint mbidxshift0 = (is < 4) ? scalesoffs : scalesoffs + 4; - const uint mbidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2; - - const uint8_t sc = uint8_t(((scales[scidx0] >> scidxshift0) & 0xF) | ((scales[0] >> scidxshift1) & 0x30)); - const uint8_t mbyte = uint8_t(((scales[mbidx0] >> mbidxshift0) & 0xF) | ((scales[1] >> mbidxshift1) & 0x30)); - - const float d = loadd.x * sc; - const float m = -loadd.y * mbyte; - - const uint qs = (a_q5_k_p32.data[ib].qs[qsi / 4] >> (b * 4)) & 0x0F0F0F0F; - const uint qh = ((a_q5_k_p32.data[ib].qh[qhi / 4] >> (iqs / 16)) & 0x01010101) << 4; - const vec4 q = vec4(unpack8(qs | qh)); - - store_a(col, k_pair, FLOAT_TYPEV2(fma(d, q.x, m), fma(d, q.y, m))); - store_a(col, k_pair + 1, FLOAT_TYPEV2(fma(d, q.z, m), fma(d, q.w, m))); - } else if (MmTypeA == GGML_TYPE_Q6_K) { - const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; - const uint k_pair = row * mm_load_vec_a() / 2; - - const uint ib = idx / 128; // 2 values per idx - const uint iqs = idx % 128; // 0..127 - - const uint n = iqs / 64; // 0,1 - const uint b = ((iqs % 64) / 32) * 4; // 0,4 - const uint is_b = (iqs % 16) / 8; // 0,1 - const uint qhshift = ((iqs % 64) / 16) * 2; // 0,2,4,6 - const uint is = 8 * n + qhshift + is_b; // 0..15 - const uint qsi = n * 32 + (iqs % 32); // 0..63 - const uint qhi = n * 16 + (iqs % 16); // 0..31 - - const float dscale = float(a_q6_k.data[ib].d) * float(a_q6_k.data[ib].scales[is]); - - const uint ql = (uint(a_q6_k_p16.data[ib].ql[qsi]) >> b) & 0x0F0F; - const uint qh = (uint(a_q6_k_p16.data[ib].qh[qhi]) >> qhshift) & 0x0303; - const vec2 q = (vec2(unpack8(ql | (qh << 4)).xy) - 32) * dscale; - - store_a(col, k_pair, FLOAT_TYPEV2(q.x, q.y)); - } else if (MmTypeA == GGML_TYPE_TQ1_0) { - const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; - - const uint ib = idx / 128; // 2 values per idx - const uint iqs = (idx % 128) * 2; // elem 0,2,4..254 - - const float d = float(a_tq1_0.data[ib].d); - vec2 v; - for (uint kk = 0u; kk < 2u; ++kk) { - const uint e = iqs + kk; - const uint bidx = tq1_0_byte_of(e); - const uint qbyte = uint(bidx < 48u ? a_tq1_0.data[ib].qs[bidx] - : a_tq1_0.data[ib].qh[bidx - 48u]); - v[kk] = d * (float(tq1_0_trit(qbyte, tq1_0_digit_of(e))) - 1.0); - } - - const uint k_pair = row * mm_load_vec_a() / 2; - store_a(col, k_pair, FLOAT_TYPEV2(v.xy)); - } else if (MmTypeA == GGML_TYPE_TQ2_0) { - const uint idx = pos_a + col * p.stride_a / mm_load_vec_a() + row; +} - const uint ib = idx / 128; // 2 values per idx - const uint iqs = (idx % 128) * 2; // elem 0,2,4..254 +#define MM_STORE_A(slot, m, kp, val) store_a(m, kp, val) + +void load_a_to_shmem(const uint pos_a, const uint row, const uint col_sh, const uint col_mem, const uint idx_m, const uint block, const uint end_k) { +#define MM_DATA_A data_a +#define MM_DATA_A_SCALAR data_a_scalar +#define MM_DATA_A_P16 data_a_packed16 +#define MM_DATA_A_P32 data_a_packed32 +#define MM_Q1_0 a_q1_0 +#define MM_Q2_0 a_q2_0 +#define MM_Q2_K a_q2_k +#define MM_Q3_K a_q3_k +#define MM_Q4_K a_q4_k +#define MM_Q5_K a_q5_k +#define MM_Q6_K a_q6_k +#define MM_TQ1_0 a_tq1_0 +#define MM_TQ2_0 a_tq2_0 +#define MM_Q4_0_P16 a_q4_0_p16 +#define MM_Q5_0_P16 a_q5_0_p16 +#define MM_Q8_0_P16 a_q8_0_p16 +#define MM_Q3_K_P16 a_q3_k_p16 +#define MM_Q6_K_P16 a_q6_k_p16 +#define MM_Q4_1_P32 a_q4_1_p32 +#define MM_Q5_1_P32 a_q5_1_p32 +#define MM_Q2_K_P32 a_q2_k_p32 +#define MM_Q4_K_P32 a_q4_k_p32 +#define MM_Q5_K_P32 a_q5_k_p32 +#include "mul_mm_load_a_body.glsl" +#undef MM_DATA_A +#undef MM_DATA_A_SCALAR +#undef MM_DATA_A_P16 +#undef MM_DATA_A_P32 +#undef MM_Q1_0 +#undef MM_Q2_0 +#undef MM_Q2_K +#undef MM_Q3_K +#undef MM_Q4_K +#undef MM_Q5_K +#undef MM_Q6_K +#undef MM_TQ1_0 +#undef MM_TQ2_0 +#undef MM_Q4_0_P16 +#undef MM_Q5_0_P16 +#undef MM_Q8_0_P16 +#undef MM_Q3_K_P16 +#undef MM_Q6_K_P16 +#undef MM_Q4_1_P32 +#undef MM_Q5_1_P32 +#undef MM_Q2_K_P32 +#undef MM_Q4_K_P32 +#undef MM_Q5_K_P32 +} - const uint qsi = (iqs / 128) * 32 + (iqs % 32); // byte pair start - const uint shift = 2 * ((iqs % 128) / 32); // 0,2,4,6 +#ifdef MM_FUSED +void load_u_to_shmem(const uint pos_a, const uint row, const uint col_sh, const uint col_mem, const uint idx_m, const uint block, const uint end_k) { +#define MM_DATA_A data_u +#define MM_DATA_A_SCALAR data_u_scalar +#define MM_DATA_A_P16 data_u_packed16 +#define MM_DATA_A_P32 data_u_packed32 +#define MM_Q1_0 u_q1_0 +#define MM_Q2_0 u_q2_0 +#define MM_Q2_K u_q2_k +#define MM_Q3_K u_q3_k +#define MM_Q4_K u_q4_k +#define MM_Q5_K u_q5_k +#define MM_Q6_K u_q6_k +#define MM_TQ1_0 u_tq1_0 +#define MM_TQ2_0 u_tq2_0 +#define MM_Q4_0_P16 u_q4_0_p16 +#define MM_Q5_0_P16 u_q5_0_p16 +#define MM_Q8_0_P16 u_q8_0_p16 +#define MM_Q3_K_P16 u_q3_k_p16 +#define MM_Q6_K_P16 u_q6_k_p16 +#define MM_Q4_1_P32 u_q4_1_p32 +#define MM_Q5_1_P32 u_q5_1_p32 +#define MM_Q2_K_P32 u_q2_k_p32 +#define MM_Q4_K_P32 u_q4_k_p32 +#define MM_Q5_K_P32 u_q5_k_p32 +#include "mul_mm_load_a_body.glsl" +#undef MM_DATA_A +#undef MM_DATA_A_SCALAR +#undef MM_DATA_A_P16 +#undef MM_DATA_A_P32 +#undef MM_Q1_0 +#undef MM_Q2_0 +#undef MM_Q2_K +#undef MM_Q3_K +#undef MM_Q4_K +#undef MM_Q5_K +#undef MM_Q6_K +#undef MM_TQ1_0 +#undef MM_TQ2_0 +#undef MM_Q4_0_P16 +#undef MM_Q5_0_P16 +#undef MM_Q8_0_P16 +#undef MM_Q3_K_P16 +#undef MM_Q6_K_P16 +#undef MM_Q4_1_P32 +#undef MM_Q5_1_P32 +#undef MM_Q2_K_P32 +#undef MM_Q4_K_P32 +#undef MM_Q5_K_P32 +} +#endif - const uvec2 qs = uvec2(a_tq2_0.data[ib].qs[qsi], a_tq2_0.data[ib].qs[qsi + 1]); - const float d = float(a_tq2_0.data[ib].d); +#undef MM_STORE_A + +#ifdef COOPMAT +#define MM_STORE_A(slot, m, kp, val) stage_v[slot] = (val) + +void stage_a_to_reg(const uint pos_a, const uint row, const uint col_sh, const uint col_mem, const uint idx_m, const uint block, const uint end_k, out FLOAT_TYPEV2 stage_v[8]) { + [[unroll]] for (uint s = 0u; s < 8u; ++s) { stage_v[s] = FLOAT_TYPEV2(0.0f); } + if (idx_m >= p.M) { return; } +#define MM_DATA_A data_a +#define MM_DATA_A_SCALAR data_a_scalar +#define MM_DATA_A_P16 data_a_packed16 +#define MM_DATA_A_P32 data_a_packed32 +#define MM_Q1_0 a_q1_0 +#define MM_Q2_0 a_q2_0 +#define MM_Q2_K a_q2_k +#define MM_Q3_K a_q3_k +#define MM_Q4_K a_q4_k +#define MM_Q5_K a_q5_k +#define MM_Q6_K a_q6_k +#define MM_TQ1_0 a_tq1_0 +#define MM_TQ2_0 a_tq2_0 +#define MM_Q4_0_P16 a_q4_0_p16 +#define MM_Q5_0_P16 a_q5_0_p16 +#define MM_Q8_0_P16 a_q8_0_p16 +#define MM_Q3_K_P16 a_q3_k_p16 +#define MM_Q6_K_P16 a_q6_k_p16 +#define MM_Q4_1_P32 a_q4_1_p32 +#define MM_Q5_1_P32 a_q5_1_p32 +#define MM_Q2_K_P32 a_q2_k_p32 +#define MM_Q4_K_P32 a_q4_k_p32 +#define MM_Q5_K_P32 a_q5_k_p32 +#include "mul_mm_load_a_body.glsl" +#undef MM_DATA_A +#undef MM_DATA_A_SCALAR +#undef MM_DATA_A_P16 +#undef MM_DATA_A_P32 +#undef MM_Q1_0 +#undef MM_Q2_0 +#undef MM_Q2_K +#undef MM_Q3_K +#undef MM_Q4_K +#undef MM_Q5_K +#undef MM_Q6_K +#undef MM_TQ1_0 +#undef MM_TQ2_0 +#undef MM_Q4_0_P16 +#undef MM_Q5_0_P16 +#undef MM_Q8_0_P16 +#undef MM_Q3_K_P16 +#undef MM_Q6_K_P16 +#undef MM_Q4_1_P32 +#undef MM_Q5_1_P32 +#undef MM_Q2_K_P32 +#undef MM_Q4_K_P32 +#undef MM_Q5_K_P32 +} - const vec2 v = d * (vec2((qs >> shift) & 3) - 1.0); +#ifdef MM_FUSED +void stage_u_to_reg(const uint pos_a, const uint row, const uint col_sh, const uint col_mem, const uint idx_m, const uint block, const uint end_k, out FLOAT_TYPEV2 stage_v[8]) { + [[unroll]] for (uint s = 0u; s < 8u; ++s) { stage_v[s] = FLOAT_TYPEV2(0.0f); } + if (idx_m >= p.M) { return; } +#define MM_DATA_A data_u +#define MM_DATA_A_SCALAR data_u_scalar +#define MM_DATA_A_P16 data_u_packed16 +#define MM_DATA_A_P32 data_u_packed32 +#define MM_Q1_0 u_q1_0 +#define MM_Q2_0 u_q2_0 +#define MM_Q2_K u_q2_k +#define MM_Q3_K u_q3_k +#define MM_Q4_K u_q4_k +#define MM_Q5_K u_q5_k +#define MM_Q6_K u_q6_k +#define MM_TQ1_0 u_tq1_0 +#define MM_TQ2_0 u_tq2_0 +#define MM_Q4_0_P16 u_q4_0_p16 +#define MM_Q5_0_P16 u_q5_0_p16 +#define MM_Q8_0_P16 u_q8_0_p16 +#define MM_Q3_K_P16 u_q3_k_p16 +#define MM_Q6_K_P16 u_q6_k_p16 +#define MM_Q4_1_P32 u_q4_1_p32 +#define MM_Q5_1_P32 u_q5_1_p32 +#define MM_Q2_K_P32 u_q2_k_p32 +#define MM_Q4_K_P32 u_q4_k_p32 +#define MM_Q5_K_P32 u_q5_k_p32 +#include "mul_mm_load_a_body.glsl" +#undef MM_DATA_A +#undef MM_DATA_A_SCALAR +#undef MM_DATA_A_P16 +#undef MM_DATA_A_P32 +#undef MM_Q1_0 +#undef MM_Q2_0 +#undef MM_Q2_K +#undef MM_Q3_K +#undef MM_Q4_K +#undef MM_Q5_K +#undef MM_Q6_K +#undef MM_TQ1_0 +#undef MM_TQ2_0 +#undef MM_Q4_0_P16 +#undef MM_Q5_0_P16 +#undef MM_Q8_0_P16 +#undef MM_Q3_K_P16 +#undef MM_Q6_K_P16 +#undef MM_Q4_1_P32 +#undef MM_Q5_1_P32 +#undef MM_Q2_K_P32 +#undef MM_Q4_K_P32 +#undef MM_Q5_K_P32 +} +#endif - const uint k_pair = row * mm_load_vec_a() / 2; - store_a(col, k_pair, FLOAT_TYPEV2(v.xy)); - } +#undef MM_STORE_A #endif -} #if !defined(MUL_MAT_ID) void load_b_to_shmem(const uint pos_b, const uint row, const uint col, const uint idx_n, const uint block, const uint end_k) { #if LOAD_VEC_B == 8 if (ALIGNED != 0) { - // Not supported for b_type bf16 because bf16mat2x4 does not exist - const uint idx = pos_b + col * p.stride_b / LOAD_VEC_B + row; - const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_B / 2; - FLOAT_TYPEV8 bb = FLOAT_TYPEV8(data_b[idx]); - buf_b[buf_idx + 0] = bb[0].xy; - buf_b[buf_idx + 1] = bb[0].zw; - buf_b[buf_idx + 2] = bb[1].xy; - buf_b[buf_idx + 3] = bb[1].zw; + const uint kp = row * (LOAD_VEC_B / 2); + if (idx_n < p.N) { + // Not supported for b_type bf16 because bf16mat2x4 does not exist + const uint idx = pos_b + col * p.stride_b / LOAD_VEC_B + row; + FLOAT_TYPEV8 bb = FLOAT_TYPEV8(data_b[idx]); + store_b(col, kp + 0, bb[0].xy); + store_b(col, kp + 1, bb[0].zw); + store_b(col, kp + 2, bb[1].xy); + store_b(col, kp + 3, bb[1].zw); + } else { + store_b(col, kp + 0, FLOAT_TYPEV2(0.0f)); + store_b(col, kp + 1, FLOAT_TYPEV2(0.0f)); + store_b(col, kp + 2, FLOAT_TYPEV2(0.0f)); + store_b(col, kp + 3, FLOAT_TYPEV2(0.0f)); + } return; } #elif LOAD_VEC_B == 4 if (ALIGNED != 0) { - const uint idx = pos_b + col * p.stride_b / LOAD_VEC_B + row; - const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_B / 2; + const uint kp = row * (LOAD_VEC_B / 2); + if (idx_n < p.N) { + const uint idx = pos_b + col * p.stride_b / LOAD_VEC_B + row; #if defined(DATA_B_BF16) - FLOAT_TYPEV4 bb = FLOAT_TYPEV4(TO_FLOAT_TYPE(data_b[idx])); + FLOAT_TYPEV4 bb = FLOAT_TYPEV4(TO_FLOAT_TYPE(data_b[idx])); #else - FLOAT_TYPEV4 bb = FLOAT_TYPEV4(data_b[idx]); + FLOAT_TYPEV4 bb = FLOAT_TYPEV4(data_b[idx]); #endif - buf_b[buf_idx + 0] = bb.xy; - buf_b[buf_idx + 1] = bb.zw; + store_b(col, kp + 0, bb.xy); + store_b(col, kp + 1, bb.zw); + } else { + store_b(col, kp + 0, FLOAT_TYPEV2(0.0f)); + store_b(col, kp + 1, FLOAT_TYPEV2(0.0f)); + } return; } #endif const uint idx = pos_b + col * p.stride_b + row * 2; - const uint buf_idx = col * SHMEM_STRIDE + row; if (idx_n < p.N && block + row * 2 + 1 < end_k) { - buf_b[buf_idx] = FLOAT_TYPEV2(TO_FLOAT_TYPE(data_b_scalar[idx]), - TO_FLOAT_TYPE(data_b_scalar[idx + 1])); + store_b(col, row, FLOAT_TYPEV2(TO_FLOAT_TYPE(data_b_scalar[idx]), + TO_FLOAT_TYPE(data_b_scalar[idx + 1]))); } else if (idx_n < p.N && block + row * 2 < end_k) { - buf_b[buf_idx] = FLOAT_TYPEV2(TO_FLOAT_TYPE(data_b_scalar[idx]), 0.0f); + store_b(col, row, FLOAT_TYPEV2(TO_FLOAT_TYPE(data_b_scalar[idx]), 0.0f)); } else { - buf_b[buf_idx] = FLOAT_TYPEV2(0.0f); + store_b(col, row, FLOAT_TYPEV2(0.0f)); } } #else @@ -721,3 +387,94 @@ void load_b_to_shmem(const uint pos_b, const uint row, const uint col, const uin } } #endif + +#ifdef COOPMAT +void stage_b_to_reg(const uint pos_b, const uint row, const uint col, const uint idx_n, out FLOAT_TYPEV2 stage_bv[4]) { +#if !defined(MUL_MAT_ID) + if (idx_n >= p.N) { + stage_bv[0] = FLOAT_TYPEV2(0.0f); + stage_bv[1] = FLOAT_TYPEV2(0.0f); + stage_bv[2] = FLOAT_TYPEV2(0.0f); + stage_bv[3] = FLOAT_TYPEV2(0.0f); + return; + } + const uint idx = pos_b + col * p.stride_b / LOAD_VEC_B + row; +#else + if (col >= _ne1) { + stage_bv[0] = FLOAT_TYPEV2(0.0f); + stage_bv[1] = FLOAT_TYPEV2(0.0f); + stage_bv[2] = FLOAT_TYPEV2(0.0f); + stage_bv[3] = FLOAT_TYPEV2(0.0f); + return; + } + const u16vec2 row_idx = row_ids[col]; + const uint idx = pos_b + row_idx.y * p.batch_stride_b / LOAD_VEC_B + (row_idx.x % p.ne11) * p.stride_b / LOAD_VEC_B + row; +#endif +#if LOAD_VEC_B == 8 + FLOAT_TYPEV8 bb = FLOAT_TYPEV8(data_b[idx]); + stage_bv[0] = bb[0].xy; + stage_bv[1] = bb[0].zw; + stage_bv[2] = bb[1].xy; + stage_bv[3] = bb[1].zw; +#elif LOAD_VEC_B == 4 +#if defined(DATA_B_BF16) + FLOAT_TYPEV4 bb = FLOAT_TYPEV4(TO_FLOAT_TYPE(data_b[idx])); +#else + FLOAT_TYPEV4 bb = FLOAT_TYPEV4(data_b[idx]); +#endif + stage_bv[0] = bb.xy; + stage_bv[1] = bb.zw; + stage_bv[2] = FLOAT_TYPEV2(0.0f); + stage_bv[3] = FLOAT_TYPEV2(0.0f); +#endif +} + +void commit_stage_b(const uint col, const uint row, const FLOAT_TYPEV2 stage_bv[4]) { +#if LOAD_VEC_B == 8 && defined(FLOAT16) && !defined(DATA_A_BF16) + if (AGX_DIRECT_STORE != 0) { + if (APPLY_SLM_A_RESHAPE) { + if (BK == 32) { + const uint odd_c = col & 1u; + const bool is_odd = odd_c != 0u; + const uint base_sh = row * ((BN << 2u) + 8u) + ((col >> 3u) << 5u) + ((col & 7u) >> 1u); + [[unroll]] for (uint q = 0u; q < 2u; ++q) { + const f16vec2 v0 = stage_bv[2u * q + 0u]; + const f16vec2 v1 = stage_bv[2u * q + 1u]; + const f16vec2 my_keep = is_odd ? f16vec2(v0.y, v1.y) : f16vec2(v0.x, v1.x); + const f16vec2 my_send = is_odd ? f16vec2(v0.x, v1.x) : f16vec2(v0.y, v1.y); + const f16vec2 partner = unpackFloat2x16(subgroupShuffleXor(packFloat2x16(my_send), 4u)); + buf_b[base_sh + (((4u * q + 0u) + odd_c) << 2u)] = is_odd ? f16vec2(partner.x, my_keep.x) : f16vec2(my_keep.x, partner.x); + buf_b[base_sh + (((4u * q + 2u) + odd_c) << 2u)] = is_odd ? f16vec2(partner.y, my_keep.y) : f16vec2(my_keep.y, partner.y); + } + return; + } + } + } +#endif +#if !defined(MUL_MAT_ID) +#if LOAD_VEC_B == 8 + const uint kp = row * 4u; + store_b(col, kp + 0u, stage_bv[0]); + store_b(col, kp + 1u, stage_bv[1]); + store_b(col, kp + 2u, stage_bv[2]); + store_b(col, kp + 3u, stage_bv[3]); +#elif LOAD_VEC_B == 4 + const uint kp = row * 2u; + store_b(col, kp + 0u, stage_bv[0]); + store_b(col, kp + 1u, stage_bv[1]); +#endif +#else +#if LOAD_VEC_B == 8 + const uint buf_idx = col * SHMEM_STRIDE + row * 4u; + buf_b[buf_idx + 0u] = stage_bv[0]; + buf_b[buf_idx + 1u] = stage_bv[1]; + buf_b[buf_idx + 2u] = stage_bv[2]; + buf_b[buf_idx + 3u] = stage_bv[3]; +#elif LOAD_VEC_B == 4 + const uint buf_idx = col * SHMEM_STRIDE + row * 2u; + buf_b[buf_idx + 0u] = stage_bv[0]; + buf_b[buf_idx + 1u] = stage_bv[1]; +#endif +#endif +} +#endif diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_load_a_body.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_load_a_body.glsl new file mode 100644 index 000000000000..550621237fb8 --- /dev/null +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_load_a_body.glsl @@ -0,0 +1,742 @@ +#if defined(DATA_A_F32) || defined(DATA_A_F16) +#if LOAD_VEC_A == 8 + if (ALIGNED != 0) { + const uint idx = pos_a + col_mem * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 2; + FLOAT_TYPEV8 aa = FLOAT_TYPEV8(MM_DATA_A[idx]); + MM_STORE_A(0, col_sh, k_pair, aa[0].xy); + MM_STORE_A(1, col_sh, k_pair + 1, aa[0].zw); + MM_STORE_A(2, col_sh, k_pair + 2, aa[1].xy); + MM_STORE_A(3, col_sh, k_pair + 3, aa[1].zw); + return; + } +#elif LOAD_VEC_A == 4 + if (ALIGNED != 0) { + const uint idx = pos_a + col_mem * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 2; + FLOAT_TYPEV4 aa = FLOAT_TYPEV4(MM_DATA_A[idx]); + MM_STORE_A(0, col_sh, k_pair, aa.xy); + MM_STORE_A(1, col_sh, k_pair + 1, aa.zw); + return; + } +#endif + const uint idx = pos_a + col_mem * p.stride_a + row * 2; + if (idx_m < p.M && block + row * 2 + 1 < end_k) { + store_a(col_sh, row, FLOAT_TYPEV2(MM_DATA_A_SCALAR[idx], + MM_DATA_A_SCALAR[idx + 1])); + } else if (idx_m < p.M && block + row * 2 < end_k) { + store_a(col_sh, row, FLOAT_TYPEV2(MM_DATA_A_SCALAR[idx], 0.0f)); + } else { + store_a(col_sh, row, FLOAT_TYPEV2(0.0f)); + } +#elif defined(DATA_A_BF16) +#if LOAD_VEC_A == 4 + if (ALIGNED != 0) { + const uint idx = pos_a + col_mem * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 2; + FLOAT_TYPEV4 aa = FLOAT_TYPEV4(TO_FLOAT_TYPE(MM_DATA_A[idx])); + store_a(col_sh, k_pair, aa.xy); + store_a(col_sh, k_pair + 1, aa.zw); + return; + } +#endif + const uint idx = pos_a + col_mem * p.stride_a + row * 2; + if (idx_m < p.M && block + row * 2 + 1 < end_k) { + store_a(col_sh, row, FLOAT_TYPEV2(TO_FLOAT_TYPE(MM_DATA_A_SCALAR[idx]), + TO_FLOAT_TYPE(MM_DATA_A_SCALAR[idx + 1]))); + } else if (idx_m < p.M && block + row * 2 < end_k) { + store_a(col_sh, row, FLOAT_TYPEV2(TO_FLOAT_TYPE(MM_DATA_A_SCALAR[idx]), 0.0f)); + } else { + store_a(col_sh, row, FLOAT_TYPEV2(0.0f)); + } +#elif defined(DATA_A_IQ1_S) + const uint idx = pos_a + col_mem * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 2; + + const uint ib = idx / 32; + const uint ib32 = (idx % 32) / 4; + const uint ib8 = idx % 32; + + const float d = float(MM_DATA_A[ib].d); + const uint qh = MM_DATA_A[ib].qh[ib32]; + const uint qs = MM_DATA_A[ib].qs[ib8]; + const float dl = d * (2 * bitfieldExtract(qh, 12, 3) + 1); + const float delta = ((qh & 0x8000) != 0) ? -IQ1S_DELTA : IQ1S_DELTA; + const int16_t grid = int16_t(iq1s_grid[qs | (bitfieldExtract(qh, 3 * int(ib8 & 3), 3) << 8)]); + + [[unroll]] for (int k = 0; k < 4; ++k) { + store_a(col_sh, k_pair + k, FLOAT_TYPEV2(dl * (bitfieldExtract(grid, 4 * k , 2) + delta), + dl * (bitfieldExtract(grid, 4 * k + 2, 2) + delta))); + } +#elif defined(DATA_A_IQ1_M) + const uint idx = pos_a + col_mem * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 2; + + const uint ib = idx / 32; + const uint ib8 = idx % 32; + const uint ib16 = ib8 / 2; + + const uint16_t[4] scales = MM_DATA_A[ib].scales; + const u16vec4 s = u16vec4(scales[0], scales[1], scales[2], scales[3]) >> 12; + const float d = float(unpackHalf2x16(s.x | (s.y << 4) | (s.z << 8) | (s.w << 12)).x); + const uint sc = scales[ib8 / 8]; + const uint qs = MM_DATA_A[ib].qs[ib8]; + const uint qh = MM_DATA_A[ib].qh[ib16] >> (4 * (ib8 & 1)); + const float dl = d * (2 * bitfieldExtract(sc, 3 * int(ib16 & 3), 3) + 1); + const float delta = ((qh & 8) != 0) ? -IQ1M_DELTA : IQ1M_DELTA; + const int16_t grid = int16_t(iq1s_grid[qs | ((qh & 7) << 8)]); + + [[unroll]] for (int k = 0; k < 4; ++k) { + store_a(col_sh, k_pair + k, FLOAT_TYPEV2(dl * (bitfieldExtract(grid, 4 * k , 2) + delta), + dl * (bitfieldExtract(grid, 4 * k + 2, 2) + delta))); + } +#elif defined(DATA_A_IQ2_XXS) + const uint idx = pos_a + col_mem * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 2; + + const uint ib = idx / 32; + const uint ib32 = (idx % 32) / 4; + const uint ib8 = idx % 4; + + const float d = float(MM_DATA_A[ib].d); + const uint qs = MM_DATA_A[ib].qs[8 * ib32 + ib8]; + const uint signs = pack32(u8vec4( + MM_DATA_A[ib].qs[8*ib32 + 4], + MM_DATA_A[ib].qs[8*ib32 + 5], + MM_DATA_A[ib].qs[8*ib32 + 6], + MM_DATA_A[ib].qs[8*ib32 + 7] + )); + const FLOAT_TYPE db = FLOAT_TYPE(d * 0.25 * (0.5 + (signs >> 28))); + const uint32_t sign7 = bitfieldExtract(signs, 7 * int(ib8), 7); + const uint sign = sign7 | (bitCount(sign7) << 7); + const uvec2 grid = iq2xxs_grid[qs]; + const vec4 grid0 = vec4(unpack8(grid.x)); + const vec4 grid1 = vec4(unpack8(grid.y)); + + store_a(col_sh, k_pair, db * FLOAT_TYPEV2((sign & 1) != 0 ? -grid0.x : grid0.x, + (sign & 2) != 0 ? -grid0.y : grid0.y)); + store_a(col_sh, k_pair + 1, db * FLOAT_TYPEV2((sign & 4) != 0 ? -grid0.z : grid0.z, + (sign & 8) != 0 ? -grid0.w : grid0.w)); + store_a(col_sh, k_pair + 2, db * FLOAT_TYPEV2((sign & 16) != 0 ? -grid1.x : grid1.x, + (sign & 32) != 0 ? -grid1.y : grid1.y)); + store_a(col_sh, k_pair + 3, db * FLOAT_TYPEV2((sign & 64) != 0 ? -grid1.z : grid1.z, + (sign & 128) != 0 ? -grid1.w : grid1.w)); +#elif defined(DATA_A_IQ2_XS) + const uint idx = pos_a + col_mem * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 2; + + const uint ib = idx / 32; + const uint ib32 = (idx % 32) / 4; + const uint ib8 = idx % 4; + + const float d = float(MM_DATA_A[ib].d); + const uint scale = (MM_DATA_A[ib].scales[ib32] >> (2 * (ib8 & 2))) & 0xf; + const FLOAT_TYPE db = FLOAT_TYPE(d * 0.25 * (0.5 + scale)); + const uint qs = MM_DATA_A[ib].qs[4 * ib32 + ib8]; + const uint sign7 = qs >> 9; + const uint sign = sign7 | (bitCount(sign7) << 7); + const uvec2 grid = iq2xs_grid[qs & 511]; + const vec4 grid0 = vec4(unpack8(grid.x)); + const vec4 grid1 = vec4(unpack8(grid.y)); + + store_a(col_sh, k_pair, db * FLOAT_TYPEV2((sign & 1) != 0 ? -grid0.x : grid0.x, + (sign & 2) != 0 ? -grid0.y : grid0.y)); + store_a(col_sh, k_pair + 1, db * FLOAT_TYPEV2((sign & 4) != 0 ? -grid0.z : grid0.z, + (sign & 8) != 0 ? -grid0.w : grid0.w)); + store_a(col_sh, k_pair + 2, db * FLOAT_TYPEV2((sign & 16) != 0 ? -grid1.x : grid1.x, + (sign & 32) != 0 ? -grid1.y : grid1.y)); + store_a(col_sh, k_pair + 3, db * FLOAT_TYPEV2((sign & 64) != 0 ? -grid1.z : grid1.z, + (sign & 128) != 0 ? -grid1.w : grid1.w)); +#elif defined(DATA_A_IQ2_S) + const uint idx = pos_a + col_mem * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 2; + + const uint ib = idx / 32; + const uint ib8 = idx % 32; + const uint ib32 = ib8 / 4; + + const uint scale = (MM_DATA_A[ib].scales[ib32] >> (2 * (ib8 & 2))) & 0xf; + const uint qs = MM_DATA_A[ib].qs[ib8]; + const uint qh = MM_DATA_A[ib].qh[ib32]; + const uint qhshift = 2 * (ib8 % 4); + const uint sign = MM_DATA_A[ib].qs[QUANT_K_IQ2_S / 8 + ib8]; + + const float d = float(MM_DATA_A[ib].d); + const FLOAT_TYPE db = FLOAT_TYPE(d * 0.25 * (0.5 + scale)); + const uvec2 grid = iq2s_grid[qs | ((qh << (8 - qhshift)) & 0x300)]; + const vec4 grid0 = vec4(unpack8(grid.x)); + const vec4 grid1 = vec4(unpack8(grid.y)); + + store_a(col_sh, k_pair, db * FLOAT_TYPEV2((sign & 1) != 0 ? -grid0.x : grid0.x, + (sign & 2) != 0 ? -grid0.y : grid0.y)); + store_a(col_sh, k_pair + 1, db * FLOAT_TYPEV2((sign & 4) != 0 ? -grid0.z : grid0.z, + (sign & 8) != 0 ? -grid0.w : grid0.w)); + store_a(col_sh, k_pair + 2, db * FLOAT_TYPEV2((sign & 16) != 0 ? -grid1.x : grid1.x, + (sign & 32) != 0 ? -grid1.y : grid1.y)); + store_a(col_sh, k_pair + 3, db * FLOAT_TYPEV2((sign & 64) != 0 ? -grid1.z : grid1.z, + (sign & 128) != 0 ? -grid1.w : grid1.w)); +#elif defined(DATA_A_IQ3_XXS) + const uint idx = pos_a + col_mem * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 2; + + const uint ib = idx / 64; + const uint iqs = idx % 64; + const uint is = QUANT_K_IQ3_XXS / 4 + 4 * (iqs / 8); + + const float d = float(MM_DATA_A[ib].d); + const uint qs = MM_DATA_A[ib].qs[iqs]; + const uint signs = pack32(u16vec2( + MM_DATA_A_P16[ib].qs[is/2], + MM_DATA_A_P16[ib].qs[is/2+1] + )); + const float db = d * 0.5 * (0.5 + (signs >> 28)); + const uint32_t sign7 = bitfieldExtract(signs, 7 * (int(iqs / 2) % 4), 7); + const uint sign = (sign7 | (bitCount(sign7) << 7)) >> (4 * (idx % 2)); + const uint grid = iq3xxs_grid[qs]; + const vec4 v = db * vec4(unpack8(grid)); + + store_a(col_sh, k_pair, FLOAT_TYPEV2((sign & 1) != 0 ? -v.x : v.x, + (sign & 2) != 0 ? -v.y : v.y)); + store_a(col_sh, k_pair + 1, FLOAT_TYPEV2((sign & 4) != 0 ? -v.z : v.z, + (sign & 8) != 0 ? -v.w : v.w)); +#elif defined(DATA_A_IQ3_S) + const uint idx = pos_a + col_mem * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 2; + + const uint ib = idx / 64; + const uint iqs = idx % 64; + const uint iqh = iqs / 8; + + const float d = float(MM_DATA_A[ib].d); + const uint qs = MM_DATA_A[ib].qs[iqs]; + const uint qh = MM_DATA_A[ib].qh[iqh]; + const int8_t sign = int8_t(MM_DATA_A[ib].signs[iqs / 2] >> (4 * (idx % 2))); + const uint scale = MM_DATA_A[ib].scales[iqs / 16]; + const float db = d * (1 + 2 * ((scale >> (4 * (iqh & 1))) & 0xf)); + const uint32_t grid = iq3s_grid[qs | ((qh << (8 - (iqs % 8))) & 256)]; + const vec4 v = db * vec4(unpack8(grid)); + + store_a(col_sh, k_pair, FLOAT_TYPEV2((sign & 1) != 0 ? -v.x : v.x, + (sign & 2) != 0 ? -v.y : v.y)); + store_a(col_sh, k_pair + 1, FLOAT_TYPEV2((sign & 4) != 0 ? -v.z : v.z, + (sign & 8) != 0 ? -v.w : v.w)); +#elif defined(DATA_A_IQ4_XS) + const uint idx = pos_a + col_mem * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 4; + + const uint ib = idx / 32; + const uint ib32 = (idx % 32) / 4; + const uint iq = 4 * ib32 + (idx % 4); + + const uint sl = (MM_DATA_A[ib].scales_l[ib32/2] >> (4 * (ib32 & 1))) & 0xF; + const uint sh = ((MM_DATA_A[ib].scales_h) >> (2 * ib32)) & 3; + const float d = float(MM_DATA_A[ib].d); + const float dl = d * float(int(sl | (sh << 4)) - 32); + const uint vui = uint(MM_DATA_A_P32[ib].qs[iq]); + + const u8vec4 qs0 = unpack8( vui & 0x0F0F0F0F); + const u8vec4 qs1 = unpack8((vui >> 4) & 0x0F0F0F0F); + const vec4 v0 = dl * vec4(kvalues_iq4nl[qs0.x], kvalues_iq4nl[qs0.y], kvalues_iq4nl[qs0.z], kvalues_iq4nl[qs0.w]); + const vec4 v1 = dl * vec4(kvalues_iq4nl[qs1.x], kvalues_iq4nl[qs1.y], kvalues_iq4nl[qs1.z], kvalues_iq4nl[qs1.w]); + + MM_STORE_A(0, col_sh, k_pair, FLOAT_TYPEV2(v0.xy)); + MM_STORE_A(1, col_sh, k_pair + 1, FLOAT_TYPEV2(v0.zw)); + MM_STORE_A(2, col_sh, k_pair + 8, FLOAT_TYPEV2(v1.xy)); + MM_STORE_A(3, col_sh, k_pair + 9, FLOAT_TYPEV2(v1.zw)); +#elif defined(DATA_A_IQ4_NL) + const uint idx = pos_a + col_mem * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 4; + + const uint ib = idx / 8; + const uint iqs = idx & 0x07; + + const FLOAT_TYPE d = FLOAT_TYPE(MM_DATA_A_P16[ib].d); + const uint vui = uint(MM_DATA_A_P16[ib].qs[iqs]); + + store_a(col_sh, k_pair, d * FLOAT_TYPEV2(kvalues_iq4nl[vui & 0xF], + kvalues_iq4nl[bitfieldExtract(vui, 8, 4)])); + store_a(col_sh, k_pair + 8, d * FLOAT_TYPEV2(kvalues_iq4nl[bitfieldExtract(vui, 4, 4)], + kvalues_iq4nl[vui >> 12])); +#elif defined(DATA_A_MXFP4) + const uint idx = pos_a + col_mem * p.stride_a / LOAD_VEC_A + row; + const uint k_pair = row * LOAD_VEC_A / 4; + + const uint ib = idx / 8; + const uint iqs = (idx & 0x07) * 2; + + const uint vui = uint(MM_DATA_A[ib].qs[iqs]); + const uint vui2 = uint(MM_DATA_A[ib].qs[iqs+1]); + +#ifdef USE_OCP_FP4 + const float d = e8m0_to_fp32(MM_DATA_A[ib].e); + const u8vec2 packed = u8vec2(vui, vui2); + store_a(col_sh, k_pair, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 0u)) * FLOAT_TYPE(d)); + store_a(col_sh, k_pair + 8, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 4u)) * FLOAT_TYPE(d)); +#else + const float d = e8m0_to_fp32(MM_DATA_A[ib].e) * 0.5; + store_a(col_sh, k_pair, FLOAT_TYPEV2(kvalues_mxfp4[vui & 0xF] * d, + kvalues_mxfp4[vui2 & 0xF] * d)); + store_a(col_sh, k_pair + 8, FLOAT_TYPEV2(kvalues_mxfp4[vui >> 4] * d, + kvalues_mxfp4[vui2 >> 4] * d)); +#endif +#elif defined(DATA_A_NVFP4) + const uint idx = pos_a + col_mem * p.stride_a / LOAD_VEC_A + row; + const uint eff_row = (row & 3) + (row & ~3) * 2; + + const uint ib = idx / 16u; + const uint sub = (idx & 0xC) >> 2; + const uint iqs = (idx & 0xF) * 2; + const uint vui = uint(MM_DATA_A[ib].qs[iqs]); + const uint vui2 = uint(MM_DATA_A[ib].qs[iqs+1]); + +#ifdef USE_OCP_FP4 + const FLOAT_TYPE d = FLOAT_TYPE(ue4m3_from_bits(MM_DATA_A[ib].d[sub])); + const u8vec2 packed = u8vec2(vui, vui2); + store_a(col_sh, eff_row, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 0u)) * d); + store_a(col_sh, eff_row + 4, FLOAT_TYPEV2(bitcastExtractfe2m1EXT(packed, 4u)) * d); +#else + const float d = ue4m3_to_fp32(MM_DATA_A[ib].d[sub]) * 0.5; + store_a(col_sh, eff_row, FLOAT_TYPEV2(kvalues_mxfp4[vui & 0xF] * d, + kvalues_mxfp4[vui2 & 0xF] * d)); + store_a(col_sh, eff_row + 4, FLOAT_TYPEV2(kvalues_mxfp4[vui >> 4] * d, + kvalues_mxfp4[vui2 >> 4] * d)); +#endif +#else + if (MmTypeA == GGML_TYPE_Q4_0) { + const uint idx = pos_a + col_mem * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 4; + + const uint ib = idx / 4; + const uint iqs = idx & 0x03; + + const float d = float(MM_Q4_0_P16.data[ib].d); + const uint vui = uint(MM_Q4_0_P16.data[ib].qs[2*iqs]) | (uint(MM_Q4_0_P16.data[ib].qs[2*iqs + 1]) << 16); + const vec4 v0 = (vec4(unpack8(vui & 0x0F0F0F0F)) - 8.0f) * d; + const vec4 v1 = (vec4(unpack8((vui >> 4) & 0x0F0F0F0F)) - 8.0f) * d; + + store_a(col_sh, k_pair, FLOAT_TYPEV2(v0.xy)); + store_a(col_sh, k_pair + 1, FLOAT_TYPEV2(v0.zw)); + store_a(col_sh, k_pair + 8, FLOAT_TYPEV2(v1.xy)); + store_a(col_sh, k_pair + 9, FLOAT_TYPEV2(v1.zw)); + } else if (MmTypeA == GGML_TYPE_Q4_1) { + const uint idx = pos_a + col_mem * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 4; + + const uint ib = idx / 4; + const uint iqs = idx & 0x03; + + const vec2 dm = vec2(MM_Q4_1_P32.data[ib].dm); + const uint vui = MM_Q4_1_P32.data[ib].qs[iqs]; + const vec4 v0 = vec4(unpack8(vui & 0x0F0F0F0F)) * dm.x + dm.y; + const vec4 v1 = vec4(unpack8((vui >> 4) & 0x0F0F0F0F)) * dm.x + dm.y; + + store_a(col_sh, k_pair, FLOAT_TYPEV2(v0.xy)); + store_a(col_sh, k_pair + 1, FLOAT_TYPEV2(v0.zw)); + store_a(col_sh, k_pair + 8, FLOAT_TYPEV2(v1.xy)); + store_a(col_sh, k_pair + 9, FLOAT_TYPEV2(v1.zw)); + } else if (MmTypeA == GGML_TYPE_Q5_0) { + const uint idx = pos_a + col_mem * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 4; + + const uint ib = idx / 8; + const uint iqs = idx & 0x07; + + const float d = float(MM_Q5_0_P16.data[ib].d); + const uint uint_qh = uint(MM_Q5_0_P16.data[ib].qh[1]) << 16 | uint(MM_Q5_0_P16.data[ib].qh[0]); + const ivec2 qh0 = ivec2(((uint_qh >> 2*iqs) << 4) & 0x10, (uint_qh >> (2*iqs + 12)) & 0x10); + const ivec2 qh1 = ivec2(((uint_qh >> (2*iqs + 1)) << 4) & 0x10, (uint_qh >> (2*iqs + 13)) & 0x10); + + const uint vui = uint(MM_Q5_0_P16.data[ib].qs[iqs]); + const vec4 v = (vec4((vui & 0xF) | qh0.x, ((vui >> 4) & 0xF) | qh0.y, ((vui >> 8) & 0xF) | qh1.x, (vui >> 12) | qh1.y) - 16.0f) * d; + + store_a(col_sh, k_pair, FLOAT_TYPEV2(v.xz)); + store_a(col_sh, k_pair + 8, FLOAT_TYPEV2(v.yw)); + } else if (MmTypeA == GGML_TYPE_Q5_1) { + const uint idx = pos_a + col_mem * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 4; + + const uint ib = idx / 4; + const uint iqs = idx & 0x03; + + const vec2 dm = vec2(MM_Q5_1_P32.data[ib].dm); + const uint uint_qh = MM_Q5_1_P32.data[ib].qh; + const uvec2 qh0 = uvec2(((uint_qh >> 4*iqs) << 4) & 0x10, (uint_qh >> (4*iqs + 12)) & 0x10); + const uvec2 qh1 = uvec2(((uint_qh >> (4*iqs + 1)) << 4) & 0x10, (uint_qh >> (4*iqs + 13)) & 0x10); + const uvec2 qh2 = uvec2(((uint_qh >> (4*iqs + 2)) << 4) & 0x10, (uint_qh >> (4*iqs + 14)) & 0x10); + const uvec2 qh3 = uvec2(((uint_qh >> (4*iqs + 3)) << 4) & 0x10, (uint_qh >> (4*iqs + 15)) & 0x10); + + const uint vui = MM_Q5_1_P32.data[ib].qs[iqs]; + const vec4 v0 = vec4((vui & 0xF) | qh0.x, ((vui >> 4) & 0xF) | qh0.y, ((vui >> 8) & 0xF) | qh1.x, ((vui >> 12) & 0xF) | qh1.y) * dm.x + dm.y; + const vec4 v1 = vec4(((vui >> 16) & 0xF) | qh2.x, ((vui >> 20) & 0xF) | qh2.y, ((vui >> 24) & 0xF) | qh3.x, ((vui >> 28) & 0xF) | qh3.y) * dm.x + dm.y; + + store_a(col_sh, k_pair, FLOAT_TYPEV2(v0.xz)); + store_a(col_sh, k_pair + 1, FLOAT_TYPEV2(v1.xz)); + store_a(col_sh, k_pair + 8, FLOAT_TYPEV2(v0.yw)); + store_a(col_sh, k_pair + 9, FLOAT_TYPEV2(v1.yw)); + } else if (MmTypeA == GGML_TYPE_Q8_0) { + if (AGX_DIRECT_STORE == 1u) { + const uint idx = pos_a + col_mem * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 2; + + const uint ib = idx / 8; + const uint iqs = idx & 0x07; + + const float d = float(MM_Q8_0_P16.data[ib].d); + const i8vec2 v0 = unpack8(int32_t(MM_Q8_0_P16.data[ib].qs[2*iqs])).xy; + const i8vec2 v1 = unpack8(int32_t(MM_Q8_0_P16.data[ib].qs[2*iqs + 1])).xy; + const vec4 v = vec4(v0.x, v0.y, v1.x, v1.y) * d; + + MM_STORE_A(0, col_sh, k_pair, FLOAT_TYPEV2(v.xy)); + MM_STORE_A(1, col_sh, k_pair + 1, FLOAT_TYPEV2(v.zw)); + } else { + const uint idx = pos_a + col_mem * (p.stride_a >> 4u) + row; + const uint k_pair = row * 8u; + + const uint ib = idx >> 1u; + const uint iqs = (idx & 1u) << 3u; + + const FLOAT_TYPE d = FLOAT_TYPE(MM_Q8_0_P16.data[ib].d); + MM_STORE_A(0, col_sh, k_pair + 0u, FLOAT_TYPEV2(unpack8(MM_Q8_0_P16.data[ib].qs[iqs + 0u])) * d); + MM_STORE_A(1, col_sh, k_pair + 1u, FLOAT_TYPEV2(unpack8(MM_Q8_0_P16.data[ib].qs[iqs + 1u])) * d); + MM_STORE_A(2, col_sh, k_pair + 2u, FLOAT_TYPEV2(unpack8(MM_Q8_0_P16.data[ib].qs[iqs + 2u])) * d); + MM_STORE_A(3, col_sh, k_pair + 3u, FLOAT_TYPEV2(unpack8(MM_Q8_0_P16.data[ib].qs[iqs + 3u])) * d); + MM_STORE_A(4, col_sh, k_pair + 4u, FLOAT_TYPEV2(unpack8(MM_Q8_0_P16.data[ib].qs[iqs + 4u])) * d); + MM_STORE_A(5, col_sh, k_pair + 5u, FLOAT_TYPEV2(unpack8(MM_Q8_0_P16.data[ib].qs[iqs + 5u])) * d); + MM_STORE_A(6, col_sh, k_pair + 6u, FLOAT_TYPEV2(unpack8(MM_Q8_0_P16.data[ib].qs[iqs + 6u])) * d); + MM_STORE_A(7, col_sh, k_pair + 7u, FLOAT_TYPEV2(unpack8(MM_Q8_0_P16.data[ib].qs[iqs + 7u])) * d); + } + } else if (MmTypeA == GGML_TYPE_Q1_0) { + const uint idx = pos_a + col_mem * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 2; + + const uint ib = idx / 16; + const uint iqs = idx & 0xfu; + + const float d = float(MM_Q1_0.data[ib].d); + const uint bits = uint(MM_Q1_0.data[ib].qs[iqs]); + + store_a(col_sh, k_pair, FLOAT_TYPEV2((bits & 0x01u) != 0u ? d : -d, (bits & 0x02u) != 0u ? d : -d)); + store_a(col_sh, k_pair + 1, FLOAT_TYPEV2((bits & 0x04u) != 0u ? d : -d, (bits & 0x08u) != 0u ? d : -d)); + store_a(col_sh, k_pair + 2, FLOAT_TYPEV2((bits & 0x10u) != 0u ? d : -d, (bits & 0x20u) != 0u ? d : -d)); + store_a(col_sh, k_pair + 3, FLOAT_TYPEV2((bits & 0x40u) != 0u ? d : -d, (bits & 0x80u) != 0u ? d : -d)); + } else if (MmTypeA == GGML_TYPE_Q2_0) { + const uint idx = pos_a + col_mem * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 2; + + const uint ib = idx / 16; + const uint iqs = idx & 0xfu; + + const FLOAT_TYPE d = FLOAT_TYPE(MM_Q2_0.data[ib].d); + const uint bits = uint(MM_Q2_0.data[ib].qs[iqs]); + + store_a(col_sh, k_pair, d * (FLOAT_TYPEV2(bits & 3u, (bits >> 2u) & 3u) - FLOAT_TYPEV2(1.0f))); + store_a(col_sh, k_pair + 1, d * (FLOAT_TYPEV2((bits >> 4u) & 3u, bits >> 6u) - FLOAT_TYPEV2(1.0f))); + } else if (MmTypeA == GGML_TYPE_Q2_K) { + const uint idx = pos_a + col_mem * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 2; + + const uint ib = idx / 64; + const uint iqs = (idx % 64) * 2; + + const uint qsi = (iqs / 64) * 16 + (iqs % 16); + const uint scalesi = iqs / 8; + const uint qsshift = ((iqs % 64) / 16) * 2; + + const vec4 qs = vec4(unpack8((MM_Q2_K_P32.data[ib].qs[qsi / 2] >> qsshift) & 0x03030303)); + const uint scales = MM_Q2_K.data[ib].scales[scalesi]; + const vec2 dm = vec2(MM_Q2_K.data[ib].dm); + + const vec4 v = dm.x * float(scales & 0xF) * qs - dm.y * float(scales >> 4); + + store_a(col_sh, k_pair, FLOAT_TYPEV2(v.xy)); + store_a(col_sh, k_pair + 1, FLOAT_TYPEV2(v.zw)); + } else if (MmTypeA == GGML_TYPE_Q3_K) { + const uint idx = pos_a + col_mem * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 2; + + const uint ib = idx / 128; + const uint iqs = idx % 128; + + const uint n = iqs / 64; + const uint qsi = n * 32 + (iqs % 16) * 2; + const uint hmi = (iqs % 16) * 2; + const uint is = iqs / 8; + const uint halfsplit = ((iqs % 64) / 16); + const uint qsshift = halfsplit * 2; + + const int8_t us = int8_t(((MM_Q3_K.data[ib].scales[is % 8] >> (4 * int(is / 8))) & 0xF) + | (((MM_Q3_K.data[ib].scales[8 + (is % 4)] >> (2 * int(is / 4))) & 3) << 4)); + const float dl = float(MM_Q3_K.data[ib].d) * float(us - 32); + + const vec2 qs = vec2(unpack8((uint(MM_Q3_K_P16.data[ib].qs[qsi / 2]) >> qsshift) & 0x0303).xy); + const vec2 hm = vec2(unpack8(((uint(MM_Q3_K_P16.data[ib].hmask[hmi / 2]) >> (4 * n + halfsplit)) & 0x0101 ^ 0x0101) << 2).xy); + + store_a(col_sh, k_pair, FLOAT_TYPEV2(dl * (qs.x - hm.x), + dl * (qs.y - hm.y))); + } else if (MmTypeA == GGML_TYPE_Q4_K) { + if (AGX_DIRECT_STORE == 1u) { + const uint idx = pos_a + col_mem * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 2; + + const uint ib = idx / 64; + const uint iqs = (idx % 64) * 2; + + const uint n = iqs / 32; + const uint b = (iqs % 32) / 16; + const uint is = 2 * n + b; + const uint qsi = n * 32 + (iqs % 16) * 2; + + const vec2 loadd = vec2(MM_Q4_K.data[ib].dm); + + const uvec3 scales = uvec3(MM_Q4_K_P32.data[ib].scales[0], + MM_Q4_K_P32.data[ib].scales[1], + MM_Q4_K_P32.data[ib].scales[2]); + const uint scalesoffs = (is & 3) * 8; + + const uint scidxshift0 = scalesoffs; + const uint scidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2; + const uint mbidxshift0 = (is < 4) ? scalesoffs : scalesoffs + 4; + const uint mbidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2; + + const uint sc_word = (is < 4) ? scales.x : scales.z; + const uint mb_word = (is < 4) ? scales.y : scales.z; + const uint8_t sc = uint8_t(((sc_word >> scidxshift0) & 0xF) | ((scales.x >> scidxshift1) & 0x30)); + const uint8_t mbyte = uint8_t(((mb_word >> mbidxshift0) & 0xF) | ((scales.y >> mbidxshift1) & 0x30)); + + const float d = loadd.x * sc; + const float m = -loadd.y * mbyte; + + const vec4 q = vec4(unpack8((MM_Q4_K_P32.data[ib].qs[qsi / 4] >> (b * 4)) & 0x0F0F0F0F)); + + MM_STORE_A(0, col_sh, k_pair, FLOAT_TYPEV2(fma(d, q.x, m), fma(d, q.y, m))); + MM_STORE_A(1, col_sh, k_pair + 1, FLOAT_TYPEV2(fma(d, q.z, m), fma(d, q.w, m))); + } else { + const uint idx = pos_a + col_mem * (p.stride_a >> 4u) + row; + const uint k_pair = row * 8u; + + const uint ib = idx >> 4u; + const uint il = idx & 15u; + const uint is = il >> 1u; + const uint b = is & 1u; + const uint n = il >> 2u; + const uint w_idx = (n << 3u) + ((il & 1u) << 2u); + + const vec2 loadd = vec2(MM_Q4_K.data[ib].dm); + + const uvec3 scales = uvec3(MM_Q4_K_P32.data[ib].scales[0], + MM_Q4_K_P32.data[ib].scales[1], + MM_Q4_K_P32.data[ib].scales[2]); + const uint scalesoffs = (is & 3u) * 8u; + const uint scidxshift1 = (is < 4u) ? scalesoffs : scalesoffs + 2u; + const uint mbidxshift0 = (is < 4u) ? scalesoffs : scalesoffs + 4u; + const uint sc_word = (is < 4u) ? scales.x : scales.z; + const uint mb_word = (is < 4u) ? scales.y : scales.z; + const uint sc = ((sc_word >> scalesoffs) & 0xFu) | ((scales.x >> scidxshift1) & 0x30u); + const uint mbyte = ((mb_word >> mbidxshift0) & 0xFu) | ((scales.y >> scidxshift1) & 0x30u); + + const float d_hi = (b == 0u) ? 1.0f : 0.0625f; + const uint q_mk = (b == 0u) ? 0x0F0F0F0Fu : 0xF0F0F0F0u; + const FLOAT_TYPEV4 d = FLOAT_TYPEV4(loadd.x * float(sc) * d_hi); + const FLOAT_TYPEV4 m = FLOAT_TYPEV4(-loadd.y * float(mbyte)); + + const FLOAT_TYPEV4 q0 = fma(d, FLOAT_TYPEV4(unpack8(MM_Q4_K_P32.data[ib].qs[w_idx + 0u] & q_mk)), m); + const FLOAT_TYPEV4 q1 = fma(d, FLOAT_TYPEV4(unpack8(MM_Q4_K_P32.data[ib].qs[w_idx + 1u] & q_mk)), m); + const FLOAT_TYPEV4 q2 = fma(d, FLOAT_TYPEV4(unpack8(MM_Q4_K_P32.data[ib].qs[w_idx + 2u] & q_mk)), m); + const FLOAT_TYPEV4 q3 = fma(d, FLOAT_TYPEV4(unpack8(MM_Q4_K_P32.data[ib].qs[w_idx + 3u] & q_mk)), m); + + MM_STORE_A(0, col_sh, k_pair + 0u, q0.xy); + MM_STORE_A(1, col_sh, k_pair + 1u, q0.zw); + MM_STORE_A(2, col_sh, k_pair + 2u, q1.xy); + MM_STORE_A(3, col_sh, k_pair + 3u, q1.zw); + MM_STORE_A(4, col_sh, k_pair + 4u, q2.xy); + MM_STORE_A(5, col_sh, k_pair + 5u, q2.zw); + MM_STORE_A(6, col_sh, k_pair + 6u, q3.xy); + MM_STORE_A(7, col_sh, k_pair + 7u, q3.zw); + } + } else if (MmTypeA == GGML_TYPE_Q5_K) { + if (AGX_DIRECT_STORE == 1u) { + const uint idx = pos_a + col_mem * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 2; + + const uint ib = idx / 64; + const uint iqs = (idx % 64) * 2; + + const uint n = iqs / 32; + const uint b = (iqs % 32) / 16; + const uint is = 2 * n + b; + const uint qsi = n * 32 + (iqs % 16) * 2; + const uint qhi = (iqs % 16) * 2; + + const vec2 loadd = vec2(MM_Q5_K.data[ib].dm); + + const uvec3 scales = uvec3(MM_Q5_K_P32.data[ib].scales[0], + MM_Q5_K_P32.data[ib].scales[1], + MM_Q5_K_P32.data[ib].scales[2]); + const uint scalesoffs = (is & 3) * 8; + + const uint scidxshift0 = scalesoffs; + const uint scidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2; + const uint mbidxshift0 = (is < 4) ? scalesoffs : scalesoffs + 4; + const uint mbidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2; + + const uint sc_word = (is < 4) ? scales.x : scales.z; + const uint mb_word = (is < 4) ? scales.y : scales.z; + const uint8_t sc = uint8_t(((sc_word >> scidxshift0) & 0xF) | ((scales.x >> scidxshift1) & 0x30)); + const uint8_t mbyte = uint8_t(((mb_word >> mbidxshift0) & 0xF) | ((scales.y >> mbidxshift1) & 0x30)); + + const float d = loadd.x * sc; + const float m = -loadd.y * mbyte; + + const uint qs = (MM_Q5_K_P32.data[ib].qs[qsi / 4] >> (b * 4)) & 0x0F0F0F0F; + const uint qh = ((MM_Q5_K_P32.data[ib].qh[qhi / 4] >> (iqs / 16)) & 0x01010101) << 4; + const vec4 q = vec4(unpack8(qs | qh)); + + MM_STORE_A(0, col_sh, k_pair, FLOAT_TYPEV2(fma(d, q.x, m), fma(d, q.y, m))); + MM_STORE_A(1, col_sh, k_pair + 1, FLOAT_TYPEV2(fma(d, q.z, m), fma(d, q.w, m))); + } else { + const uint idx = pos_a + col_mem * (p.stride_a >> 4u) + row; + const uint k_pair = row * 8u; + + const uint ib = idx >> 4u; + const uint il = idx & 15u; + const uint is = il >> 1u; + const uint b = is & 1u; + const uint n = il >> 2u; + const uint w_idx = (n << 3u) + ((il & 1u) << 2u); + const uint qh_idx = (il & 1u) << 2u; + + const vec2 loadd = vec2(MM_Q5_K.data[ib].dm); + + const uvec3 scales = uvec3(MM_Q5_K_P32.data[ib].scales[0], + MM_Q5_K_P32.data[ib].scales[1], + MM_Q5_K_P32.data[ib].scales[2]); + const uint scalesoffs = (is & 3u) * 8u; + const uint scidxshift1 = (is < 4u) ? scalesoffs : scalesoffs + 2u; + const uint mbidxshift0 = (is < 4u) ? scalesoffs : scalesoffs + 4u; + const uint sc_word = (is < 4u) ? scales.x : scales.z; + const uint mb_word = (is < 4u) ? scales.y : scales.z; + const uint sc = ((sc_word >> scalesoffs) & 0xFu) | ((scales.x >> scidxshift1) & 0x30u); + const uint mbyte = ((mb_word >> mbidxshift0) & 0xFu) | ((scales.y >> scidxshift1) & 0x30u); + + const FLOAT_TYPEV4 d = FLOAT_TYPEV4(loadd.x * float(sc)); + const FLOAT_TYPEV4 m = FLOAT_TYPEV4(-loadd.y * float(mbyte)); + + const uint shift = b * 4u; + const uint qs0 = (MM_Q5_K_P32.data[ib].qs[w_idx + 0u] >> shift) & 0x0F0F0F0Fu; + const uint qs1 = (MM_Q5_K_P32.data[ib].qs[w_idx + 1u] >> shift) & 0x0F0F0F0Fu; + const uint qs2 = (MM_Q5_K_P32.data[ib].qs[w_idx + 2u] >> shift) & 0x0F0F0F0Fu; + const uint qs3 = (MM_Q5_K_P32.data[ib].qs[w_idx + 3u] >> shift) & 0x0F0F0F0Fu; + const uint qh0 = ((MM_Q5_K_P32.data[ib].qh[qh_idx + 0u] >> is) & 0x01010101u) << 4u; + const uint qh1 = ((MM_Q5_K_P32.data[ib].qh[qh_idx + 1u] >> is) & 0x01010101u) << 4u; + const uint qh2 = ((MM_Q5_K_P32.data[ib].qh[qh_idx + 2u] >> is) & 0x01010101u) << 4u; + const uint qh3 = ((MM_Q5_K_P32.data[ib].qh[qh_idx + 3u] >> is) & 0x01010101u) << 4u; + const FLOAT_TYPEV4 q0 = fma(d, FLOAT_TYPEV4(unpack8(qs0 | qh0)), m); + const FLOAT_TYPEV4 q1 = fma(d, FLOAT_TYPEV4(unpack8(qs1 | qh1)), m); + const FLOAT_TYPEV4 q2 = fma(d, FLOAT_TYPEV4(unpack8(qs2 | qh2)), m); + const FLOAT_TYPEV4 q3 = fma(d, FLOAT_TYPEV4(unpack8(qs3 | qh3)), m); + + MM_STORE_A(0, col_sh, k_pair + 0u, q0.xy); + MM_STORE_A(1, col_sh, k_pair + 1u, q0.zw); + MM_STORE_A(2, col_sh, k_pair + 2u, q1.xy); + MM_STORE_A(3, col_sh, k_pair + 3u, q1.zw); + MM_STORE_A(4, col_sh, k_pair + 4u, q2.xy); + MM_STORE_A(5, col_sh, k_pair + 5u, q2.zw); + MM_STORE_A(6, col_sh, k_pair + 6u, q3.xy); + MM_STORE_A(7, col_sh, k_pair + 7u, q3.zw); + } + } else if (MmTypeA == GGML_TYPE_Q6_K) { + if (AGX_DIRECT_STORE == 1u) { + const uint idx = pos_a + col_mem * p.stride_a / mm_load_vec_a() + row; + const uint k_pair = row * mm_load_vec_a() / 2; + + const uint ib = idx / 128; + const uint iqs = idx % 128; + + const uint n = iqs / 64; + const uint b = ((iqs % 64) / 32) * 4; + const uint is_b = (iqs % 16) / 8; + const uint qhshift = ((iqs % 64) / 16) * 2; + const uint is = 8 * n + qhshift + is_b; + const uint qsi = n * 32 + (iqs % 32); + const uint qhi = n * 16 + (iqs % 16); + + const float dscale = float(MM_Q6_K.data[ib].d) * float(MM_Q6_K.data[ib].scales[is]); + + const uint ql = (uint(MM_Q6_K_P16.data[ib].ql[qsi]) >> b) & 0x0F0F; + const uint qh = (uint(MM_Q6_K_P16.data[ib].qh[qhi]) >> qhshift) & 0x0303; + const vec2 q = (vec2(unpack8(ql | (qh << 4)).xy) - 32) * dscale; + + MM_STORE_A(0, col_sh, k_pair, FLOAT_TYPEV2(q.x, q.y)); + } else { + const uint idx = pos_a + col_mem * (p.stride_a >> 4u) + row; + const uint k_pair = row * 8u; + + const uint ib = idx >> 4u; + const uint il = idx & 15u; + const uint b = ((il >> 2u) & 1u) * 4u; + const uint qhshift = ((il >> 1u) & 3u) * 2u; + const uint qsi = ((il & 8u) << 2u) + ((il & 3u) << 3u); + const uint qhi = ((il & 8u) << 1u) + ((il & 1u) << 3u); + + const float dscale = float(MM_Q6_K.data[ib].d) * float(MM_Q6_K.data[ib].scales[il]); + + const uint ql01 = ((uint(MM_Q6_K_P16.data[ib].ql[qsi + 0u]) | (uint(MM_Q6_K_P16.data[ib].ql[qsi + 1u]) << 16u)) >> b) & 0x0F0F0F0Fu; + const uint ql23 = ((uint(MM_Q6_K_P16.data[ib].ql[qsi + 2u]) | (uint(MM_Q6_K_P16.data[ib].ql[qsi + 3u]) << 16u)) >> b) & 0x0F0F0F0Fu; + const uint ql45 = ((uint(MM_Q6_K_P16.data[ib].ql[qsi + 4u]) | (uint(MM_Q6_K_P16.data[ib].ql[qsi + 5u]) << 16u)) >> b) & 0x0F0F0F0Fu; + const uint ql67 = ((uint(MM_Q6_K_P16.data[ib].ql[qsi + 6u]) | (uint(MM_Q6_K_P16.data[ib].ql[qsi + 7u]) << 16u)) >> b) & 0x0F0F0F0Fu; + const uint qh01 = ((uint(MM_Q6_K_P16.data[ib].qh[qhi + 0u]) | (uint(MM_Q6_K_P16.data[ib].qh[qhi + 1u]) << 16u)) >> qhshift) & 0x03030303u; + const uint qh23 = ((uint(MM_Q6_K_P16.data[ib].qh[qhi + 2u]) | (uint(MM_Q6_K_P16.data[ib].qh[qhi + 3u]) << 16u)) >> qhshift) & 0x03030303u; + const uint qh45 = ((uint(MM_Q6_K_P16.data[ib].qh[qhi + 4u]) | (uint(MM_Q6_K_P16.data[ib].qh[qhi + 5u]) << 16u)) >> qhshift) & 0x03030303u; + const uint qh67 = ((uint(MM_Q6_K_P16.data[ib].qh[qhi + 6u]) | (uint(MM_Q6_K_P16.data[ib].qh[qhi + 7u]) << 16u)) >> qhshift) & 0x03030303u; + const FLOAT_TYPEV4 d = FLOAT_TYPEV4(dscale); + const FLOAT_TYPEV4 m = FLOAT_TYPEV4(-32.0f * dscale); + const FLOAT_TYPEV4 q0 = fma(FLOAT_TYPEV4(unpack8(ql01 | (qh01 << 4u))), d, m); + const FLOAT_TYPEV4 q1 = fma(FLOAT_TYPEV4(unpack8(ql23 | (qh23 << 4u))), d, m); + const FLOAT_TYPEV4 q2 = fma(FLOAT_TYPEV4(unpack8(ql45 | (qh45 << 4u))), d, m); + const FLOAT_TYPEV4 q3 = fma(FLOAT_TYPEV4(unpack8(ql67 | (qh67 << 4u))), d, m); + + MM_STORE_A(0, col_sh, k_pair + 0u, q0.xy); + MM_STORE_A(1, col_sh, k_pair + 1u, q0.zw); + MM_STORE_A(2, col_sh, k_pair + 2u, q1.xy); + MM_STORE_A(3, col_sh, k_pair + 3u, q1.zw); + MM_STORE_A(4, col_sh, k_pair + 4u, q2.xy); + MM_STORE_A(5, col_sh, k_pair + 5u, q2.zw); + MM_STORE_A(6, col_sh, k_pair + 6u, q3.xy); + MM_STORE_A(7, col_sh, k_pair + 7u, q3.zw); + } + } else if (MmTypeA == GGML_TYPE_TQ1_0) { + const uint idx = pos_a + col_mem * p.stride_a / mm_load_vec_a() + row; + + const uint ib = idx / 128; + const uint iqs = (idx % 128) * 2; + + const float d = float(MM_TQ1_0.data[ib].d); + vec2 v; + for (uint kk = 0u; kk < 2u; ++kk) { + const uint e = iqs + kk; + const uint bidx = tq1_0_byte_of(e); + const uint qbyte = uint(bidx < 48u ? MM_TQ1_0.data[ib].qs[bidx] + : MM_TQ1_0.data[ib].qh[bidx - 48u]); + v[kk] = d * (float(tq1_0_trit(qbyte, tq1_0_digit_of(e))) - 1.0); + } + + const uint k_pair = row * mm_load_vec_a() / 2; + store_a(col_sh, k_pair, FLOAT_TYPEV2(v.xy)); + } else if (MmTypeA == GGML_TYPE_TQ2_0) { + const uint idx = pos_a + col_mem * p.stride_a / mm_load_vec_a() + row; + + const uint ib = idx / 128; + const uint iqs = (idx % 128) * 2; + + const uint qsi = (iqs / 128) * 32 + (iqs % 32); + const uint shift = 2 * ((iqs % 128) / 32); + + const uvec2 qs = uvec2(MM_TQ2_0.data[ib].qs[qsi], MM_TQ2_0.data[ib].qs[qsi + 1]); + const float d = float(MM_TQ2_0.data[ib].d); + + const vec2 v = d * (vec2((qs >> shift) & 3) - 1.0); + + const uint k_pair = row * mm_load_vec_a() / 2; + store_a(col_sh, k_pair, FLOAT_TYPEV2(v.xy)); + } +#endif diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mv_mma.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mv_mma.comp new file mode 100644 index 000000000000..61670df24821 --- /dev/null +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mv_mma.comp @@ -0,0 +1,1828 @@ +#version 450 + +#extension GL_EXT_control_flow_attributes : enable +#extension GL_EXT_shader_16bit_storage : require +#extension GL_EXT_shader_8bit_storage : require +#extension GL_EXT_shader_explicit_arithmetic_types : require +#extension GL_EXT_shader_explicit_arithmetic_types_float16 : require +#extension GL_EXT_shader_explicit_arithmetic_types_int16 : require +#extension GL_EXT_shader_explicit_arithmetic_types_int8 : require +#extension GL_EXT_shader_explicit_arithmetic_types_int32 : require +#extension GL_KHR_shader_subgroup_basic : require +#extension GL_KHR_shader_subgroup_shuffle : enable +#extension GL_KHR_memory_scope_semantics : enable +#extension GL_KHR_cooperative_matrix : enable + +#ifndef FLOAT_TYPE +#define FLOAT_TYPE float16_t +#endif + +#include "types.glsl" + +// Zero-spill register-direct 8x8x8 cooperative-matrix few-column matmul / MoE matmul_id for Apple AGX (G17P). +// Subgroup layout (AGX3B 8x8 coopmat): +// lane l in [0..31] owns: +// fm = ((l >> 1) & 3) | ((l >> 2) & 4) in [0..7] +// fn = ((l & 1) << 1) | ((l >> 1) & 4) in {0, 2, 4, 6} (elements e=0,1 -> fn, fn+1) + +layout(local_size_x = 32, local_size_y_id = 0, local_size_z = 1) in; + +layout(constant_id = 0) const uint WG_SG = 1; // total subgroups per workgroup = NSG * MSG * CSG +layout(constant_id = 1) const uint NT = 2; // 8*NT M-rows per subgroup +layout(constant_id = 2) const uint RT = 1; // 8*RT N-cols per subgroup (1, 2, 4) +layout(constant_id = 3) const uint NSG = 1; // K-split subgroups +// 0 = plain, 1 = single-op epilogue, 2 = PAIR_MM (dense only), 3 = PAIR_GLU, 4 = fast epilogue +layout(constant_id = 4) const uint FUSE_MODE = 0; +layout(constant_id = 5) const uint USE_SH_B = 1; +layout(constant_id = 6) const uint K_PAIRS = 1; +layout(constant_id = 7) const uint CSG = 1; // column-split subgroups (MUL_MAT_ID cooperative dequant) +layout(constant_id = 8) const uint B_IS_F16 = 0; // compile-time FP16 B buffer specialization +layout(constant_id = 9) const uint FUSE_LORA = 0; // compile-time fused dynamic LoRA specialization + +const uint MSG = WG_SG / (NSG * CSG); +const uint WG_THREADS = 32u * WG_SG; +const uint BN_COLS = 8u * RT * CSG; + +#if defined(DATA_A_F32) +#define MMA_FTYPE float +#define MMA_FVEC4 vec4 +#define MMA_FVEC2 vec2 +#define MMA_ELEM_SHIFT 0u +#elif defined(DATA_A_F32_F16) +#define MMA_FTYPE float16_t +#define MMA_FVEC4 f16vec4 +#define MMA_FVEC2 f16vec2 +#define MMA_ELEM_SHIFT 0u +#elif defined(DATA_A_F16) +#define MMA_FTYPE float16_t +#define MMA_FVEC4 f16vec4 +#define MMA_FVEC2 f16vec2 +#define MMA_ELEM_SHIFT 0u +#elif defined(DATA_A_Q4_K) || defined(DATA_A_Q5_K) || defined(DATA_A_Q6_K) || defined(DATA_A_IQ4_XS) +#define MMA_FTYPE float16_t +#define MMA_FVEC4 f16vec4 +#define MMA_FVEC2 f16vec2 +#define MMA_ELEM_SHIFT 8u +#else +#define MMA_FTYPE float16_t +#define MMA_FVEC4 f16vec4 +#define MMA_FVEC2 f16vec2 +#define MMA_ELEM_SHIFT 5u +#endif + +#ifdef MUL_MAT_ID +#define MMA_BIND_U 5 +#define MMA_BIND_AUX0 6 +#define MMA_BIND_AUX1 7 +#else +// Matches mul_mat_vec 7-binding layout: {A=0, B=1, D=2, F0/U=3, F1=4, D2=5, F2=6} +#define MMA_BIND_U 3 +#define MMA_BIND_AUX0 3 +#define MMA_BIND_AUX1 4 +#endif + +// Per-type SSBO bindings (only the active type's descriptors are declared) +#if defined(DATA_A_F32) || defined(DATA_A_F32_F16) +layout(binding = 0) readonly buffer BUF_A_F32 { float data[]; } a_f32; +layout(binding = 0) readonly buffer BUF_A_F32V4 { vec4 data[]; } a_f32v4; +layout(binding = MMA_BIND_U) readonly buffer BUF_U_F32 { float data[]; } u_f32; +layout(binding = MMA_BIND_U) readonly buffer BUF_U_F32V4 { vec4 data[]; } u_f32v4; +#elif defined(DATA_A_F16) +layout(binding = 0) readonly buffer BUF_A_F16 { float16_t data[]; } a_f16; +layout(binding = 0) readonly buffer BUF_A_F16V4 { f16vec4 data[]; } a_f16v4; +layout(binding = MMA_BIND_U) readonly buffer BUF_U_F16 { float16_t data[]; } u_f16; +layout(binding = MMA_BIND_U) readonly buffer BUF_U_F16V4 { f16vec4 data[]; } u_f16v4; +#elif defined(DATA_A_Q8_0) +layout(binding = 0) readonly buffer BUF_A_Q8_0 { block_q8_0_packed16 data[]; } a_q8_0; +layout(binding = 0) readonly buffer BUF_A_U32 { uint data[]; } a_u32; +layout(binding = MMA_BIND_U) readonly buffer BUF_U_Q8_0 { block_q8_0_packed16 data[]; } u_q8_0; +layout(binding = MMA_BIND_U) readonly buffer BUF_U_U32 { uint data[]; } u_u32; +#elif defined(DATA_A_Q4_0) +layout(binding = 0) readonly buffer BUF_A_Q4_0 { block_q4_0_packed16 data[]; } a_q4_0; +layout(binding = MMA_BIND_U) readonly buffer BUF_U_Q4_0 { block_q4_0_packed16 data[]; } u_q4_0; +#elif defined(DATA_A_Q4_1) +layout(binding = 0) readonly buffer BUF_A_Q4_1 { block_q4_1_packed32 data[]; } a_q4_1; +layout(binding = MMA_BIND_U) readonly buffer BUF_U_Q4_1 { block_q4_1_packed32 data[]; } u_q4_1; +#elif defined(DATA_A_Q5_0) +layout(binding = 0) readonly buffer BUF_A_Q5_0 { block_q5_0_packed16 data[]; } a_q5_0; +layout(binding = MMA_BIND_U) readonly buffer BUF_U_Q5_0 { block_q5_0_packed16 data[]; } u_q5_0; +#elif defined(DATA_A_Q5_1) +layout(binding = 0) readonly buffer BUF_A_Q5_1 { block_q5_1_packed32 data[]; } a_q5_1; +layout(binding = MMA_BIND_U) readonly buffer BUF_U_Q5_1 { block_q5_1_packed32 data[]; } u_q5_1; +#elif defined(DATA_A_Q4_K) +layout(binding = 0) readonly buffer BUF_A_Q4_K { block_q4_K data[]; } a_q4_k; +layout(binding = 0) readonly buffer BUF_A_Q4_KP32 { block_q4_K_packed32 data[]; } a_q4_k_p32; +layout(binding = MMA_BIND_U) readonly buffer BUF_U_Q4_K { block_q4_K data[]; } u_q4_k; +layout(binding = MMA_BIND_U) readonly buffer BUF_U_Q4_KP32 { block_q4_K_packed32 data[]; } u_q4_k_p32; +#elif defined(DATA_A_Q5_K) +layout(binding = 0) readonly buffer BUF_A_Q5_K { block_q5_K data[]; } a_q5_k; +layout(binding = 0) readonly buffer BUF_A_Q5_KP32 { block_q5_K_packed32 data[]; } a_q5_k_p32; +layout(binding = MMA_BIND_U) readonly buffer BUF_U_Q5_K { block_q5_K data[]; } u_q5_k; +layout(binding = MMA_BIND_U) readonly buffer BUF_U_Q5_KP32 { block_q5_K_packed32 data[]; } u_q5_k_p32; +#elif defined(DATA_A_Q6_K) +layout(binding = 0) readonly buffer BUF_A_Q6_K { block_q6_K data[]; } a_q6_k; +layout(binding = 0) readonly buffer BUF_A_Q6_KP16 { block_q6_K_packed16 data[]; } a_q6_k_p16; +layout(binding = MMA_BIND_U) readonly buffer BUF_U_Q6_K { block_q6_K data[]; } u_q6_k; +layout(binding = MMA_BIND_U) readonly buffer BUF_U_Q6_KP16 { block_q6_K_packed16 data[]; } u_q6_k_p16; +#elif defined(DATA_A_IQ4_XS) +layout(binding = 0) readonly buffer BUF_A_IQ4_XS { block_iq4_xs data[]; } a_iq4_xs; +layout(binding = 0) readonly buffer BUF_A_IQ4XSP32 { block_iq4_xs_packed32 data[]; } a_iq4_xs_p32; +layout(binding = MMA_BIND_U) readonly buffer BUF_U_IQ4_XS { block_iq4_xs data[]; } u_iq4_xs; +layout(binding = MMA_BIND_U) readonly buffer BUF_U_IQ4XSP32 { block_iq4_xs_packed32 data[]; } u_iq4_xs_p32; +#elif defined(DATA_A_IQ4_NL) +layout(binding = 0) readonly buffer BUF_A_IQ4_NL { block_iq4_nl_packed16 data[]; } a_iq4_nl; +layout(binding = MMA_BIND_U) readonly buffer BUF_U_IQ4_NL { block_iq4_nl_packed16 data[]; } u_iq4_nl; +#endif + +// B bindings (binding = 1) +layout(binding = 1) readonly buffer BUF_B_F32 { float data_b[]; }; +layout(binding = 1) readonly buffer BUF_B_F32V2 { vec2 data_b_v2[]; }; +layout(binding = 1) readonly buffer BUF_B_F32V4 { vec4 data_b_v4[]; }; +layout(binding = 1) readonly buffer BUF_B_F16 { float16_t data_b_f16[]; }; +layout(binding = 1) readonly buffer BUF_B_F16V2 { f16vec2 data_b_f16v2[]; }; +layout(binding = 1) readonly buffer BUF_B_F16V4 { f16vec4 data_b_f16v4[]; }; + +// D binding (binding = 2) +layout(binding = 2) writeonly buffer BUF_D_F32 { float data_d[]; }; + +#ifdef MUL_MAT_ID +layout(binding = 3) readonly buffer BUF_IDS { int data_ids[]; }; +layout(binding = 4) readonly buffer BUF_ROW_MAP { uint data_row_map[]; }; +layout(binding = MMA_BIND_AUX1) writeonly buffer BUF_D_F16 { float16_t data_d_f16[]; }; +#else +layout(binding = 5) writeonly buffer BUF_D2_F32 { float data_d2[]; }; +layout(binding = 6) readonly buffer BUF_F2_F32 { float data_aux2[]; }; +layout(binding = 7) readonly buffer BUF_LORA_B0 { float data[]; } lora_b0; +layout(binding = 7) readonly buffer BUF_LORA_B0_V4 { vec4 data[]; } lora_b0_v4; +layout(binding = 8) readonly buffer BUF_LORA_H0 { float data[]; } lora_h0; +layout(binding = 8) readonly buffer BUF_LORA_H0_V4 { vec4 data[]; } lora_h0_v4; +layout(binding = 9) readonly buffer BUF_LORA_B1 { float data[]; } lora_b1; +layout(binding = 9) readonly buffer BUF_LORA_B1_V4 { vec4 data[]; } lora_b1_v4; +layout(binding = 10) readonly buffer BUF_LORA_H1 { float data[]; } lora_h1; +layout(binding = 10) readonly buffer BUF_LORA_H1_V4 { vec4 data[]; } lora_h1_v4; +#endif + +layout(binding = MMA_BIND_AUX0) readonly buffer BUF_AUX0 { float data_aux0[]; }; +layout(binding = MMA_BIND_AUX1) readonly buffer BUF_AUX1 { float data_aux1[]; }; + +#define MAT_VEC_FUSION_FLAGS_BIAS0 0x1u +#define MAT_VEC_FUSION_FLAGS_BIAS1 0x2u +#define MAT_VEC_FUSION_FLAGS_SCALE0 0x4u +#define MAT_VEC_FUSION_FLAGS_SCALE1 0x8u +#define MAT_VEC_FUSION_FLAGS_GLU_A 0x10u +#define MAT_VEC_FUSION_FLAGS_GLU_B 0x20u +#define MAT_VEC_FUSION_FLAGS_SIGMOID 0x80u +#define MAT_VEC_FUSION_FLAGS_SOFTPLUS_GATE 0x100u +#define MAT_VEC_FUSION_FLAGS_ROW_SCALE0 0x200u +#define MAT_VEC_FUSION_FLAGS_PAIR_GLU 0x400u +#define MAT_VEC_FUSION_FLAGS_PAIR_MM 0x800u +#define MAT_VEC_FUSION_FLAGS_PAIR_MM_SIGMOID1 0x1000u +#define MAT_VEC_FUSION_FLAGS_SCALE0_TOK 0x2000u +#define MAT_VEC_FUSION_FLAGS_SCALE1_TOK 0x4000u +#define MAT_VEC_FUSION_FLAGS_B_F16 0x10000u +#define MAT_VEC_FUSION_FLAGS_D_F16_AUX1 0x20000u +#define MAT_VEC_FUSION_FLAGS_SKIP_F32_D 0x40000u +#define MAT_VEC_FUSION_FLAGS_LORA0 0x80000u +#define MAT_VEC_FUSION_FLAGS_LORA1 0x100000u + +layout(push_constant) uniform parameter { +#ifdef MUL_MAT_ID + uint M; + uint N; + uint K; + uint stride_a; + uint stride_b; + uint stride_d; + uint batch_stride_a; + uint batch_stride_b; + uint batch_stride_d; + uint nei0; + uint nei1; + uint nbi1; + uint ne11; + uint n_experts; + uint row_ids_offset; + uint fusion_flags; +#else + uint M; + uint N; + uint K; + uint stride_a; + uint stride_b; + uint stride_d; + uint batch_stride_a; + uint batch_stride_b; + uint batch_stride_d; + uint fusion_flags; + uint base_work_group_z; + uint ne02; + uint ne12; + uint broadcast2; + uint broadcast3; + uint pair_m0; + uint stride_d2; + uint batch_stride_d2; + uint lora_r; + float lora_scale0; + float lora_scale1; +#endif +} p; + +#ifndef MUL_MAT_ID +const uint RED_SLOTS = (NSG > 1u) ? (NSG * MSG * RT * NT * 64u * ((FUSE_MODE == 3u) ? 2u : 1u)) : 1u; +shared float red_shmem[RED_SLOTS]; +#endif + +// Cooperative B staging tile when MSG > 1 && USE_SH_B != 0 +const bool COOP_SH_B = (MSG > 1u) && (USE_SH_B != 0u); +const uint SH_B_COLS = COOP_SH_B ? BN_COLS : 1u; +const uint SH_B_K4 = COOP_SH_B ? (16u * K_PAIRS) : 1u; +shared MMA_FVEC4 sh_b[SH_B_COLS][SH_B_K4]; + +#ifdef MUL_MAT_ID +const bool USE_SH_IDS = (SH_B_COLS * SH_B_K4 * 8u) < 16384u; +shared uint sh_b_base[USE_SH_IDS ? BN_COLS : 1u]; +shared uint sh_r_ids[USE_SH_IDS ? BN_COLS : 1u]; +shared float sh_col_scale[(USE_SH_IDS && FUSE_MODE == 4u) ? BN_COLS : 1u]; + +uint mma_id_col_packed(uint row_begin, uint tile_ne1, uint col_i) { + if (USE_SH_IDS) { + return sh_r_ids[min(col_i, BN_COLS - 1u)]; + } + return data_row_map[p.row_ids_offset + row_begin + min(col_i, tile_ne1 - 1u)]; +} +uint mma_id_col_b_base(uint row_begin, uint tile_ne1, uint col_i) { + if (USE_SH_IDS) { + return sh_b_base[min(col_i, BN_COLS - 1u)]; + } + const uint packed = mma_id_col_packed(row_begin, tile_ne1, col_i); + return (packed >> 16u) * p.batch_stride_b + ((packed & 0xFFFFu) % p.ne11) * p.stride_b; +} +#endif + +uint mma_lane_fm(uint l) { return ((l >> 1u) & 3u) | ((l >> 2u) & 4u); } +uint mma_lane_fn(uint l) { return ((l & 1u) << 1u) | ((l >> 1u) & 4u); } + +#if defined(DATA_A_IQ4_XS) || defined(DATA_A_IQ4_NL) +float iq4nl_lut(uint idx) { + switch (idx) { + case 0u: return -127.0f; + case 1u: return -104.0f; + case 2u: return -83.0f; + case 3u: return -65.0f; + case 4u: return -49.0f; + case 5u: return -35.0f; + case 6u: return -22.0f; + case 7u: return -10.0f; + case 8u: return 1.0f; + case 9u: return 13.0f; + case 10u: return 25.0f; + case 11u: return 38.0f; + case 12u: return 53.0f; + case 13u: return 69.0f; + case 14u: return 89.0f; + default: return 113.0f; + } +} +#endif + +MMA_FVEC4 load_b_mvec4(uint elem_idx, bool v4_ok, bool b_is_f16) { + if (b_is_f16) { + if (v4_ok) { + return MMA_FVEC4(data_b_f16v4[elem_idx >> 2u]); + } + return MMA_FVEC4(data_b_f16[elem_idx + 0u], data_b_f16[elem_idx + 1u], data_b_f16[elem_idx + 2u], data_b_f16[elem_idx + 3u]); + } + if (v4_ok) { + return MMA_FVEC4(data_b_v4[elem_idx >> 2u]); + } + return MMA_FVEC4(data_b[elem_idx + 0u], data_b[elem_idx + 1u], data_b[elem_idx + 2u], data_b[elem_idx + 3u]); +} + +MMA_FVEC2 load_b_mvec2(uint elem_idx, bool v2_ok, bool b_is_f16) { + if (b_is_f16) { + if (v2_ok) { + return MMA_FVEC2(data_b_f16v2[elem_idx >> 1u]); + } + return MMA_FVEC2(data_b_f16[elem_idx + 0u], data_b_f16[elem_idx + 1u]); + } + if (v2_ok) { + return MMA_FVEC2(data_b_v2[elem_idx >> 1u]); + } + return MMA_FVEC2(data_b[elem_idx + 0u], data_b[elem_idx + 1u]); +} + +// Unified single-body dequant16 (prevents duplicate temporary live ranges in PAIR_GLU / PAIR_MM!) +void dequant16(bool use_u, uint row_unit_base, uint ci, out MMA_FVEC4 w0, out MMA_FVEC4 w1, out MMA_FVEC4 w2, out MMA_FVEC4 w3) { +#if defined(DATA_A_F32) || defined(DATA_A_F32_F16) + const uint base = row_unit_base + (ci << 4u); + if ((base & 3u) == 0u) { + const uint v4 = base >> 2u; + if (use_u) { + w0 = MMA_FVEC4(u_f32v4.data[v4 + 0u]); w1 = MMA_FVEC4(u_f32v4.data[v4 + 1u]); w2 = MMA_FVEC4(u_f32v4.data[v4 + 2u]); w3 = MMA_FVEC4(u_f32v4.data[v4 + 3u]); + } else { + w0 = MMA_FVEC4(a_f32v4.data[v4 + 0u]); w1 = MMA_FVEC4(a_f32v4.data[v4 + 1u]); w2 = MMA_FVEC4(a_f32v4.data[v4 + 2u]); w3 = MMA_FVEC4(a_f32v4.data[v4 + 3u]); + } + } else if (use_u) { + w0 = MMA_FVEC4(u_f32.data[base+0u], u_f32.data[base+1u], u_f32.data[base+2u], u_f32.data[base+3u]); + w1 = MMA_FVEC4(u_f32.data[base+4u], u_f32.data[base+5u], u_f32.data[base+6u], u_f32.data[base+7u]); + w2 = MMA_FVEC4(u_f32.data[base+8u], u_f32.data[base+9u], u_f32.data[base+10u], u_f32.data[base+11u]); + w3 = MMA_FVEC4(u_f32.data[base+12u], u_f32.data[base+13u], u_f32.data[base+14u], u_f32.data[base+15u]); + } else { + w0 = MMA_FVEC4(a_f32.data[base+0u], a_f32.data[base+1u], a_f32.data[base+2u], a_f32.data[base+3u]); + w1 = MMA_FVEC4(a_f32.data[base+4u], a_f32.data[base+5u], a_f32.data[base+6u], a_f32.data[base+7u]); + w2 = MMA_FVEC4(a_f32.data[base+8u], a_f32.data[base+9u], a_f32.data[base+10u], a_f32.data[base+11u]); + w3 = MMA_FVEC4(a_f32.data[base+12u], a_f32.data[base+13u], a_f32.data[base+14u], a_f32.data[base+15u]); + } +#elif defined(DATA_A_F16) + const uint base = row_unit_base + (ci << 4u); + if ((base & 3u) == 0u) { + const uint v4 = base >> 2u; + if (use_u) { + w0 = u_f16v4.data[v4 + 0u]; w1 = u_f16v4.data[v4 + 1u]; w2 = u_f16v4.data[v4 + 2u]; w3 = u_f16v4.data[v4 + 3u]; + } else { + w0 = a_f16v4.data[v4 + 0u]; w1 = a_f16v4.data[v4 + 1u]; w2 = a_f16v4.data[v4 + 2u]; w3 = a_f16v4.data[v4 + 3u]; + } + } else if (use_u) { + w0 = f16vec4(u_f16.data[base+0u], u_f16.data[base+1u], u_f16.data[base+2u], u_f16.data[base+3u]); + w1 = f16vec4(u_f16.data[base+4u], u_f16.data[base+5u], u_f16.data[base+6u], u_f16.data[base+7u]); + w2 = f16vec4(u_f16.data[base+8u], u_f16.data[base+9u], u_f16.data[base+10u], u_f16.data[base+11u]); + w3 = f16vec4(u_f16.data[base+12u], u_f16.data[base+13u], u_f16.data[base+14u], u_f16.data[base+15u]); + } else { + w0 = f16vec4(a_f16.data[base+0u], a_f16.data[base+1u], a_f16.data[base+2u], a_f16.data[base+3u]); + w1 = f16vec4(a_f16.data[base+4u], a_f16.data[base+5u], a_f16.data[base+6u], a_f16.data[base+7u]); + w2 = f16vec4(a_f16.data[base+8u], a_f16.data[base+9u], a_f16.data[base+10u], a_f16.data[base+11u]); + w3 = f16vec4(a_f16.data[base+12u], a_f16.data[base+13u], a_f16.data[base+14u], a_f16.data[base+15u]); + } +#elif defined(DATA_A_Q8_0) + const uint ib = row_unit_base + (ci >> 1u); + const uint iqs = (ci & 1u) << 3u; + float16_t d; + uint q01, q23, q45, q67; + if (use_u) { + d = u_q8_0.data[ib].d; + q01 = pack32(u16vec2(uint16_t(u_q8_0.data[ib].qs[iqs + 0u]), uint16_t(u_q8_0.data[ib].qs[iqs + 1u]))) ^ 0x80808080u; + q23 = pack32(u16vec2(uint16_t(u_q8_0.data[ib].qs[iqs + 2u]), uint16_t(u_q8_0.data[ib].qs[iqs + 3u]))) ^ 0x80808080u; + q45 = pack32(u16vec2(uint16_t(u_q8_0.data[ib].qs[iqs + 4u]), uint16_t(u_q8_0.data[ib].qs[iqs + 5u]))) ^ 0x80808080u; + q67 = pack32(u16vec2(uint16_t(u_q8_0.data[ib].qs[iqs + 6u]), uint16_t(u_q8_0.data[ib].qs[iqs + 7u]))) ^ 0x80808080u; + } else { + d = a_q8_0.data[ib].d; + q01 = pack32(u16vec2(uint16_t(a_q8_0.data[ib].qs[iqs + 0u]), uint16_t(a_q8_0.data[ib].qs[iqs + 1u]))) ^ 0x80808080u; + q23 = pack32(u16vec2(uint16_t(a_q8_0.data[ib].qs[iqs + 2u]), uint16_t(a_q8_0.data[ib].qs[iqs + 3u]))) ^ 0x80808080u; + q45 = pack32(u16vec2(uint16_t(a_q8_0.data[ib].qs[iqs + 4u]), uint16_t(a_q8_0.data[ib].qs[iqs + 5u]))) ^ 0x80808080u; + q67 = pack32(u16vec2(uint16_t(a_q8_0.data[ib].qs[iqs + 6u]), uint16_t(a_q8_0.data[ib].qs[iqs + 7u]))) ^ 0x80808080u; + } + const f16vec2 bias16 = f16vec2(float16_t(1152.0)); + const f16vec2 e0 = unpackFloat2x16((q01 & 0x00FF00FFu) | 0x64006400u) - bias16; + const f16vec2 o0 = unpackFloat2x16(((q01 >> 8u) & 0x00FF00FFu) | 0x64006400u) - bias16; + const f16vec2 e1 = unpackFloat2x16((q23 & 0x00FF00FFu) | 0x64006400u) - bias16; + const f16vec2 o1 = unpackFloat2x16(((q23 >> 8u) & 0x00FF00FFu) | 0x64006400u) - bias16; + const f16vec2 e2 = unpackFloat2x16((q45 & 0x00FF00FFu) | 0x64006400u) - bias16; + const f16vec2 o2 = unpackFloat2x16(((q45 >> 8u) & 0x00FF00FFu) | 0x64006400u) - bias16; + const f16vec2 e3 = unpackFloat2x16((q67 & 0x00FF00FFu) | 0x64006400u) - bias16; + const f16vec2 o3 = unpackFloat2x16(((q67 >> 8u) & 0x00FF00FFu) | 0x64006400u) - bias16; + w0 = d * f16vec4(e0.x, o0.x, e0.y, o0.y); + w1 = d * f16vec4(e1.x, o1.x, e1.y, o1.y); + w2 = d * f16vec4(e2.x, o2.x, e2.y, o2.y); + w3 = d * f16vec4(e3.x, o3.x, e3.y, o3.y); +#elif defined(DATA_A_Q4_0) + const uint ib = row_unit_base + (ci >> 1u); + const uint shift = (ci & 1u) * 4u; + float d; + uint u0, u1, u2, u3; + if (use_u) { + d = float(u_q4_0.data[ib].d); + u0 = ((uint(u_q4_0.data[ib].qs[0]) | (uint(u_q4_0.data[ib].qs[1]) << 16u)) >> shift) & 0x0F0F0F0Fu; + u1 = ((uint(u_q4_0.data[ib].qs[2]) | (uint(u_q4_0.data[ib].qs[3]) << 16u)) >> shift) & 0x0F0F0F0Fu; + u2 = ((uint(u_q4_0.data[ib].qs[4]) | (uint(u_q4_0.data[ib].qs[5]) << 16u)) >> shift) & 0x0F0F0F0Fu; + u3 = ((uint(u_q4_0.data[ib].qs[6]) | (uint(u_q4_0.data[ib].qs[7]) << 16u)) >> shift) & 0x0F0F0F0Fu; + } else { + d = float(a_q4_0.data[ib].d); + u0 = ((uint(a_q4_0.data[ib].qs[0]) | (uint(a_q4_0.data[ib].qs[1]) << 16u)) >> shift) & 0x0F0F0F0Fu; + u1 = ((uint(a_q4_0.data[ib].qs[2]) | (uint(a_q4_0.data[ib].qs[3]) << 16u)) >> shift) & 0x0F0F0F0Fu; + u2 = ((uint(a_q4_0.data[ib].qs[4]) | (uint(a_q4_0.data[ib].qs[5]) << 16u)) >> shift) & 0x0F0F0F0Fu; + u3 = ((uint(a_q4_0.data[ib].qs[6]) | (uint(a_q4_0.data[ib].qs[7]) << 16u)) >> shift) & 0x0F0F0F0Fu; + } + const f16vec4 dv = f16vec4(d); + const f16vec4 mv = f16vec4(-8.0f * d); + w0 = fma(dv, f16vec4(unpack8(u0)), mv); + w1 = fma(dv, f16vec4(unpack8(u1)), mv); + w2 = fma(dv, f16vec4(unpack8(u2)), mv); + w3 = fma(dv, f16vec4(unpack8(u3)), mv); +#elif defined(DATA_A_Q4_1) + const uint ib = row_unit_base + (ci >> 1u); + const uint shift = (ci & 1u) * 4u; + vec2 dm; + uint q0, q1, q2, q3; + if (use_u) { + dm = vec2(u_q4_1.data[ib].dm); + q0 = u_q4_1.data[ib].qs[0]; q1 = u_q4_1.data[ib].qs[1]; q2 = u_q4_1.data[ib].qs[2]; q3 = u_q4_1.data[ib].qs[3]; + } else { + dm = vec2(a_q4_1.data[ib].dm); + q0 = a_q4_1.data[ib].qs[0]; q1 = a_q4_1.data[ib].qs[1]; q2 = a_q4_1.data[ib].qs[2]; q3 = a_q4_1.data[ib].qs[3]; + } + const vec4 d = vec4(dm.x), m = vec4(dm.y); + w0 = f16vec4(fma(d, vec4(unpack8((q0 >> shift) & 0x0F0F0F0Fu)), m)); + w1 = f16vec4(fma(d, vec4(unpack8((q1 >> shift) & 0x0F0F0F0Fu)), m)); + w2 = f16vec4(fma(d, vec4(unpack8((q2 >> shift) & 0x0F0F0F0Fu)), m)); + w3 = f16vec4(fma(d, vec4(unpack8((q3 >> shift) & 0x0F0F0F0Fu)), m)); +#elif defined(DATA_A_Q5_0) + const uint ib = row_unit_base + (ci >> 1u); + const uint half_i = ci & 1u; + const uint shift = half_i * 4u; + float d; + uint qh, u0, u1, u2, u3; + if (use_u) { + d = float(u_q5_0.data[ib].d); + qh = (uint(u_q5_0.data[ib].qh[0]) | (uint(u_q5_0.data[ib].qh[1]) << 16u)) >> (half_i * 16u); + u0 = ((uint(u_q5_0.data[ib].qs[0]) | (uint(u_q5_0.data[ib].qs[1]) << 16u)) >> shift) & 0x0F0F0F0Fu; + u1 = ((uint(u_q5_0.data[ib].qs[2]) | (uint(u_q5_0.data[ib].qs[3]) << 16u)) >> shift) & 0x0F0F0F0Fu; + u2 = ((uint(u_q5_0.data[ib].qs[4]) | (uint(u_q5_0.data[ib].qs[5]) << 16u)) >> shift) & 0x0F0F0F0Fu; + u3 = ((uint(u_q5_0.data[ib].qs[6]) | (uint(u_q5_0.data[ib].qs[7]) << 16u)) >> shift) & 0x0F0F0F0Fu; + } else { + d = float(a_q5_0.data[ib].d); + qh = (uint(a_q5_0.data[ib].qh[0]) | (uint(a_q5_0.data[ib].qh[1]) << 16u)) >> (half_i * 16u); + u0 = ((uint(a_q5_0.data[ib].qs[0]) | (uint(a_q5_0.data[ib].qs[1]) << 16u)) >> shift) & 0x0F0F0F0Fu; + u1 = ((uint(a_q5_0.data[ib].qs[2]) | (uint(a_q5_0.data[ib].qs[3]) << 16u)) >> shift) & 0x0F0F0F0Fu; + u2 = ((uint(a_q5_0.data[ib].qs[4]) | (uint(a_q5_0.data[ib].qs[5]) << 16u)) >> shift) & 0x0F0F0F0Fu; + u3 = ((uint(a_q5_0.data[ib].qs[6]) | (uint(a_q5_0.data[ib].qs[7]) << 16u)) >> shift) & 0x0F0F0F0Fu; + } + const float m = -16.0f * d; + const vec4 h0 = vec4(((qh >> 0u) & 1u) << 4u, ((qh >> 1u) & 1u) << 4u, ((qh >> 2u) & 1u) << 4u, ((qh >> 3u) & 1u) << 4u); + const vec4 h1 = vec4(((qh >> 4u) & 1u) << 4u, ((qh >> 5u) & 1u) << 4u, ((qh >> 6u) & 1u) << 4u, ((qh >> 7u) & 1u) << 4u); + const vec4 h2 = vec4(((qh >> 8u) & 1u) << 4u, ((qh >> 9u) & 1u) << 4u, ((qh >>10u) & 1u) << 4u, ((qh >>11u) & 1u) << 4u); + const vec4 h3 = vec4(((qh >>12u) & 1u) << 4u, ((qh >>13u) & 1u) << 4u, ((qh >>14u) & 1u) << 4u, ((qh >>15u) & 1u) << 4u); + w0 = f16vec4(fma(vec4(d), vec4(unpack8(u0)) + h0, vec4(m))); + w1 = f16vec4(fma(vec4(d), vec4(unpack8(u1)) + h1, vec4(m))); + w2 = f16vec4(fma(vec4(d), vec4(unpack8(u2)) + h2, vec4(m))); + w3 = f16vec4(fma(vec4(d), vec4(unpack8(u3)) + h3, vec4(m))); +#elif defined(DATA_A_Q5_1) + const uint ib = row_unit_base + (ci >> 1u); + const uint half_i = ci & 1u; + const uint shift = half_i * 4u; + vec2 dm; + uint qh, q0, q1, q2, q3; + if (use_u) { + dm = vec2(u_q5_1.data[ib].dm); qh = u_q5_1.data[ib].qh >> (half_i * 16u); + q0 = u_q5_1.data[ib].qs[0]; q1 = u_q5_1.data[ib].qs[1]; q2 = u_q5_1.data[ib].qs[2]; q3 = u_q5_1.data[ib].qs[3]; + } else { + dm = vec2(a_q5_1.data[ib].dm); qh = a_q5_1.data[ib].qh >> (half_i * 16u); + q0 = a_q5_1.data[ib].qs[0]; q1 = a_q5_1.data[ib].qs[1]; q2 = a_q5_1.data[ib].qs[2]; q3 = a_q5_1.data[ib].qs[3]; + } + const vec4 d = vec4(dm.x), m = vec4(dm.y); + const vec4 h0 = vec4(((qh >> 0u) & 1u) << 4u, ((qh >> 1u) & 1u) << 4u, ((qh >> 2u) & 1u) << 4u, ((qh >> 3u) & 1u) << 4u); + const vec4 h1 = vec4(((qh >> 4u) & 1u) << 4u, ((qh >> 5u) & 1u) << 4u, ((qh >> 6u) & 1u) << 4u, ((qh >> 7u) & 1u) << 4u); + const vec4 h2 = vec4(((qh >> 8u) & 1u) << 4u, ((qh >> 9u) & 1u) << 4u, ((qh >>10u) & 1u) << 4u, ((qh >>11u) & 1u) << 4u); + const vec4 h3 = vec4(((qh >>12u) & 1u) << 4u, ((qh >>13u) & 1u) << 4u, ((qh >>14u) & 1u) << 4u, ((qh >>15u) & 1u) << 4u); + w0 = f16vec4(fma(d, vec4(unpack8((q0 >> shift) & 0x0F0F0F0Fu)) + h0, m)); + w1 = f16vec4(fma(d, vec4(unpack8((q1 >> shift) & 0x0F0F0F0Fu)) + h1, m)); + w2 = f16vec4(fma(d, vec4(unpack8((q2 >> shift) & 0x0F0F0F0Fu)) + h2, m)); + w3 = f16vec4(fma(d, vec4(unpack8((q3 >> shift) & 0x0F0F0F0Fu)) + h3, m)); +#elif defined(DATA_A_Q4_K) + const uint ib = row_unit_base + (ci >> 4u); + const uint il = ci & 15u; + const uint is = il >> 1u; + const uint b = is & 1u; + const uint n = il >> 2u; + const uint w_idx = (n << 3u) + ((il & 1u) << 2u); + vec2 loadd; + uvec3 scales; + uint q0, q1, q2, q3; + if (use_u) { + loadd = vec2(u_q4_k.data[ib].dm); + scales = uvec3(u_q4_k_p32.data[ib].scales[0], u_q4_k_p32.data[ib].scales[1], u_q4_k_p32.data[ib].scales[2]); + q0 = u_q4_k_p32.data[ib].qs[w_idx + 0u]; q1 = u_q4_k_p32.data[ib].qs[w_idx + 1u]; + q2 = u_q4_k_p32.data[ib].qs[w_idx + 2u]; q3 = u_q4_k_p32.data[ib].qs[w_idx + 3u]; + } else { + loadd = vec2(a_q4_k.data[ib].dm); + scales = uvec3(a_q4_k_p32.data[ib].scales[0], a_q4_k_p32.data[ib].scales[1], a_q4_k_p32.data[ib].scales[2]); + q0 = a_q4_k_p32.data[ib].qs[w_idx + 0u]; q1 = a_q4_k_p32.data[ib].qs[w_idx + 1u]; + q2 = a_q4_k_p32.data[ib].qs[w_idx + 2u]; q3 = a_q4_k_p32.data[ib].qs[w_idx + 3u]; + } + const uint scalesoffs = (is & 3u) * 8u; + const uint scidxshift1 = (is < 4u) ? scalesoffs : (scalesoffs + 2u); + const uint mbidxshift0 = (is < 4u) ? scalesoffs : (scalesoffs + 4u); + const uint sc_word = (is < 4u) ? scales.x : scales.z; + const uint mb_word = (is < 4u) ? scales.y : scales.z; + const uint sc = ((sc_word >> scalesoffs) & 0xFu) | ((scales.x >> scidxshift1) & 0x30u); + const uint mbyte = ((mb_word >> mbidxshift0) & 0xFu) | ((scales.y >> scidxshift1) & 0x30u); + const float d_hi = (b == 0u) ? loadd.x : (loadd.x * 0.0625f); + const uint q_mk = (b == 0u) ? 0x0F0F0F0Fu : 0xF0F0F0F0u; + const f16vec4 d = f16vec4(d_hi * float(sc)); + const f16vec4 m = f16vec4(-loadd.y * float(mbyte)); + w0 = fma(d, f16vec4(unpack8(q0 & q_mk)), m); + w1 = fma(d, f16vec4(unpack8(q1 & q_mk)), m); + w2 = fma(d, f16vec4(unpack8(q2 & q_mk)), m); + w3 = fma(d, f16vec4(unpack8(q3 & q_mk)), m); +#elif defined(DATA_A_Q5_K) + const uint ib = row_unit_base + (ci >> 4u); + const uint il = ci & 15u; + const uint is = il >> 1u; + const uint b = is & 1u; + const uint n = il >> 2u; + const uint w_idx = (n << 3u) + ((il & 1u) << 2u); + const uint qh_idx = (il & 1u) << 2u; + vec2 loadd; + uvec3 scales; + uint ql0, ql1, ql2, ql3, qh0, qh1, qh2, qh3; + if (use_u) { + loadd = vec2(u_q5_k.data[ib].dm); + scales = uvec3(u_q5_k_p32.data[ib].scales[0], u_q5_k_p32.data[ib].scales[1], u_q5_k_p32.data[ib].scales[2]); + ql0 = u_q5_k_p32.data[ib].qs[w_idx + 0u]; ql1 = u_q5_k_p32.data[ib].qs[w_idx + 1u]; + ql2 = u_q5_k_p32.data[ib].qs[w_idx + 2u]; ql3 = u_q5_k_p32.data[ib].qs[w_idx + 3u]; + qh0 = u_q5_k_p32.data[ib].qh[qh_idx + 0u]; qh1 = u_q5_k_p32.data[ib].qh[qh_idx + 1u]; + qh2 = u_q5_k_p32.data[ib].qh[qh_idx + 2u]; qh3 = u_q5_k_p32.data[ib].qh[qh_idx + 3u]; + } else { + loadd = vec2(a_q5_k.data[ib].dm); + scales = uvec3(a_q5_k_p32.data[ib].scales[0], a_q5_k_p32.data[ib].scales[1], a_q5_k_p32.data[ib].scales[2]); + ql0 = a_q5_k_p32.data[ib].qs[w_idx + 0u]; ql1 = a_q5_k_p32.data[ib].qs[w_idx + 1u]; + ql2 = a_q5_k_p32.data[ib].qs[w_idx + 2u]; ql3 = a_q5_k_p32.data[ib].qs[w_idx + 3u]; + qh0 = a_q5_k_p32.data[ib].qh[qh_idx + 0u]; qh1 = a_q5_k_p32.data[ib].qh[qh_idx + 1u]; + qh2 = a_q5_k_p32.data[ib].qh[qh_idx + 2u]; qh3 = a_q5_k_p32.data[ib].qh[qh_idx + 3u]; + } + const uint scalesoffs = (is & 3u) * 8u; + const uint scidxshift1 = (is < 4u) ? scalesoffs : (scalesoffs + 2u); + const uint mbidxshift0 = (is < 4u) ? scalesoffs : (scalesoffs + 4u); + const uint sc_word = (is < 4u) ? scales.x : scales.z; + const uint mb_word = (is < 4u) ? scales.y : scales.z; + const uint sc = ((sc_word >> scalesoffs) & 0xFu) | ((scales.x >> scidxshift1) & 0x30u); + const uint mbyte = ((mb_word >> mbidxshift0) & 0xFu) | ((scales.y >> scidxshift1) & 0x30u); + const f16vec4 d = f16vec4(loadd.x * float(sc)); + const f16vec4 m = f16vec4(-loadd.y * float(mbyte)); + const uint shift = b * 4u; + const bool qh_left = (is <= 4u); + const uint qh_sh = qh_left ? (4u - is) : (is - 4u); + const uint qhb0 = (qh_left ? (qh0 << qh_sh) : (qh0 >> qh_sh)) & 0x10101010u; + const uint qhb1 = (qh_left ? (qh1 << qh_sh) : (qh1 >> qh_sh)) & 0x10101010u; + const uint qhb2 = (qh_left ? (qh2 << qh_sh) : (qh2 >> qh_sh)) & 0x10101010u; + const uint qhb3 = (qh_left ? (qh3 << qh_sh) : (qh3 >> qh_sh)) & 0x10101010u; + w0 = fma(d, f16vec4(unpack8(((ql0 >> shift) & 0x0F0F0F0Fu) | qhb0)), m); + w1 = fma(d, f16vec4(unpack8(((ql1 >> shift) & 0x0F0F0F0Fu) | qhb1)), m); + w2 = fma(d, f16vec4(unpack8(((ql2 >> shift) & 0x0F0F0F0Fu) | qhb2)), m); + w3 = fma(d, f16vec4(unpack8(((ql3 >> shift) & 0x0F0F0F0Fu) | qhb3)), m); +#elif defined(DATA_A_Q6_K) + const uint ib = row_unit_base + (ci >> 4u); + const uint il = ci & 15u; + const uint b = ((il >> 2u) & 1u) * 4u; + const uint qhshift = ((il >> 1u) & 3u) * 2u; + const uint qsi = ((il & 8u) << 2u) + ((il & 3u) << 3u); + const uint qhi = ((il & 8u) << 1u) + ((il & 1u) << 3u); + float dscale; + uint ql01, ql23, ql45, ql67, qh01, qh23, qh45, qh67; + if (use_u) { + dscale = float(u_q6_k.data[ib].d) * float(u_q6_k.data[ib].scales[il]); + ql01 = ((uint(u_q6_k_p16.data[ib].ql[qsi + 0u]) | (uint(u_q6_k_p16.data[ib].ql[qsi + 1u]) << 16u)) >> b) & 0x0F0F0F0Fu; + ql23 = ((uint(u_q6_k_p16.data[ib].ql[qsi + 2u]) | (uint(u_q6_k_p16.data[ib].ql[qsi + 3u]) << 16u)) >> b) & 0x0F0F0F0Fu; + ql45 = ((uint(u_q6_k_p16.data[ib].ql[qsi + 4u]) | (uint(u_q6_k_p16.data[ib].ql[qsi + 5u]) << 16u)) >> b) & 0x0F0F0F0Fu; + ql67 = ((uint(u_q6_k_p16.data[ib].ql[qsi + 6u]) | (uint(u_q6_k_p16.data[ib].ql[qsi + 7u]) << 16u)) >> b) & 0x0F0F0F0Fu; + qh01 = ((uint(u_q6_k_p16.data[ib].qh[qhi + 0u]) | (uint(u_q6_k_p16.data[ib].qh[qhi + 1u]) << 16u)) >> qhshift) & 0x03030303u; + qh23 = ((uint(u_q6_k_p16.data[ib].qh[qhi + 2u]) | (uint(u_q6_k_p16.data[ib].qh[qhi + 3u]) << 16u)) >> qhshift) & 0x03030303u; + qh45 = ((uint(u_q6_k_p16.data[ib].qh[qhi + 4u]) | (uint(u_q6_k_p16.data[ib].qh[qhi + 5u]) << 16u)) >> qhshift) & 0x03030303u; + qh67 = ((uint(u_q6_k_p16.data[ib].qh[qhi + 6u]) | (uint(u_q6_k_p16.data[ib].qh[qhi + 7u]) << 16u)) >> qhshift) & 0x03030303u; + } else { + dscale = float(a_q6_k.data[ib].d) * float(a_q6_k.data[ib].scales[il]); + ql01 = ((uint(a_q6_k_p16.data[ib].ql[qsi + 0u]) | (uint(a_q6_k_p16.data[ib].ql[qsi + 1u]) << 16u)) >> b) & 0x0F0F0F0Fu; + ql23 = ((uint(a_q6_k_p16.data[ib].ql[qsi + 2u]) | (uint(a_q6_k_p16.data[ib].ql[qsi + 3u]) << 16u)) >> b) & 0x0F0F0F0Fu; + ql45 = ((uint(a_q6_k_p16.data[ib].ql[qsi + 4u]) | (uint(a_q6_k_p16.data[ib].ql[qsi + 5u]) << 16u)) >> b) & 0x0F0F0F0Fu; + ql67 = ((uint(a_q6_k_p16.data[ib].ql[qsi + 6u]) | (uint(a_q6_k_p16.data[ib].ql[qsi + 7u]) << 16u)) >> b) & 0x0F0F0F0Fu; + qh01 = ((uint(a_q6_k_p16.data[ib].qh[qhi + 0u]) | (uint(a_q6_k_p16.data[ib].qh[qhi + 1u]) << 16u)) >> qhshift) & 0x03030303u; + qh23 = ((uint(a_q6_k_p16.data[ib].qh[qhi + 2u]) | (uint(a_q6_k_p16.data[ib].qh[qhi + 3u]) << 16u)) >> qhshift) & 0x03030303u; + qh45 = ((uint(a_q6_k_p16.data[ib].qh[qhi + 4u]) | (uint(a_q6_k_p16.data[ib].qh[qhi + 5u]) << 16u)) >> qhshift) & 0x03030303u; + qh67 = ((uint(a_q6_k_p16.data[ib].qh[qhi + 6u]) | (uint(a_q6_k_p16.data[ib].qh[qhi + 7u]) << 16u)) >> qhshift) & 0x03030303u; + } + const f16vec4 dv = f16vec4(dscale); + const f16vec4 mv = f16vec4(-32.0f * dscale); + w0 = fma(f16vec4(unpack8(ql01 | (qh01 << 4u))), dv, mv); + w1 = fma(f16vec4(unpack8(ql23 | (qh23 << 4u))), dv, mv); + w2 = fma(f16vec4(unpack8(ql45 | (qh45 << 4u))), dv, mv); + w3 = fma(f16vec4(unpack8(ql67 | (qh67 << 4u))), dv, mv); +#elif defined(DATA_A_IQ4_XS) + const uint ib = row_unit_base + (ci >> 4u); + const uint il = ci & 15u; + const uint ib32 = il >> 1u; + const uint shift = (il & 1u) * 4u; + const uint qbase = ib32 << 2u; + float dl; + uint r0, r1, r2, r3; + if (use_u) { + const uint sl = (uint(u_iq4_xs.data[ib].scales_l[ib32 >> 1u]) >> (4u * (ib32 & 1u))) & 0xFu; + const uint sh = (uint(u_iq4_xs.data[ib].scales_h) >> (2u * ib32)) & 3u; + dl = float(u_iq4_xs.data[ib].d) * float(int(sl | (sh << 4u)) - 32); + r0 = u_iq4_xs_p32.data[ib].qs[qbase + 0u]; r1 = u_iq4_xs_p32.data[ib].qs[qbase + 1u]; + r2 = u_iq4_xs_p32.data[ib].qs[qbase + 2u]; r3 = u_iq4_xs_p32.data[ib].qs[qbase + 3u]; + } else { + const uint sl = (uint(a_iq4_xs.data[ib].scales_l[ib32 >> 1u]) >> (4u * (ib32 & 1u))) & 0xFu; + const uint sh = (uint(a_iq4_xs.data[ib].scales_h) >> (2u * ib32)) & 3u; + dl = float(a_iq4_xs.data[ib].d) * float(int(sl | (sh << 4u)) - 32); + r0 = a_iq4_xs_p32.data[ib].qs[qbase + 0u]; r1 = a_iq4_xs_p32.data[ib].qs[qbase + 1u]; + r2 = a_iq4_xs_p32.data[ib].qs[qbase + 2u]; r3 = a_iq4_xs_p32.data[ib].qs[qbase + 3u]; + } + const u8vec4 q0 = unpack8((r0 >> shift) & 0x0F0F0F0Fu); + const u8vec4 q1 = unpack8((r1 >> shift) & 0x0F0F0F0Fu); + const u8vec4 q2 = unpack8((r2 >> shift) & 0x0F0F0F0Fu); + const u8vec4 q3 = unpack8((r3 >> shift) & 0x0F0F0F0Fu); + w0 = f16vec4(dl * vec4(iq4nl_lut(q0.x), iq4nl_lut(q0.y), iq4nl_lut(q0.z), iq4nl_lut(q0.w))); + w1 = f16vec4(dl * vec4(iq4nl_lut(q1.x), iq4nl_lut(q1.y), iq4nl_lut(q1.z), iq4nl_lut(q1.w))); + w2 = f16vec4(dl * vec4(iq4nl_lut(q2.x), iq4nl_lut(q2.y), iq4nl_lut(q2.z), iq4nl_lut(q2.w))); + w3 = f16vec4(dl * vec4(iq4nl_lut(q3.x), iq4nl_lut(q3.y), iq4nl_lut(q3.z), iq4nl_lut(q3.w))); +#elif defined(DATA_A_IQ4_NL) + const uint ib = row_unit_base + (ci >> 1u); + const uint shift = (ci & 1u) * 4u; + float d; + uint r0, r1, r2, r3; + if (use_u) { + d = float(u_iq4_nl.data[ib].d); + r0 = (uint(u_iq4_nl.data[ib].qs[0]) | (uint(u_iq4_nl.data[ib].qs[1]) << 16u)); + r1 = (uint(u_iq4_nl.data[ib].qs[2]) | (uint(u_iq4_nl.data[ib].qs[3]) << 16u)); + r2 = (uint(u_iq4_nl.data[ib].qs[4]) | (uint(u_iq4_nl.data[ib].qs[5]) << 16u)); + r3 = (uint(u_iq4_nl.data[ib].qs[6]) | (uint(u_iq4_nl.data[ib].qs[7]) << 16u)); + } else { + d = float(a_iq4_nl.data[ib].d); + r0 = (uint(a_iq4_nl.data[ib].qs[0]) | (uint(a_iq4_nl.data[ib].qs[1]) << 16u)); + r1 = (uint(a_iq4_nl.data[ib].qs[2]) | (uint(a_iq4_nl.data[ib].qs[3]) << 16u)); + r2 = (uint(a_iq4_nl.data[ib].qs[4]) | (uint(a_iq4_nl.data[ib].qs[5]) << 16u)); + r3 = (uint(a_iq4_nl.data[ib].qs[6]) | (uint(a_iq4_nl.data[ib].qs[7]) << 16u)); + } + const u8vec4 q0 = unpack8((r0 >> shift) & 0x0F0F0F0Fu); + const u8vec4 q1 = unpack8((r1 >> shift) & 0x0F0F0F0Fu); + const u8vec4 q2 = unpack8((r2 >> shift) & 0x0F0F0F0Fu); + const u8vec4 q3 = unpack8((r3 >> shift) & 0x0F0F0F0Fu); + w0 = f16vec4(d * vec4(iq4nl_lut(q0.x), iq4nl_lut(q0.y), iq4nl_lut(q0.z), iq4nl_lut(q0.w))); + w1 = f16vec4(d * vec4(iq4nl_lut(q1.x), iq4nl_lut(q1.y), iq4nl_lut(q1.z), iq4nl_lut(q1.w))); + w2 = f16vec4(d * vec4(iq4nl_lut(q2.x), iq4nl_lut(q2.y), iq4nl_lut(q2.z), iq4nl_lut(q2.w))); + w3 = f16vec4(d * vec4(iq4nl_lut(q3.x), iq4nl_lut(q3.y), iq4nl_lut(q3.z), iq4nl_lut(q3.w))); +#endif +} + +#ifdef MUL_MAT_ID +float mma_apply_id_epilogue(float val, uint expert_idx, u16vec2 row_idx, uint d_row, uint d_idx) { + if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS0) != 0u) { + val += data_aux0[expert_idx * p.stride_d + d_row]; + } + if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_SCALE0) != 0u) { + const uint s0_idx = uint(row_idx.x) + ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_SCALE0_TOK) != 0u ? uint(row_idx.y) * p.nei0 : 0u); + val *= data_aux0[s0_idx]; + } + if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_SCALE1) != 0u) { + const uint s1_idx = uint(row_idx.x) + ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_SCALE1_TOK) != 0u ? uint(row_idx.y) * p.nei0 : 0u); + val *= data_aux1[s1_idx]; + } + if ((p.fusion_flags & (MAT_VEC_FUSION_FLAGS_GLU_A | MAT_VEC_FUSION_FLAGS_GLU_B)) != 0u) { + const float o = data_aux0[d_idx]; + val = ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_GLU_A) != 0u) + ? (val / (1.0f + exp(-val)) * o) + : (o / (1.0f + exp(-o)) * val); + } + return val; +} +#else +float mma_apply_dense_epilogue(float val, uint d_row, uint d_col, uint d_idx, bool is_pair_second) { + if (is_pair_second) { + if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_PAIR_MM_SIGMOID1) != 0u) { + val = 1.0f / (1.0f + exp(-val)); + } + return val; + } + if (FUSE_MODE == 2u) { + if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_SOFTPLUS_GATE) != 0u) { + const float x = val + data_aux1[d_row]; + const float sp = (x > 20.0f) ? x : log(1.0f + exp(x)); + val = sp * data_aux2[d_row]; + } + if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_SIGMOID) != 0u) { + val = 1.0f / (1.0f + exp(-val)); + } + return val; + } + if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS0) != 0u) { + val += data_aux0[d_idx]; + } + if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS1) != 0u) { + val += data_aux1[d_idx]; + } + if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_SIGMOID) != 0u) { + val = 1.0f / (1.0f + exp(-val)); + } + if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_SOFTPLUS_GATE) != 0u) { + const float x = val + data_aux0[d_row]; + const float sp = (x > 20.0f) ? x : log(1.0f + exp(x)); + val = sp * data_aux1[d_row]; + } + if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_ROW_SCALE0) != 0u) { + val *= data_aux0[d_col]; + } + if ((p.fusion_flags & (MAT_VEC_FUSION_FLAGS_GLU_A | MAT_VEC_FUSION_FLAGS_GLU_B)) != 0u) { + const float o = data_aux0[d_idx]; + val = ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_GLU_A) != 0u) + ? (val / (1.0f + exp(-val)) * o) + : (o / (1.0f + exp(-o)) * val); + } + return val; +} + +float mma_eval_lora0(uint row, uint col) { + if (FUSE_LORA == 0u) { return 0.0f; } + float acc = 0.0f; + if ((p.lora_r & 3u) == 0u) { + const uint r4 = p.lora_r >> 2u; + const uint b_off = row * r4; + const uint h_off = col * r4; + for (uint k4 = 0u; k4 < r4; ++k4) { + acc += dot(lora_b0_v4.data[b_off + k4], lora_h0_v4.data[h_off + k4]); + } + } else { + const uint b_off = row * p.lora_r; + const uint h_off = col * p.lora_r; + for (uint k = 0u; k < p.lora_r; ++k) { + acc = fma(lora_b0.data[b_off + k], lora_h0.data[h_off + k], acc); + } + } + return acc * p.lora_scale0; +} + +float mma_eval_lora1(uint row, uint col) { + if (FUSE_LORA == 0u) { return 0.0f; } + float acc = 0.0f; + if ((p.lora_r & 3u) == 0u) { + const uint r4 = p.lora_r >> 2u; + const uint b_off = row * r4; + const uint h_off = col * r4; + for (uint k4 = 0u; k4 < r4; ++k4) { + acc += dot(lora_b1_v4.data[b_off + k4], lora_h1_v4.data[h_off + k4]); + } + } else { + const uint b_off = row * p.lora_r; + const uint h_off = col * p.lora_r; + for (uint k = 0u; k < p.lora_r; ++k) { + acc = fma(lora_b1.data[b_off + k], lora_h1.data[h_off + k], acc); + } + } + return acc * p.lora_scale1; +} +#endif + +void main() { + const uint lane = gl_LocalInvocationID.x; + const uint sgitg = gl_LocalInvocationID.y; + const uint tid = sgitg * 32u + lane; + const uint sg_k = (CSG > 1u) ? 0u : (sgitg % NSG); + const uint sg_c = (CSG > 1u) ? (sgitg % CSG) : 0u; + const uint sg_m = (CSG > 1u) ? (sgitg / CSG) : (sgitg / NSG); + const uint sg_c_off = sg_c * (8u * RT); + const uint fm = mma_lane_fm(lane); + const uint fn = mma_lane_fn(lane); + + const uint wg_m0 = gl_WorkGroupID.x * (8u * NT * MSG); + const uint sg_m0 = wg_m0 + sg_m * (8u * NT); + + const uint stride_a_u = p.stride_a >> MMA_ELEM_SHIFT; + +#ifdef MUL_MAT_ID + const uint tile_idx = gl_WorkGroupID.y + gl_WorkGroupID.z * gl_NumWorkGroups.y; + if (tile_idx >= data_row_map[0]) { + return; + } + const uint expert_idx = data_row_map[1u + 2u * tile_idx]; + const uint row_begin = tile_idx * BN_COLS; + const uint tile_ne1 = data_row_map[2u + 2u * tile_idx]; + if (tile_ne1 == 0u || wg_m0 >= p.M) { + return; + } + const uint n_valid = tile_ne1; + const uint a_batch_base = expert_idx * (p.batch_stride_a >> MMA_ELEM_SHIFT); + if (USE_SH_IDS) { + for (uint c_it = tid; c_it < BN_COLS; c_it += WG_THREADS) { + const uint c = min(c_it, tile_ne1 - 1u); + const uint packed = data_row_map[p.row_ids_offset + row_begin + c]; + const uint slot = packed & 0xFFFFu; + const uint tok = packed >> 16u; + sh_r_ids[c_it] = packed; + sh_b_base[c_it] = tok * p.batch_stride_b + (slot % p.ne11) * p.stride_b; + if (FUSE_MODE == 4u) { + const uint s0_idx = slot + (((p.fusion_flags & MAT_VEC_FUSION_FLAGS_SCALE0_TOK) != 0u) ? (tok * p.nei0) : 0u); + sh_col_scale[c_it] = data_aux0[s0_idx]; + } + } + barrier(); + } +#else + if (wg_m0 >= p.M) { + return; + } + const uint wg_n0 = gl_WorkGroupID.y * (8u * RT); + const uint n_valid = (wg_n0 < p.N) ? min(8u * RT, p.N - wg_n0) : 0u; + if (n_valid == 0u) { + return; + } + const uint batch_idx = gl_WorkGroupID.z + p.base_work_group_z; + const uint i13 = batch_idx / p.ne12; + const uint i12 = batch_idx % p.ne12; + const uint i03 = i13 / p.broadcast3; + const uint i02 = i12 / p.broadcast2; + const uint batch_idx_a = i03 * p.ne02 + i02; + const uint a_batch_base = batch_idx_a * (p.batch_stride_a >> MMA_ELEM_SHIFT); + const uint b_batch_base = batch_idx * p.batch_stride_b; +#endif + + uint a_row_base[NT]; + bool tile_second[NT]; + [[unroll]] for (uint t = 0u; t < NT; ++t) { + const uint tile_r = sg_m0 + 8u * t; +#ifndef MUL_MAT_ID + if (FUSE_MODE == 2u && tile_r >= p.pair_m0) { + tile_second[t] = true; + const uint m1 = p.M - p.pair_m0; + const uint r1 = min(tile_r - p.pair_m0 + fm, m1 - 1u); + a_row_base[t] = a_batch_base + r1 * stride_a_u; + } else if (FUSE_MODE == 2u) { + tile_second[t] = false; + const uint r0 = min(tile_r + fm, p.pair_m0 - 1u); + a_row_base[t] = a_batch_base + r0 * stride_a_u; + } else +#endif + { + tile_second[t] = false; + const uint r0 = min(tile_r + fm, p.M - 1u); + a_row_base[t] = a_batch_base + r0 * stride_a_u; + } + } + + coopmat mc[RT][NT]; + coopmat mc_u[RT][NT]; + [[unroll]] for (uint rt = 0u; rt < RT; ++rt) { + [[unroll]] for (uint t = 0u; t < NT; ++t) { + mc[rt][t] = coopmat(0.0f); + mc_u[rt][t] = coopmat(0.0f); + } + } + + const bool b_is_f16 = (B_IS_F16 != 0u); + + // Cooperative B-tile path when MSG > 1 && USE_SH_B != 0 (NSG == 1, CSG == 1, K % 64 == 0) + if (MSG > 1u && USE_SH_B != 0u) { + const bool v4_ok = ((p.stride_b & 3u) == 0u) && ((p.batch_stride_b & 3u) == 0u); + const uint k4_lane_base = 4u * (fm >> 1u) + 2u * (fm & 1u); + +#define MMA_SG_FENCE() controlBarrier(gl_ScopeSubgroup, gl_ScopeSubgroup, gl_StorageSemanticsShared, gl_SemanticsAcquireRelease) +#define MMA_NO_FENCE() do {} while (false) + +#define MMA_ACCUM_SHB_1D(ACC_VEC, SUB_OFF, W0, W1, W2, W3) \ + do { \ + [[unroll]] for (uint rt = 0u; rt < RT; ++rt) { \ + if (rt == 0u || (8u * rt < n_valid)) { \ + const uint c0 = 8u * rt + fn + 0u; \ + const uint c1 = 8u * rt + fn + 1u; \ + const MMA_FVEC4 ba0 = sh_b[c0][(SUB_OFF) + k4_lane_base + 0u]; \ + const MMA_FVEC4 ba1 = sh_b[c0][(SUB_OFF) + k4_lane_base + 1u]; \ + const MMA_FVEC4 bb0 = sh_b[c1][(SUB_OFF) + k4_lane_base + 0u]; \ + const MMA_FVEC4 bb1 = sh_b[c1][(SUB_OFF) + k4_lane_base + 1u]; \ + [[unroll]] for (uint s = 0u; s < 4u; ++s) { \ + coopmat ma0, ma1; \ + coopmat mb0, mb1; \ + ma0[0] = (W0)[s]; ma0[1] = (W2)[s]; \ + ma1[0] = (W1)[s]; ma1[1] = (W3)[s]; \ + mb0[0] = ba0[s]; mb0[1] = bb0[s]; \ + mb1[0] = ba1[s]; mb1[1] = bb1[s]; \ + ACC_VEC[rt] = coopMatMulAdd(ma0, mb0, ACC_VEC[rt]); \ + ACC_VEC[rt] = coopMatMulAdd(ma1, mb1, ACC_VEC[rt]); \ + } \ + } \ + } \ + } while (false) + +#define MMA_ACCUM_SHB(MC_ARR, SUB_OFF, W0, W1, W2, W3) \ + do { \ + [[unroll]] for (uint rt = 0u; rt < RT; ++rt) { \ + if (use_sg_fence) { MMA_SG_FENCE(); } \ + if (rt == 0u || (8u * rt < n_valid)) { \ + const uint c0 = 8u * rt + fn + 0u; \ + const uint c1 = 8u * rt + fn + 1u; \ + const MMA_FVEC4 ba0 = sh_b[c0][(SUB_OFF) + k4_lane_base + 0u]; \ + const MMA_FVEC4 ba1 = sh_b[c0][(SUB_OFF) + k4_lane_base + 1u]; \ + const MMA_FVEC4 bb0 = sh_b[c1][(SUB_OFF) + k4_lane_base + 0u]; \ + const MMA_FVEC4 bb1 = sh_b[c1][(SUB_OFF) + k4_lane_base + 1u]; \ + [[unroll]] for (uint s = 0u; s < 4u; ++s) { \ + coopmat ma0, ma1; \ + coopmat mb0, mb1; \ + ma0[0] = (W0)[s]; ma0[1] = (W2)[s]; \ + ma1[0] = (W1)[s]; ma1[1] = (W3)[s]; \ + mb0[0] = ba0[s]; mb0[1] = bb0[s]; \ + mb1[0] = ba1[s]; mb1[1] = bb1[s]; \ + MC_ARR[rt][t] = coopMatMulAdd(ma0, mb0, MC_ARR[rt][t]); \ + MC_ARR[rt][t] = coopMatMulAdd(ma1, mb1, MC_ARR[rt][t]); \ + } \ + } \ + } \ + } while (false) + +#if defined(DATA_A_Q4_K) +#define MMA_ACCUM_KPAIRS_GEN(LOOP_ATTR, FENCE_MACRO, TARGET_MACRO, TARGET_ARR, USE_U, ROW_BASE) \ + do { \ + FENCE_MACRO(); \ + const uint ib = (ROW_BASE) + ((g * K_PAIRS) >> 2u); \ + const uint sub_base = (g * K_PAIRS) & 3u; \ + const uint fn2 = fn >> 1u; \ + const uint b = fn2 >> 1u; \ + const float d_hi = (b == 0u) ? 1.0f : 0.0625f; \ + const uint q_mk = (b == 0u) ? 0x0F0F0F0Fu : 0xF0F0F0F0u; \ + const uint w_lo = (fn2 & 1u) << 2u; \ + vec2 loadd; \ + uvec3 scales; \ + if (USE_U) { \ + loadd = vec2(u_q4_k.data[ib].dm); \ + scales = uvec3(u_q4_k_p32.data[ib].scales[0], u_q4_k_p32.data[ib].scales[1], u_q4_k_p32.data[ib].scales[2]); \ + } else { \ + loadd = vec2(a_q4_k.data[ib].dm); \ + scales = uvec3(a_q4_k_p32.data[ib].scales[0], a_q4_k_p32.data[ib].scales[1], a_q4_k_p32.data[ib].scales[2]); \ + } \ + const float dx = loadd.x * d_hi; \ + LOOP_ATTR for (uint s_kp = 0u; s_kp < K_PAIRS; ++s_kp) { \ + FENCE_MACRO(); \ + const uint sub = sub_base + s_kp; \ + const uint is = (sub << 1u) | b; \ + const uint scalesoffs = (is & 3u) * 8u; \ + const uint scidxshift1 = (is < 4u) ? scalesoffs : (scalesoffs + 2u); \ + const uint mbidxshift0 = (is < 4u) ? scalesoffs : (scalesoffs + 4u); \ + const uint sc_word = (is < 4u) ? scales.x : scales.z; \ + const uint mb_word = (is < 4u) ? scales.y : scales.z; \ + const uint sc = ((sc_word >> scalesoffs) & 0xFu) | ((scales.x >> scidxshift1) & 0x30u); \ + const uint mbyte = ((mb_word >> mbidxshift0) & 0xFu) | ((scales.y >> scidxshift1) & 0x30u); \ + const f16vec4 d = f16vec4(dx * float(sc)); \ + const f16vec4 m = f16vec4(-loadd.y * float(mbyte)); \ + const uint w_idx = (sub << 3u) + w_lo; \ + uint q0, q1, q2, q3; \ + if (USE_U) { \ + q0 = u_q4_k_p32.data[ib].qs[w_idx + 0u]; q1 = u_q4_k_p32.data[ib].qs[w_idx + 1u]; \ + q2 = u_q4_k_p32.data[ib].qs[w_idx + 2u]; q3 = u_q4_k_p32.data[ib].qs[w_idx + 3u]; \ + } else { \ + q0 = a_q4_k_p32.data[ib].qs[w_idx + 0u]; q1 = a_q4_k_p32.data[ib].qs[w_idx + 1u]; \ + q2 = a_q4_k_p32.data[ib].qs[w_idx + 2u]; q3 = a_q4_k_p32.data[ib].qs[w_idx + 3u]; \ + } \ + const f16vec4 w0 = fma(d, f16vec4(unpack8(q0 & q_mk)), m); \ + const f16vec4 w1 = fma(d, f16vec4(unpack8(q1 & q_mk)), m); \ + const f16vec4 w2 = fma(d, f16vec4(unpack8(q2 & q_mk)), m); \ + const f16vec4 w3 = fma(d, f16vec4(unpack8(q3 & q_mk)), m); \ + FENCE_MACRO(); \ + TARGET_MACRO(TARGET_ARR, (s_kp << 4u), w0, w1, w2, w3); \ + } \ + } while (false) +#elif defined(DATA_A_Q5_K) +#define MMA_ACCUM_KPAIRS_GEN(LOOP_ATTR, FENCE_MACRO, TARGET_MACRO, TARGET_ARR, USE_U, ROW_BASE) \ + do { \ + FENCE_MACRO(); \ + const uint ib = (ROW_BASE) + ((g * K_PAIRS) >> 2u); \ + const uint sub_base = (g * K_PAIRS) & 3u; \ + const uint fn2 = fn >> 1u; \ + const uint shift = (fn2 >> 1u) * 4u; \ + const uint w_lo = (fn2 & 1u) << 2u; \ + vec2 loadd; \ + uvec3 scales; \ + uint qh0, qh1, qh2, qh3; \ + if (USE_U) { \ + loadd = vec2(u_q5_k.data[ib].dm); \ + scales = uvec3(u_q5_k_p32.data[ib].scales[0], u_q5_k_p32.data[ib].scales[1], u_q5_k_p32.data[ib].scales[2]); \ + qh0 = u_q5_k_p32.data[ib].qh[w_lo + 0u]; qh1 = u_q5_k_p32.data[ib].qh[w_lo + 1u]; \ + qh2 = u_q5_k_p32.data[ib].qh[w_lo + 2u]; qh3 = u_q5_k_p32.data[ib].qh[w_lo + 3u]; \ + } else { \ + loadd = vec2(a_q5_k.data[ib].dm); \ + scales = uvec3(a_q5_k_p32.data[ib].scales[0], a_q5_k_p32.data[ib].scales[1], a_q5_k_p32.data[ib].scales[2]); \ + qh0 = a_q5_k_p32.data[ib].qh[w_lo + 0u]; qh1 = a_q5_k_p32.data[ib].qh[w_lo + 1u]; \ + qh2 = a_q5_k_p32.data[ib].qh[w_lo + 2u]; qh3 = a_q5_k_p32.data[ib].qh[w_lo + 3u]; \ + } \ + LOOP_ATTR for (uint s_kp = 0u; s_kp < K_PAIRS; ++s_kp) { \ + FENCE_MACRO(); \ + const uint sub = sub_base + s_kp; \ + const uint is = (sub << 1u) | (fn2 >> 1u); \ + const uint scalesoffs = (is & 3u) * 8u; \ + const uint scidxshift1 = (is < 4u) ? scalesoffs : (scalesoffs + 2u); \ + const uint mbidxshift0 = (is < 4u) ? scalesoffs : (scalesoffs + 4u); \ + const uint sc_word = (is < 4u) ? scales.x : scales.z; \ + const uint mb_word = (is < 4u) ? scales.y : scales.z; \ + const uint sc = ((sc_word >> scalesoffs) & 0xFu) | ((scales.x >> scidxshift1) & 0x30u); \ + const uint mbyte = ((mb_word >> mbidxshift0) & 0xFu) | ((scales.y >> scidxshift1) & 0x30u); \ + const f16vec4 d = f16vec4(loadd.x * float(sc)); \ + const f16vec4 m = f16vec4(-loadd.y * float(mbyte)); \ + const uint w_idx = (sub << 3u) + w_lo; \ + uint ql0, ql1, ql2, ql3; \ + if (USE_U) { \ + ql0 = u_q5_k_p32.data[ib].qs[w_idx + 0u]; ql1 = u_q5_k_p32.data[ib].qs[w_idx + 1u]; \ + ql2 = u_q5_k_p32.data[ib].qs[w_idx + 2u]; ql3 = u_q5_k_p32.data[ib].qs[w_idx + 3u]; \ + } else { \ + ql0 = a_q5_k_p32.data[ib].qs[w_idx + 0u]; ql1 = a_q5_k_p32.data[ib].qs[w_idx + 1u]; \ + ql2 = a_q5_k_p32.data[ib].qs[w_idx + 2u]; ql3 = a_q5_k_p32.data[ib].qs[w_idx + 3u]; \ + } \ + const bool qh_left = (is <= 4u); \ + const uint qh_sh = qh_left ? (4u - is) : (is - 4u); \ + const uint qhb0 = (qh_left ? (qh0 << qh_sh) : (qh0 >> qh_sh)) & 0x10101010u; \ + const uint qhb1 = (qh_left ? (qh1 << qh_sh) : (qh1 >> qh_sh)) & 0x10101010u; \ + const uint qhb2 = (qh_left ? (qh2 << qh_sh) : (qh2 >> qh_sh)) & 0x10101010u; \ + const uint qhb3 = (qh_left ? (qh3 << qh_sh) : (qh3 >> qh_sh)) & 0x10101010u; \ + const f16vec4 w0 = fma(d, f16vec4(unpack8(((ql0 >> shift) & 0x0F0F0F0Fu) | qhb0)), m); \ + const f16vec4 w1 = fma(d, f16vec4(unpack8(((ql1 >> shift) & 0x0F0F0F0Fu) | qhb1)), m); \ + const f16vec4 w2 = fma(d, f16vec4(unpack8(((ql2 >> shift) & 0x0F0F0F0Fu) | qhb2)), m); \ + const f16vec4 w3 = fma(d, f16vec4(unpack8(((ql3 >> shift) & 0x0F0F0F0Fu) | qhb3)), m); \ + FENCE_MACRO(); \ + TARGET_MACRO(TARGET_ARR, (s_kp << 4u), w0, w1, w2, w3); \ + } \ + } while (false) +#elif defined(DATA_A_Q6_K) +#define MMA_ACCUM_KPAIRS_GEN(LOOP_ATTR, FENCE_MACRO, TARGET_MACRO, TARGET_ARR, USE_U, ROW_BASE) \ + do { \ + FENCE_MACRO(); \ + const uint ib = (ROW_BASE) + ((g * K_PAIRS) >> 2u); \ + const uint sub_base = (g * K_PAIRS) & 3u; \ + const uint fn2 = fn >> 1u; \ + const float d_sb = float((USE_U) ? u_q6_k.data[ib].d : a_q6_k.data[ib].d); \ + LOOP_ATTR for (uint s_kp = 0u; s_kp < K_PAIRS; ++s_kp) { \ + FENCE_MACRO(); \ + const uint il = ((sub_base + s_kp) << 2u) | fn2; \ + const uint b = ((il >> 2u) & 1u) * 4u; \ + const uint qhshift = ((il >> 1u) & 3u) * 2u; \ + const uint qsi = ((il & 8u) << 2u) + ((il & 3u) << 3u); \ + const uint qhi = ((il & 8u) << 1u) + ((il & 1u) << 3u); \ + float dscale; \ + uint ql01, ql23, ql45, ql67, qh01, qh23, qh45, qh67; \ + if (USE_U) { \ + dscale = d_sb * float(u_q6_k.data[ib].scales[il]); \ + ql01 = ((uint(u_q6_k_p16.data[ib].ql[qsi + 0u]) | (uint(u_q6_k_p16.data[ib].ql[qsi + 1u]) << 16u)) >> b) & 0x0F0F0F0Fu; \ + ql23 = ((uint(u_q6_k_p16.data[ib].ql[qsi + 2u]) | (uint(u_q6_k_p16.data[ib].ql[qsi + 3u]) << 16u)) >> b) & 0x0F0F0F0Fu; \ + ql45 = ((uint(u_q6_k_p16.data[ib].ql[qsi + 4u]) | (uint(u_q6_k_p16.data[ib].ql[qsi + 5u]) << 16u)) >> b) & 0x0F0F0F0Fu; \ + ql67 = ((uint(u_q6_k_p16.data[ib].ql[qsi + 6u]) | (uint(u_q6_k_p16.data[ib].ql[qsi + 7u]) << 16u)) >> b) & 0x0F0F0F0Fu; \ + qh01 = ((uint(u_q6_k_p16.data[ib].qh[qhi + 0u]) | (uint(u_q6_k_p16.data[ib].qh[qhi + 1u]) << 16u)) >> qhshift) & 0x03030303u; \ + qh23 = ((uint(u_q6_k_p16.data[ib].qh[qhi + 2u]) | (uint(u_q6_k_p16.data[ib].qh[qhi + 3u]) << 16u)) >> qhshift) & 0x03030303u; \ + qh45 = ((uint(u_q6_k_p16.data[ib].qh[qhi + 4u]) | (uint(u_q6_k_p16.data[ib].qh[qhi + 5u]) << 16u)) >> qhshift) & 0x03030303u; \ + qh67 = ((uint(u_q6_k_p16.data[ib].qh[qhi + 6u]) | (uint(u_q6_k_p16.data[ib].qh[qhi + 7u]) << 16u)) >> qhshift) & 0x03030303u; \ + } else { \ + dscale = d_sb * float(a_q6_k.data[ib].scales[il]); \ + ql01 = ((uint(a_q6_k_p16.data[ib].ql[qsi + 0u]) | (uint(a_q6_k_p16.data[ib].ql[qsi + 1u]) << 16u)) >> b) & 0x0F0F0F0Fu; \ + ql23 = ((uint(a_q6_k_p16.data[ib].ql[qsi + 2u]) | (uint(a_q6_k_p16.data[ib].ql[qsi + 3u]) << 16u)) >> b) & 0x0F0F0F0Fu; \ + ql45 = ((uint(a_q6_k_p16.data[ib].ql[qsi + 4u]) | (uint(a_q6_k_p16.data[ib].ql[qsi + 5u]) << 16u)) >> b) & 0x0F0F0F0Fu; \ + ql67 = ((uint(a_q6_k_p16.data[ib].ql[qsi + 6u]) | (uint(a_q6_k_p16.data[ib].ql[qsi + 7u]) << 16u)) >> b) & 0x0F0F0F0Fu; \ + qh01 = ((uint(a_q6_k_p16.data[ib].qh[qhi + 0u]) | (uint(a_q6_k_p16.data[ib].qh[qhi + 1u]) << 16u)) >> qhshift) & 0x03030303u; \ + qh23 = ((uint(a_q6_k_p16.data[ib].qh[qhi + 2u]) | (uint(a_q6_k_p16.data[ib].qh[qhi + 3u]) << 16u)) >> qhshift) & 0x03030303u; \ + qh45 = ((uint(a_q6_k_p16.data[ib].qh[qhi + 4u]) | (uint(a_q6_k_p16.data[ib].qh[qhi + 5u]) << 16u)) >> qhshift) & 0x03030303u; \ + qh67 = ((uint(a_q6_k_p16.data[ib].qh[qhi + 6u]) | (uint(a_q6_k_p16.data[ib].qh[qhi + 7u]) << 16u)) >> qhshift) & 0x03030303u; \ + } \ + const f16vec4 dv = f16vec4(dscale); \ + const f16vec4 mv = f16vec4(-32.0f * dscale); \ + const f16vec4 w0 = fma(f16vec4(unpack8(ql01 | (qh01 << 4u))), dv, mv); \ + const f16vec4 w1 = fma(f16vec4(unpack8(ql23 | (qh23 << 4u))), dv, mv); \ + const f16vec4 w2 = fma(f16vec4(unpack8(ql45 | (qh45 << 4u))), dv, mv); \ + const f16vec4 w3 = fma(f16vec4(unpack8(ql67 | (qh67 << 4u))), dv, mv); \ + FENCE_MACRO(); \ + TARGET_MACRO(TARGET_ARR, (s_kp << 4u), w0, w1, w2, w3); \ + } \ + } while (false) +#else +#define MMA_ACCUM_KPAIRS_GEN(LOOP_ATTR, FENCE_MACRO, TARGET_MACRO, TARGET_ARR, USE_U, ROW_BASE) \ + do { \ + FENCE_MACRO(); \ + LOOP_ATTR for (uint s_kp = 0u; s_kp < K_PAIRS; ++s_kp) { \ + FENCE_MACRO(); \ + MMA_FVEC4 w0, w1, w2, w3; \ + dequant16((USE_U), (ROW_BASE), ((g * K_PAIRS + s_kp) << 2u) + (fn >> 1u), w0, w1, w2, w3); \ + FENCE_MACRO(); \ + TARGET_MACRO(TARGET_ARR, (s_kp << 4u), w0, w1, w2, w3); \ + } \ + } while (false) +#endif + +#define MMA_ACCUM_KPAIRS(MC_ARR, USE_U, ROW_BASE) \ + MMA_ACCUM_KPAIRS_GEN([[dont_unroll]], MMA_NO_FENCE, MMA_ACCUM_SHB, MC_ARR, USE_U, ROW_BASE) +#define MMA_ACCUM_KPAIRS_UNROLLED(ACC_VEC, USE_U, ROW_BASE) \ + MMA_ACCUM_KPAIRS_GEN([[unroll]], MMA_NO_FENCE, MMA_ACCUM_SHB_1D, ACC_VEC, USE_U, ROW_BASE) +#define MMA_ACCUM_KPAIRS_FENCED(MC_ARR, USE_U, ROW_BASE) \ + MMA_ACCUM_KPAIRS_GEN([[unroll]], MMA_SG_FENCE, MMA_ACCUM_SHB, MC_ARR, USE_U, ROW_BASE) +#define MMA_ACCUM_PRELOADED_MB(MC_ARR, SUB_OFF, W0, W1, W2, W3) \ + do { \ + const uint _kp_i = (SUB_OFF) >> 4u; \ + [[unroll]] for (uint rt = 0u; rt < RT; ++rt) { \ + if (rt == 0u || (8u * rt < n_valid)) { \ + [[unroll]] for (uint s = 0u; s < 4u; ++s) { \ + coopmat ma0, ma1; \ + ma0[0] = (W0)[s]; ma0[1] = (W2)[s]; \ + ma1[0] = (W1)[s]; ma1[1] = (W3)[s]; \ + MC_ARR[rt][t] = coopMatMulAdd(ma0, mb0_all[_kp_i][rt][s], MC_ARR[rt][t]); \ + MC_ARR[rt][t] = coopMatMulAdd(ma1, mb1_all[_kp_i][rt][s], MC_ARR[rt][t]); \ + } \ + } \ + } \ + } while (false) + + const uint K4_PER_COL = 16u * K_PAIRS; + const uint K_CHUNK = 64u * K_PAIRS; + const uint nch = p.K / K_CHUNK; + const uint B_ELEMS = K4_PER_COL * 8u * RT; + const uint B_LOADS = (B_ELEMS + WG_THREADS - 1u) / WG_THREADS; + + const bool stage_tmp_b = (USE_SH_B == 1u) ? true : ((USE_SH_B == 5u) ? true : (USE_SH_B == 8u)); + const bool use_sg_fence = (USE_SH_B == 5u) ? true : (USE_SH_B == 6u); + const bool load_mb_once = (USE_SH_B == 7u) ? true : ((USE_SH_B == 8u) ? true : (USE_SH_B == 9u)); + const bool fence_mb_once = (USE_SH_B != 9u); + const bool preload_mb_all = (USE_SH_B == 10u) ? true : (USE_SH_B == 11u); + + for (uint g = 0u; g < nch; ++g) { + if (stage_tmp_b) { + MMA_FVEC4 tmp_b[B_LOADS]; + [[unroll]] for (uint li = 0u; li < B_LOADS; ++li) { + tmp_b[li] = MMA_FVEC4(0.0f); + } + [[unroll]] for (uint li = 0u; li < B_LOADS; ++li) { + const uint col_min = (li * WG_THREADS) / K4_PER_COL; + if (li == 0u || col_min < n_valid) { + const uint idx = li * WG_THREADS + tid; + const uint col_in_tile = idx / K4_PER_COL; + const uint k4 = idx % K4_PER_COL; + if (idx < B_ELEMS && col_in_tile < n_valid) { +#ifdef MUL_MAT_ID + const uint b_base = mma_id_col_b_base(row_begin, tile_ne1, col_in_tile); +#else + const uint b_base = b_batch_base + (wg_n0 + col_in_tile) * p.stride_b; +#endif + tmp_b[li] = load_b_mvec4(b_base + g * K_CHUNK + (k4 << 2u), v4_ok, b_is_f16); + } + } + } + + if (g > 0u) { + barrier(); + } + + [[unroll]] for (uint li = 0u; li < B_LOADS; ++li) { +#ifdef MUL_MAT_ID + const uint col_min = (li * WG_THREADS) / K4_PER_COL; + if (li == 0u || col_min < n_valid) { + const uint idx = li * WG_THREADS + tid; + if (idx < B_ELEMS) { + sh_b[idx / K4_PER_COL][idx % K4_PER_COL] = tmp_b[li]; + } + } +#else + const uint idx = li * WG_THREADS + tid; + if (idx < B_ELEMS) { + sh_b[idx / K4_PER_COL][idx % K4_PER_COL] = tmp_b[li]; + } +#endif + } + barrier(); + } else { +#ifndef MUL_MAT_ID + if (B_LOADS * WG_THREADS == B_ELEMS && n_valid == 8u * RT && v4_ok) { + MMA_FVEC4 bv_arr[B_LOADS]; + [[unroll]] for (uint li = 0u; li < B_LOADS; ++li) { + const uint idx = li * WG_THREADS + tid; + const uint col_in_tile = idx / K4_PER_COL; + const uint k4 = idx % K4_PER_COL; + const uint b4 = (b_batch_base + (wg_n0 + col_in_tile) * p.stride_b + g * K_CHUNK) >> 2u; + bv_arr[li] = b_is_f16 ? MMA_FVEC4(data_b_f16v4[b4 + k4]) : MMA_FVEC4(data_b_v4[b4 + k4]); + } + if (g > 0u) { + barrier(); + } + [[unroll]] for (uint li = 0u; li < B_LOADS; ++li) { + const uint idx = li * WG_THREADS + tid; + sh_b[idx / K4_PER_COL][idx % K4_PER_COL] = bv_arr[li]; + } + barrier(); + } else +#endif + { + if (g > 0u) { + barrier(); + } + [[unroll]] for (uint li = 0u; li < B_LOADS; ++li) { + const uint idx = li * WG_THREADS + tid; + if (idx < B_ELEMS) { + const uint col_in_tile = idx / K4_PER_COL; + const uint k4 = idx % K4_PER_COL; + MMA_FVEC4 bv = MMA_FVEC4(0.0f); + if (col_in_tile < n_valid) { +#ifdef MUL_MAT_ID + const uint b_base = mma_id_col_b_base(row_begin, tile_ne1, col_in_tile); +#else + const uint b_base = b_batch_base + (wg_n0 + col_in_tile) * p.stride_b; +#endif + bv = load_b_mvec4(b_base + g * K_CHUNK + (k4 << 2u), v4_ok, b_is_f16); + } + sh_b[col_in_tile][k4] = bv; + } + } + barrier(); + } + } + + if (load_mb_once) { + [[unroll]] for (uint s_kp = 0u; s_kp < K_PAIRS; ++s_kp) { + if (fence_mb_once) { MMA_SG_FENCE(); } + const uint sub_off = (s_kp << 4u) + k4_lane_base; + coopmat mb0[RT][4]; + coopmat mb1[RT][4]; + [[unroll]] for (uint rt = 0u; rt < RT; ++rt) { + if (rt == 0u || (8u * rt < n_valid)) { + const uint c0 = 8u * rt + fn + 0u; + const uint c1 = 8u * rt + fn + 1u; + const MMA_FVEC4 ba0 = sh_b[c0][sub_off + 0u]; + const MMA_FVEC4 ba1 = sh_b[c0][sub_off + 1u]; + const MMA_FVEC4 bb0 = sh_b[c1][sub_off + 0u]; + const MMA_FVEC4 bb1 = sh_b[c1][sub_off + 1u]; + [[unroll]] for (uint s = 0u; s < 4u; ++s) { + mb0[rt][s][0] = ba0[s]; mb0[rt][s][1] = bb0[s]; + mb1[rt][s][0] = ba1[s]; mb1[rt][s][1] = bb1[s]; + } + } + } + const uint ci_kp = ((g * K_PAIRS + s_kp) << 2u) + (fn >> 1u); + [[unroll]] for (uint t = 0u; t < NT; ++t) { + if (fence_mb_once) { MMA_SG_FENCE(); } + MMA_FVEC4 w0, w1, w2, w3; + dequant16(FUSE_MODE == 2u && tile_second[t], a_row_base[t], ci_kp, w0, w1, w2, w3); + if (fence_mb_once) { MMA_SG_FENCE(); } + [[unroll]] for (uint rt = 0u; rt < RT; ++rt) { + if (rt == 0u || (8u * rt < n_valid)) { + [[unroll]] for (uint s = 0u; s < 4u; ++s) { + coopmat ma0, ma1; + ma0[0] = w0[s]; ma0[1] = w2[s]; + ma1[0] = w1[s]; ma1[1] = w3[s]; + mc[rt][t] = coopMatMulAdd(ma0, mb0[rt][s], mc[rt][t]); + mc[rt][t] = coopMatMulAdd(ma1, mb1[rt][s], mc[rt][t]); + } + } + } + if (FUSE_MODE == 3u) { + if (fence_mb_once) { MMA_SG_FENCE(); } + MMA_FVEC4 uw0, uw1, uw2, uw3; + dequant16(true, a_row_base[t], ci_kp, uw0, uw1, uw2, uw3); + if (fence_mb_once) { MMA_SG_FENCE(); } + [[unroll]] for (uint rt = 0u; rt < RT; ++rt) { + if (rt == 0u || (8u * rt < n_valid)) { + [[unroll]] for (uint s = 0u; s < 4u; ++s) { + coopmat uma0, uma1; + uma0[0] = uw0[s]; uma0[1] = uw2[s]; + uma1[0] = uw1[s]; uma1[1] = uw3[s]; + mc_u[rt][t] = coopMatMulAdd(uma0, mb0[rt][s], mc_u[rt][t]); + mc_u[rt][t] = coopMatMulAdd(uma1, mb1[rt][s], mc_u[rt][t]); + } + } + } + } + } + } + } else if (preload_mb_all) { + coopmat mb0_all[K_PAIRS][RT][4]; + coopmat mb1_all[K_PAIRS][RT][4]; + [[unroll]] for (uint s_kp = 0u; s_kp < K_PAIRS; ++s_kp) { + const uint sub_off = (s_kp << 4u) + k4_lane_base; + [[unroll]] for (uint rt = 0u; rt < RT; ++rt) { + if (rt == 0u || (8u * rt < n_valid)) { + const uint c0 = 8u * rt + fn + 0u; + const uint c1 = 8u * rt + fn + 1u; + const MMA_FVEC4 ba0 = sh_b[c0][sub_off + 0u]; + const MMA_FVEC4 ba1 = sh_b[c0][sub_off + 1u]; + const MMA_FVEC4 bb0 = sh_b[c1][sub_off + 0u]; + const MMA_FVEC4 bb1 = sh_b[c1][sub_off + 1u]; + [[unroll]] for (uint s = 0u; s < 4u; ++s) { + mb0_all[s_kp][rt][s][0] = ba0[s]; mb0_all[s_kp][rt][s][1] = bb0[s]; + mb1_all[s_kp][rt][s][0] = ba1[s]; mb1_all[s_kp][rt][s][1] = bb1[s]; + } + } + } + } + if (USE_SH_B == 11u) { + [[unroll]] for (uint t = 0u; t < NT; ++t) { + MMA_ACCUM_KPAIRS_GEN([[unroll]], MMA_SG_FENCE, MMA_ACCUM_PRELOADED_MB, mc, (FUSE_MODE == 2u && tile_second[t]), a_row_base[t]); + if (FUSE_MODE == 3u) { + MMA_ACCUM_KPAIRS_GEN([[unroll]], MMA_SG_FENCE, MMA_ACCUM_PRELOADED_MB, mc_u, true, a_row_base[t]); + } + } + } else { + [[unroll]] for (uint t = 0u; t < NT; ++t) { + MMA_ACCUM_KPAIRS_GEN([[unroll]], MMA_NO_FENCE, MMA_ACCUM_PRELOADED_MB, mc, (FUSE_MODE == 2u && tile_second[t]), a_row_base[t]); + if (FUSE_MODE == 3u) { + MMA_ACCUM_KPAIRS_GEN([[unroll]], MMA_NO_FENCE, MMA_ACCUM_PRELOADED_MB, mc_u, true, a_row_base[t]); + } + } + } + } else if (USE_SH_B == 4u) { + const uint total_mats = NT * ((FUSE_MODE == 3u) ? 2u : 1u); + [[dont_unroll]] for (uint mi = 0u; mi < total_mats; ++mi) { + const bool is_u = (FUSE_MODE == 3u) && ((mi & 1u) != 0u); + const uint ti = (FUSE_MODE == 3u) ? (mi >> 1u) : mi; + const uint rbase = (ti == 0u) ? a_row_base[0] : a_row_base[NT - 1u]; + const bool use_u = is_u || (FUSE_MODE == 2u && ((ti == 0u) ? tile_second[0] : tile_second[NT - 1u])); + coopmat acc_p[RT]; + [[unroll]] for (uint rt = 0u; rt < RT; ++rt) { + acc_p[rt] = is_u + ? ((ti == 0u) ? mc_u[rt][0] : mc_u[rt][NT - 1u]) + : ((ti == 0u) ? mc[rt][0] : mc[rt][NT - 1u]); + } + MMA_ACCUM_KPAIRS_UNROLLED(acc_p, use_u, rbase); + [[unroll]] for (uint rt = 0u; rt < RT; ++rt) { + if (is_u) { + if (ti == 0u) { mc_u[rt][0] = acc_p[rt]; } else if (NT > 1u) { mc_u[rt][1] = acc_p[rt]; } + } else { + if (ti == 0u) { mc[rt][0] = acc_p[rt]; } else if (NT > 1u) { mc[rt][1] = acc_p[rt]; } + } + } + } + } else if (use_sg_fence) { + [[unroll]] for (uint t = 0u; t < NT; ++t) { + MMA_ACCUM_KPAIRS_FENCED(mc, (FUSE_MODE == 2u && tile_second[t]), a_row_base[t]); + if (FUSE_MODE == 3u) { + MMA_ACCUM_KPAIRS_FENCED(mc_u, true, a_row_base[t]); + } + } + } else if (USE_SH_B == 12u) { + [[unroll]] for (uint s_kp = 0u; s_kp < K_PAIRS; ++s_kp) { + const uint ci_kp = ((g * K_PAIRS + s_kp) << 2u) + (fn >> 1u); + const uint sub_off = (s_kp << 4u) + k4_lane_base; + if (FUSE_MODE == 3u) { + [[unroll]] for (uint t = 0u; t < NT; ++t) { + MMA_FVEC4 w0, w1, w2, w3; + MMA_FVEC4 uw0, uw1, uw2, uw3; + dequant16(false, a_row_base[t], ci_kp, w0, w1, w2, w3); + dequant16(true, a_row_base[t], ci_kp, uw0, uw1, uw2, uw3); + [[unroll]] for (uint rt = 0u; rt < RT; ++rt) { + const uint c0 = 8u * rt + fn + 0u; + const uint c1 = 8u * rt + fn + 1u; + const MMA_FVEC4 ba0 = sh_b[c0][sub_off + 0u]; + const MMA_FVEC4 ba1 = sh_b[c0][sub_off + 1u]; + const MMA_FVEC4 bb0 = sh_b[c1][sub_off + 0u]; + const MMA_FVEC4 bb1 = sh_b[c1][sub_off + 1u]; + [[unroll]] for (uint s = 0u; s < 4u; ++s) { + coopmat ma0, ma1, uma0, uma1; + coopmat mb0, mb1; + ma0[0] = w0[s]; ma0[1] = w2[s]; + ma1[0] = w1[s]; ma1[1] = w3[s]; + uma0[0] = uw0[s]; uma0[1] = uw2[s]; + uma1[0] = uw1[s]; uma1[1] = uw3[s]; + mb0[0] = ba0[s]; mb0[1] = bb0[s]; + mb1[0] = ba1[s]; mb1[1] = bb1[s]; + mc[rt][t] = coopMatMulAdd(ma0, mb0, mc[rt][t]); + mc[rt][t] = coopMatMulAdd(ma1, mb1, mc[rt][t]); + mc_u[rt][t] = coopMatMulAdd(uma0, mb0, mc_u[rt][t]); + mc_u[rt][t] = coopMatMulAdd(uma1, mb1, mc_u[rt][t]); + } + } + } + } else if (NT == 2u) { + MMA_FVEC4 w0_0, w1_0, w2_0, w3_0; + MMA_FVEC4 w0_1, w1_1, w2_1, w3_1; + dequant16(FUSE_MODE == 2u && tile_second[0], a_row_base[0], ci_kp, w0_0, w1_0, w2_0, w3_0); + dequant16(FUSE_MODE == 2u && tile_second[1], a_row_base[1], ci_kp, w0_1, w1_1, w2_1, w3_1); + [[unroll]] for (uint rt = 0u; rt < RT; ++rt) { + const uint c0 = 8u * rt + fn + 0u; + const uint c1 = 8u * rt + fn + 1u; + const MMA_FVEC4 ba0 = sh_b[c0][sub_off + 0u]; + const MMA_FVEC4 ba1 = sh_b[c0][sub_off + 1u]; + const MMA_FVEC4 bb0 = sh_b[c1][sub_off + 0u]; + const MMA_FVEC4 bb1 = sh_b[c1][sub_off + 1u]; + [[unroll]] for (uint s = 0u; s < 4u; ++s) { + coopmat ma0_0, ma1_0, ma0_1, ma1_1; + coopmat mb0, mb1; + ma0_0[0] = w0_0[s]; ma0_0[1] = w2_0[s]; + ma1_0[0] = w1_0[s]; ma1_0[1] = w3_0[s]; + ma0_1[0] = w0_1[s]; ma0_1[1] = w2_1[s]; + ma1_1[0] = w1_1[s]; ma1_1[1] = w3_1[s]; + mb0[0] = ba0[s]; mb0[1] = bb0[s]; + mb1[0] = ba1[s]; mb1[1] = bb1[s]; + mc[rt][0] = coopMatMulAdd(ma0_0, mb0, mc[rt][0]); + mc[rt][0] = coopMatMulAdd(ma1_0, mb1, mc[rt][0]); + mc[rt][1] = coopMatMulAdd(ma0_1, mb0, mc[rt][1]); + mc[rt][1] = coopMatMulAdd(ma1_1, mb1, mc[rt][1]); + } + } + } else { + [[unroll]] for (uint t = 0u; t < NT; ++t) { + MMA_ACCUM_KPAIRS(mc, (FUSE_MODE == 2u && tile_second[t]), a_row_base[t]); + } + } + } + } else { + [[unroll]] for (uint t = 0u; t < NT; ++t) { + MMA_ACCUM_KPAIRS(mc, (FUSE_MODE == 2u && tile_second[t]), a_row_base[t]); + if (FUSE_MODE == 3u) { + MMA_ACCUM_KPAIRS(mc_u, true, a_row_base[t]); + } + } + } + } +#undef MMA_ACCUM_PRELOADED_MB +#undef MMA_ACCUM_KPAIRS_FENCED +#undef MMA_ACCUM_KPAIRS_UNROLLED +#undef MMA_ACCUM_KPAIRS +#undef MMA_ACCUM_KPAIRS_GEN +#undef MMA_NO_FENCE +#undef MMA_SG_FENCE +#undef MMA_ACCUM_SHB +#undef MMA_ACCUM_SHB_1D + } +#if defined(DATA_A_Q8_0) + else if (RT == 1u || (p.K & 63u) != 0u) { + const bool v2_ok = ((p.stride_b & 1u) == 0u) && ((p.batch_stride_b & 1u) == 0u); + const uint b_lane_off = 8u * (fm >> 1u) + 2u * (fm & 1u); + const uint nb = p.K >> 5u; + + for (uint ib = sg_k; ib < nb; ib += NSG) { + MMA_FVEC2 b0[RT][2]; + MMA_FVEC2 b1[RT][2]; + [[unroll]] for (uint rt = 0u; rt < RT; ++rt) { + if (rt == 0u || (8u * rt < n_valid)) { + [[unroll]] for (uint e = 0u; e < 2u; ++e) { +#ifdef MUL_MAT_ID + const uint b_base = mma_id_col_b_base(row_begin, tile_ne1, min(8u * rt + fn + e, n_valid - 1u)); +#else + const uint b_base = b_batch_base + (wg_n0 + min(8u * rt + fn + e, n_valid - 1u)) * p.stride_b; +#endif + const uint base = b_base + (ib << 5u) + b_lane_off; + b0[rt][e] = load_b_mvec2(base, v2_ok, b_is_f16); + b1[rt][e] = load_b_mvec2(base + 4u, v2_ok, b_is_f16); + } + } + } + + [[unroll]] for (uint t = 0u; t < NT; ++t) { + const uint blk_idx = a_row_base[t] + ib; + const uint q_off = 2u * fn; + float d_a; + i8vec4 v0_a, v1_a; + if (FUSE_MODE == 2u && tile_second[t]) { + d_a = float(u_q8_0.data[blk_idx].d); + v0_a = unpack8(int32_t((uint(int(u_q8_0.data[blk_idx].qs[q_off + 0u])) & 0xFFFFu) | (uint(int(u_q8_0.data[blk_idx].qs[q_off + 1u])) << 16u))); + v1_a = unpack8(int32_t((uint(int(u_q8_0.data[blk_idx].qs[q_off + 2u])) & 0xFFFFu) | (uint(int(u_q8_0.data[blk_idx].qs[q_off + 3u])) << 16u))); + } else { + d_a = float(a_q8_0.data[blk_idx].d); + v0_a = unpack8(int32_t((uint(int(a_q8_0.data[blk_idx].qs[q_off + 0u])) & 0xFFFFu) | (uint(int(a_q8_0.data[blk_idx].qs[q_off + 1u])) << 16u))); + v1_a = unpack8(int32_t((uint(int(a_q8_0.data[blk_idx].qs[q_off + 2u])) & 0xFFFFu) | (uint(int(a_q8_0.data[blk_idx].qs[q_off + 3u])) << 16u))); + } + + [[unroll]] for (uint rt = 0u; rt < RT; ++rt) { + if (rt == 0u || (8u * rt < n_valid)) { + coopmat mp = coopmat(0.0f); + coopmat ma; + coopmat mb; + ma[0] = MMA_FTYPE(v0_a.x); ma[1] = MMA_FTYPE(v0_a.z); mb[0] = b0[rt][0].x; mb[1] = b0[rt][1].x; mp = coopMatMulAdd(ma, mb, mp); + ma[0] = MMA_FTYPE(v0_a.y); ma[1] = MMA_FTYPE(v0_a.w); mb[0] = b0[rt][0].y; mb[1] = b0[rt][1].y; mp = coopMatMulAdd(ma, mb, mp); + ma[0] = MMA_FTYPE(v1_a.x); ma[1] = MMA_FTYPE(v1_a.z); mb[0] = b1[rt][0].x; mb[1] = b1[rt][1].x; mp = coopMatMulAdd(ma, mb, mp); + ma[0] = MMA_FTYPE(v1_a.y); ma[1] = MMA_FTYPE(v1_a.w); mb[0] = b1[rt][0].y; mb[1] = b1[rt][1].y; mp = coopMatMulAdd(ma, mb, mp); + mc[rt][t][0] = fma(d_a, mp[0], mc[rt][t][0]); + mc[rt][t][1] = fma(d_a, mp[1], mc[rt][t][1]); + } + } + + if (FUSE_MODE == 3u) { + const float d_u = float(u_q8_0.data[blk_idx].d); + const i8vec4 v0_u = unpack8(int32_t((uint(int(u_q8_0.data[blk_idx].qs[q_off + 0u])) & 0xFFFFu) | (uint(int(u_q8_0.data[blk_idx].qs[q_off + 1u])) << 16u))); + const i8vec4 v1_u = unpack8(int32_t((uint(int(u_q8_0.data[blk_idx].qs[q_off + 2u])) & 0xFFFFu) | (uint(int(u_q8_0.data[blk_idx].qs[q_off + 3u])) << 16u))); + [[unroll]] for (uint rt = 0u; rt < RT; ++rt) { + if (rt == 0u || (8u * rt < n_valid)) { + coopmat mpu = coopmat(0.0f); + coopmat ma; + coopmat mb; + ma[0] = MMA_FTYPE(v0_u.x); ma[1] = MMA_FTYPE(v0_u.z); mb[0] = b0[rt][0].x; mb[1] = b0[rt][1].x; mpu = coopMatMulAdd(ma, mb, mpu); + ma[0] = MMA_FTYPE(v0_u.y); ma[1] = MMA_FTYPE(v0_u.w); mb[0] = b0[rt][0].y; mb[1] = b0[rt][1].y; mpu = coopMatMulAdd(ma, mb, mpu); + ma[0] = MMA_FTYPE(v1_u.x); ma[1] = MMA_FTYPE(v1_u.z); mb[0] = b1[rt][0].x; mb[1] = b1[rt][1].x; mpu = coopMatMulAdd(ma, mb, mpu); + ma[0] = MMA_FTYPE(v1_u.y); ma[1] = MMA_FTYPE(v1_u.w); mb[0] = b1[rt][0].y; mb[1] = b1[rt][1].y; mpu = coopMatMulAdd(ma, mb, mpu); + mc_u[rt][t][0] = fma(d_u, mpu[0], mc_u[rt][t][0]); + mc_u[rt][t][1] = fma(d_u, mpu[1], mc_u[rt][t][1]); + } + } + } + } + } + } +#elif defined(DATA_A_Q4_0) + else if ((p.K & 63u) != 0u) { + const bool v2_ok = ((p.stride_b & 1u) == 0u) && ((p.batch_stride_b & 1u) == 0u); + const uint b_lane_off = 2u * fm; + const uint nb = p.K >> 5u; + + for (uint ib = sg_k; ib < nb; ib += NSG) { + MMA_FVEC2 b0[RT][2]; + MMA_FVEC2 b1[RT][2]; + [[unroll]] for (uint rt = 0u; rt < RT; ++rt) { + if (rt == 0u || (8u * rt < n_valid)) { + [[unroll]] for (uint e = 0u; e < 2u; ++e) { +#ifdef MUL_MAT_ID + const uint b_base = mma_id_col_b_base(row_begin, tile_ne1, min(8u * rt + fn + e, n_valid - 1u)); +#else + const uint b_base = b_batch_base + (wg_n0 + min(8u * rt + fn + e, n_valid - 1u)) * p.stride_b; +#endif + const uint base = b_base + (ib << 5u) + b_lane_off; + b0[rt][e] = load_b_mvec2(base, v2_ok, b_is_f16); + b1[rt][e] = load_b_mvec2(base + 16u, v2_ok, b_is_f16); + } + } + } + + [[unroll]] for (uint t = 0u; t < NT; ++t) { + const uint blk_idx = a_row_base[t] + ib; + float d_a; + uint raw_u; + if (FUSE_MODE == 2u && tile_second[t]) { + d_a = float(u_q4_0.data[blk_idx].d); + raw_u = uint(u_q4_0.data[blk_idx].qs[fn]) | (uint(u_q4_0.data[blk_idx].qs[fn + 1u]) << 16u); + } else { + d_a = float(a_q4_0.data[blk_idx].d); + raw_u = uint(a_q4_0.data[blk_idx].qs[fn]) | (uint(a_q4_0.data[blk_idx].qs[fn + 1u]) << 16u); + } + const u8vec4 lo = unpack8( raw_u & 0x0F0F0F0Fu); + const u8vec4 hi = unpack8((raw_u >> 4u) & 0x0F0F0F0Fu); + + [[unroll]] for (uint rt = 0u; rt < RT; ++rt) { + if (rt == 0u || (8u * rt < n_valid)) { + coopmat mp = coopmat(0.0f); + coopmat ma; + coopmat mb; + ma[0] = MMA_FTYPE(int(lo.x) - 8); ma[1] = MMA_FTYPE(int(lo.z) - 8); mb[0] = b0[rt][0].x; mb[1] = b0[rt][1].x; mp = coopMatMulAdd(ma, mb, mp); + ma[0] = MMA_FTYPE(int(lo.y) - 8); ma[1] = MMA_FTYPE(int(lo.w) - 8); mb[0] = b0[rt][0].y; mb[1] = b0[rt][1].y; mp = coopMatMulAdd(ma, mb, mp); + ma[0] = MMA_FTYPE(int(hi.x) - 8); ma[1] = MMA_FTYPE(int(hi.z) - 8); mb[0] = b1[rt][0].x; mb[1] = b1[rt][1].x; mp = coopMatMulAdd(ma, mb, mp); + ma[0] = MMA_FTYPE(int(hi.y) - 8); ma[1] = MMA_FTYPE(int(hi.w) - 8); mb[0] = b1[rt][0].y; mb[1] = b1[rt][1].y; mp = coopMatMulAdd(ma, mb, mp); + mc[rt][t][0] = fma(d_a, mp[0], mc[rt][t][0]); + mc[rt][t][1] = fma(d_a, mp[1], mc[rt][t][1]); + } + } + + if (FUSE_MODE == 3u) { + const float d_u = float(u_q4_0.data[blk_idx].d); + const uint uraw = uint(u_q4_0.data[blk_idx].qs[fn]) | (uint(u_q4_0.data[blk_idx].qs[fn + 1u]) << 16u); + const u8vec4 ulo = unpack8( uraw & 0x0F0F0F0Fu); + const u8vec4 uhi = unpack8((uraw >> 4u) & 0x0F0F0F0Fu); + [[unroll]] for (uint rt = 0u; rt < RT; ++rt) { + if (rt == 0u || (8u * rt < n_valid)) { + coopmat mpu = coopmat(0.0f); + coopmat ma; + coopmat mb; + ma[0] = MMA_FTYPE(int(ulo.x) - 8); ma[1] = MMA_FTYPE(int(ulo.z) - 8); mb[0] = b0[rt][0].x; mb[1] = b0[rt][1].x; mpu = coopMatMulAdd(ma, mb, mpu); + ma[0] = MMA_FTYPE(int(ulo.y) - 8); ma[1] = MMA_FTYPE(int(ulo.w) - 8); mb[0] = b0[rt][0].y; mb[1] = b0[rt][1].y; mpu = coopMatMulAdd(ma, mb, mpu); + ma[0] = MMA_FTYPE(int(uhi.x) - 8); ma[1] = MMA_FTYPE(int(uhi.z) - 8); mb[0] = b1[rt][0].x; mb[1] = b1[rt][1].x; mpu = coopMatMulAdd(ma, mb, mpu); + ma[0] = MMA_FTYPE(int(uhi.y) - 8); ma[1] = MMA_FTYPE(int(uhi.w) - 8); mb[0] = b1[rt][0].y; mb[1] = b1[rt][1].y; mpu = coopMatMulAdd(ma, mb, mpu); + mc_u[rt][t][0] = fma(d_u, mpu[0], mc_u[rt][t][0]); + mc_u[rt][t][1] = fma(d_u, mpu[1], mc_u[rt][t][1]); + } + } + } + } + } + } +#endif + else { + // K-split (MSG == 1, NSG >= 1) 64-weight chunk loop with on-the-fly mb construction + const bool v4_ok = ((p.stride_b & 3u) == 0u) && ((p.batch_stride_b & 3u) == 0u); + const uint b_lane_off = 16u * (fm >> 1u) + 8u * (fm & 1u); + const uint nch = p.K >> 6u; + + uint base0_rt[RT], base1_rt[RT]; + [[unroll]] for (uint rt = 0u; rt < RT; ++rt) { + if (rt == 0u || (8u * rt < n_valid)) { +#ifdef MUL_MAT_ID + base0_rt[rt] = mma_id_col_b_base(row_begin, tile_ne1, min(8u * rt + fn + 0u, n_valid - 1u)) + b_lane_off; + base1_rt[rt] = mma_id_col_b_base(row_begin, tile_ne1, min(8u * rt + fn + 1u, n_valid - 1u)) + b_lane_off; +#else + base0_rt[rt] = b_batch_base + (wg_n0 + min(8u * rt + fn + 0u, n_valid - 1u)) * p.stride_b + b_lane_off; + base1_rt[rt] = b_batch_base + (wg_n0 + min(8u * rt + fn + 1u, n_valid - 1u)) * p.stride_b + b_lane_off; +#endif + } else { + base0_rt[rt] = 0u; + base1_rt[rt] = 0u; + } + } + + for (uint g = sg_k; g < nch; g += NSG) { + MMA_FVEC4 ba0[RT], ba1[RT], bb0[RT], bb1[RT]; + [[unroll]] for (uint rt = 0u; rt < RT; ++rt) { + if (rt == 0u || (8u * rt < n_valid)) { + const uint base0 = base0_rt[rt] + (g << 6u); + const uint base1 = base1_rt[rt] + (g << 6u); + ba0[rt] = load_b_mvec4(base0, v4_ok, b_is_f16); + ba1[rt] = load_b_mvec4(base0 + 4u, v4_ok, b_is_f16); + bb0[rt] = load_b_mvec4(base1, v4_ok, b_is_f16); + bb1[rt] = load_b_mvec4(base1 + 4u, v4_ok, b_is_f16); + } + } + + const uint ci = (g << 2u) + (fn >> 1u); + + [[unroll]] for (uint t = 0u; t < NT; ++t) { + MMA_FVEC4 w0, w1, w2, w3; + dequant16(FUSE_MODE == 2u && tile_second[t], a_row_base[t], ci, w0, w1, w2, w3); + + [[unroll]] for (uint s = 0u; s < 4u; ++s) { + coopmat ma0, ma1; + ma0[0] = w0[s]; ma0[1] = w2[s]; + ma1[0] = w1[s]; ma1[1] = w3[s]; + [[unroll]] for (uint rt = 0u; rt < RT; ++rt) { + if (rt == 0u || (8u * rt < n_valid)) { + coopmat mb0, mb1; + mb0[0] = ba0[rt][s]; mb0[1] = bb0[rt][s]; + mb1[0] = ba1[rt][s]; mb1[1] = bb1[rt][s]; + mc[rt][t] = coopMatMulAdd(ma0, mb0, mc[rt][t]); + mc[rt][t] = coopMatMulAdd(ma1, mb1, mc[rt][t]); + } + } + } + + if (FUSE_MODE == 3u) { + MMA_FVEC4 uw0, uw1, uw2, uw3; + dequant16(true, a_row_base[t], ci, uw0, uw1, uw2, uw3); + [[unroll]] for (uint s = 0u; s < 4u; ++s) { + coopmat mu0, mu1; + mu0[0] = uw0[s]; mu0[1] = uw2[s]; + mu1[0] = uw1[s]; mu1[1] = uw3[s]; + [[unroll]] for (uint rt = 0u; rt < RT; ++rt) { + if (rt == 0u || (8u * rt < n_valid)) { + coopmat mb0, mb1; + mb0[0] = ba0[rt][s]; mb0[1] = bb0[rt][s]; + mb1[0] = ba1[rt][s]; mb1[1] = bb1[rt][s]; + mc_u[rt][t] = coopMatMulAdd(mu0, mb0, mc_u[rt][t]); + mc_u[rt][t] = coopMatMulAdd(mu1, mb1, mc_u[rt][t]); + } + } + } + } + } + } + } + + // Store path: when NSG == 1, write directly from registers with ZERO reduction shmem and ZERO barriers! + if (NSG == 1u) { + [[unroll]] for (uint rt = 0u; rt < RT; ++rt) { + if ((CSG == 1u && rt == 0u) || (sg_c_off + 8u * rt < n_valid)) { + [[unroll]] for (uint e = 0u; e < 2u; ++e) { +#ifdef MUL_MAT_ID + const uint col_in_tile = sg_c_off + 8u * rt + fn + e; + if (col_in_tile < tile_ne1) { + const uint packed = mma_id_col_packed(row_begin, tile_ne1, col_in_tile); + const u16vec2 rid = u16vec2(packed & 0xFFFFu, packed >> 16u); + const uint d_col_base = uint(rid.y) * p.batch_stride_d + uint(rid.x) * p.stride_d; + float col_scale = 1.0f; + if (FUSE_MODE == 4u) { + if (USE_SH_IDS) { + col_scale = sh_col_scale[col_in_tile]; + } else { + const uint s0_idx = uint(rid.x) + (((p.fusion_flags & MAT_VEC_FUSION_FLAGS_SCALE0_TOK) != 0u) ? (uint(rid.y) * p.nei0) : 0u); + col_scale = data_aux0[s0_idx]; + } + } + [[unroll]] for (uint t = 0u; t < NT; ++t) { + const uint r0 = sg_m0 + 8u * t + fm; + if (r0 < p.M) { + const uint d_idx = d_col_base + r0; + float val = mc[rt][t][e]; + if (FUSE_MODE == 3u) { + const float u = mc_u[rt][t][e]; + val = (val / (1.0f + exp(-val))) * u; + if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_D_F16_AUX1) != 0u) { + data_d_f16[d_idx] = float16_t(val); + } + if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_SKIP_F32_D) == 0u) { + data_d[d_idx] = val; + } + } else { + if (FUSE_MODE == 4u) { + val *= col_scale; + } else if (FUSE_MODE == 1u) { + val = mma_apply_id_epilogue(val, expert_idx, rid, r0, d_idx); + } + data_d[d_idx] = val; + } + } + } + } +#else + const uint col = wg_n0 + 8u * rt + fn + e; + if (col < p.N) { + const uint tok_col = batch_idx * p.N + col; + [[unroll]] for (uint t = 0u; t < NT; ++t) { + const uint r_global = sg_m0 + 8u * t + fm; + if (FUSE_MODE == 2u && tile_second[t]) { + const uint r1 = r_global - p.pair_m0; + const uint m1 = p.M - p.pair_m0; + if (r1 < m1) { + const uint d2_idx = batch_idx * p.batch_stride_d2 + col * p.stride_d2 + r1; + float v2 = mc[rt][t][e]; + if (FUSE_LORA != 0u && (p.fusion_flags & MAT_VEC_FUSION_FLAGS_LORA1) != 0u) { + v2 += mma_eval_lora1(r1, tok_col); + } + data_d2[d2_idx] = mma_apply_dense_epilogue(v2, r1, col, d2_idx, true); + } + } else { + const uint m_lim = (FUSE_MODE == 2u) ? p.pair_m0 : p.M; + if (r_global < m_lim) { + const uint d_idx = batch_idx * p.batch_stride_d + col * p.stride_d + r_global; + float val = mc[rt][t][e]; + if (FUSE_LORA != 0u && FUSE_MODE != 0u && (p.fusion_flags & MAT_VEC_FUSION_FLAGS_LORA0) != 0u) { + val += mma_eval_lora0(r_global, tok_col); + } + if (FUSE_MODE == 3u) { + float u = mc_u[rt][t][e]; + if (FUSE_LORA != 0u && (p.fusion_flags & MAT_VEC_FUSION_FLAGS_LORA1) != 0u) { + u += mma_eval_lora1(r_global, tok_col); + } + val = (val / (1.0f + exp(-val))) * u; + } else if (FUSE_MODE == 4u) { + if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS0) != 0u) { + val += data_aux0[d_idx]; + } + if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS1) != 0u) { + val += data_aux1[d_idx]; + } + } else if (FUSE_MODE == 1u || FUSE_MODE == 2u) { + val = mma_apply_dense_epilogue(val, r_global, col, d_idx, false); + } + data_d[d_idx] = val; + } + } + } + } +#endif + } + } + } + return; + } + +#ifndef MUL_MAT_ID + // NSG > 1 (MSG == 1) reduction through shared memory (dense only) + const uint stride_sg = RT * NT * 64u; + [[unroll]] for (uint rt = 0u; rt < RT; ++rt) { + [[unroll]] for (uint t = 0u; t < NT; ++t) { + const uint base = sg_k * stride_sg + (rt * NT + t) * 64u + 2u * lane; + red_shmem[base + 0u] = mc[rt][t][0]; + red_shmem[base + 1u] = mc[rt][t][1]; + if (FUSE_MODE == 3u) { + const uint ubase = NSG * stride_sg + base; + red_shmem[ubase + 0u] = mc_u[rt][t][0]; + red_shmem[ubase + 1u] = mc_u[rt][t][1]; + } + } + } + barrier(); + + for (uint idx = tid; idx < stride_sg; idx += WG_THREADS) { + const uint rt = idx / (NT * 64u); + if (rt > 0u && (8u * rt >= n_valid)) { + continue; + } + const uint t = (idx / 64u) % NT; + const uint l = (idx & 63u) >> 1u; + const uint e = idx & 1u; + + float sum = 0.0f; + float sum_u = 0.0f; + [[unroll]] for (uint sg = 0u; sg < NSG; ++sg) { + sum += red_shmem[sg * stride_sg + idx]; + if (FUSE_MODE == 3u) { + sum_u += red_shmem[(NSG + sg) * stride_sg + idx]; + } + } + + const uint l_fm = mma_lane_fm(l); + const uint l_fn = mma_lane_fn(l); + + const uint col = wg_n0 + 8u * rt + l_fn + e; + if (col < p.N) { + const uint tok_col = batch_idx * p.N + col; + const uint tile_r = sg_m0 + 8u * t; + const uint r_global = tile_r + l_fm; + if (FUSE_MODE == 2u && tile_r >= p.pair_m0) { + const uint r1 = r_global - p.pair_m0; + const uint m1 = p.M - p.pair_m0; + if (r1 < m1) { + const uint d2_idx = batch_idx * p.batch_stride_d2 + col * p.stride_d2 + r1; + float v2 = sum; + if (FUSE_LORA != 0u && (p.fusion_flags & MAT_VEC_FUSION_FLAGS_LORA1) != 0u) { + v2 += mma_eval_lora1(r1, tok_col); + } + data_d2[d2_idx] = mma_apply_dense_epilogue(v2, r1, col, d2_idx, true); + } + } else { + const uint m_lim = (FUSE_MODE == 2u) ? p.pair_m0 : p.M; + if (r_global < m_lim) { + const uint d_idx = batch_idx * p.batch_stride_d + col * p.stride_d + r_global; + float val = sum; + if (FUSE_LORA != 0u && FUSE_MODE != 0u && (p.fusion_flags & MAT_VEC_FUSION_FLAGS_LORA0) != 0u) { + val += mma_eval_lora0(r_global, tok_col); + } + if (FUSE_MODE == 3u) { + float u = sum_u; + if (FUSE_LORA != 0u && (p.fusion_flags & MAT_VEC_FUSION_FLAGS_LORA1) != 0u) { + u += mma_eval_lora1(r_global, tok_col); + } + val = (val / (1.0f + exp(-val))) * u; + } else if (FUSE_MODE == 4u) { + if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS0) != 0u) { + val += data_aux0[d_idx]; + } + if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS1) != 0u) { + val += data_aux1[d_idx]; + } + } else if (FUSE_MODE == 1u || FUSE_MODE == 2u) { + val = mma_apply_dense_epilogue(val, r_global, col, d_idx, false); + } + data_d[d_idx] = val; + } + } + } + } +#endif +} diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/multi_add_rms_mul.comp b/ggml/src/ggml-vulkan/vulkan-shaders/multi_add_rms_mul.comp new file mode 100644 index 000000000000..48dba71a79ff --- /dev/null +++ b/ggml/src/ggml-vulkan/vulkan-shaders/multi_add_rms_mul.comp @@ -0,0 +1,365 @@ +#version 450 + +#extension GL_EXT_control_flow_attributes : enable +#extension GL_EXT_shader_16bit_storage : require +#extension GL_EXT_shader_explicit_arithmetic_types_float16 : require +#extension GL_KHR_shader_subgroup_basic : require +#extension GL_KHR_shader_subgroup_arithmetic : require +#extension GL_KHR_shader_subgroup_shuffle : require + +layout(constant_id = 0) const uint NUM_SRCS = 2; +layout(constant_id = 1) const uint RMS_TREE_MODE = 0; +layout(constant_id = 2) const uint DO_MUL = 1; +layout(constant_id = 3) const uint BLOCK_SIZE = 512; + +layout(local_size_x_id = 3, local_size_y = 1, local_size_z = 1) in; + +layout(binding = 0) readonly buffer A0 { float data_a0[]; }; +layout(binding = 0) readonly buffer A0V4 { vec4 data_a0_v4[]; }; +layout(binding = 1) readonly buffer A1 { float data_a1[]; }; +layout(binding = 1) readonly buffer A1V4 { vec4 data_a1_v4[]; }; +layout(binding = 2) readonly buffer A2 { float data_a2[]; }; +layout(binding = 2) readonly buffer A2V4 { vec4 data_a2_v4[]; }; +layout(binding = 3) readonly buffer A3 { float data_a3[]; }; +layout(binding = 3) readonly buffer A3V4 { vec4 data_a3_v4[]; }; +layout(binding = 4) readonly buffer A4 { float data_a4[]; }; +layout(binding = 4) readonly buffer A4V4 { vec4 data_a4_v4[]; }; +layout(binding = 5) readonly buffer A5 { float data_a5[]; }; +layout(binding = 5) readonly buffer A5V4 { vec4 data_a5_v4[]; }; +layout(binding = 6) readonly buffer A6 { float data_a6[]; }; +layout(binding = 6) readonly buffer A6V4 { vec4 data_a6_v4[]; }; +layout(binding = 7) readonly buffer A7 { float data_a7[]; }; +layout(binding = 7) readonly buffer A7V4 { vec4 data_a7_v4[]; }; +layout(binding = 8) readonly buffer A8 { float data_a8[]; }; +layout(binding = 8) readonly buffer A8V4 { vec4 data_a8_v4[]; }; +layout(binding = 9) readonly buffer A9 { float data_a9[]; }; +layout(binding = 9) readonly buffer A9V4 { vec4 data_a9_v4[]; }; +layout(binding = 10) writeonly buffer AddDst { float add_dst[]; }; +layout(binding = 10) writeonly buffer AddDstV4 { vec4 add_dst_v4[]; }; +layout(binding = 11) readonly buffer Weight { float data_w[]; }; +layout(binding = 11) readonly buffer WeightV4 { vec4 data_w_v4[]; }; +layout(binding = 12) writeonly buffer RmsDst { float rms_dst[]; }; +layout(binding = 12) writeonly buffer RmsDstV4 { vec4 rms_dst_v4[]; }; +layout(binding = 13) writeonly buffer RmsDstF16V4 { f16vec4 rms_dst_f16_v4[]; }; + +layout(push_constant) uniform PushConstants { + uint ne0; + uint nrows; + float eps; + uint write_add_dst; + uint nb1[10]; +} p; + +const uint SUMSH_SIZE = (RMS_TREE_MODE == 2u) ? 16u : 512u; +shared float sumsh[SUMSH_SIZE]; + +vec4 load_sum_v4(const uint row, const uint v4_col) { + vec4 s = data_a0_v4[row * (p.nb1[0] >> 2u) + v4_col] + data_a1_v4[row * (p.nb1[1] >> 2u) + v4_col]; + if (NUM_SRCS > 2) s += data_a2_v4[row * (p.nb1[2] >> 2u) + v4_col]; + if (NUM_SRCS > 3) s += data_a3_v4[row * (p.nb1[3] >> 2u) + v4_col]; + if (NUM_SRCS > 4) s += data_a4_v4[row * (p.nb1[4] >> 2u) + v4_col]; + if (NUM_SRCS > 5) s += data_a5_v4[row * (p.nb1[5] >> 2u) + v4_col]; + if (NUM_SRCS > 6) s += data_a6_v4[row * (p.nb1[6] >> 2u) + v4_col]; + if (NUM_SRCS > 7) s += data_a7_v4[row * (p.nb1[7] >> 2u) + v4_col]; + if (NUM_SRCS > 8) s += data_a8_v4[row * (p.nb1[8] >> 2u) + v4_col]; + if (NUM_SRCS > 9) s += data_a9_v4[row * (p.nb1[9] >> 2u) + v4_col]; + return s; +} + +float load_sum(const uint row, const uint col) { + precise float s; + if (NUM_SRCS == 2) { + s = data_a0[row * p.nb1[0] + col] + data_a1[row * p.nb1[1] + col]; + } else { + s = 0.0f; + s += data_a0[row * p.nb1[0] + col]; + s += data_a1[row * p.nb1[1] + col]; + if (NUM_SRCS > 2) s += data_a2[row * p.nb1[2] + col]; + if (NUM_SRCS > 3) s += data_a3[row * p.nb1[3] + col]; + if (NUM_SRCS > 4) s += data_a4[row * p.nb1[4] + col]; + if (NUM_SRCS > 5) s += data_a5[row * p.nb1[5] + col]; + if (NUM_SRCS > 6) s += data_a6[row * p.nb1[6] + col]; + if (NUM_SRCS > 7) s += data_a7[row * p.nb1[7] + col]; + if (NUM_SRCS > 8) s += data_a8[row * p.nb1[8] + col]; + if (NUM_SRCS > 9) s += data_a9[row * p.nb1[9] + col]; + } + return s; +} + +void main() { + const uint row = gl_WorkGroupID.x + gl_WorkGroupID.y * gl_NumWorkGroups.x; + if (row >= p.nrows) { + return; + } + + const uint tid = gl_LocalInvocationID.x; + const uint row_off = row * p.ne0; + + if (RMS_TREE_MODE == 2u) { + const uint ne0_v4 = p.ne0 >> 2u; + const uint row_v4 = row_off >> 2u; + vec4 vals_v4[8]; + float sum = 0.0f; + + const bool do_add = (p.write_add_dst & 1u) != 0u; + const bool do_f16 = (p.write_add_dst & 2u) != 0u; + + if (ne0_v4 <= 4u * BLOCK_SIZE) { + [[unroll]] for (uint idx = 0u; idx < 4u; ++idx) { + const uint v4_col = tid + idx * BLOCK_SIZE; + vec4 xv = vec4(0.0f); + if (v4_col < ne0_v4) { + xv = load_sum_v4(row, v4_col); + if (do_add) { + add_dst_v4[row_v4 + v4_col] = xv; + } + } + vals_v4[idx] = xv; + sum += dot(xv, xv); + } + } else { + [[unroll]] for (uint idx = 0u; idx < 8u; ++idx) { + const uint v4_col = tid + idx * BLOCK_SIZE; + vec4 xv = vec4(0.0f); + if (v4_col < ne0_v4) { + xv = load_sum_v4(row, v4_col); + if (do_add) { + add_dst_v4[row_v4 + v4_col] = xv; + } + } + vals_v4[idx] = xv; + sum += dot(xv, xv); + } + } + + sum = subgroupAdd(sum); + if (subgroupElect()) { + sumsh[gl_SubgroupID] = sum; + } + barrier(); + + float total_sq = 0.0f; + if (gl_NumSubgroups <= 4u) { + [[unroll]] for (uint i = 0u; i < gl_NumSubgroups; ++i) { + total_sq += sumsh[i]; + } + } else { + for (uint i = gl_SubgroupInvocationID; i < gl_NumSubgroups; i += gl_SubgroupSize) { + total_sq += sumsh[i]; + } + total_sq = subgroupAdd(total_sq); + } + + const float scale = inversesqrt(total_sq / float(p.ne0) + p.eps); + if (ne0_v4 <= 4u * BLOCK_SIZE) { + [[unroll]] for (uint idx = 0u; idx < 4u; ++idx) { + const uint v4_col = tid + idx * BLOCK_SIZE; + if (v4_col < ne0_v4) { + const vec4 out_v4 = (DO_MUL != 0u) + ? (scale * vals_v4[idx] * data_w_v4[v4_col]) + : (scale * vals_v4[idx]); + rms_dst_v4[row_v4 + v4_col] = out_v4; + if (do_f16) { + rms_dst_f16_v4[row_v4 + v4_col] = f16vec4(out_v4); + } + } + } + } else { + [[unroll]] for (uint idx = 0u; idx < 8u; ++idx) { + const uint v4_col = tid + idx * BLOCK_SIZE; + if (v4_col < ne0_v4) { + const vec4 out_v4 = (DO_MUL != 0u) + ? (scale * vals_v4[idx] * data_w_v4[v4_col]) + : (scale * vals_v4[idx]); + rms_dst_v4[row_v4 + v4_col] = out_v4; + if (do_f16) { + rms_dst_f16_v4[row_v4 + v4_col] = f16vec4(out_v4); + } + } + } + } + return; + } + + if (RMS_TREE_MODE == 0) { + float vals[8]; + float sum = 0.0f; + + if (p.ne0 <= 2048u) { + [[unroll]] for (uint idx = 0; idx < 4; ++idx) { + const uint col = tid + idx * BLOCK_SIZE; + float xi = 0.0f; + if (col < p.ne0) { + xi = load_sum(row, col); + if (p.write_add_dst != 0) { + add_dst[row_off + col] = xi; + } + } + vals[idx] = xi; + sum += xi * xi; + } + } else { + [[unroll]] for (uint idx = 0; idx < 8; ++idx) { + const uint col = tid + idx * BLOCK_SIZE; + float xi = 0.0f; + if (col < p.ne0) { + xi = load_sum(row, col); + if (p.write_add_dst != 0) { + add_dst[row_off + col] = xi; + } + } + vals[idx] = xi; + sum += xi * xi; + } + } + + sumsh[tid] = sum; + barrier(); + + if (gl_SubgroupSize == 32) { + if (gl_SubgroupID == 0) { + const uint lane = gl_SubgroupInvocationID; + precise float p0_a = sumsh[lane] + sumsh[lane + 256u]; + precise float p0_b = sumsh[lane + 128u] + sumsh[lane + 384u]; + precise float p1_a = sumsh[lane + 32u] + sumsh[lane + 288u]; + precise float p1_b = sumsh[lane + 160u] + sumsh[lane + 416u]; + precise float p2_a = sumsh[lane + 64u] + sumsh[lane + 320u]; + precise float p2_b = sumsh[lane + 192u] + sumsh[lane + 448u]; + precise float p3_a = sumsh[lane + 96u] + sumsh[lane + 352u]; + precise float p3_b = sumsh[lane + 224u] + sumsh[lane + 480u]; + precise float p0 = p0_a + p0_b; + precise float p1 = p1_a + p1_b; + precise float p2 = p2_a + p2_b; + precise float p3 = p3_a + p3_b; + precise float s_lo = p0 + p2; + precise float s_hi = p1 + p3; + precise float s32 = s_lo + s_hi; + precise float s16 = s32 + subgroupShuffleXor(s32, 16); + precise float s8 = s16 + subgroupShuffleXor(s16, 8); + precise float s4 = s8 + subgroupShuffleXor(s8, 4); + precise float s2 = s4 + subgroupShuffleXor(s4, 2); + precise float s1 = s2 + subgroupShuffleXor(s2, 1); + if (lane == 0) { + sumsh[0] = s1; + } + } + barrier(); + } else { + [[unroll]] for (int s = int(BLOCK_SIZE) / 2; s > 0; s >>= 1) { + if (int(tid) < s) { + sum += sumsh[tid + uint(s)]; + sumsh[tid] = sum; + } + barrier(); + } + } + + const float mean = sumsh[0] / float(p.ne0); + const float scale = inversesqrt(mean + p.eps); + + if (p.ne0 <= 2048u) { + [[unroll]] for (uint idx = 0; idx < 4; ++idx) { + const uint col = tid + idx * BLOCK_SIZE; + if (col < p.ne0) { + const float value = (DO_MUL != 0) ? (scale * vals[idx] * data_w[col]) : (scale * vals[idx]); + rms_dst[row_off + col] = value; + } + } + } else { + [[unroll]] for (uint idx = 0; idx < 8; ++idx) { + const uint col = tid + idx * BLOCK_SIZE; + if (col < p.ne0) { + const float value = (DO_MUL != 0) ? (scale * vals[idx] * data_w[col]) : (scale * vals[idx]); + rms_dst[row_off + col] = value; + } + } + } + } else { + const uint num_partials = p.ne0 / 512u; + const uint num_iters = (num_partials + 1u) >> 1; + const uint team = tid >> 8; + const uint t256 = tid & 255u; + const uint num_sub_per_team = 256u / gl_SubgroupSize; + const uint sg_in_team = t256 / gl_SubgroupSize; + + float vals_lo[4]; + float vals_hi[4]; + + [[unroll]] for (uint k = 0; k < 4; ++k) { + if (k < num_iters) { + const uint c = k * 2u + team; + float sum_sq = 0.0f; + float s0 = 0.0f; + float s1 = 0.0f; + if (c < num_partials) { + const uint col_lo = c * 512u + t256; + const uint c_off = row_off + c * 512u; + s0 = load_sum(row, col_lo); + if (p.write_add_dst != 0) { + add_dst[c_off + t256] = s0; + } + s1 = load_sum(row, col_lo + 256u); + if (p.write_add_dst != 0) { + add_dst[c_off + t256 + 256u] = s1; + } + precise float sq0 = s0 * s0; + sum_sq = fma(s1, s1, sq0); + } + vals_lo[k] = s0; + vals_hi[k] = s1; + + sum_sq = subgroupAdd(sum_sq); + if (gl_SubgroupInvocationID == 0 && c < num_partials) { + sumsh[c * 16u + sg_in_team] = sum_sq; + } + } + } + barrier(); + + if (gl_SubgroupID == 0) { + const uint lane = gl_SubgroupInvocationID; + float psum = 0.0f; + if (gl_SubgroupSize == 32) { + if (lane < num_partials) { + const uint base = lane * 16u; + precise float a0 = sumsh[base + 0u] + sumsh[base + 4u]; + precise float a1 = sumsh[base + 1u] + sumsh[base + 5u]; + precise float a2 = sumsh[base + 2u] + sumsh[base + 6u]; + precise float a3 = sumsh[base + 3u] + sumsh[base + 7u]; + precise float b0 = a0 + a2; + precise float b1 = a1 + a3; + psum += (b0 + b1); + } + } else { + for (uint i = lane; i < num_partials; i += gl_SubgroupSize) { + const uint base = i * 16u; + float c_sum = 0.0f; + for (uint s = 0u; s < num_sub_per_team; ++s) { + c_sum += sumsh[base + s]; + } + psum += c_sum; + } + } + psum = subgroupAdd(psum); + if (lane == 0u) { + sumsh[0] = psum; + } + } + barrier(); + + const float mean = sumsh[0] / float(p.ne0); + const float scale = inversesqrt(mean + p.eps); + + [[unroll]] for (uint k = 0; k < 4; ++k) { + if (k < num_iters) { + const uint c = k * 2u + team; + if (c < num_partials) { + const uint col_lo = c * 512u + t256; + const uint col_hi = col_lo + 256u; + rms_dst[row_off + col_lo] = (DO_MUL != 0) ? (scale * vals_lo[k] * data_w[col_lo]) : (scale * vals_lo[k]); + rms_dst[row_off + col_hi] = (DO_MUL != 0) ? (scale * vals_hi[k] * data_w[col_hi]) : (scale * vals_hi[k]); + } + } + } + } +} diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp b/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp index 6a2525849c00..9f7959837508 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp @@ -7,6 +7,10 @@ #if USE_SUBGROUP_ADD #extension GL_KHR_shader_subgroup_basic : require #extension GL_KHR_shader_subgroup_arithmetic : require +#extension GL_EXT_shader_explicit_arithmetic_types_float16 : require +#else +#extension GL_KHR_shader_subgroup_basic : enable +#extension GL_KHR_shader_subgroup_shuffle : enable #endif layout (constant_id = 3) const int BLOCK_SIZE = 512; @@ -36,6 +40,7 @@ layout (binding = 6) readonly buffer R_I {uvec2 rope_data_i[];}; // indices for #define GGML_ROPE_TYPE_NEOX 2 #define GGML_ROPE_TYPE_MROPE 8 #define GGML_ROPE_TYPE_VISION 24 +#define GGML_ROPE_TYPE_IMROPE 40 #elif RMS_NORM_ADD_FUSION @@ -46,6 +51,34 @@ layout (binding = 4) readonly buffer E {float data_e[];}; layout (binding = 3) readonly buffer I {uvec2 data_i[];}; +#elif RMS_NORM_SILU_GATE_FUSION + +// gen: (rms_norm(x) * w) * silu(z), z laid out like dst (the Gated-DeltaNet output norm) +layout (binding = 3) readonly buffer Z {float data_z[];}; +#if USE_SUBGROUP_ADD +layout (binding = 3) readonly buffer Z_V4 {vec4 data_z_v4[];}; +layout (binding = 4) writeonly buffer D_F16_V4 {f16vec4 data_d_f16_v4[];}; +#endif + +#elif RMS_NORM_SCALE_FUSION + +layout (binding = 3) writeonly buffer D2 {D_TYPE data_d2[];}; +#if USE_SUBGROUP_ADD +layout (binding = 3) writeonly buffer D2_V4 {vec4 data_d2_v4[];}; +#endif + +#else + +#if USE_SUBGROUP_ADD +layout (binding = 3) writeonly buffer D_F16_V4 {f16vec4 data_d_f16_v4[];}; +#endif + +#endif + +#if USE_SUBGROUP_ADD && !RMS_NORM_ROPE_FUSION && !RMS_NORM_SET_ROWS_FUSION && !RMS_NORM_ADD_FUSION +layout (binding = 0) readonly buffer A_V4 {vec4 data_a_v4[];}; +layout (binding = 1) readonly buffer B_V4 {vec4 data_b_v4[];}; +layout (binding = 2) writeonly buffer D_V4 {vec4 data_d_v4[];}; #endif layout (constant_id = 1) const bool do_multiply = false; @@ -53,9 +86,53 @@ layout (constant_id = 1) const bool do_multiply = false; layout (constant_id = 2) const bool do_post_multiply = false; #endif +#if USE_SUBGROUP_ADD +const int SUMSH_SIZE = (BLOCK_SIZE <= 32) ? 1 : (BLOCK_SIZE / 32); +shared FLOAT_TYPE sumsh[SUMSH_SIZE]; +#else shared FLOAT_TYPE sumsh[BLOCK_SIZE]; +#endif + +void write_rms_col(const uint col, const FLOAT_TYPE scale, const uint32_t a_offset, const uint32_t b_offset, const uint32_t d_offset, const bool use_fastmod) { + if (do_multiply) { + const uint b_col = use_fastmod ? fastmod(col, p.ne10) : col; + FLOAT_TYPE value = scale * FLOAT_TYPE(data_a[a_offset + col]) * FLOAT_TYPE(data_b[b_offset + b_col]); +#if RMS_NORM_ADD_FUSION + value += FLOAT_TYPE(data_c[d_offset + col]); + if (do_post_multiply) { + value *= FLOAT_TYPE(data_e[0]); + } +#endif +#if RMS_NORM_SILU_GATE_FUSION + { + // same expressions and roundings as rms_norm_mul (store) then silu_mul (unary.comp op_silu) + precise float gen_n = float(D_TYPE(value)); + precise float gen_zv = data_z[d_offset + col]; + precise float gen_g = gen_zv / (1.0f + exp(-gen_zv)); + precise FLOAT_TYPE gen_o = FLOAT_TYPE(gen_g) * FLOAT_TYPE(gen_n); + value = gen_o; + } +#endif + data_d[d_offset + col] = D_TYPE(value); + } else { +#if RMS_NORM_SCALE_FUSION + // gen: rms_norm followed by scale(param2, bias 0), same expressions as scale.comp + // precise: no contraction or reassociation, so the two roundings match rms_norm + scale run separately + precise FLOAT_TYPE gen_v = scale * FLOAT_TYPE(data_a[a_offset + col]); + precise FLOAT_TYPE gen_s = gen_v * FLOAT_TYPE(p.param2); + precise FLOAT_TYPE gen_o = gen_s + FLOAT_TYPE(0.0f); + if ((p.param3 & 1) != 0 && gl_WorkGroupID.x >= p.ne21) { + data_d2[d_offset + col] = D_TYPE(gen_o); + } else { + data_d[d_offset + col] = D_TYPE(gen_o); + } +#else + data_d[d_offset + col] = D_TYPE(scale * FLOAT_TYPE(data_a[a_offset + col])); +#endif + } +} -void rms_norm(uint num_iters) { +void main() { const uint ncols = p.ne00; const uint nrows = gl_NumWorkGroups.x; const uint nchannels = p.ne02; @@ -67,11 +144,14 @@ void rms_norm(uint num_iters) { const uint stride_row = p.nb01; const uint stride_channel = p.nb02; const uint stride_sample = p.nb03; + const uint num_blocks = (ncols + BLOCK_SIZE - 1) / BLOCK_SIZE; // grid.y/z are clamped to the device workgroup limit, iterate over the excess channels/samples for (uint samp = gl_WorkGroupID.z; samp < nsamples; samp += gl_NumWorkGroups.z) { for (uint channel = gl_WorkGroupID.y; channel < nchannels; channel += gl_NumWorkGroups.y) { - barrier(); + if (BLOCK_SIZE > 32 && (samp != gl_WorkGroupID.z || channel != gl_WorkGroupID.y)) { + barrier(); + } uint32_t a_offset = samp*stride_sample + channel*stride_channel + row*stride_row + get_aoffset(); uint32_t b_offset = src1_idx(0, row, channel, samp) + get_boffset(); @@ -80,18 +160,186 @@ void rms_norm(uint num_iters) { uint32_t d_offset = 0; #elif RMS_NORM_SET_ROWS_FUSION uint32_t d_offset = data_i[channel].x*p.nb21 + row*ncols + get_doffset(); +#elif RMS_NORM_SCALE_FUSION + const bool is_d2 = (p.param3 & 1) != 0 && row >= p.ne21; + const uint eff_nrows = (p.param3 & 1) != 0 ? p.ne21 : nrows; + const uint eff_row = is_d2 ? (row - p.ne21) : row; + uint32_t d_offset = ((samp*nchannels + channel)*eff_nrows + eff_row)*ncols + (is_d2 ? uint(p.param3 >> 8) : get_doffset()); #else uint32_t d_offset = ((samp*nchannels + channel)*nrows + row)*ncols + get_doffset(); #endif + +#if USE_SUBGROUP_ADD && !RMS_NORM_ROPE_FUSION && !RMS_NORM_SET_ROWS_FUSION && !RMS_NORM_ADD_FUSION + if ((uint(p.param3) & 2u) != 0u) { + const uint a_v4 = a_offset >> 2u; + const uint b_v4 = b_offset >> 2u; + const uint d_v4 = d_offset >> 2u; + const uint ncols_v4 = ncols >> 2u; + const uint ne10_v4 = p.ne10 >> 2u; + const bool same_w = (ncols_v4 == ne10_v4); +#if !RMS_NORM_SCALE_FUSION + const bool do_f16 = (uint(p.param3) & 4u) != 0u; + const bool skip_f32 = (uint(p.param3) & 8u) != 0u; +#endif + if (ncols_v4 == uint(BLOCK_SIZE)) { + const vec4 xv = data_a_v4[a_v4 + tid]; + float sum_sq = subgroupAdd(dot(xv, xv)); + if (BLOCK_SIZE > 32) { + if (subgroupElect()) { + sumsh[gl_SubgroupID] = sum_sq; + } + barrier(); + sum_sq = 0.0f; + [[unroll]] for (uint i = 0u; i < gl_NumSubgroups; ++i) { + sum_sq += sumsh[i]; + } + } + const float scale = inversesqrt(sum_sq / float(ncols) + p.param1); + if (do_multiply) { + const uint w_idx = same_w ? tid : (tid % ne10_v4); + vec4 out_v4 = (scale * xv) * data_b_v4[b_v4 + w_idx]; +#if RMS_NORM_SILU_GATE_FUSION + const vec4 zv = data_z_v4[d_v4 + tid]; + out_v4 = (zv / (vec4(1.0f) + exp(-zv))) * out_v4; +#endif +#if !RMS_NORM_SCALE_FUSION + if (do_f16) { + data_d_f16_v4[d_v4 + tid] = f16vec4(out_v4); + } + if (!skip_f32) { + data_d_v4[d_v4 + tid] = out_v4; + } +#endif + } else { +#if RMS_NORM_SCALE_FUSION + const vec4 out_v4 = (scale * xv) * p.param2; + if (is_d2) { + data_d2_v4[d_v4 + tid] = out_v4; + } else { + data_d_v4[d_v4 + tid] = out_v4; + } +#else + const vec4 out_v4 = scale * xv; + if (do_f16) { + data_d_f16_v4[d_v4 + tid] = f16vec4(out_v4); + } + if (!skip_f32) { + data_d_v4[d_v4 + tid] = out_v4; + } +#endif + } + } else { + vec4 xv[4]; + float sum_sq = 0.0f; + [[unroll]] for (uint idx = 0u; idx < 4u; ++idx) { + const uint v4_col = tid + idx * uint(BLOCK_SIZE); + xv[idx] = data_a_v4[a_v4 + v4_col]; + sum_sq += dot(xv[idx], xv[idx]); + } + sum_sq = subgroupAdd(sum_sq); + if (BLOCK_SIZE > 32) { + if (subgroupElect()) { + sumsh[gl_SubgroupID] = sum_sq; + } + barrier(); + sum_sq = 0.0f; + if (gl_NumSubgroups <= 4u) { + [[unroll]] for (uint i = 0u; i < gl_NumSubgroups; ++i) { + sum_sq += sumsh[i]; + } + } else { + for (uint i = gl_SubgroupInvocationID; i < gl_NumSubgroups; i += gl_SubgroupSize) { + sum_sq += sumsh[i]; + } + sum_sq = subgroupAdd(sum_sq); + } + } + const float scale = inversesqrt(sum_sq / float(ncols) + p.param1); + [[unroll]] for (uint idx = 0u; idx < 4u; ++idx) { + const uint v4_col = tid + idx * uint(BLOCK_SIZE); + if (do_multiply) { + const uint w_idx = same_w ? v4_col : (v4_col % ne10_v4); + vec4 out_v4 = (scale * xv[idx]) * data_b_v4[b_v4 + w_idx]; +#if RMS_NORM_SILU_GATE_FUSION + const vec4 zv = data_z_v4[d_v4 + v4_col]; + out_v4 = (zv / (vec4(1.0f) + exp(-zv))) * out_v4; +#endif +#if !RMS_NORM_SCALE_FUSION + if (do_f16) { + data_d_f16_v4[d_v4 + v4_col] = f16vec4(out_v4); + } + if (!skip_f32) { + data_d_v4[d_v4 + v4_col] = out_v4; + } +#endif + } else { +#if RMS_NORM_SCALE_FUSION + const vec4 out_v4 = (scale * xv[idx]) * p.param2; + if (is_d2) { + data_d2_v4[d_v4 + v4_col] = out_v4; + } else { + data_d_v4[d_v4 + v4_col] = out_v4; + } +#else + const vec4 out_v4 = scale * xv[idx]; + if (do_f16) { + data_d_f16_v4[d_v4 + v4_col] = f16vec4(out_v4); + } + if (!skip_f32) { + data_d_v4[d_v4 + v4_col] = out_v4; + } +#endif + } + } + } + continue; + } +#endif FLOAT_TYPE sum = FLOAT_TYPE(0.0f); // partial sum for thread in warp - [[unroll]] for (uint col = tid, idx = 0; idx < num_iters; col += BLOCK_SIZE, ++idx) { +#if RMS_NORM_ROPE_FUSION + if (num_blocks == 1) { FLOAT_TYPE xi = FLOAT_TYPE(0); - if (col < ncols) { - xi = FLOAT_TYPE(data_a[a_offset + col]); + if (tid < ncols) { + xi = FLOAT_TYPE(data_a[a_offset + tid]); } sum += xi * xi; + } else { + [[unroll]] for (uint idx = 0; idx < 2; ++idx) { + const uint col = tid + idx * BLOCK_SIZE; + FLOAT_TYPE xi = FLOAT_TYPE(0); + if (col < ncols) { + xi = FLOAT_TYPE(data_a[a_offset + col]); + } + sum += xi * xi; + } } +#else + if (num_blocks == 1) { + FLOAT_TYPE xi = FLOAT_TYPE(0); + if (tid < ncols) { + xi = FLOAT_TYPE(data_a[a_offset + tid]); + } + sum += xi * xi; + } else if (num_blocks == 4) { + [[unroll]] for (uint idx = 0; idx < 4; ++idx) { + const uint col = tid + idx * BLOCK_SIZE; + FLOAT_TYPE xi = FLOAT_TYPE(0); + if (col < ncols) { + xi = FLOAT_TYPE(data_a[a_offset + col]); + } + sum += xi * xi; + } + } else { + [[dont_unroll]] for (uint col = tid, idx = 0; idx < num_blocks; col += BLOCK_SIZE, ++idx) { + FLOAT_TYPE xi = FLOAT_TYPE(0); + if (col < ncols) { + xi = FLOAT_TYPE(data_a[a_offset + col]); + } + sum += xi * xi; + } + } +#endif #if USE_SUBGROUP_ADD sum = subgroupAdd(sum); @@ -116,57 +364,153 @@ void rms_norm(uint num_iters) { sumsh[tid] = sum; // sum up partial sums and write back result barrier(); - [[unroll]] for (int s = BLOCK_SIZE / 2; s > 0; s >>= 1) { - if (tid < s) { - sum += sumsh[tid + s]; - sumsh[tid] = sum; + if (BLOCK_SIZE == 512 && gl_SubgroupSize == 32) { + if (gl_SubgroupID == 0) { + const uint lane = gl_SubgroupInvocationID; + precise FLOAT_TYPE p0_a = sumsh[lane] + sumsh[lane + 256u]; + precise FLOAT_TYPE p0_b = sumsh[lane + 128u] + sumsh[lane + 384u]; + precise FLOAT_TYPE p1_a = sumsh[lane + 32u] + sumsh[lane + 288u]; + precise FLOAT_TYPE p1_b = sumsh[lane + 160u] + sumsh[lane + 416u]; + precise FLOAT_TYPE p2_a = sumsh[lane + 64u] + sumsh[lane + 320u]; + precise FLOAT_TYPE p2_b = sumsh[lane + 192u] + sumsh[lane + 448u]; + precise FLOAT_TYPE p3_a = sumsh[lane + 96u] + sumsh[lane + 352u]; + precise FLOAT_TYPE p3_b = sumsh[lane + 224u] + sumsh[lane + 480u]; + precise FLOAT_TYPE p0 = p0_a + p0_b; + precise FLOAT_TYPE p1 = p1_a + p1_b; + precise FLOAT_TYPE p2 = p2_a + p2_b; + precise FLOAT_TYPE p3 = p3_a + p3_b; + precise FLOAT_TYPE s_lo = p0 + p2; + precise FLOAT_TYPE s_hi = p1 + p3; + precise FLOAT_TYPE s32 = s_lo + s_hi; + precise FLOAT_TYPE s16 = s32 + subgroupShuffleXor(s32, 16); + precise FLOAT_TYPE s8 = s16 + subgroupShuffleXor(s16, 8); + precise FLOAT_TYPE s4 = s8 + subgroupShuffleXor(s8, 4); + precise FLOAT_TYPE s2 = s4 + subgroupShuffleXor(s4, 2); + precise FLOAT_TYPE s1 = s2 + subgroupShuffleXor(s2, 1); + if (lane == 0) { + sumsh[0] = s1; + } + } + barrier(); + } else if (BLOCK_SIZE == 256 && gl_SubgroupSize == 32) { + if (gl_SubgroupID == 0) { + const uint lane = gl_SubgroupInvocationID; + precise FLOAT_TYPE p0 = sumsh[lane] + sumsh[lane + 128u]; + precise FLOAT_TYPE p1 = sumsh[lane + 32u] + sumsh[lane + 160u]; + precise FLOAT_TYPE p2 = sumsh[lane + 64u] + sumsh[lane + 192u]; + precise FLOAT_TYPE p3 = sumsh[lane + 96u] + sumsh[lane + 224u]; + precise FLOAT_TYPE s_lo = p0 + p2; + precise FLOAT_TYPE s_hi = p1 + p3; + precise FLOAT_TYPE s32 = s_lo + s_hi; + precise FLOAT_TYPE s16 = s32 + subgroupShuffleXor(s32, 16); + precise FLOAT_TYPE s8 = s16 + subgroupShuffleXor(s16, 8); + precise FLOAT_TYPE s4 = s8 + subgroupShuffleXor(s8, 4); + precise FLOAT_TYPE s2 = s4 + subgroupShuffleXor(s4, 2); + precise FLOAT_TYPE s1 = s2 + subgroupShuffleXor(s2, 1); + if (lane == 0) { + sumsh[0] = s1; + } + } + barrier(); + } else if (BLOCK_SIZE >= 128 && gl_SubgroupSize == 32) { + [[unroll]] for (int s = BLOCK_SIZE / 2; s >= 128; s >>= 1) { + if (tid < s) { + sum += sumsh[tid + s]; + sumsh[tid] = sum; + } + barrier(); + } + // Exact 0-ULP match to the s = 64, 32, 16, 8, 4, 2, 1 shared-memory tree reduction + if (gl_SubgroupID == 0) { + const uint lane = gl_SubgroupInvocationID; + precise FLOAT_TYPE s_lo = sumsh[lane] + sumsh[lane + 64]; + precise FLOAT_TYPE s_hi = sumsh[lane + 32] + sumsh[lane + 96]; + precise FLOAT_TYPE s32 = s_lo + s_hi; + precise FLOAT_TYPE s16 = s32 + subgroupShuffleXor(s32, 16); + precise FLOAT_TYPE s8 = s16 + subgroupShuffleXor(s16, 8); + precise FLOAT_TYPE s4 = s8 + subgroupShuffleXor(s8, 4); + precise FLOAT_TYPE s2 = s4 + subgroupShuffleXor(s4, 2); + precise FLOAT_TYPE s1 = s2 + subgroupShuffleXor(s2, 1); + if (lane == 0) { + sumsh[0] = s1; + } } barrier(); + } else { + [[unroll]] for (int s = BLOCK_SIZE / 2; s > 0; s >>= 1) { + if (tid < s) { + sum += sumsh[tid + s]; + sumsh[tid] = sum; + } + barrier(); + } } sum = sumsh[0]; #endif const FLOAT_TYPE mean = sum / FLOAT_TYPE(ncols); const FLOAT_TYPE scale = inversesqrt(mean + FLOAT_TYPE(p.param1)); + const bool use_fastmod = (ncols > p.ne10); - if (do_multiply) { - if (ncols > p.ne10) { - [[unroll]] for (uint col = tid, idx = 0; idx < num_iters; col += BLOCK_SIZE, ++idx) { - if (col >= ncols) { - continue; - } - FLOAT_TYPE value = scale * FLOAT_TYPE(data_a[a_offset + col]) * FLOAT_TYPE(data_b[b_offset + fastmod(col, p.ne10)]); -#if RMS_NORM_ADD_FUSION - value += FLOAT_TYPE(data_c[d_offset + col]); - if (do_post_multiply) { - value *= FLOAT_TYPE(data_e[0]); +#if RMS_NORM_ROPE_FUSION + if (num_blocks == 1) { + if (tid < ncols) { + write_rms_col(tid, scale, a_offset, b_offset, d_offset, use_fastmod); + } + } else { + if (use_fastmod) { + [[unroll]] for (uint idx = 0; idx < 2; ++idx) { + const uint col = tid + idx * BLOCK_SIZE; + if (col < ncols) { + write_rms_col(col, scale, a_offset, b_offset, d_offset, true); } -#endif - data_d[d_offset + col] = D_TYPE(value); } } else { - [[unroll]] for (uint col = tid, idx = 0; idx < num_iters; col += BLOCK_SIZE, ++idx) { - if (col >= ncols) { - continue; + [[unroll]] for (uint idx = 0; idx < 2; ++idx) { + const uint col = tid + idx * BLOCK_SIZE; + if (col < ncols) { + write_rms_col(col, scale, a_offset, b_offset, d_offset, false); } - FLOAT_TYPE value = scale * FLOAT_TYPE(data_a[a_offset + col]) * FLOAT_TYPE(data_b[b_offset + col]); -#if RMS_NORM_ADD_FUSION - value += FLOAT_TYPE(data_c[d_offset + col]); - if (do_post_multiply) { - value *= FLOAT_TYPE(data_e[0]); + } + } + } +#else + if (num_blocks == 1) { + if (tid < ncols) { + write_rms_col(tid, scale, a_offset, b_offset, d_offset, use_fastmod); + } + } else if (num_blocks == 4) { + if (use_fastmod) { + [[unroll]] for (uint idx = 0; idx < 4; ++idx) { + const uint col = tid + idx * BLOCK_SIZE; + if (col < ncols) { + write_rms_col(col, scale, a_offset, b_offset, d_offset, true); + } + } + } else { + [[unroll]] for (uint idx = 0; idx < 4; ++idx) { + const uint col = tid + idx * BLOCK_SIZE; + if (col < ncols) { + write_rms_col(col, scale, a_offset, b_offset, d_offset, false); } -#endif - data_d[d_offset + col] = D_TYPE(value); } } } else { - [[unroll]] for (uint col = tid, idx = 0; idx < num_iters; col += BLOCK_SIZE, ++idx) { - if (col >= ncols) { - continue; + if (use_fastmod) { + [[dont_unroll]] for (uint col = tid, idx = 0; idx < num_blocks; col += BLOCK_SIZE, ++idx) { + if (col < ncols) { + write_rms_col(col, scale, a_offset, b_offset, d_offset, true); + } + } + } else { + [[dont_unroll]] for (uint col = tid, idx = 0; idx < num_blocks; col += BLOCK_SIZE, ++idx) { + if (col < ncols) { + write_rms_col(col, scale, a_offset, b_offset, d_offset, false); + } } - data_d[d_offset + col] = D_TYPE(scale * FLOAT_TYPE(data_a[a_offset + col])); } } +#endif #if RMS_NORM_ROPE_FUSION barrier(); rope_params rp = p.rope; @@ -175,36 +519,11 @@ void rms_norm(uint num_iters) { rope_neox(t, row, channel, samp, rp); } else if (rp.rope_mode == GGML_ROPE_TYPE_NORMAL) { rope_norm(t, row, channel, samp, rp); + } else if (rp.rope_mode == GGML_ROPE_TYPE_MROPE || rp.rope_mode == GGML_ROPE_TYPE_IMROPE) { + rope_multi(t, row, channel, samp, rp); } } #endif } } } - -void main() { - // instantiate the rms_norm function for several different - // dimensions, to allow loop unrolling - uint num_blocks = (p.ne00 + BLOCK_SIZE - 1) / BLOCK_SIZE; - if (num_blocks > 32) { - rms_norm(num_blocks); - } else if (num_blocks > 16) { - rms_norm(32); - } else if (num_blocks > 12) { - rms_norm(16); - } else if (num_blocks > 10) { - rms_norm(12); - } else if (num_blocks > 8) { - rms_norm(10); - } else if (num_blocks > 4) { - rms_norm(8); - } else if (num_blocks == 4) { - rms_norm(4); - } else if (num_blocks == 3) { - rms_norm(3); - } else if (num_blocks == 2) { - rms_norm(2); - } else if (num_blocks == 1) { - rms_norm(1); - } -} diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_partials.comp b/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_partials.comp index cf7ab21f261d..3318c301bb76 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_partials.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_partials.comp @@ -23,16 +23,13 @@ layout (binding = 4) readonly buffer C {float data_c[];}; layout (binding = 5) readonly buffer E {float data_e[];}; #endif -shared FLOAT_TYPE sumsh[BLOCK_SIZE]; - void main() { const uint ncols = p.ne00; - const uint nrows = gl_NumWorkGroups.x; - const uint nchannels = gl_NumWorkGroups.y; - - const uint row = 0; - const uint channel = gl_WorkGroupID.y; - const uint samp = gl_WorkGroupID.z; + // gen: one workgroup row (y) per tensor row (rows > 1 only in GGML_VK_BATCH_INVARIANT mode); a row's partials are + // num_partials consecutive values after the previous rows' + const uint row = gl_WorkGroupID.y; + const uint channel = 0; + const uint samp = 0; // The work is split across multiple workgroups in the x dimension. Each invocation // processes one element const uint tid = gl_GlobalInvocationID.x; @@ -43,13 +40,13 @@ void main() { uint32_t a_offset = samp*stride_sample + channel*stride_channel + row*stride_row + get_aoffset(); uint32_t b_offset = src1_idx(0, row, channel, samp) + get_boffset(); - uint32_t d_offset = ((samp*nchannels + channel)*nrows + row)*ncols + get_doffset(); + uint32_t d_offset = row*ncols + get_doffset(); FLOAT_TYPE sum = FLOAT_TYPE(0.0f); // partial sum for thread in warp uint32_t num_partials = p.param3; for (uint32_t i = gl_SubgroupInvocationID; i < num_partials; i += gl_SubgroupSize) { - sum += partial_sums[i]; + sum += partial_sums[row * num_partials + i]; } sum = subgroupAdd(sum); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/scale.comp b/ggml/src/ggml-vulkan/vulkan-shaders/scale.comp index 35ec726a01c6..a3f7875e7d78 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/scale.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/scale.comp @@ -5,6 +5,8 @@ const uint num_threads = 128; +layout(constant_id = 0) const bool ZERO_FILL = false; + layout(local_size_x = num_threads, local_size_y = 1, local_size_z = 1) in; void main() { @@ -18,7 +20,11 @@ void main() { continue; } - data_d[get_doffset() + idx] = D_TYPE(FLOAT_TYPE(data_a[get_aoffset() + idx]) * FLOAT_TYPE(p.param1) + FLOAT_TYPE(p.param2)); + if (ZERO_FILL) { + data_d[get_doffset() + idx] = D_TYPE(0); + } else { + data_d[get_doffset() + idx] = D_TYPE(FLOAT_TYPE(data_a[get_aoffset() + idx]) * FLOAT_TYPE(p.param1) + FLOAT_TYPE(p.param2)); + } idx += num_threads; } } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/ssm_conv.comp b/ggml/src/ggml-vulkan/vulkan-shaders/ssm_conv.comp index 4cd9b8da3596..c7d7388f7f3b 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/ssm_conv.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/ssm_conv.comp @@ -8,6 +8,9 @@ layout(constant_id = 0) const uint BLOCK_SIZE = 32; layout(constant_id = 1) const uint TOKENS_PER_WG = 16; layout(constant_id = 2) const bool APPLY_BIAS = false; layout(constant_id = 3) const bool APPLY_SILU = false; +layout(constant_id = 4) const bool WRITE_STATE_OUT = false; +layout(constant_id = 5) const bool CONCAT_SLIDING = false; +layout(constant_id = 6) const uint TOK_PER_THREAD = 1; layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z = 1) in; @@ -15,19 +18,120 @@ layout(binding = 0) readonly buffer Src0 { float src0[]; }; layout(binding = 1) readonly buffer Src1 { float src1[]; }; layout(binding = 2) readonly buffer Bias { float bias[]; }; layout(binding = 3) buffer Dst { float dst[]; }; +layout(binding = 4) writeonly buffer StateOut { float state_out[]; }; layout(push_constant) uniform PushConstants { uint nb01; uint nb02; uint nb11; uint dst_nb0; uint dst_nb1; uint dst_nb2; uint nc; uint ncs; uint nr; uint n_t; uint n_s; + uint st_nb1; uint st_nb2; + uint qkv_nb0; uint qkv_nb1; uint qkv_nb2; + uint st_k; uint st_slot_nb; }; void main() { const uint i1 = gl_GlobalInvocationID.x; - const uint i2 = gl_WorkGroupID.y * TOKENS_PER_WG + gl_LocalInvocationID.y; const uint i3 = gl_WorkGroupID.z; + if (CONCAT_SLIDING) { + const bool zero_state = (st_k & 0x80000000u) != 0u; + const uint eff_st_k = st_k & 0x7FFFFFFFu; + if (TOKENS_PER_WG > 1u) { + const uint i2_base = (gl_WorkGroupID.y * TOKENS_PER_WG + gl_LocalInvocationID.y) * TOK_PER_THREAD; + if (i1 >= nr || i2_base >= n_t || i3 >= n_s) { + return; + } + const uint os_base = i3 * (nb02 / 4) + i1 * (nb01 / 4); + const uint q_base = i3 * (qkv_nb2 / 4) + i1 * (qkv_nb1 / 4); + const uint q_step = qkv_nb0 / 4; + const uint w_base = i1 * (nb11 / 4); + const vec4 w = vec4(src1[w_base], src1[w_base + 1], src1[w_base + 2], src1[w_base + 3]); + const uint d_base = i3 * (dst_nb2 / 4) + i1 * (dst_nb0 / 4); + const uint d_step = dst_nb1 / 4; + const uint st_base = i3 * (st_nb2 / 4) + i1 * (st_nb1 / 4); + + float s0 = (i2_base < 3u) ? (zero_state ? 0.0f : src0[os_base + i2_base]) : bias[q_base + (i2_base - 3u) * q_step]; + float s1 = (i2_base + 1u < 3u) ? (zero_state ? 0.0f : src0[os_base + i2_base + 1u]) : bias[q_base + (i2_base - 2u) * q_step]; + float s2 = (i2_base + 2u < 3u) ? (zero_state ? 0.0f : src0[os_base + i2_base + 2u]) : bias[q_base + (i2_base - 1u) * q_step]; + + [[unroll]] for (uint tt = 0u; tt < TOK_PER_THREAD; ++tt) { + const uint i2 = i2_base + tt; + if (i2 < n_t) { + const float x3 = bias[q_base + i2 * q_step]; + float sum = dot(vec4(s0, s1, s2, x3), w); + if (APPLY_SILU) { + sum = sum / (1.0f + exp(-sum)); + } + dst[d_base + i2 * d_step] = sum; + s0 = s1; + s1 = s2; + s2 = x3; + + if (eff_st_k > 0u) { + const int target_slot = int(n_t) - 1 - int(i2); + if (target_slot >= 0 && target_slot < int(eff_st_k)) { + const uint st_dst = uint(target_slot) * (st_slot_nb / 4) + st_base; + state_out[st_dst ] = s0; + state_out[st_dst + 1] = s1; + state_out[st_dst + 2] = s2; + } + } else if (i2 + 1u == n_t) { + state_out[st_base ] = s0; + state_out[st_base + 1] = s1; + state_out[st_base + 2] = s2; + } + } + } + return; + } + if (i1 >= nr || i3 >= n_s) { + return; + } + const uint os_base = i3 * (nb02 / 4) + i1 * (nb01 / 4); + const uint w_base = i1 * (nb11 / 4); + const vec4 w = vec4(src1[w_base], src1[w_base + 1], src1[w_base + 2], src1[w_base + 3]); + float s0 = zero_state ? 0.0f : src0[os_base]; + float s1 = zero_state ? 0.0f : src0[os_base + 1]; + float s2 = zero_state ? 0.0f : src0[os_base + 2]; + + const uint q_base = i3 * (qkv_nb2 / 4) + i1 * (qkv_nb1 / 4); + const uint q_step = qkv_nb0 / 4; + const uint d_base = i3 * (dst_nb2 / 4) + i1 * (dst_nb0 / 4); + const uint d_step = dst_nb1 / 4; + + const uint st_base = i3 * (st_nb2 / 4) + i1 * (st_nb1 / 4); + for (uint t = 0; t < n_t; ++t) { + const float x = bias[q_base + t * q_step]; + float sum = dot(vec4(s0, s1, s2, x), w); + if (APPLY_SILU) { + sum = sum / (1.0f + exp(-sum)); + } + dst[d_base + t * d_step] = sum; + s0 = s1; + s1 = s2; + s2 = x; + if (eff_st_k > 0u) { + const int target_slot = int(n_t) - 1 - int(t); + if (target_slot >= 0 && target_slot < int(eff_st_k)) { + const uint st_dst = uint(target_slot) * (st_slot_nb / 4) + st_base; + state_out[st_dst ] = s0; + state_out[st_dst + 1] = s1; + state_out[st_dst + 2] = s2; + } + } + } + + if (eff_st_k == 0u) { + state_out[st_base ] = s0; + state_out[st_base + 1] = s1; + state_out[st_base + 2] = s2; + } + return; + } + + const uint i2 = gl_WorkGroupID.y * TOKENS_PER_WG + gl_LocalInvocationID.y; + if (i1 >= nr || i2 >= n_t || i3 >= n_s) { return; } @@ -38,14 +142,31 @@ void main() { float sum = 0.0; if (nc == 4) { + const float x0 = src0[src0_base]; + const float x1 = src0[src0_base + 1]; + const float x2 = src0[src0_base + 2]; + const float x3 = src0[src0_base + 3]; sum = dot( - vec4(src0[src0_base], src0[src0_base + 1], src0[src0_base + 2], src0[src0_base + 3]), + vec4(x0, x1, x2, x3), vec4(src1[src1_base], src1[src1_base + 1], src1[src1_base + 2], src1[src1_base + 3]) ); + if (WRITE_STATE_OUT && i2 + 1 == n_t) { + const uint st_dst = i3 * (st_nb2 / 4) + i1 * (st_nb1 / 4); + state_out[st_dst ] = x1; + state_out[st_dst + 1] = x2; + state_out[st_dst + 2] = x3; + } } else { [[unroll]] for (uint i0 = 0; i0 < nc; i0++) { sum += src0[src0_base + i0] * src1[src1_base + i0]; } + if (WRITE_STATE_OUT && i2 + 1 == n_t) { + const uint st_src = i3 * (nb02 / 4) + n_t + i1 * (nb01 / 4); + const uint st_dst = i3 * (st_nb2 / 4) + i1 * (st_nb1 / 4); + for (uint c = 0; c + 1 < nc; ++c) { + state_out[st_dst + c] = src0[st_src + c]; + } + } } if (APPLY_BIAS) { diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/topk_moe.comp b/ggml/src/ggml-vulkan/vulkan-shaders/topk_moe.comp index d219201fda0a..2f99488214fb 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/topk_moe.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/topk_moe.comp @@ -103,39 +103,34 @@ void main() { float probs[experts_per_thread]; [[unroll]] - for (int i = 0; i < experts_per_thread; i++) { - probs[i] = -INFINITY; - } - - [[unroll]] - for (uint i = 0; i < n_experts; i += WARP_SIZE) { - const uint expert = i + lane; - probs[i / WARP_SIZE] = (n_experts % WARP_SIZE == 0 || expert < n_experts) ? logits[logits_offset + expert] : -INFINITY; + for (uint idx = 0; idx < experts_per_thread; idx++) { + const uint expert = idx * WARP_SIZE + lane; + probs[idx] = ((!nexperts_use_push && (n_experts_spec % WARP_SIZE == 0)) || expert < n_experts) ? logits[logits_offset + expert] : -INFINITY; } if (gating_func == GATING_FUNC_SOFTMAX) { softmax_warp_inplace(probs, n_experts, lane, nexperts_use_push); } else if (gating_func == GATING_FUNC_SIGMOID) { [[unroll]] - for (uint i = 0; i < n_experts; i += WARP_SIZE) { - const uint expert = i + lane; - probs[i / WARP_SIZE] = (n_experts % WARP_SIZE == 0 || expert < n_experts) ? 1.f / (1.f + exp(-probs[i / WARP_SIZE])) : -INFINITY; + for (uint idx = 0; idx < experts_per_thread; idx++) { + const uint expert = idx * WARP_SIZE + lane; + probs[idx] = ((!nexperts_use_push && (n_experts_spec % WARP_SIZE == 0)) || expert < n_experts) ? 1.f / (1.f + exp(-probs[idx])) : -INFINITY; } } else if (gating_func == GATING_FUNC_SQRT_SOFTPLUS) { [[unroll]] - for (uint i = 0; i < n_experts; i += WARP_SIZE) { - const uint expert = i + lane; - const float val = probs[i / WARP_SIZE]; - probs[i / WARP_SIZE] = (n_experts % WARP_SIZE == 0 || expert < n_experts) ? sqrt(val > 20.0f ? val : log(1.0f + exp(val))) : -INFINITY; + for (uint idx = 0; idx < experts_per_thread; idx++) { + const uint expert = idx * WARP_SIZE + lane; + const float val = probs[idx]; + probs[idx] = ((!nexperts_use_push && (n_experts_spec % WARP_SIZE == 0)) || expert < n_experts) ? sqrt(val > 20.0f ? val : log(1.0f + exp(val))) : -INFINITY; } } float selection_probs[experts_per_thread]; if (has_bias != 0) { [[unroll]] - for (uint i = 0; i < n_experts; i += WARP_SIZE) { - const uint expert = i + lane; - selection_probs[i / WARP_SIZE] = (n_experts % WARP_SIZE == 0 || expert < n_experts) ? probs[i / WARP_SIZE] + bias[bias_offset + expert] : -INFINITY; + for (uint idx = 0; idx < experts_per_thread; idx++) { + const uint expert = idx * WARP_SIZE + lane; + selection_probs[idx] = ((!nexperts_use_push && (n_experts_spec % WARP_SIZE == 0)) || expert < n_experts) ? probs[idx] + bias[bias_offset + expert] : -INFINITY; } } else { [[unroll]] @@ -163,11 +158,11 @@ void main() { uint max_expert = lane; [[unroll]] - for (uint i = WARP_SIZE; i < n_experts; i += WARP_SIZE) { - const uint expert = i + lane; - if ((n_experts % WARP_SIZE == 0 || expert < n_experts) && selection_probs[i / WARP_SIZE] > max_val_s) { - max_val = probs[i / WARP_SIZE]; - max_val_s = selection_probs[i / WARP_SIZE]; + for (uint idx = 1; idx < experts_per_thread; idx++) { + const uint expert = idx * WARP_SIZE + lane; + if (((!nexperts_use_push && (n_experts_spec % WARP_SIZE == 0)) || expert < n_experts) && selection_probs[idx] > max_val_s) { + max_val = probs[idx]; + max_val_s = selection_probs[idx]; max_expert = expert; } } @@ -184,13 +179,17 @@ void main() { } } - if ((k & (WARP_SIZE - 1)) == lane) { - output_weights[k / WARP_SIZE] = max_val; + [[unroll]] + for (uint idx = 0; idx < experts_per_thread; idx++) { + if (uint(k) == idx * WARP_SIZE + lane) { + output_weights[idx] = max_val; + } + if (max_expert == idx * WARP_SIZE + lane) { + selection_probs[idx] = -INFINITY; + } } if ((max_expert & (WARP_SIZE - 1)) == lane) { - selection_probs[max_expert / WARP_SIZE] = -INFINITY; - ids[ids_offset + k] = max_expert; wt_sum += max_val; } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp index 3952030fb73c..5338efb4a0c1 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp @@ -565,10 +565,18 @@ void matmul_shaders(bool fp16, MatMulIdType matmul_id_type, bool coopmat, bool c {"FLOAT_TYPEV8", FLOAT_TYPE(8, "f16")}, }; + const bool emit_cm1_fused = coopmat && !coopmat2 && !f16acc && !dot2; + // Shaders with f16 B_TYPE string_to_spv(shader_name + "_f32_f16" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, float_type_dict_f16), {{"DATA_A_F32", "1"}, {"LOAD_VEC_A", load_vec}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f16}, {"B_TYPE_SCALAR", "float16_t"}, {"B_TYPEV4", "f16vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); + if (emit_cm1_fused) { + string_to_spv(shader_name + "_f32_f16_fused", source_name, merge_maps(merge_maps(base_dict, float_type_dict_f16), {{"MM_FUSED", "1"}, {"DATA_A_F32", "1"}, {"LOAD_VEC_A", load_vec}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f16}, {"B_TYPE_SCALAR", "float16_t"}, {"B_TYPEV4", "f16vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); + } string_to_spv(shader_name + "_f16" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, float_type_dict_f16), {{"DATA_A_F16", "1"}, {"LOAD_VEC_A", load_vec}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f16}, {"B_TYPE_SCALAR", "float16_t"}, {"B_TYPEV4", "f16vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); + if (emit_cm1_fused) { + string_to_spv(shader_name + "_f16_fused", source_name, merge_maps(merge_maps(base_dict, float_type_dict_f16), {{"MM_FUSED", "1"}, {"DATA_A_F16", "1"}, {"LOAD_VEC_A", load_vec}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f16}, {"B_TYPE_SCALAR", "float16_t"}, {"B_TYPEV4", "f16vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); + } // bf16 { @@ -613,6 +621,9 @@ void matmul_shaders(bool fp16, MatMulIdType matmul_id_type, bool coopmat, bool c if (!coopmat2) { string_to_spv(shader_name + "_" + tname + "_f32" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"LOAD_VEC_A", load_vec}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f32}, {"B_TYPE_SCALAR", "float"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); + if (emit_cm1_fused) { + string_to_spv(shader_name + "_" + tname + "_f32_fused", source_name, merge_maps(merge_maps(base_dict, float_type_dict), {{"MM_FUSED", "1"}, {data_a_key, "1"}, {"LOAD_VEC_A", load_vec}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f32}, {"B_TYPE_SCALAR", "float"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); + } } continue; } @@ -640,9 +651,15 @@ void matmul_shaders(bool fp16, MatMulIdType matmul_id_type, bool coopmat, bool c std::string lva = lut_load_vec_a(tname); string_to_spv(shader_name + "_" + tname + "_f16" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"LOAD_VEC_A", lva}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f16}, {"B_TYPE_SCALAR", "float16_t"}, {"B_TYPEV4", "f16vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); + if (emit_cm1_fused && (tname == "iq4_xs" || tname == "iq4_nl")) { + string_to_spv(shader_name + "_" + tname + "_f16_fused", source_name, merge_maps(merge_maps(base_dict, float_type_dict), {{"MM_FUSED", "1"}, {data_a_key, "1"}, {"LOAD_VEC_A", lva}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f16}, {"B_TYPE_SCALAR", "float16_t"}, {"B_TYPEV4", "f16vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); + } if (!coopmat2) { string_to_spv(shader_name + "_" + tname + "_f32" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"LOAD_VEC_A", lva}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f32}, {"B_TYPE_SCALAR", "float"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); + if (emit_cm1_fused && (tname == "iq4_xs" || tname == "iq4_nl")) { + string_to_spv(shader_name + "_" + tname + "_f32_fused", source_name, merge_maps(merge_maps(base_dict, float_type_dict), {{"MM_FUSED", "1"}, {data_a_key, "1"}, {"LOAD_VEC_A", lva}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f32}, {"B_TYPE_SCALAR", "float"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); + } } #if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT) @@ -672,9 +689,15 @@ void matmul_shaders(bool fp16, MatMulIdType matmul_id_type, bool coopmat, bool c }; string_to_spv(shader_name + "_quant_f16" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, quant_float_type_dict), {{"MULMAT_QUANT", "1"}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f16}, {"B_TYPE_SCALAR", "float16_t"}, {"B_TYPEV4", "f16vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); + if (emit_cm1_fused) { + string_to_spv(shader_name + "_quant_f16_fused", source_name, merge_maps(merge_maps(base_dict, quant_float_type_dict), {{"MM_FUSED", "1"}, {"MULMAT_QUANT", "1"}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f16}, {"B_TYPE_SCALAR", "float16_t"}, {"B_TYPEV4", "f16vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); + } if (!coopmat2) { string_to_spv(shader_name + "_quant_f32" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, quant_float_type_dict), {{"MULMAT_QUANT", "1"}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f32}, {"B_TYPE_SCALAR", "float"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); + if (emit_cm1_fused) { + string_to_spv(shader_name + "_quant_f32_fused", source_name, merge_maps(merge_maps(base_dict, quant_float_type_dict), {{"MM_FUSED", "1"}, {"MULMAT_QUANT", "1"}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f32}, {"B_TYPE_SCALAR", "float"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); + } } } } @@ -738,11 +761,14 @@ void process_shaders() { #if defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT) string_to_spv("flash_attn_f32_f16", "flash_attn_cm1.comp", merge_maps(fa_base_dict, {{"Q_TYPE", "float"}, {"D_TYPE", "float"}, {"D_TYPEV4", "vec4"}, {"COOPMAT", "1"}}), fp16, true, false, f16acc); - std::map fa_flip_dict = fa_base_dict; fa_flip_dict.erase("DATA_A_IQ4_NL"); string_to_spv("flash_attn_f32_f16_flip", "flash_attn_cm1_flip.comp", merge_maps(fa_flip_dict, {{"Q_TYPE", "float"}, {"D_TYPE", "float"}, {"D_TYPEV4", "vec4"}, {"COOPMAT", "1"}}), fp16, true, false, f16acc); + if (!f16acc) { + string_to_spv("flash_attn_f32_f16_8x8", "flash_attn_cm1.comp", + merge_maps(fa_base_dict, {{"Q_TYPE", "float"}, {"D_TYPE", "float"}, {"D_TYPEV4", "vec4"}, {"COOPMAT", "1"}, {"COOPMAT_8X8", "1"}}), fp16, true, false, false); + } #endif } @@ -778,6 +804,18 @@ void process_shaders() { string_to_spv("flash_attn_f32_f16_bf16_flip", "flash_attn_cm1_flip.comp", merge_maps(fa_bf16_dict, {{"Q_TYPE", "float"}, {"D_TYPE", "float"}, {"D_TYPEV4", "vec4"}, {"COOPMAT", "1"}}), true, true, false, false); + string_to_spv("flash_attn_f32_f16_bf16_8x8", "flash_attn_cm1.comp", + merge_maps(fa_bf16_dict, {{"Q_TYPE", "float"}, {"D_TYPE", "float"}, {"D_TYPEV4", "vec4"}, {"COOPMAT", "1"}, {"COOPMAT_8X8", "1"}}), + true, true, false, false); +#endif + +#if defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT) + string_to_spv("mul_mv_mma_f32_f16", "mul_mv_mma.comp", {{"FLOAT16", "1"}, {"DATA_A_F32_F16", "1"}}, true, true, false, false); + for (const std::string & mma_t : {"f32", "f16", "q4_0", "q4_1", "q5_0", "q5_1", "q8_0", "q4_k", "q5_k", "q6_k", "iq4_xs", "iq4_nl"}) { + const std::string mma_def = "DATA_A_" + to_uppercase(mma_t); + string_to_spv("mul_mv_mma_" + mma_t, "mul_mv_mma.comp", {{"FLOAT16", "1"}, {mma_def, "1"}}, true, true, false, false); + string_to_spv("mul_mv_mma_id_" + mma_t, "mul_mv_mma.comp", {{"FLOAT16", "1"}, {"MUL_MAT_ID", "1"}, {mma_def, "1"}}, true, true, false, false); + } #endif #if defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT) && defined(GGML_VULKAN_COOPMAT2_GLSLC_SUPPORT) @@ -865,7 +903,9 @@ void process_shaders() { const std::string suffix = use_subgroups ? "_subgroup" : ""; const auto rms_dict = merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"USE_SUBGROUP_ADD", use_subgroups ? "1" : "0"}}); string_to_spv("rms_norm_f32" + suffix, "rms_norm.comp", rms_dict); + string_to_spv("rms_norm_scale_f32" + suffix, "rms_norm.comp", merge_maps(rms_dict, {{"RMS_NORM_SCALE_FUSION", "1"}})); string_to_spv("rms_norm_mul_add_f32" + suffix, "rms_norm.comp", merge_maps(rms_dict, {{"RMS_NORM_ADD_FUSION", "1"}})); + string_to_spv("rms_norm_mul_silu_f32" + suffix, "rms_norm.comp", merge_maps(rms_dict, {{"RMS_NORM_SILU_GATE_FUSION", "1"}})); string_to_spv("rms_norm_set_rows_f32_f32" + suffix, "rms_norm.comp", merge_maps(rms_dict, {{"RMS_NORM_SET_ROWS_FUSION", "1"}})); string_to_spv("rms_norm_set_rows_f32_f16" + suffix, "rms_norm.comp", merge_maps({{"D_TYPE", "float16_t"}, {"RMS_NORM_SET_ROWS_FUSION", "1"}}, rms_dict)); // keep rms_dict last so 'merge_maps' overwrites the value for key 'D_TYPE' string_to_spv("rms_norm_mul_rope_f32_f32" + suffix, "rms_norm.comp", merge_maps(rms_dict, {{"ROPE_D_TYPE", "float"}, {"RMS_NORM_ROPE_FUSION", "1"}})); @@ -1178,6 +1218,9 @@ void process_shaders() { string_to_spv("gated_delta_net_f32", "gated_delta_net.comp", merge_maps(base_dict, {{"FLOAT_TYPE", "float"}, {"USE_SUBGROUP_ADD", "1"}, {"USE_SUBGROUP_CLUSTERED", "1"}})); string_to_spv("gated_delta_net_f32_nocluster", "gated_delta_net.comp", merge_maps(base_dict, {{"FLOAT_TYPE", "float"}, {"USE_SUBGROUP_ADD", "1"}, {"USE_SUBGROUP_CLUSTERED", "0"}})); string_to_spv("gated_delta_net_f32_shmem", "gated_delta_net.comp", merge_maps(base_dict, {{"FLOAT_TYPE", "float"}, {"USE_SUBGROUP_ADD", "0"}, {"USE_SUBGROUP_CLUSTERED", "0"}})); + string_to_spv("gated_delta_net_f32_sout", "gated_delta_net.comp", merge_maps(base_dict, {{"FLOAT_TYPE", "float"}, {"USE_SUBGROUP_ADD", "1"}, {"USE_SUBGROUP_CLUSTERED", "1"}, {"STATE_OUT", "1"}})); + string_to_spv("gated_delta_net_f32_nocluster_sout", "gated_delta_net.comp", merge_maps(base_dict, {{"FLOAT_TYPE", "float"}, {"USE_SUBGROUP_ADD", "1"}, {"USE_SUBGROUP_CLUSTERED", "0"}, {"STATE_OUT", "1"}})); + string_to_spv("gated_delta_net_f32_shmem_sout", "gated_delta_net.comp", merge_maps(base_dict, {{"FLOAT_TYPE", "float"}, {"USE_SUBGROUP_ADD", "0"}, {"USE_SUBGROUP_CLUSTERED", "0"}, {"STATE_OUT", "1"}})); string_to_spv("opt_step_adamw_f32", "opt_step_adamw.comp", merge_maps(base_dict, {{"A_TYPE", "float"}})); string_to_spv("opt_step_sgd_f32", "opt_step_sgd.comp", merge_maps(base_dict, {{"A_TYPE", "float"}})); @@ -1249,6 +1292,7 @@ void process_shaders() { string_to_spv("multi_add_f32", "multi_add.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}, {"ADD_RMS" , "0"}}); string_to_spv("multi_add_rms_f32", "multi_add.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}, {"ADD_RMS" , "1"}}); + string_to_spv("multi_add_rms_mul_f32", "multi_add_rms_mul.comp", {}); string_to_spv("ssm_scan_f32", "ssm_scan.comp", {{"A_TYPE", "float"}}); string_to_spv("ssm_scan_subgroup_f32", "ssm_scan.comp", {{"A_TYPE", "float"}, {"USE_SUBGROUP_ADD", "1"}}); diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index 01354c36cccd..17df121aeb24 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -2,6 +2,7 @@ #include "llama-impl.h" #include "llama-model.h" +#include "llama-model-loader.h" #include "llama-moe-cache.h" #include "llama-batch.h" #include "llama-cparams.h" @@ -498,7 +499,7 @@ void llm_graph_input_rs::set_input(const llama_ubatch * ubatch) { const int64_t n_rs = mctx->get_n_rs(); - if (s_copy) { + if (s_copy && s_copy->buffer) { // gen: unused (no buffer) when every state is viewed in place GGML_ASSERT(ggml_backend_buffer_is_host(s_copy->buffer)); int32_t * data = (int32_t *) s_copy->data; @@ -509,6 +510,27 @@ void llm_graph_input_rs::set_input(const llama_ubatch * ubatch) { } } +int llm_graph_input_rs::gen_compute_identity(const llama_memory_recurrent_context * mctx, int64_t n_seqs, uint32_t n_rs_seq) { + static const bool enabled = getenv("GGML_GEN_GDN_INPLACE") ? (atoi(getenv("GGML_GEN_GDN_INPLACE")) != 0) : true; + if (!enabled) { + return -1; + } + // with rollback snapshots (speculative decoding) s_copy() consumes the per-sequence rollback index, so it must + // only be called by set_input; the in-place view is not used in that mode anyway + if (n_rs_seq != 0) { + return 0; + } + if ((int64_t) mctx->get_n_rs() != n_seqs) { + return 0; + } + for (int64_t i = 0; i < n_seqs; i++) { + if (mctx->s_copy((int) i) != (int32_t) (mctx->get_head() + i)) { + return 0; + } + } + return 1; +} + bool llm_graph_input_rs::can_reuse(const llm_graph_params & params) { const auto * mctx = static_cast(params.mctx); @@ -516,6 +538,8 @@ bool llm_graph_input_rs::can_reuse(const llm_graph_params & params) { bool res = true; + res &= gen_identity == gen_compute_identity(mctx, params.ubatch.n_seqs, params.cparams.n_rs_seq); + res &= s_copy->ne[0] == mctx->get_n_rs(); res &= s_copy_main->ne[0] == params.ubatch.n_seqs; @@ -1212,7 +1236,7 @@ void llm_graph_input_mem_hybrid::set_input(const llama_ubatch * ubatch) { const int64_t n_rs = mctx->get_recr()->get_n_rs(); - if (inp_rs->s_copy) { + if (inp_rs->s_copy && inp_rs->s_copy->buffer) { // gen: unused when states are viewed in place GGML_ASSERT(ggml_backend_buffer_is_host(inp_rs->s_copy->buffer)); int32_t * data = (int32_t *) inp_rs->s_copy->data; @@ -1241,6 +1265,7 @@ bool llm_graph_input_mem_hybrid::can_reuse(const llm_graph_params & params) { res &= inp_rs->head == mctx->get_recr()->get_head(); res &= inp_rs->rs_z == mctx->get_recr()->get_rs_z(); + res &= inp_rs->gen_identity == llm_graph_input_rs::gen_compute_identity(mctx->get_recr(), params.ubatch.n_seqs, params.cparams.n_rs_seq); return res; } @@ -1258,7 +1283,7 @@ void llm_graph_input_mem_hybrid_k::set_input(const llama_ubatch * ubatch) { const int64_t n_rs = mctx->get_recr()->get_n_rs(); - if (inp_rs->s_copy) { + if (inp_rs->s_copy && inp_rs->s_copy->buffer) { // gen: unused when states are viewed in place GGML_ASSERT(ggml_backend_buffer_is_host(inp_rs->s_copy->buffer)); int32_t * data = (int32_t *) inp_rs->s_copy->data; @@ -1286,6 +1311,7 @@ bool llm_graph_input_mem_hybrid_k::can_reuse(const llm_graph_params & params) { res &= inp_rs->head == mctx->get_recr()->get_head(); res &= inp_rs->rs_z == mctx->get_recr()->get_rs_z(); + res &= inp_rs->gen_identity == llm_graph_input_rs::gen_compute_identity(mctx->get_recr(), params.ubatch.n_seqs, params.cparams.n_rs_seq); return res; } @@ -1315,7 +1341,7 @@ void llm_graph_input_mem_hybrid_iswa::set_input(const llama_ubatch * ubatch) { const int64_t n_rs = mctx->get_recr()->get_n_rs(); - if (inp_rs->s_copy) { + if (inp_rs->s_copy && inp_rs->s_copy->buffer) { // gen: unused when states are viewed in place GGML_ASSERT(ggml_backend_buffer_is_host(inp_rs->s_copy->buffer)); int32_t * data = (int32_t *) inp_rs->s_copy->data; @@ -1357,6 +1383,7 @@ bool llm_graph_input_mem_hybrid_iswa::can_reuse(const llm_graph_params & params) res &= inp_rs->head == mctx->get_recr()->get_head(); res &= inp_rs->rs_z == mctx->get_recr()->get_rs_z(); + res &= inp_rs->gen_identity == llm_graph_input_rs::gen_compute_identity(mctx->get_recr(), params.ubatch.n_seqs, params.cparams.n_rs_seq); return res; } @@ -1686,6 +1713,14 @@ ggml_tensor * llm_graph_context::build_lora_mm( ggml_tensor * w_s) const { // a Hadamard-folded weight reads the transformed input; a LoRA delta lives in the primal basis and reads `cur` ggml_tensor * cur_mm = build_hadamard_input(w, cur); + // gen: weights the loader stored as BF16 only because they are bf16-exact (GGML_GEN_BF16_EXACT) go through + // the matrix-vector kernels as BF16 (same per-row arithmetic as F32, half the bytes). Above + // GGML_GEN_BF16_CAST_ABOVE columns (default 8, <= the backend's mat-vec limit) they are cast back to F32 so + // the tiled matmul sees exactly the original F32 operands. + if (w->type == GGML_TYPE_BF16 && cur_mm->ne[1]*cur_mm->ne[2]*cur_mm->ne[3] > llama_gen_bf16_cast_above() && + llama_gen_bf16_converted(ggml_get_name(w))) { + w = ggml_cast(ctx0, w, GGML_TYPE_F32); + } ggml_tensor * res = ggml_mul_mat(ctx0, w, cur_mm); if (prec_policy) { @@ -4048,7 +4083,8 @@ ggml_tensor * llm_graph_context::build_rs( static std::unique_ptr build_rs_inp_impl( ggml_context * ctx0, const llama_ubatch & ubatch, - const llama_memory_recurrent_context * mctx_cur) { + const llama_memory_recurrent_context * mctx_cur, + uint32_t n_rs_seq) { auto inp = std::make_unique(mctx_cur); @@ -4064,6 +4100,7 @@ static std::unique_ptr build_rs_inp_impl( inp->head = mctx_cur->get_head(); inp->rs_z = mctx_cur->get_rs_z(); + inp->gen_identity = llm_graph_input_rs::gen_compute_identity(mctx_cur, n_seqs, n_rs_seq); return inp; } @@ -4071,7 +4108,7 @@ static std::unique_ptr build_rs_inp_impl( llm_graph_input_rs * llm_graph_context::build_rs_inp() const { const auto * mctx_cur = static_cast(mctx); - auto inp = build_rs_inp_impl(ctx0, ubatch, mctx_cur); + auto inp = build_rs_inp_impl(ctx0, ubatch, mctx_cur, cparams.n_rs_seq); return (llm_graph_input_rs *) res->add_input(std::move(inp)); } @@ -4084,6 +4121,17 @@ ggml_tensor * llm_graph_context::build_rs( const llm_graph_get_rows_fn & get_state_rows) const { const auto * kv_state = inp->mctx; + // gen (GGML_GEN_GDN_INPLACE): when every state is read from its own cell, view the cache rows instead of + // gathering them (same values); the backend can then also update them in place + if (inp->gen_identity == 1 && !get_state_rows && cparams.n_rs_seq == 0 && + (int64_t) kv_state->get_n_rs() == (int64_t) n_seqs) { + ggml_tensor * states = ggml_reshape_2d(ctx0, s, state_size, s->ne[1]); + const int32_t rs_zero = kv_state->get_rs_z(); + ggml_tensor * state_zero = ggml_view_1d(ctx0, states, state_size*(rs_zero >= 0), rs_zero*states->nb[1]*(rs_zero >= 0)); + ggml_build_forward_expand(gf, ggml_scale_inplace(ctx0, state_zero, 0)); + return ggml_view_2d(ctx0, states, state_size, n_seqs, states->nb[1], (size_t) kv_state->get_head()*states->nb[1]); + } + // a custom getter reads the states of the ubatch straight from the cache, so the gather skips the first // state: it still holds the n_rs - n_seqs extra states and copies no state of a single sequence ubatch ggml_tensor * state_copy = get_state_rows ? inp->s_copy_tail : inp->s_copy; @@ -4198,7 +4246,7 @@ ggml_tensor * llm_graph_context::build_rwkv_token_shift_store( llm_graph_input_mem_hybrid * llm_graph_context::build_inp_mem_hybrid() const { const auto * mctx_cur = static_cast(mctx); - auto inp_rs = build_rs_inp_impl (ctx0, ubatch, mctx_cur->get_recr()); + auto inp_rs = build_rs_inp_impl (ctx0, ubatch, mctx_cur->get_recr(), cparams.n_rs_seq); auto inp_attn = build_attn_inp_kv_impl(ctx0, ubatch, hparams, cparams, mctx_cur->get_attn()); auto inp = std::make_unique(cparams, std::move(inp_attn), std::move(inp_rs), mctx_cur); @@ -4209,7 +4257,7 @@ llm_graph_input_mem_hybrid * llm_graph_context::build_inp_mem_hybrid() const { llm_graph_input_mem_hybrid_k * llm_graph_context::build_inp_mem_hybrid_k() const { const auto * mctx_cur = static_cast(mctx); - auto inp_rs = build_rs_inp_impl (ctx0, ubatch, mctx_cur->get_recr()); + auto inp_rs = build_rs_inp_impl (ctx0, ubatch, mctx_cur->get_recr(), cparams.n_rs_seq); auto inp_attn = build_attn_inp_k_impl(ctx0, ubatch, hparams, cparams, mctx_cur->get_attn()); auto inp = std::make_unique(cparams, std::move(inp_attn), std::move(inp_rs), mctx_cur); @@ -4220,7 +4268,7 @@ llm_graph_input_mem_hybrid_k * llm_graph_context::build_inp_mem_hybrid_k() const llm_graph_input_mem_hybrid_iswa * llm_graph_context::build_inp_mem_hybrid_iswa() const { const auto * mctx_cur = static_cast(mctx); - auto inp_rs = build_rs_inp_impl(ctx0, ubatch, mctx_cur->get_recr()); + auto inp_rs = build_rs_inp_impl(ctx0, ubatch, mctx_cur->get_recr(), cparams.n_rs_seq); // build iswa attention input const auto * attn_ctx = mctx_cur->get_attn(); diff --git a/src/llama-graph.h b/src/llama-graph.h index 2d8c65718fee..56cae1ac5e65 100644 --- a/src/llama-graph.h +++ b/src/llama-graph.h @@ -314,6 +314,11 @@ class llm_graph_input_rs : public llm_graph_input_i { // used in view offsets, need to match for valid graph reuse uint32_t head; int32_t rs_z; + + // gen (GGML_GEN_GDN_INPLACE): 1 when every state of the ubatch is read from its own cell (identity copy), + // so build_rs can view the cache instead of gathering; -1 when the feature is off + int gen_identity = -1; + static int gen_compute_identity(const llama_memory_recurrent_context * mctx, int64_t n_seqs, uint32_t n_rs_seq); }; class llm_graph_input_cross_embd : public llm_graph_input_i { diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp index 548ade1019a0..a7fe4e221d76 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -1,5 +1,7 @@ #include "llama-model-loader.h" +#include + #include "ggml-alloc.h" #include "ggml.h" #include "gguf.h" @@ -1208,6 +1210,19 @@ bool llama_model_loader::lazy_read::add(const std::string & name, const ggml_ten return true; } +static std::mutex g_gen_bf16_mutex; +static std::set g_gen_bf16_names; + +bool llama_gen_bf16_converted(const char * name) { + std::lock_guard lock(g_gen_bf16_mutex); + return g_gen_bf16_names.count(name) != 0; +} + +int64_t llama_gen_bf16_cast_above() { + static const int64_t v = getenv("GGML_GEN_BF16_CAST_ABOVE") ? atoll(getenv("GGML_GEN_BF16_CAST_ABOVE")) : 8; + return v; +} + struct ggml_tensor * llama_model_loader::create_tensor( const llama_hparams & hparams, const buft_list_t * buft_list_cpu, const buft_list_t * buft_list_input, const buft_list_t * buft_list_output, const buft_list_t * buft_list_layer, const LLM_TN_IMPL & tn, const std::initializer_list & ne, int flags) { @@ -1455,6 +1470,47 @@ struct ggml_tensor * llama_model_loader::create_tensor( GGML_ASSERT(ggml_nbytes(&t_meta) == ggml_nbytes(cur)); + // gen: GGML_GEN_BF16_EXACT=1 stores the F32 router / shared-expert gate / ssm_alpha / ssm_beta weights as + // BF16 when every value is exactly representable (low 16 bits zero), checked here per tensor from the file. + // The values are unchanged, so the matrix-vector product gives the same result with half the bytes. + bool gen_to_bf16 = false; + { + static const bool gen_bf16_exact = getenv("GGML_GEN_BF16_EXACT") != nullptr && atoi(getenv("GGML_GEN_BF16_EXACT")) != 0; + const std::string name = tn.str(); + auto ends_with = [&](const char * suf) { + const size_t n = strlen(suf); + return name.size() >= n && name.compare(name.size() - n, n, suf) == 0; + }; + if (gen_bf16_exact && !use_mmap && !is_lazy && !(flags & TENSOR_DUPLICATED) && t_meta.type == GGML_TYPE_F32 && + t_meta.ne[0] % 2 == 0 && + (ends_with(".ffn_gate_inp.weight") || ends_with(".ffn_gate_inp_shexp.weight") || + ends_with(".ssm_alpha.weight") || ends_with(".ssm_beta.weight") || + ends_with(".ssm_beta_alpha.weight") || ends_with(".ssm_ba.weight"))) { + const auto & w = require_weight(name.c_str()); + std::vector vals(ggml_nelements(&t_meta)); + files.at(w.idx)->seek(w.offs, SEEK_SET); + files.at(w.idx)->read_raw(vals.data(), ggml_nbytes(&t_meta)); + bool exact = true; + for (uint32_t v : vals) { + if (v & 0xffffu) { + exact = false; + break; + } + } + if (exact) { + t_meta.type = GGML_TYPE_BF16; + t_meta.nb[0] = ggml_type_size(GGML_TYPE_BF16); + t_meta.nb[1] = ggml_row_size(GGML_TYPE_BF16, t_meta.ne[0]); + for (int dim = 2; dim < GGML_MAX_DIMS; dim++) { + t_meta.nb[dim] = t_meta.ne[dim-1]*t_meta.nb[dim-1]; + } + gen_to_bf16 = true; + } else { + LLAMA_LOG_WARN("%s: %s is not bf16-exact, kept as F32\n", __func__, name.c_str()); + } + } + } + ggml_backend_buffer_type_t buft = buft_for_tensor(&t_meta); if (buft == nullptr) { return nullptr; @@ -1475,6 +1531,12 @@ struct ggml_tensor * llama_model_loader::create_tensor( struct ggml_tensor * tensor = ggml_dup_tensor(ctx, &t_meta); ggml_set_name(tensor, ggml_get_name(&t_meta)); + if (gen_to_bf16) { + gen_bf16_tensors.insert(ggml_get_name(tensor)); + std::lock_guard lock(g_gen_bf16_mutex); + g_gen_bf16_names.insert(ggml_get_name(tensor)); + } + if (duplicated) { size_data += ggml_nbytes(&t_meta); } else { @@ -1658,6 +1720,10 @@ bool llama_model_loader::load_all_data( } GGML_ASSERT(size_data != 0 && "call init_mappings() first"); + if (!gen_bf16_tensors.empty()) { + fprintf(stderr, "gen: %zu bf16-exact F32 weight tensors stored as BF16 (GGML_GEN_BF16_EXACT)\n", gen_bf16_tensors.size()); + } + std::vector>> validation_result; // 4 staging buffers for async uploads, each sized 1MB seems to be a good default for single NVMe drives. @@ -1793,6 +1859,26 @@ bool llama_model_loader::load_all_data( size_t n_size = ggml_nbytes(cur); + if (!gen_bf16_tensors.empty() && gen_bf16_tensors.count(ggml_get_name(cur))) { + // gen: the file holds F32; convert (exactly, see create_tensor) and upload as BF16 + const auto & file = files.at(weight->idx); + const int64_t n = ggml_nelements(cur); + std::vector f32(n); + std::vector bf16(n); + file->seek(weight->offs, SEEK_SET); + file->read_raw(f32.data(), n * sizeof(uint32_t)); + for (int64_t i = 0; i < n; i++) { + GGML_ASSERT((f32[i] & 0xffffu) == 0); + bf16[i] = (uint16_t) (f32[i] >> 16); + } + if (upload_backend) { + ggml_backend_synchronize(upload_backend); + } + ggml_backend_tensor_set(cur, bf16.data(), 0, n_size); + size_done += n * sizeof(uint32_t); + continue; + } + const bool from_mapping = use_mmap || lazy.has(cur); if (from_mapping) { diff --git a/src/llama-model-loader.h b/src/llama-model-loader.h index ed59079a4ead..01a862ba545a 100644 --- a/src/llama-model-loader.h +++ b/src/llama-model-loader.h @@ -27,6 +27,10 @@ enum llama_fver { const char * llama_file_version_name(llama_fver version); +// gen: names of F32 weights stored as BF16 by GGML_GEN_BF16_EXACT (filled at load, read during graph build) +bool llama_gen_bf16_converted(const char * name); +int64_t llama_gen_bf16_cast_above(); + struct llama_model_loader { // Holds information on a model weight struct llama_tensor_weight { @@ -222,6 +226,9 @@ struct llama_model_loader { const llama_tensor_weight & require_weight(const char * name) const; + // gen: F32 weights stored as BF16 because every value is exactly representable (GGML_GEN_BF16_EXACT=1) + std::set gen_bf16_tensors; + struct ggml_tensor * get_tensor_meta(const char * name) const; struct ggml_tensor * require_tensor_meta(const std::string & name) const;