From 93c9973af05b4ad619a8747eec00f8df05baacc2 Mon Sep 17 00:00:00 2001 From: mrciffa Date: Mon, 24 Aug 2026 00:55:15 +0200 Subject: [PATCH 1/3] qwen35: span the whole KV pool for slot-mapped verify attention With --kvflash the KV cache lives at pool slots and verify_batch builds its attention mask in slot space over the entire pool. The flash-attention view, however, was still sized from the logical context length (kv_start + n_tokens, rounded to the 256 stride). Those two disagree: slot indices are not ordered by logical position, so the view could end below slots the mask still marked visible. Attention then read rows that were never written and the softmax row degenerated, which surfaced as an argmax of -1 for every verify row past the first. The symptom was a hard failure. do_spec_decode saw the invalid seed, fell back to plain decode, and that fallback inherited the same state and failed too, so the request returned decode_failed. Reproduced on a Radeon AI PRO R9700 with Qwen3.8-27B and --kvflash auto (16384-token pool): prompts of 1556 and 6208 tokens died on the first speculative step, while 13148 and 26728 happened to survive because their logical extent covered the slots in use. Span the pool instead. The mask is sized from the same pool and is what restricts which slots are readable, so this is the bound that matches the caller's contract. The condition is scoped to the slot-mapped path: a set_rows KV write together with an explicit mask is a pair only kvflash verify produces, since the non-kvflash step-invariant write requires no mask and the paged path never reaches this branch. Measured on the same box, --kvflash auto, block-16 DFlash2, greedy, prompts that previously failed now complete: 1556 tokens 59.5 tok/s, 6208 tokens 43.3 tok/s, both recalling a label planted at the top of the context; 13148 and 26728 are unchanged at 39.0 and 38.6. Zero invalid-seed events across the sweep. The default (non-kvflash) path is untouched: HumanEval-10 is 144.61 tok/s with output sha a4467e9d, identical to before the change. --- server/src/qwen35/qwen35_target_graph.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/server/src/qwen35/qwen35_target_graph.cpp b/server/src/qwen35/qwen35_target_graph.cpp index 66069e104..fb9dd27d0 100644 --- a/server/src/qwen35/qwen35_target_graph.cpp +++ b/server/src/qwen35/qwen35_target_graph.cpp @@ -1330,6 +1330,18 @@ static ggml_tensor * build_full_attn_block( // Never view past the read tensor (its rows may not be 256-aligned). win_len_padded = std::min(win_len_padded, (int)cache_k->ne[1]); } + // kvflash: KV lives at pool SLOTS, and the caller's mask is built in + // slot space over the whole pool. Slot indices are not bounded by the + // logical context length, so a view sized from kv_start can end below + // slots the mask still marks visible: those rows fall outside the + // view and the softmax row degenerates, which surfaces as an argmax + // of -1 for every verify row past the first. Span the whole pool + // instead; the mask, sized from that same pool, is what decides which + // slots are readable. Detect the mode by the pair only slot-mapped + // verify sets: a set_rows KV write together with an explicit mask. + if (kv_write_rows != nullptr && attn_mask != nullptr) { + win_len_padded = (int)cache_k->ne[1]; + } // K and V from cache: a windowed view starting at win_start. ggml_tensor * Kfa = ggml_view_3d(ctx, cache_k, From a522451b0905ea1bc5c65726c525653d12fef1f4 Mon Sep 17 00:00:00 2001 From: mrciffa <49000955+davide221@users.noreply.github.com> Date: Mon, 24 Aug 2026 14:18:36 +0200 Subject: [PATCH 2/3] fix: address Cubic review findings for PR 652 --- .../llama.cpp/ggml/src/ggml-cpu/ggml-cpu.cpp | 14 ++-- server/scripts/convert_dflash_to_gguf.py | 4 ++ server/src/common/dflash2_head.cpp | 13 +++- server/src/common/dspark_head.cpp | 11 ++- server/src/draft/draft_gguf_loader.cpp | 34 +++++++++- server/src/qwen35/qwen35_target_graph.cpp | 12 +++- server/test/test_batched_gdn.cpp | 68 +++++++++++++++++++ 7 files changed, 136 insertions(+), 20 deletions(-) diff --git a/server/deps/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.cpp b/server/deps/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.cpp index 5d6e93a68..c2a194d60 100644 --- a/server/deps/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.cpp +++ b/server/deps/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.cpp @@ -473,14 +473,16 @@ static bool ggml_backend_cpu_device_supports_op(ggml_backend_dev_t dev, const st case GGML_OP_PAGED_ATTN: return false; case GGML_OP_SSM_CONV: - // The Specla layout (op param 0 == 1) needs the packed HLD state and - // is only supported by the CUDA kernel; the generic CPU kernel would - // silently compute garbage. - return ggml_get_op_params_i32(op, 0) != 1; + // Every nonzero mode is a dflash CUDA/HIP extension (SpecLA, + // fused step, or dynamic conv). The generic CPU kernel only + // implements the original mode and asserts if one reaches it. + return ggml_get_op_params_i32(op, 0) == 0; case GGML_OP_GATED_DELTA_NET: // The Specla GDN variant (op param 2 == 1) is stateful via HLD and is - // only supported by the CUDA kernel. - return ggml_get_op_params_i32(op, 2) != 1; + // only supported by CUDA/HIP. Raw-gate mode is also CUDA/HIP-only: + // the CPU kernel expects beta/g to have already been transformed. + return ggml_get_op_params_i32(op, 2) != 1 && + ggml_get_op_params_i32(op, 10) == 0 && op->src[9] == nullptr; case GGML_OP_OUT_PROD: return (src0->type == GGML_TYPE_F32 || (ggml_is_quantized(src0->type) && src0->ne[2] == src1->ne[2] && src0->ne[3] == src1->ne[3])) && src1->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32; diff --git a/server/scripts/convert_dflash_to_gguf.py b/server/scripts/convert_dflash_to_gguf.py index e5fd2213a..f1f7d86d4 100644 --- a/server/scripts/convert_dflash_to_gguf.py +++ b/server/scripts/convert_dflash_to_gguf.py @@ -122,6 +122,9 @@ def pick(*keys): a["yarn_orig_ctx"] = int(rp.get("original_max_position_embeddings") or c.get("original_max_position_embeddings") or 0) + attn_factor = rp.get("attention_factor") + a["yarn_attn_factor"] = float( + attn_factor if attn_factor is not None else 1.0) a["yarn_beta_fast"] = float(rp.get("beta_fast", 32.0)) a["yarn_beta_slow"] = float(rp.get("beta_slow", 1.0)) if dfc.get("mask_token_id") is not None: @@ -528,6 +531,7 @@ def main(): writer.add_string(f"{ARCH}.rope.scaling.type", "yarn") writer.add_float32(f"{ARCH}.rope.scaling.factor", a["yarn_factor"]) writer.add_uint32(f"{ARCH}.rope.scaling.original_context_length", a["yarn_orig_ctx"]) + writer.add_float32(f"{ARCH}.rope.scaling.attn_factor", a["yarn_attn_factor"]) writer.add_float32(f"{ARCH}.rope.scaling.beta_fast", a["yarn_beta_fast"]) writer.add_float32(f"{ARCH}.rope.scaling.beta_slow", a["yarn_beta_slow"]) diff --git a/server/src/common/dflash2_head.cpp b/server/src/common/dflash2_head.cpp index 8dde6290a..fda03fafe 100644 --- a/server/src/common/dflash2_head.cpp +++ b/server/src/common/dflash2_head.cpp @@ -115,8 +115,17 @@ bool dflash2_score_candidates(const DraftWeights & dw, ggml_set_input(g.inp_succ); ggml_set_input(g.inp_pred); g.hproj = ggml_mul_mat(g.ctx, sel.hproj, g.inp_hidden); // [rank, n_cand] - g.succ = ggml_get_rows(g.ctx, sel.succ_cb, g.inp_succ); // [rank, n_cand*K] f32 - g.pred = ggml_get_rows(g.ctx, sel.pred_cb, g.inp_pred); // [rank, 1+n_cand*K] f32 + // Current ggml_get_rows dequantizes floating-point/quantized sources + // to F32. Keep the explicit fallback so float readback remains safe + // if that API later starts preserving F16/BF16 source types. + auto get_rows_f32 = [&](ggml_tensor * codebook, ggml_tensor * ids) { + ggml_tensor * rows = ggml_get_rows(g.ctx, codebook, ids); + return rows->type == GGML_TYPE_F32 + ? rows : ggml_cast(g.ctx, rows, GGML_TYPE_F32); + }; + g.succ = get_rows_f32(sel.succ_cb, g.inp_succ); // [rank, n_cand*K] f32 + g.pred = get_rows_f32(sel.pred_cb, g.inp_pred); // [rank, 1+n_cand*K] f32 + GGML_ASSERT(g.succ->type == GGML_TYPE_F32 && g.pred->type == GGML_TYPE_F32); ggml_set_output(g.hproj); ggml_set_output(g.succ); ggml_set_output(g.pred); diff --git a/server/src/common/dspark_head.cpp b/server/src/common/dspark_head.cpp index f0df52c10..135d86e1e 100644 --- a/server/src/common/dspark_head.cpp +++ b/server/src/common/dspark_head.cpp @@ -361,10 +361,9 @@ bool dspark_markov_correct_greedy_chain_fused(const DraftWeights & dw, draft_tok.assign((size_t)q_len, 0); draft_tok[0] = last_tok; // One synchronize instead of n_cand blocking readbacks. - int32_t t_out[16]; - float c_out[16] = {}; - const int n_get = n_cand < 16 ? n_cand : 16; - for (int i = 0; i < n_get; ++i) { + std::vector t_out((size_t)n_cand); + std::vector c_out(want_confidence ? (size_t)n_cand : 0); + for (int i = 0; i < n_cand; ++i) { ggml_backend_tensor_get_async(backend, g.toks[(size_t)i], &t_out[i], 0, sizeof(int32_t)); if (want_confidence && g.confidence[(size_t)i]) { ggml_backend_tensor_get_async( @@ -372,11 +371,11 @@ bool dspark_markov_correct_greedy_chain_fused(const DraftWeights & dw, } } ggml_backend_synchronize(backend); - for (int i = 0; i < n_get; ++i) { + for (int i = 0; i < n_cand; ++i) { draft_tok[(size_t)i + 1] = t_out[i]; } if (want_confidence && !g.confidence.empty() && g.confidence[0]) { - confidence_out->assign(c_out, c_out + n_get); + *confidence_out = std::move(c_out); } ggml_free(g.ctx); return true; diff --git a/server/src/draft/draft_gguf_loader.cpp b/server/src/draft/draft_gguf_loader.cpp index 7af2b4bce..cf6e91b21 100644 --- a/server/src/draft/draft_gguf_loader.cpp +++ b/server/src/draft/draft_gguf_loader.cpp @@ -522,11 +522,39 @@ bool load_draft_gguf(const std::string & path, } out.conv_kernel_size = conv_k; out.conv_group_size = (int)read_u32("dflash.dflash2.conv_group_size", 16); - const DraftLayer & L0 = out.layers[0]; + if (out.conv_group_size <= 0 || out.n_embd % out.conv_group_size != 0) { + char b[192]; + std::snprintf(b, sizeof(b), + "draft GGUF: dflash.dflash2.conv_group_size=%d " + "must be positive and divide embedding_length=%d", + out.conv_group_size, out.n_embd); + set_last_error(b); + ggml_free(meta_ctx); out.ctx = nullptr; gguf_free(gctx); + return false; + } const int64_t groups = out.n_embd / out.conv_group_size; char shape_err[192]; - if (!check_shape_3d(L0.attn_conv.base, out.n_embd, conv_k, 2, "attn_conv.base", shape_err, sizeof(shape_err)) || - !check_shape_2d(L0.attn_conv.proj, out.n_embd, 2 * conv_k * groups, "attn_conv.proj", shape_err, sizeof(shape_err))) { + bool shapes_ok = true; + for (int il = 0; il < out.n_layer && shapes_ok; ++il) { + const DraftLayer & L = out.layers[(size_t)il]; + const DraftConvWeights * convs[] = {&L.attn_conv, &L.mlp_conv}; + const char * kinds[] = {"attn_conv", "ffn_conv"}; + for (int ci = 0; ci < 2 && shapes_ok; ++ci) { + char base_name[64]; + char proj_name[64]; + std::snprintf(base_name, sizeof(base_name), + "blk.%d.%s.base", il, kinds[ci]); + std::snprintf(proj_name, sizeof(proj_name), + "blk.%d.%s.proj", il, kinds[ci]); + shapes_ok = + check_shape_3d(convs[ci]->base, out.n_embd, conv_k, 2, + base_name, shape_err, sizeof(shape_err)) && + check_shape_2d(convs[ci]->proj, out.n_embd, + 2 * conv_k * groups, proj_name, + shape_err, sizeof(shape_err)); + } + } + if (!shapes_ok) { set_last_error(shape_err); ggml_free(meta_ctx); out.ctx = nullptr; gguf_free(gctx); return false; diff --git a/server/src/qwen35/qwen35_target_graph.cpp b/server/src/qwen35/qwen35_target_graph.cpp index 28ee72d4b..7a429344b 100644 --- a/server/src/qwen35/qwen35_target_graph.cpp +++ b/server/src/qwen35/qwen35_target_graph.cpp @@ -41,6 +41,7 @@ #include "common/specla_mode.h" #include "ggml-alloc.h" +#include "ggml-cuda.h" #include #include @@ -1410,6 +1411,7 @@ static ggml_tensor * build_delta_net_block( DeltaNetCapture * cap, // optional: populated on capture_delta_intermediate ggml_tensor * parent_ids, // optional [n_tokens] i32; tree mode when non-null bool skip_gdn_intermediate, + bool fused_kernel_backend, // CUDA/HIP backend implements fused conv/raw gates // Supported shapes are one sequence with any number of timesteps // (prefill/verify), or compact decode with one timestep per mapped row. int n_seqs = 1, @@ -1537,7 +1539,8 @@ static ggml_tensor * build_delta_net_block( }(); const bool chunked_call = chunked_env_on && can_skip_gdn_intermediate && !ragged && !active_slot_ids && !use_specla_factorized && !use_specla_hld && n_tokens > 1; - const bool fused_plain = fused_kernels_env && !parent_ids && !ragged && !active_slot_ids && + const bool fused_plain = fused_kernels_env && fused_kernel_backend && + !parent_ids && !ragged && !active_slot_ids && !use_specla_factorized && !use_specla_hld; const bool fused_conv = fused_plain; const bool raw_gates = fused_plain && !chunked_call && L.ssm_gate_ba != nullptr; @@ -2088,7 +2091,8 @@ static ggml_tensor * build_single_layer( cur = build_delta_net_block(ctx, gf, w, L, cur, cache.conv_state[dn_idx], cache.ssm_state[dn_idx], n_tokens, cap_ptr, parent_ids, - /*skip_gdn_intermediate=*/true); + /*skip_gdn_intermediate=*/true, + ggml_backend_is_cuda(cache.backend)); } cur = ggml_add(ctx, cur, inpSA); @@ -2282,6 +2286,7 @@ QwenGraphOutputs build_qwen35_graph( conv_st, ssm_st, n_tokens, cap_ptr, in.parent_ids, /*skip_gdn_intermediate=*/true, + ggml_backend_is_cuda(cache.backend), in.n_seqs, in.prefill_segments, in.n_prefill_segments, @@ -2489,7 +2494,8 @@ QwenLayerPrefnOutputs build_qwen35_layer_prefn( cur = build_delta_net_block(ctx, gf, w, L, cur, cache.conv_state[dn_idx], cache.ssm_state[dn_idx], n_tokens, nullptr, nullptr, - skip_gdn_intermediate); + skip_gdn_intermediate, + ggml_backend_is_cuda(cache.backend)); } cur = ggml_add(ctx, cur, inpSA); diff --git a/server/test/test_batched_gdn.cpp b/server/test/test_batched_gdn.cpp index 1e639e049..4e9090ed1 100644 --- a/server/test/test_batched_gdn.cpp +++ b/server/test/test_batched_gdn.cpp @@ -212,6 +212,71 @@ bool run_conv(ggml_backend_t backend, int n_seqs, return ok; } +bool test_cpu_rejects_gpu_only_extensions(ggml_backend_t backend) { + ggml_init_params params{}; + params.mem_size = 2 * 1024 * 1024; + params.no_alloc = true; + ggml_context * ctx = ggml_init(params); + if (!ctx) return false; + + ggml_tensor * sx = ggml_new_tensor_3d( + ctx, GGML_TYPE_F32, D_CONV, CONV_CHANNELS, 1); + ggml_tensor * conv_w = ggml_new_tensor_2d( + ctx, GGML_TYPE_F32, D_CONV, CONV_CHANNELS); + ggml_tensor * plain_conv = ggml_ssm_conv(ctx, sx, conv_w); + + ggml_tensor * step_x = ggml_new_tensor_3d( + ctx, GGML_TYPE_F32, CONV_CHANNELS, 1, 1); + ggml_tensor * step_state = ggml_new_tensor_3d( + ctx, GGML_TYPE_F32, D_CONV - 1, CONV_CHANNELS, 1); + ggml_tensor * fused_step = ggml_ssm_conv_step( + ctx, step_x, conv_w, step_state, nullptr); + + constexpr int DYN_HIDDEN = 32; + constexpr int DYN_GROUP = 16; + constexpr int DYN_TOKENS = 2; + ggml_tensor * dyn_x = ggml_new_tensor_2d( + ctx, GGML_TYPE_F32, DYN_HIDDEN, DYN_TOKENS); + ggml_tensor * dyn_base = ggml_new_tensor_3d( + ctx, GGML_TYPE_F32, DYN_HIDDEN, D_CONV, 2); + ggml_tensor * dyn_weights = ggml_new_tensor_2d( + ctx, GGML_TYPE_F32, + 2 * D_CONV * (DYN_HIDDEN / DYN_GROUP), DYN_TOKENS); + ggml_tensor * fused_dyn = ggml_dflash_dyn_conv( + ctx, dyn_x, dyn_base, dyn_weights, 0, D_CONV, DYN_GROUP); + + ggml_tensor * q = ggml_new_tensor_4d( + ctx, GGML_TYPE_F32, S_V, N_HEAD, 1, 1); + ggml_tensor * k = ggml_new_tensor_4d( + ctx, GGML_TYPE_F32, S_V, N_HEAD, 1, 1); + ggml_tensor * v = ggml_new_tensor_4d( + ctx, GGML_TYPE_F32, S_V, N_HEAD, 1, 1); + ggml_tensor * g = ggml_new_tensor_4d( + ctx, GGML_TYPE_F32, 1, N_HEAD, 1, 1); + ggml_tensor * beta = ggml_new_tensor_4d( + ctx, GGML_TYPE_F32, 1, N_HEAD, 1, 1); + ggml_tensor * state = ggml_new_tensor_4d( + ctx, GGML_TYPE_F32, S_V, S_V, N_HEAD, 1); + ggml_tensor * plain_gdn = ggml_gated_delta_net( + ctx, q, k, v, g, beta, state); + const bool plain_gdn_supported = + ggml_backend_supports_op(backend, plain_gdn); + ggml_tensor * gate_ba = ggml_new_tensor_1d( + ctx, GGML_TYPE_F32, 2 * N_HEAD); + ggml_gated_delta_net_set_raw_gates(plain_gdn, gate_ba); + + const bool ok = + ggml_backend_supports_op(backend, plain_conv) && + !ggml_backend_supports_op(backend, fused_step) && + !ggml_backend_supports_op(backend, fused_dyn) && + plain_gdn_supported && + !ggml_backend_supports_op(backend, plain_gdn); + std::printf("batched gdn CPU capability guards %s\n", + ok ? "PASS" : "FAIL"); + ggml_free(ctx); + return ok; +} + bool test_masked_set_rows(ggml_backend_t backend) { constexpr int ROW_WIDTH = 4; constexpr int DEST_ROWS = 4; @@ -530,6 +595,9 @@ int main(int argc, char ** argv) { ok = test_gdn_active_slots(backend, rng) && ok; ok = test_conv(backend, rng) && ok; ok = test_masked_set_rows(backend) && ok; + if (cpu) { + ok = test_cpu_rejects_gpu_only_extensions(backend) && ok; + } ggml_backend_free(backend); return ok ? 0 : 1; From 8a8b719d855a690fe37eb3eb9e8397d75906f086 Mon Sep 17 00:00:00 2001 From: mrciffa <49000955+davide221@users.noreply.github.com> Date: Mon, 24 Aug 2026 14:27:55 +0200 Subject: [PATCH 3/3] fix: address Cubic follow-up review --- server/scripts/convert_dflash_to_gguf.py | 12 +++++++++-- server/src/qwen35/qwen35_target_graph.cpp | 25 ++++++++++++++++++++--- server/test/test_batched_gdn.cpp | 6 ++++-- 3 files changed, 36 insertions(+), 7 deletions(-) diff --git a/server/scripts/convert_dflash_to_gguf.py b/server/scripts/convert_dflash_to_gguf.py index f1f7d86d4..c95c89587 100644 --- a/server/scripts/convert_dflash_to_gguf.py +++ b/server/scripts/convert_dflash_to_gguf.py @@ -29,6 +29,7 @@ import argparse import json +import math import struct import sys from pathlib import Path @@ -123,8 +124,15 @@ def pick(*keys): or c.get("original_max_position_embeddings") or 0) attn_factor = rp.get("attention_factor") - a["yarn_attn_factor"] = float( - attn_factor if attn_factor is not None else 1.0) + # GGML applies YaRN's standard 1 + 0.1*log(factor) magnitude + # internally. HF attention_factor is the final magnitude, so + # normalize only an explicit override; 1.0 retains GGML's + # standard default when the config omits it. + ggml_mscale = (1.0 + 0.1 * math.log(a["yarn_factor"]) + if a["yarn_factor"] > 1.0 else 1.0) + a["yarn_attn_factor"] = ( + float(attn_factor) / ggml_mscale + if attn_factor is not None else 1.0) a["yarn_beta_fast"] = float(rp.get("beta_fast", 32.0)) a["yarn_beta_slow"] = float(rp.get("beta_slow", 1.0)) if dfc.get("mask_token_id") is not None: diff --git a/server/src/qwen35/qwen35_target_graph.cpp b/server/src/qwen35/qwen35_target_graph.cpp index 7a429344b..3629530b6 100644 --- a/server/src/qwen35/qwen35_target_graph.cpp +++ b/server/src/qwen35/qwen35_target_graph.cpp @@ -41,6 +41,7 @@ #include "common/specla_mode.h" #include "ggml-alloc.h" +#include "ggml-backend-impl.h" #include "ggml-cuda.h" #include @@ -76,6 +77,24 @@ constexpr float EPS = 1e-6f; constexpr float ROPE_THETA = 10000000.0f; } // namespace q35 +// CUDA and ROCm share ggml's CUDA backend interface. Tensor-parallel caches +// use a meta backend, so inspect every rank-local backend before enabling ops +// that have no CPU/Metal/Vulkan implementation. +static bool supports_qwen35_fused_kernels(ggml_backend_t backend) { + if (ggml_backend_is_cuda(backend)) return true; + if (!ggml_backend_is_meta(backend)) return false; + + const size_t n_backends = ggml_backend_meta_n_backends(backend); + if (n_backends == 0) return false; + for (size_t i = 0; i < n_backends; ++i) { + if (!ggml_backend_is_cuda( + ggml_backend_meta_simple_backend(backend, i))) { + return false; + } + } + return true; +} + // ─── TargetCache allocation ───────────────────────────────────────── bool create_target_cache(const TargetWeights & w, @@ -2092,7 +2111,7 @@ static ggml_tensor * build_single_layer( cache.conv_state[dn_idx], cache.ssm_state[dn_idx], n_tokens, cap_ptr, parent_ids, /*skip_gdn_intermediate=*/true, - ggml_backend_is_cuda(cache.backend)); + supports_qwen35_fused_kernels(cache.backend)); } cur = ggml_add(ctx, cur, inpSA); @@ -2286,7 +2305,7 @@ QwenGraphOutputs build_qwen35_graph( conv_st, ssm_st, n_tokens, cap_ptr, in.parent_ids, /*skip_gdn_intermediate=*/true, - ggml_backend_is_cuda(cache.backend), + supports_qwen35_fused_kernels(cache.backend), in.n_seqs, in.prefill_segments, in.n_prefill_segments, @@ -2495,7 +2514,7 @@ QwenLayerPrefnOutputs build_qwen35_layer_prefn( cache.conv_state[dn_idx], cache.ssm_state[dn_idx], n_tokens, nullptr, nullptr, skip_gdn_intermediate, - ggml_backend_is_cuda(cache.backend)); + supports_qwen35_fused_kernels(cache.backend)); } cur = ggml_add(ctx, cur, inpSA); diff --git a/server/test/test_batched_gdn.cpp b/server/test/test_batched_gdn.cpp index 4e9090ed1..150f32ed3 100644 --- a/server/test/test_batched_gdn.cpp +++ b/server/test/test_batched_gdn.cpp @@ -261,16 +261,18 @@ bool test_cpu_rejects_gpu_only_extensions(ggml_backend_t backend) { ctx, q, k, v, g, beta, state); const bool plain_gdn_supported = ggml_backend_supports_op(backend, plain_gdn); + ggml_tensor * raw_gdn = ggml_gated_delta_net( + ctx, q, k, v, g, beta, state); ggml_tensor * gate_ba = ggml_new_tensor_1d( ctx, GGML_TYPE_F32, 2 * N_HEAD); - ggml_gated_delta_net_set_raw_gates(plain_gdn, gate_ba); + ggml_gated_delta_net_set_raw_gates(raw_gdn, gate_ba); const bool ok = ggml_backend_supports_op(backend, plain_conv) && !ggml_backend_supports_op(backend, fused_step) && !ggml_backend_supports_op(backend, fused_dyn) && plain_gdn_supported && - !ggml_backend_supports_op(backend, plain_gdn); + !ggml_backend_supports_op(backend, raw_gdn); std::printf("batched gdn CPU capability guards %s\n", ok ? "PASS" : "FAIL"); ggml_free(ctx);