qwen35: DSpark speculative decoding (Qwen3.8-27B drafters)

README.md

@@ -57,11 +57,10 @@ All speedups measured vs vendored llama.cpp (`-fa 1`, matching KV quant). Combin
 | Model | Speedup |
 |-------|:-------:|
 | Qwen 3.5 0.8B (Megakernel) | **~2×** |
-| Qwen 3.6 27B + PFlash | **~5.6×** |
-| Qwen 3.6 27B + DDTree | **4.84×** |
+| Qwen 3.8 27B + DFlash2 (R9700) | **6.1×** decode |
+| Qwen 3.8 27B + DFlash2 vs llama.cpp speculating with the same drafter | **3.5×** decode |
 | Laguna XS 2.1 33B + PFlash | **8.2×** @256K |
 | Laguna XS 2.1 33B + DFlash | **1.7×** @256K |
-| Qwen 3.6 27B HIP | **~2.6×** |
 | Gemma 4 26B-A4B | **1.31×** |
 | Gemma 4 31B IT | **3.2×** |
 | [`DeepSeek V4 Flash ROCMFPX HIP`](https://huggingface.co/Lucebox/DeepSeek-V4-Flash-0731-ROCmFP3) | **2×** |
@@ -71,7 +70,7 @@ All speedups measured vs vendored llama.cpp (`-fa 1`, matching KV quant). Combin
 
 | Drafter | Phase |
 |---------|:-----:|
-| [`Qwen3.6 27B`](https://huggingface.co/Lucebox/Qwen3.6-27B-DFlash-GGUF) | decode |
+| [`Qwen3.8 27B DFlash2`](https://huggingface.co/incoai/Qwen3.8-27B-DFlash2) | decode |
 | [`gemma 4 26B A4B`](https://huggingface.co/Lucebox/gemma-4-26B-A4B-it-DFlash-GGUF) | decode |
 | [`gemma 4 31B`](https://huggingface.co/Lucebox/gemma-4-31B-it-DFlash-GGUF) | decode |
 | [`Laguna XS 2.1 33B`](https://huggingface.co/Lucebox/Laguna-XS-2.1-DFlash-GGUF) | decode |
@@ -110,7 +109,7 @@ Entries are `dflash_server` settings unless the cell shows another command.
 
 | Model | RTX 3090 (24 GB) | Strix Halo `gfx1151` | Strix Halo + R9700 `gfx1201` |
 |---|---|---|---|
-| **Qwen 3.5/3.6 27B Q4_K_M** | `DFLASH27B_KV_TQ3=1`<br>`--target-device cuda:0`<br>`--draft-device cuda:0`<br>`--ddtree`<br>`--ddtree-budget 22`<br>`--draft-residency auto`<br>`--prefill-compression auto`<br>`--prefill-drafter <path>`<br>`--kvflash auto` | `--target-device hip:0`<br>`--draft-device hip:0`<br>`--ddtree`<br>`--ddtree-budget 22`<br>`--draft-residency persistent`<br>`--prefill-compression auto`<br>`--prefill-drafter <path>`<br>`--kvflash auto` | `HIP_VISIBLE_DEVICES=<r9700-index>`<br>`--target-device hip:0`<br>`--draft-device hip:0`<br>`--ddtree`<br>`--ddtree-budget 22`<br>`--draft-residency persistent`<br>`--prefill-compression auto`<br>`--prefill-drafter <path>`<br>`--kvflash auto` |
+| **Qwen 3.8 27B IQ4_XS** | `--target-device cuda:0`<br>`--draft-device cuda:0`<br>`--draft-block-size 16`<br>`--cache-type-k q8_0`<br>`--cache-type-v q8_0` | `--target-device hip:0`<br>`--draft-device hip:0`<br>`--draft-block-size 16`<br>`--cache-type-k q8_0`<br>`--cache-type-v q8_0` | `HIP_VISIBLE_DEVICES=<r9700-index>`<br>`--target-device hip:0`<br>`--draft-device hip:0`<br>`--draft-block-size 16`<br>`--max-ctx 131072`<br>`--cache-type-k q8_0`<br>`--cache-type-v q8_0` |
 | **Qwen 3.6 35B-A3B Q4_K_M** | `--target-device cuda:0`<br>`--spark`<br>`--kvflash auto` | `--target-device hip:0`<br>`--kvflash auto` | `HIP_VISIBLE_DEVICES=<r9700-index>`<br>`--target-device hip:0`<br>`--kvflash auto` |
 | **Laguna XS 2.1 33B Q4_K_M** | `--target-device cuda:0`<br>`--draft <path>`<br>`--prefill-drafter <path>`<br>`--max-ctx 262144`<br>`--kvflash 8192`<br>`--chunk 1024` | `--target-device hip:0`<br>`--kvflash auto` | `HIP_VISIBLE_DEVICES=<r9700-index>`<br>`--target-device hip:0`<br>`--kvflash auto` |
 | **Gemma 4 26B-A4B / 31B** | `--target-device cuda:0`<br>`--draft-device cuda:0`<br>`--kvflash auto` | `--target-device hip:0`<br>`--draft-device hip:0`<br>`--kvflash auto` | `HIP_VISIBLE_DEVICES=<r9700-index>`<br>`--target-device hip:0`<br>`--draft-device hip:0`<br>`--kvflash auto` |
@@ -152,9 +151,9 @@ All launchers spawn the native C++ HTTP server (`dflash_server`). Override defau
 
 ```bash
 DFLASH_SERVER_BIN=server/build/dflash_server \
-DFLASH_TARGET=server/models/Qwen3.6-27B-Q4_K_M.gguf \
-DFLASH_DRAFT=server/models/draft/dflash-draft-3.6-q4_k_m.gguf \
-MAX_CTX=32768 BUDGET=22 VERIFY_MODE=ddtree \
+DFLASH_TARGET=server/models/Qwen3.8-27B-UD-IQ4_XS.gguf \
+DFLASH_DRAFT=server/models/draft/qwen38-dflash2-q8_0.gguf \
+MAX_CTX=32768 \
 harness/clients/run_codex.sh
 ```
 
@@ -214,10 +213,13 @@ docker pull ghcr.io/luce-org/lucebox-hub:rocm     # AMD
 # 2. Download a target model into server/models/ and the DFlash draft
 #    into server/models/draft/ (the entrypoint only auto-discovers the
 #    draft there; without it the server runs slower, target-only)
-hf download unsloth/Qwen3.6-27B-GGUF Qwen3.6-27B-Q4_K_M.gguf \
+hf download unsloth/Qwen3.8-27B-GGUF Qwen3.8-27B-UD-IQ4_XS.gguf \
   --local-dir server/models/
-hf download Lucebox/Qwen3.6-27B-DFlash-GGUF dflash-draft-3.6-q4_k_m.gguf \
-  --local-dir server/models/draft/
+#    Drafter: convert the z-lab checkpoint once with the two converter
+#    commands from the source-build quickstart below and place the q8_0
+#    GGUF in server/models/draft/. (The pre-made incoai GGUF is for
+#    llama.cpp PR#27342 and uses a different layout this server does not
+#    load yet.)
 
 # 3a. NVIDIA (CUDA 12+)
 docker run --rm --gpus all -p 8000:8080 \
@@ -235,23 +237,29 @@ Then hit `:8000/v1/chat/completions` (OpenAI-compatible).
 
 ## Run the Server
 
-Default: Qwen 3.6-27B Q4_K_M target + Lucebox Q4_K_M DFlash drafter on RTX 3090. DDTree budget=22, TQ3_0 KV cache, full attention. OpenAI-compatible HTTP on `:8000`.
+Default: Qwen 3.8-27B IQ4_XS target + the DFlash2 block-diffusion drafter. Full attention, q8_0 KV cache, OpenAI-compatible HTTP on `:8000`. The tuned decode path is the default, so no environment variables are needed.
 
 ```bash
 # build (CUDA 12+, CMake 3.18+)
 git clone --recurse-submodules https://github.com/Luce-Org/lucebox-hub && cd lucebox-hub
 cmake -B server/build -S server -DCMAKE_BUILD_TYPE=Release
 cmake --build server/build --target dflash_server -j
 
-# default weights (~18 GB)
-hf download unsloth/Qwen3.6-27B-GGUF Qwen3.6-27B-Q4_K_M.gguf --local-dir server/models/
-hf download Lucebox/Qwen3.6-27B-DFlash-GGUF dflash-draft-3.6-q4_k_m.gguf --local-dir server/models/draft/
-
-# run (TQ3_0 KV auto-enabled; set =0 to disable)
-DFLASH27B_KV_TQ3=1 \
-./server/build/dflash_server server/models/Qwen3.6-27B-Q4_K_M.gguf \
-  --draft server/models/draft/dflash-draft-3.6-q4_k_m.gguf \
-  --ddtree --ddtree-budget 22 --port 8000
+# default weights (~16 GB)
+hf download unsloth/Qwen3.8-27B-GGUF Qwen3.8-27B-UD-IQ4_XS.gguf --local-dir server/models/
+hf download incoai/Qwen3.8-27B-DFlash2 --local-dir server/models/dflash2-hf
+python server/scripts/convert_dflash_to_gguf.py \
+  server/models/dflash2-hf/model.safetensors server/models/draft/qwen38-dflash2-f16.gguf
+python server/scripts/quantize_dflash_draft.py \
+  server/models/draft/qwen38-dflash2-f16.gguf server/models/draft/qwen38-dflash2-q8_0.gguf --scheme q8_0
+
+# run
+./server/build/dflash_server server/models/Qwen3.8-27B-UD-IQ4_XS.gguf \
+  --draft server/models/draft/qwen38-dflash2-q8_0.gguf \
+  --draft-block-size 16 \
+  --max-ctx 131072 \
+  --cache-type-k q8_0 --cache-type-v q8_0 \
+  --port 8000
 ```
 
 ### Making requests
@@ -267,7 +275,7 @@ curl :8000/v1/chat/completions -H 'Content-Type: application/json' -d '{
 }'
 ```
 
-Requests that omit `temperature` use the model card's sampling (Qwen3.6: `temperature: 1.0`,
+Requests that omit `temperature` use the model card's sampling (Qwen3.8: `temperature: 1.0`,
 `top_p: 0.95`, `top_k: 20`).
 
 ### Server flags
@@ -415,7 +423,7 @@ Pages the attention KV cache through a fixed pool of GPU slots; cold 64-token ch
 
 Tensor parallelism uses NCCL collectives between the selected devices and does
 not include other visible GPUs in its communicator. For example, this runs the
-Qwen3.6 target on GPU 1 and GPU 2 while leaving GPU 0 available:
+Qwen3.8 target on GPU 1 and GPU 2 while leaving GPU 0 available:
 
 ```bash
 dflash_server model.gguf \

server/CMakeLists.txt

@@ -454,6 +454,7 @@ add_library(dflash_common STATIC
     src/common/dynamic_backend.cpp
     src/common/domino_head.cpp
     src/common/dspark_head.cpp
+    src/common/dflash2_head.cpp
     src/common/target_shard_ipc.cpp
     src/common/target_shard_ipc_daemon.cpp
     src/common/dflash_feature_ring.cpp

server/README.md

@@ -521,6 +521,26 @@ Same DFlash + PFlash stack on AMD GPUs. PR #119 ports the Phase 2 rocWMMA flashp
 
 **RDNA4 — Radeon AI PRO R9700 (`gfx1201`, 32 GB).** First-class RDNA4 target as of this build. Qwen3.6-27B Q4_K_M + DFlash draft (`dflash-draft-3.6-q4_k_m.gguf`), `--ddtree-budget=22`: **54.65 tok/s mean DFlash decode** across the 10-prompt HumanEval suite (`bench_he.py --n-gen 256`, AL 7.14, range 36.9–93.0 tok/s) on ROCm 7.1.1. The rocWMMA Phase 2 flashprefill kernels are numerically correct on RDNA4 — ROCm 7.1's rocWMMA handles the gfx12 WMMA operand-format change internally, so no kernel changes are needed (`test_flashprefill_kernels` PASS on `gfx1201`: max diff 5e-4, e2e `flash_prefill_forward_bf16` at S=8192 in 10.7 ms/iter). Note `gfx1200` (RX 9060) and `gfx1201` (RX 9070 / R9700) are **not** code-object compatible — build for `gfx1201` explicitly for the R9700.
 
+For Qwen3.8-27B IQ4_XS with the Q8_0 DFlash2 drafter, the checkpoint declares
+a block size of 8: one seed token plus seven speculative tokens. Greedy chain
+verification keeps the output byte-identical to plain decode at any width, so
+`--draft-block-size` accepts values from 2 up to 2x the checkpoint metadata;
+widening only risks acceptance depth, and on this checkpoint it extrapolates:
+completions stayed byte-identical across widths 8/10/12/16 while per-step
+commits grew.
+
+On the R9700 (`gfx1201`), chain verification with `--draft-block-size 16` is
+the fastest measured code/math configuration: HumanEval ten-prompt decode went
+from 143.6 tok/s (block 8) to 204.1 average with per-request peaks above 230,
+and a five-prompt math set from 129.2 to 152.6. Prose stays on block 8 (low
+acceptance pays the wider draft without commit gains). Widths past 16 measured
+a step-time cliff with no commit gain and stay rejected. The block-8
+`--specla --ddtree-budget 8` configuration is a quality-validated alternative
+(the out-of-the-box default is the plain chain at the checkpoint's block 8):
+163.9 versus 159.0 aggregate decode tok/s across repeated ten-prompt
+HumanEval runs (+3.1%), HumanEval+ pass@1 145/164 versus 143/164, with all ten
+short A/B replies and 133/164 full-suite replies byte-identical.
+
 ```bash
 git clone --recurse-submodules https://github.com/Luce-Org/lucebox-hub && cd lucebox-hub/server
 

server/deps/llama.cpp/ggml/include/ggml.h

@@ -2741,6 +2741,37 @@ extern "C" {
             struct ggml_tensor  * c,
             struct ggml_tensor  * parent_ids);
 
+    // dflash extension: fused causal-conv step for recurrent decode/verify.
+    // Replaces transpose + concat(state, x) + ssm_conv + silu + state
+    // write-back with one kernel.
+    //   x:              [C, T, S] f32, rows contiguous (token stride may be
+    //                   larger than C, e.g. a row-slice of a stacked GEMV)
+    //   c:              [K, C]    f32 depthwise conv weights
+    //   conv_state:     [K-1, C, S] f32 history; READ, then OVERWRITTEN in
+    //                   place with the last K-1 conv inputs
+    //   conv_input_out: optional [>= K-1+T, C, S] f32; receives the full
+    //                   conv window (history rows then x rows) per channel,
+    //                   for speculative-decode rollback. May be a view.
+    // Returns silu(conv(x)) as [C, T, S]. CUDA/HIP only.
+    GGML_API struct ggml_tensor * ggml_ssm_conv_step(
+            struct ggml_context * ctx,
+            struct ggml_tensor  * x,
+            struct ggml_tensor  * c,
+            struct ggml_tensor  * conv_state,
+            struct ggml_tensor  * conv_input_out);
+
+    // dflash2 grouped dynamic block conv (draft graph), one fused node for
+    //   out[c,l] = sum_k (dyn[(site*K+k)*G + c/gs, l] + base[c, site*K+k]) * x[c, l-k]
+    // x [C, T] f32 contiguous, base [C, >= (site+1)*K] f32, dyn [>= 2K*G, T]
+    // f32 contiguous; x is zero-padded below l < k. CUDA/HIP only.
+    GGML_API struct ggml_tensor * ggml_dflash_dyn_conv(
+            struct ggml_context * ctx,
+            struct ggml_tensor  * x,
+            struct ggml_tensor  * base,
+            struct ggml_tensor  * dyn,
+            int                   site,
+            int                   kernel,
+            int                   group_size);
     // SpecLA heavy-light convolution. Applies compact accepted inputs to the
     // durable conv state, then verifies the current tree without committing
     // speculative inputs. Current input factors are written directly to the
@@ -2907,6 +2938,18 @@ extern "C" {
             struct ggml_tensor * tensor,
             bool                 skip_intermediate);
 
+    // dflash extension: let the kernel derive the gates from the raw
+    // projections instead of graph-side sigmoid/softplus ops:
+    //   beta_val = sigmoid(beta_raw)
+    //   g_val    = exp(softplus(alpha_raw + dt_bias[h]) * A[h])
+    // `g` then carries alpha_raw and `beta` carries beta_raw (both [1,H,T,S]);
+    // gate_ba is a contiguous f32 [2*H] tensor holding [dt_bias | A]
+    // (src[9], op_params[10] = 1). Only for the non-tree, non-KDA,
+    // non-SpecLA CUDA/HIP path.
+    GGML_API void ggml_gated_delta_net_set_raw_gates(
+            struct ggml_tensor * tensor,
+            struct ggml_tensor * gate_ba);
+
     // dflash extension: tree-mode gated delta net for DDTree-style
     // speculative decoding verify. `parent_ids` is an int32 tensor of shape
     // [n_tokens, n_seqs] where entry [t, s] is the index within sequence s of

server/deps/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.cpp

@@ -473,14 +473,16 @@ static bool ggml_backend_cpu_device_supports_op(ggml_backend_dev_t dev, const st
         case GGML_OP_PAGED_ATTN:
             return false;
         case GGML_OP_SSM_CONV:
-            // The Specla layout (op param 0 == 1) needs the packed HLD state and
-            // is only supported by the CUDA kernel; the generic CPU kernel would
-            // silently compute garbage.
-            return ggml_get_op_params_i32(op, 0) != 1;
+            // Every nonzero mode is a dflash CUDA/HIP extension (SpecLA,
+            // fused step, or dynamic conv). The generic CPU kernel only
+            // implements the original mode and asserts if one reaches it.
+            return ggml_get_op_params_i32(op, 0) == 0;
         case GGML_OP_GATED_DELTA_NET:
             // The Specla GDN variant (op param 2 == 1) is stateful via HLD and is
-            // only supported by the CUDA kernel.
-            return ggml_get_op_params_i32(op, 2) != 1;
+            // only supported by CUDA/HIP. Raw-gate mode is also CUDA/HIP-only:
+            // the CPU kernel expects beta/g to have already been transformed.
+            return ggml_get_op_params_i32(op, 2) != 1 &&
+                   ggml_get_op_params_i32(op, 10) == 0 && op->src[9] == nullptr;
         case GGML_OP_OUT_PROD:
             return (src0->type == GGML_TYPE_F32 || (ggml_is_quantized(src0->type) && src0->ne[2] == src1->ne[2] && src0->ne[3] == src1->ne[3])) &&
                 src1->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32;

server/deps/llama.cpp/ggml/src/ggml-cpu/ops.cpp

@@ -9332,6 +9332,8 @@ void ggml_compute_forward_flash_attn_back(
 static void ggml_compute_forward_ssm_conv_f32(
         const ggml_compute_params * params,
         ggml_tensor * dst) {
+    // dflash: the fused step mode (ggml_ssm_conv_step) is CUDA/HIP only
+    GGML_ASSERT(ggml_get_op_params_i32(dst, 0) == 0 && "ggml_ssm_conv_step is not supported on CPU");
     const ggml_tensor * src0 = dst->src[0]; // conv_x
     const ggml_tensor * src1 = dst->src[1]; // conv1d.weight
 
@@ -10716,6 +10718,12 @@ static void ggml_compute_forward_gated_delta_net_f32(
 void ggml_compute_forward_gated_delta_net(
         const ggml_compute_params * params,
         ggml_tensor * dst) {
+    // Raw-gate mode (loader-built [dt_bias|A] in src[9], op_params[10])
+    // is CUDA/HIP only; the CPU path would silently use the raw values
+    // as final gates.
+    GGML_ASSERT(dst->op_params[10] == 0 && dst->src[9] == NULL &&
+                "raw-gate gated_delta_net is CUDA/HIP only");
+
     const ggml_tensor * src0 = dst->src[0];
 
     switch (src0->type) {

server/deps/llama.cpp/ggml/src/ggml-cuda/common.cuh

@@ -1418,7 +1418,7 @@ struct ggml_backend_cuda_context {
     std::string name;
     cudaEvent_t copy_event = nullptr;
 
-    // LUCE_Q8_MEMO=1: memoize q8_1-quantized src1 activations across the
+    // Memoize q8_1-quantized src1 activations across the
     // quantized matmuls of ONE graph evaluation (Q/K/V/gate/router/shexp all
     // re-quantize the same hidden state otherwise). Entries are freed in LIFO
     // order at the start of the next evaluation to keep the pool discipline.

server/deps/llama.cpp/ggml/src/ggml-cuda/fattn-vec.cuh

@@ -534,7 +534,10 @@ void ggml_cuda_flash_attn_ext_vec_case_impl(ggml_backend_cuda_context & ctx, ggm
     const bool need_f16_K = type_K == GGML_TYPE_F16;
     const bool need_f16_V = type_V == GGML_TYPE_F16;
     constexpr size_t nbytes_shared = 0;
-    launch_fattn<D, cols_per_block, 1>(ctx, dst, fattn_kernel, nwarps, nbytes_shared, D, need_f16_K, need_f16_V, false);
+    // The kernel walks the KV sequence in steps of nthreads (not D); telling
+    // launch_fattn so lets it split a short KV span (e.g. a 256-token window
+    // at head_dim 256) across two blocks per head instead of one.
+    launch_fattn<D, cols_per_block, 1>(ctx, dst, fattn_kernel, nwarps, nbytes_shared, nthreads, need_f16_K, need_f16_V, false);
 }
 
 template <int D, ggml_type type_K, ggml_type type_V>