perf(qwen35): batch draft projections across lanes for concurrency

.github/workflows/ci.yml

@@ -213,6 +213,7 @@ jobs:
             test_ds4_mix_registry_teardown
             test_model_smoke
             test_batched_gdn
+            test_gdn_transition_journal
             test_concat_transpose
           )
           if [[ "$RUN_SPARSE_ATTENTION_TEST" == "true" ]]; then
@@ -233,7 +234,7 @@ jobs:
       - name: Run concurrent-serving kernel tests
         run: |
           ctest --test-dir server/build --output-on-failure \
-            -R '^(test_model_smoke\.PagedAttention\.|batched_gdn$|concat_transpose$)' --no-tests=error
+            -R '^(test_model_smoke\.PagedAttention\.|batched_gdn$|gdn_transition_journal_preflight$|concat_transpose$)' --no-tests=error
 
       # Optional model-backed end-to-end smoke (real spec-decode on the 3090),
       # disabled by default because it builds dflash_server and lazy-loads the

README.md

@@ -57,11 +57,10 @@ All speedups measured vs vendored llama.cpp (`-fa 1`, matching KV quant). Combin
 | Model | Speedup |
 |-------|:-------:|
 | Qwen 3.5 0.8B (Megakernel) | **~2×** |
-| Qwen 3.6 27B + PFlash | **~5.6×** |
-| Qwen 3.6 27B + DDTree | **4.84×** |
+| Qwen 3.8 27B + DFlash2 (R9700) | **6.1×** decode |
+| Qwen 3.8 27B + DFlash2 vs llama.cpp speculating with the same drafter | **3.5×** decode |
 | Laguna XS 2.1 33B + PFlash | **8.2×** @256K |
 | Laguna XS 2.1 33B + DFlash | **1.7×** @256K |
-| Qwen 3.6 27B HIP | **~2.6×** |
 | Gemma 4 26B-A4B | **1.31×** |
 | Gemma 4 31B IT | **3.2×** |
 | [`DeepSeek V4 Flash ROCMFPX HIP`](https://huggingface.co/Lucebox/DeepSeek-V4-Flash-0731-ROCmFP3) | **2×** |
@@ -71,7 +70,7 @@ All speedups measured vs vendored llama.cpp (`-fa 1`, matching KV quant). Combin
 
 | Drafter | Phase |
 |---------|:-----:|
-| [`Qwen3.6 27B`](https://huggingface.co/Lucebox/Qwen3.6-27B-DFlash-GGUF) | decode |
+| [`Qwen3.8 27B DFlash2`](https://huggingface.co/incoai/Qwen3.8-27B-DFlash2) | decode |
 | [`gemma 4 26B A4B`](https://huggingface.co/Lucebox/gemma-4-26B-A4B-it-DFlash-GGUF) | decode |
 | [`gemma 4 31B`](https://huggingface.co/Lucebox/gemma-4-31B-it-DFlash-GGUF) | decode |
 | [`Laguna XS 2.1 33B`](https://huggingface.co/Lucebox/Laguna-XS-2.1-DFlash-GGUF) | decode |
@@ -110,7 +109,7 @@ Entries are `dflash_server` settings unless the cell shows another command.
 
 | Model | RTX 3090 (24 GB) | Strix Halo `gfx1151` | Strix Halo + R9700 `gfx1201` |
 |---|---|---|---|
-| **Qwen 3.5/3.6 27B Q4_K_M** | `DFLASH27B_KV_TQ3=1`<br>`--target-device cuda:0`<br>`--draft-device cuda:0`<br>`--ddtree`<br>`--ddtree-budget 22`<br>`--draft-residency auto`<br>`--prefill-compression auto`<br>`--prefill-drafter <path>`<br>`--kvflash auto` | `--target-device hip:0`<br>`--draft-device hip:0`<br>`--ddtree`<br>`--ddtree-budget 22`<br>`--draft-residency persistent`<br>`--prefill-compression auto`<br>`--prefill-drafter <path>`<br>`--kvflash auto` | `HIP_VISIBLE_DEVICES=<r9700-index>`<br>`--target-device hip:0`<br>`--draft-device hip:0`<br>`--ddtree`<br>`--ddtree-budget 22`<br>`--draft-residency persistent`<br>`--prefill-compression auto`<br>`--prefill-drafter <path>`<br>`--kvflash auto` |
+| **Qwen 3.8 27B IQ4_XS** | `--target-device cuda:0`<br>`--draft-device cuda:0`<br>`--draft-block-size 16`<br>`--cache-type-k q8_0`<br>`--cache-type-v q8_0` | `--target-device hip:0`<br>`--draft-device hip:0`<br>`--draft-block-size 16`<br>`--cache-type-k q8_0`<br>`--cache-type-v q8_0` | `HIP_VISIBLE_DEVICES=<r9700-index>`<br>`--target-device hip:0`<br>`--draft-device hip:0`<br>`--draft-block-size 16`<br>`--max-ctx 131072`<br>`--cache-type-k q8_0`<br>`--cache-type-v q8_0` |
 | **Qwen 3.6 35B-A3B Q4_K_M** | `--target-device cuda:0`<br>`--spark`<br>`--kvflash auto` | `--target-device hip:0`<br>`--kvflash auto` | `HIP_VISIBLE_DEVICES=<r9700-index>`<br>`--target-device hip:0`<br>`--kvflash auto` |
 | **Laguna XS 2.1 33B Q4_K_M** | `--target-device cuda:0`<br>`--draft <path>`<br>`--prefill-drafter <path>`<br>`--max-ctx 262144`<br>`--kvflash 8192`<br>`--chunk 1024` | `--target-device hip:0`<br>`--kvflash auto` | `HIP_VISIBLE_DEVICES=<r9700-index>`<br>`--target-device hip:0`<br>`--kvflash auto` |
 | **Gemma 4 26B-A4B / 31B** | `--target-device cuda:0`<br>`--draft-device cuda:0`<br>`--kvflash auto` | `--target-device hip:0`<br>`--draft-device hip:0`<br>`--kvflash auto` | `HIP_VISIBLE_DEVICES=<r9700-index>`<br>`--target-device hip:0`<br>`--draft-device hip:0`<br>`--kvflash auto` |
@@ -152,9 +151,9 @@ All launchers spawn the native C++ HTTP server (`dflash_server`). Override defau
 
 ```bash
 DFLASH_SERVER_BIN=server/build/dflash_server \
-DFLASH_TARGET=server/models/Qwen3.6-27B-Q4_K_M.gguf \
-DFLASH_DRAFT=server/models/draft/dflash-draft-3.6-q4_k_m.gguf \
-MAX_CTX=32768 BUDGET=22 VERIFY_MODE=ddtree \
+DFLASH_TARGET=server/models/Qwen3.8-27B-UD-IQ4_XS.gguf \
+DFLASH_DRAFT=server/models/draft/qwen38-dflash2-q8_0.gguf \
+MAX_CTX=32768 \
 harness/clients/run_codex.sh
 ```
 
@@ -214,10 +213,13 @@ docker pull ghcr.io/luce-org/lucebox-hub:rocm     # AMD
 # 2. Download a target model into server/models/ and the DFlash draft
 #    into server/models/draft/ (the entrypoint only auto-discovers the
 #    draft there; without it the server runs slower, target-only)
-hf download unsloth/Qwen3.6-27B-GGUF Qwen3.6-27B-Q4_K_M.gguf \
+hf download unsloth/Qwen3.8-27B-GGUF Qwen3.8-27B-UD-IQ4_XS.gguf \
   --local-dir server/models/
-hf download Lucebox/Qwen3.6-27B-DFlash-GGUF dflash-draft-3.6-q4_k_m.gguf \
-  --local-dir server/models/draft/
+#    Drafter: convert the z-lab checkpoint once with the two converter
+#    commands from the source-build quickstart below and place the q8_0
+#    GGUF in server/models/draft/. (The pre-made incoai GGUF is for
+#    llama.cpp PR#27342 and uses a different layout this server does not
+#    load yet.)
 
 # 3a. NVIDIA (CUDA 12+)
 docker run --rm --gpus all -p 8000:8080 \
@@ -235,23 +237,29 @@ Then hit `:8000/v1/chat/completions` (OpenAI-compatible).
 
 ## Run the Server
 
-Default: Qwen 3.6-27B Q4_K_M target + Lucebox Q4_K_M DFlash drafter on RTX 3090. DDTree budget=22, TQ3_0 KV cache, full attention. OpenAI-compatible HTTP on `:8000`.
+Default: Qwen 3.8-27B IQ4_XS target + the DFlash2 block-diffusion drafter. Full attention, q8_0 KV cache, OpenAI-compatible HTTP on `:8000`. The tuned decode path is the default, so no environment variables are needed.
 
 ```bash
 # build (CUDA 12+, CMake 3.18+)
 git clone --recurse-submodules https://github.com/Luce-Org/lucebox-hub && cd lucebox-hub
 cmake -B server/build -S server -DCMAKE_BUILD_TYPE=Release
 cmake --build server/build --target dflash_server -j
 
-# default weights (~18 GB)
-hf download unsloth/Qwen3.6-27B-GGUF Qwen3.6-27B-Q4_K_M.gguf --local-dir server/models/
-hf download Lucebox/Qwen3.6-27B-DFlash-GGUF dflash-draft-3.6-q4_k_m.gguf --local-dir server/models/draft/
-
-# run (TQ3_0 KV auto-enabled; set =0 to disable)
-DFLASH27B_KV_TQ3=1 \
-./server/build/dflash_server server/models/Qwen3.6-27B-Q4_K_M.gguf \
-  --draft server/models/draft/dflash-draft-3.6-q4_k_m.gguf \
-  --ddtree --ddtree-budget 22 --port 8000
+# default weights (~16 GB)
+hf download unsloth/Qwen3.8-27B-GGUF Qwen3.8-27B-UD-IQ4_XS.gguf --local-dir server/models/
+hf download incoai/Qwen3.8-27B-DFlash2 --local-dir server/models/dflash2-hf
+python server/scripts/convert_dflash_to_gguf.py \
+  server/models/dflash2-hf/model.safetensors server/models/draft/qwen38-dflash2-f16.gguf
+python server/scripts/quantize_dflash_draft.py \
+  server/models/draft/qwen38-dflash2-f16.gguf server/models/draft/qwen38-dflash2-q8_0.gguf --scheme q8_0
+
+# run
+./server/build/dflash_server server/models/Qwen3.8-27B-UD-IQ4_XS.gguf \
+  --draft server/models/draft/qwen38-dflash2-q8_0.gguf \
+  --draft-block-size 16 \
+  --max-ctx 131072 \
+  --cache-type-k q8_0 --cache-type-v q8_0 \
+  --port 8000
 ```
 
 ### Making requests
@@ -267,7 +275,7 @@ curl :8000/v1/chat/completions -H 'Content-Type: application/json' -d '{
 }'
 ```
 
-Requests that omit `temperature` use the model card's sampling (Qwen3.6: `temperature: 1.0`,
+Requests that omit `temperature` use the model card's sampling (Qwen3.8: `temperature: 1.0`,
 `top_p: 0.95`, `top_k: 20`).
 
 ### Server flags
@@ -415,7 +423,7 @@ Pages the attention KV cache through a fixed pool of GPU slots; cold 64-token ch
 
 Tensor parallelism uses NCCL collectives between the selected devices and does
 not include other visible GPUs in its communicator. For example, this runs the
-Qwen3.6 target on GPU 1 and GPU 2 while leaving GPU 0 available:
+Qwen3.8 target on GPU 1 and GPU 2 while leaving GPU 0 available:
 
 ```bash
 dflash_server model.gguf \

server/CMakeLists.txt

@@ -454,6 +454,8 @@ add_library(dflash_common STATIC
     src/common/dynamic_backend.cpp
     src/common/domino_head.cpp
     src/common/dspark_head.cpp
+    src/common/dflash2_head.cpp
+    src/common/dflash2_batch.cpp
     src/common/target_shard_ipc.cpp
     src/common/target_shard_ipc_daemon.cpp
     src/common/dflash_feature_ring.cpp
@@ -463,6 +465,7 @@ add_library(dflash_common STATIC
     src/common/pflash_drafter_ipc.cpp
     src/common/dflash_draft_graph.cpp
     src/common/dflash_draft_kv.cpp
+    src/common/draft_swa.cpp
     src/common/dflash_spec_decode.cpp
     src/common/concurrency/paged_kv_pool.cpp
     src/qwen35/concurrency/qwen35_slot_manager.cpp
@@ -1434,6 +1437,52 @@ if(DFLASH27B_TESTS)
             ${CMAKE_CURRENT_SOURCE_DIR}/test)
         list(APPEND _raw_unit_test_targets test_seq_engine_contract)
     endif()
+    if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/test_ddtree_path.cpp")
+        # Pure host-side accepted-path/pending-token contract tests.
+        add_executable(test_ddtree_path
+            test/test_ddtree_path.cpp
+            src/common/ddtree.cpp)
+        target_include_directories(test_ddtree_path PRIVATE
+            ${CMAKE_CURRENT_SOURCE_DIR}/src)
+        list(APPEND _raw_unit_test_targets test_ddtree_path)
+    endif()
+    if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/test_chain_spec_shapes.cpp")
+        # Pure host-side DFlash2 chain topology and mixed-launch arithmetic.
+        add_executable(test_chain_spec_shapes
+            test/test_chain_spec_shapes.cpp
+            src/common/ddtree.cpp)
+        target_include_directories(test_chain_spec_shapes PRIVATE
+            ${CMAKE_CURRENT_SOURCE_DIR}/src
+            ${CMAKE_CURRENT_SOURCE_DIR}/test)
+        list(APPEND _raw_unit_test_targets test_chain_spec_shapes)
+    endif()
+    if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/test_dflash2_selector_validation.cpp")
+        # Pure host-side selector metadata/layout validation: no GPU.
+        add_executable(test_dflash2_selector_validation
+            test/test_dflash2_selector_validation.cpp)
+        target_include_directories(test_dflash2_selector_validation PRIVATE
+            ${CMAKE_CURRENT_SOURCE_DIR}/src
+            ${CMAKE_CURRENT_SOURCE_DIR}/test)
+        list(APPEND _raw_unit_test_targets test_dflash2_selector_validation)
+    endif()
+    if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/test_draft_swa.cpp")
+        # Pure host-side draft SWA metadata/override policy tests.
+        add_executable(test_draft_swa test/test_draft_swa.cpp)
+        target_include_directories(test_draft_swa PRIVATE
+            ${CMAKE_CURRENT_SOURCE_DIR}/src)
+        target_link_libraries(test_draft_swa PRIVATE dflash_common)
+    endif()
+    if((DFLASH27B_GPU_BACKEND STREQUAL "cuda" OR
+        DFLASH27B_GPU_BACKEND STREQUAL "hip") AND
+       EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/test_draft_swa_multilane.cpp")
+        # Opt-in real-model proof for post-window lane masks and packed draft parity.
+        add_executable(test_draft_swa_multilane test/test_draft_swa_multilane.cpp)
+        target_include_directories(test_draft_swa_multilane PRIVATE
+            ${DFLASH27B_SRC_INCLUDE_DIRS})
+        target_link_libraries(test_draft_swa_multilane PRIVATE
+            dflash_common ggml ${DFLASH27B_GGML_BACKEND_TARGET})
+        list(APPEND _raw_unit_test_targets test_draft_swa_multilane)
+    endif()
     if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/test_seq_batch_plan.cpp")
         # Pure-host tests for model-neutral token-budget/FIFO planning.
         add_executable(test_seq_batch_plan test/test_seq_batch_plan.cpp)
@@ -1773,6 +1822,7 @@ if(DFLASH27B_TESTS)
         test_model_smoke)
     set(_new_cppunit_test_targets
         test_platform_compat
+        test_draft_swa
         test_feature_gate
         test_qwen35_split_tree_guard
         test_recurrent_snapshot
@@ -1934,6 +1984,23 @@ if(DFLASH27B_TESTS)
             add_dependencies(check test_batched_gdn)
         endif()
     endif()
+    if((DFLASH27B_GPU_BACKEND STREQUAL "cuda" OR
+        DFLASH27B_GPU_BACKEND STREQUAL "hip")
+       AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/test_gdn_replay_log.cpp")
+        dflash_add_ggml_gpu_executable(
+            test_gdn_replay_log
+            test/test_gdn_replay_log.cpp)
+        add_test(NAME gdn_replay_log COMMAND test_gdn_replay_log)
+        add_test(
+            NAME gdn_replay_log_gallocr
+            COMMAND test_gdn_replay_log --gallocr-only)
+        add_test(
+            NAME gdn_replay_log_preflight
+            COMMAND test_gdn_replay_log --preflight-only)
+        if(TARGET check)
+            add_dependencies(check test_gdn_replay_log)
+        endif()
+    endif()
     if((DFLASH27B_GPU_BACKEND STREQUAL "cuda" OR
         DFLASH27B_GPU_BACKEND STREQUAL "hip")
        AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/test_concat_transpose.cpp")

server/README.md

@@ -521,6 +521,26 @@ Same DFlash + PFlash stack on AMD GPUs. PR #119 ports the Phase 2 rocWMMA flashp
 
 **RDNA4 — Radeon AI PRO R9700 (`gfx1201`, 32 GB).** First-class RDNA4 target as of this build. Qwen3.6-27B Q4_K_M + DFlash draft (`dflash-draft-3.6-q4_k_m.gguf`), `--ddtree-budget=22`: **54.65 tok/s mean DFlash decode** across the 10-prompt HumanEval suite (`bench_he.py --n-gen 256`, AL 7.14, range 36.9–93.0 tok/s) on ROCm 7.1.1. The rocWMMA Phase 2 flashprefill kernels are numerically correct on RDNA4 — ROCm 7.1's rocWMMA handles the gfx12 WMMA operand-format change internally, so no kernel changes are needed (`test_flashprefill_kernels` PASS on `gfx1201`: max diff 5e-4, e2e `flash_prefill_forward_bf16` at S=8192 in 10.7 ms/iter). Note `gfx1200` (RX 9060) and `gfx1201` (RX 9070 / R9700) are **not** code-object compatible — build for `gfx1201` explicitly for the R9700.
 
+For Qwen3.8-27B IQ4_XS with the Q8_0 DFlash2 drafter, the checkpoint declares
+a block size of 8: one seed token plus seven speculative tokens. Greedy chain
+verification keeps the output byte-identical to plain decode at any width, so
+`--draft-block-size` accepts values from 2 up to 2x the checkpoint metadata;
+widening only risks acceptance depth, and on this checkpoint it extrapolates:
+completions stayed byte-identical across widths 8/10/12/16 while per-step
+commits grew.
+
+On the R9700 (`gfx1201`), chain verification with `--draft-block-size 16` is
+the fastest measured code/math configuration: HumanEval ten-prompt decode went
+from 143.6 tok/s (block 8) to 204.1 average with per-request peaks above 230,
+and a five-prompt math set from 129.2 to 152.6. Prose stays on block 8 (low
+acceptance pays the wider draft without commit gains). Widths past 16 measured
+a step-time cliff with no commit gain and stay rejected. The block-8
+`--specla --ddtree-budget 8` configuration is a quality-validated alternative
+(the out-of-the-box default is the plain chain at the checkpoint's block 8):
+163.9 versus 159.0 aggregate decode tok/s across repeated ten-prompt
+HumanEval runs (+3.1%), HumanEval+ pass@1 145/164 versus 143/164, with all ten
+short A/B replies and 133/164 full-suite replies byte-identical.
+
 ```bash
 git clone --recurse-submodules https://github.com/Luce-Org/lucebox-hub && cd lucebox-hub/server
 

server/deps/llama.cpp/ggml/include/ggml-cuda.h

@@ -99,6 +99,50 @@ GGML_BACKEND_API ggml_backend_reg_t ggml_backend_cuda_reg(void);
 GGML_BACKEND_API bool ggml_backend_cuda_topk_rows(const struct ggml_tensor * logits, int k,
                                                   float * probs_out, int32_t * ids_out);
 
+// Batched concurrent-tree commit. Validation is fail-closed before any kernel
+// launches; all layer replay logs and convolution windows commit on one device
+// synchronization.
+GGML_BACKEND_API bool ggml_backend_cuda_gdn_replay_log_commit_many(
+        const struct ggml_tensor * const * replay_logs,
+        struct ggml_tensor * const * states,
+        const struct ggml_tensor * const * conv_inputs,
+        struct ggml_tensor * const * conv_states,
+        int n_layers,
+        const struct ggml_tensor * accepted_prefixes,
+        const struct ggml_tensor * active_slot_ids);
+
+// Promote accepted packed-tree K/V scratch rows into pager-owned rows.
+GGML_BACKEND_API bool ggml_backend_cuda_tree_cache_commit_many(
+        struct ggml_tensor * const * caches, int n_caches,
+        const struct ggml_tensor * commit_rows,
+        const struct ggml_tensor * active_slot_ids,
+        int tree_scratch_base, int tree_scratch_stride);
+
+// Promote accepted BF16 tree feature rows into slot-local feature rings.
+GGML_BACKEND_API bool ggml_backend_cuda_tree_feature_commit(
+        const struct ggml_tensor * source, struct ggml_tensor * destination,
+        const struct ggml_tensor * destination_rows);
+
+// Validate every packed-tree destination before changing any cache. Once the
+// first kernel launches, a device failure is fatal because fallback cannot
+// recover from a partially committed state.
+GGML_BACKEND_API bool ggml_backend_cuda_tree_commit_transaction(
+        struct ggml_tensor * const * caches,
+        int n_caches,
+        const struct ggml_tensor * feature_source,
+        struct ggml_tensor * feature_destination,
+        const struct ggml_tensor * feature_destination_rows,
+        const struct ggml_tensor * const * replay_logs,
+        struct ggml_tensor * const * states,
+        const struct ggml_tensor * const * conv_inputs,
+        struct ggml_tensor * const * conv_states,
+        int n_layers,
+        const struct ggml_tensor * commit_rows,
+        const struct ggml_tensor * accepted_prefixes,
+        const struct ggml_tensor * active_slot_ids,
+        int tree_scratch_base,
+        int tree_scratch_stride);
+
 // Attach learned per-expert decode tables to a mixed-precision tensor. The
 // host variants copy the tables to the device that owns `base`. Call the
 // matching unregister function before releasing the tensor's backing buffer.