From 11d4280f6f5872fff038e20dbee824b5a6947962 Mon Sep 17 00:00:00 2001 From: Graffioh Date: Tue, 11 Aug 2026 13:11:00 +0000 Subject: [PATCH 1/5] feat(ds4): add monolithic Strix Halo concurrent serving --- server/docs/DS4.md | 37 ++ .../common/concurrency/seq_slot_manager.cpp | 271 +++++++++ .../src/common/concurrency/seq_slot_manager.h | 131 ++++ server/src/deepseek4/deepseek4_backend.cpp | 143 ++++- server/src/deepseek4/deepseek4_backend.h | 5 + .../src/deepseek4/deepseek4_fused_verify.inc | 145 ++++- server/src/deepseek4/deepseek4_graph.cpp | 563 +++++++++++++++--- server/src/deepseek4/deepseek4_internal.h | 46 ++ server/src/deepseek4/deepseek4_page_layout.h | 55 ++ .../src/deepseek4/deepseek4_paged_cache.cpp | 219 +++++++ server/src/deepseek4/deepseek4_paged_cache.h | 63 ++ server/src/deepseek4/deepseek4_seq_engine.cpp | 272 +++++++++ server/src/deepseek4/deepseek4_seq_engine.h | 41 ++ server/test/test_deepseek4_page_layout.cpp | 53 ++ server/test/test_deepseek4_paged_cache.cpp | 86 +++ 15 files changed, 2018 insertions(+), 112 deletions(-) create mode 100644 server/src/common/concurrency/seq_slot_manager.cpp create mode 100644 server/src/common/concurrency/seq_slot_manager.h create mode 100644 server/src/deepseek4/deepseek4_page_layout.h create mode 100644 server/src/deepseek4/deepseek4_paged_cache.cpp create mode 100644 server/src/deepseek4/deepseek4_paged_cache.h create mode 100644 server/src/deepseek4/deepseek4_seq_engine.cpp create mode 100644 server/src/deepseek4/deepseek4_seq_engine.h create mode 100644 server/test/test_deepseek4_page_layout.cpp create mode 100644 server/test/test_deepseek4_paged_cache.cpp diff --git a/server/docs/DS4.md b/server/docs/DS4.md index d80c4b33e..086da1683 100644 --- a/server/docs/DS4.md +++ b/server/docs/DS4.md @@ -291,6 +291,43 @@ performance profile held 48.1 tok/s median on the deterministic 128-token workload. The all-6-expert reference-exact mode is a correctness profile, not a throughput profile. +### Strix Halo concurrent serving + +DeepSeek4 paged concurrency is deliberately a single-device path: one local +HIP target on Strix Halo (`gfx1151`), with the complete model and every expert +resident on that device. It does not use layer splitting, CUDA/HIP expert +ownership, host-streamed experts, or DSpark. + +The backend keeps raw MLA rows, compressed rows, indexer state, sequence +lengths, and block tables in a persistent 128-token paged cache. The shared +HTTP scheduler performs admission, cancellation, slow-client isolation, and +fair continuous batching. DeepSeek4 lowers each scheduler plan into one exact +gathered graph with up to 16 independent lanes. Decode rows share the weight +pass; each selected prompt advances by one exact token because the graph must +not contain two rows from the same sequence. + +```bash +cmake -S . -B build-hip \ + -DDFLASH27B_GPU_BACKEND=hip \ + -DDFLASH27B_HIP_ARCHITECTURES=gfx1151 \ + -DDFLASH27B_SERVER=ON +cmake --build build-hip -j + +./build-hip/dflash_server /path/to/deepseek4-target.gguf \ + --target-device hip:0 \ + --paged-attention \ + --max-concurrency 16 \ + --kv-pool-tokens 8192 \ + --max-ctx 4096 \ + --ds4-prefill exact \ + --prefix-cache-slots 0 +``` + +This mode fails closed for non-gfx1151 devices, CUDA, layer or remote target +splits, `DFLASH_DS4_MOE_TP`, drafts/DSpark, DDTree, PFlash/KVFlash, fused +decode, approximate prefill, windowed attention, and prefix-cache parking. +There is no automatic fallback to a slower or asymmetric execution mode. + ### Local single-shard If the adapter decides all 43 layers fit on one CUDA GPU, it loads a single shard locally and no IPC daemon is involved. diff --git a/server/src/common/concurrency/seq_slot_manager.cpp b/server/src/common/concurrency/seq_slot_manager.cpp new file mode 100644 index 000000000..1ff91ed8d --- /dev/null +++ b/server/src/common/concurrency/seq_slot_manager.cpp @@ -0,0 +1,271 @@ +#include "common/concurrency/seq_slot_manager.h" + +#include +#include + +namespace dflash::common { + +SeqSlotManager::SeqSlotManager(PagedKvPool & pool, int max_ctx) + : pool_(pool), max_ctx_(max_ctx) { + slots_.assign(pool.max_sequences(), SeqSlot{}); +} + +int SeqSlotManager::decoding_count() const { + int n = 0; + for (const SeqSlot & s : slots_) { + n += s.decoding() ? 1 : 0; + } + return n; +} + +uint32_t SeqSlotManager::decode_headroom_capacity(int logical_tokens) const { + const uint64_t extended = + static_cast(std::max(0, logical_tokens)) + + pool_.block_size(); + return static_cast(std::min( + static_cast(max_ctx_), extended)); +} + +bool SeqSlotManager::capacity_fits_pool(uint32_t token_capacity) const { + const uint64_t blocks = token_capacity == 0 ? 0 : + 1 + (static_cast(token_capacity) - 1) / + pool_.block_size(); + return blocks <= pool_.physical_block_count(); +} + +PagedKvStatus SeqSlotManager::protect_decode_headroom() { + struct TopUp { + PagedKvSequenceHandle handle; + uint32_t token_capacity = 0; + }; + + std::vector topups; + topups.reserve(slots_.size()); + uint64_t total_additional = 0; + const uint64_t block_size = pool_.block_size(); + for (const SeqSlot & slot : slots_) { + if (!slot.decoding()) continue; + const uint32_t capacity = decode_headroom_capacity(slot.cur_pos); + if (!capacity_fits_pool(capacity)) continue; + + uint32_t owned_blocks = 0; + const PagedKvStatus status = + pool_.owned_block_count(slot.handle, owned_blocks); + if (status != PagedKvStatus::Ok) return status; + const uint64_t target_blocks = capacity == 0 ? 0 : + 1 + (static_cast(capacity) - 1) / block_size; + if (target_blocks <= owned_blocks) continue; + const uint32_t additional = + static_cast(target_blocks - owned_blocks); + total_additional += additional; + topups.push_back({slot.handle, capacity}); + } + + // Preflight the whole cohort before moving a block, so a failed admission + // attempt cannot protect only whichever decoder happened to be visited + // first. + if (total_additional > pool_.free_block_count()) { + return PagedKvStatus::BlocksExhausted; + } + for (const TopUp & topup : topups) { + const PagedKvStatus status = + pool_.reserve_capacity(topup.handle, topup.token_capacity); + if (status != PagedKvStatus::Ok) return status; + } + return PagedKvStatus::Ok; +} + +bool SeqSlotManager::is_active(int slot) const { + return slot >= 0 && slot < (int)slots_.size() && + slots_[(size_t)slot].active(); +} + +bool SeqSlotManager::is_prefilling(int slot) const { + return is_active(slot) && slots_[(size_t)slot].prefilling(); +} + +SeqEngine::AdmitResult SeqSlotManager::admit( + uint64_t request_id, const std::vector & prompt, + const SamplerCfg & sampler) { + using AdmitStatus = SeqEngine::AdmitResult::Status; + SeqEngine::AdmitResult r; + if (prompt.empty()) { + r.error = "empty prompt"; + return r; + } + if (prompt.size() > static_cast(max_ctx_)) { + r.error = "prompt exceeds max_ctx"; + return r; + } + const int prompt_len = static_cast(prompt.size()); + + // A prompt larger than the whole pool can NEVER be admitted; waiting + // for other sequences to drain would stall the queue forever and then + // fail anyway. Hard-fail it up front instead of reporting busy. + const uint64_t pool_capacity = + (uint64_t)pool_.physical_block_count() * pool_.block_size(); + if ((uint64_t)prompt_len > pool_capacity) { + r.error = "prompt needs " + std::to_string(prompt_len) + + " KV tokens but the pool holds " + + std::to_string(pool_capacity) + + "; raise --kv-pool-tokens or shorten the prompt"; + return r; + } + + int slot = -1; + for (int i = 0; i < (int)slots_.size(); i++) { + if (!slots_[(size_t)i].active()) { slot = i; break; } + } + if (slot < 0) { + r.status = AdmitStatus::busy; + r.error = "all decode slots are busy"; + return r; + } + + // A newly freed block belongs to any older decoder missing its rolling + // next-page reserve before it can belong to this admission. + const PagedKvStatus headroom_status = protect_decode_headroom(); + if (headroom_status != PagedKvStatus::Ok) { + r.status = headroom_status == PagedKvStatus::BlocksExhausted + ? AdmitStatus::busy : AdmitStatus::failed; + r.error = r.status == AdmitStatus::busy + ? "existing decoders need the available KV headroom" + : paged_kv_status_string(headroom_status); + return r; + } + + PagedKvSequenceHandle handle; + uint32_t reservation_capacity = + decode_headroom_capacity(prompt_len); + if (!capacity_fits_pool(reservation_capacity)) { + // The prompt itself fits, but this physical pool can never hold its + // following page. Preserve useful prompt-only behavior and report + // decode exhaustion later if the sequence reaches that boundary. + reservation_capacity = static_cast(prompt_len); + } + const PagedKvStatus status = pool_.acquire_reserved( + request_id, reservation_capacity, handle); + if (status != PagedKvStatus::Ok) { + r.status = status == PagedKvStatus::SequenceSlotsExhausted || + status == PagedKvStatus::BlocksExhausted + ? AdmitStatus::busy : AdmitStatus::failed; + r.error = status == PagedKvStatus::BlocksExhausted + ? "not enough unreserved KV blocks for the prompt and decode headroom" + : paged_kv_status_string(status); + return r; + } + + SeqSlot & s = slots_[(size_t)slot]; + s.phase = SeqSlotPhase::prefill; + s.handle = handle; + s.cur_pos = 0; + s.prompt = prompt; + s.sampler = sampler; + s.sample_history = prompt; + // Same predicate the engine uses to pick CPU sampling over GPU argmax: + // a seed only means anything when the sampler actually draws. + if (sampler.needs_logit_processing() && sampler.seed != 0) { + s.rng.seed(sampler.seed); + } else { + s.rng.seed(std::random_device{}()); + } + + r.status = AdmitStatus::admitted; + r.slot = slot; + return r; +} + +SeqSlotManager::PrefillChunk SeqSlotManager::append_prefill( + int slot, int n_tokens) { + PrefillChunk out; + if (!is_prefilling(slot) || n_tokens < 1) return out; + + SeqSlot & s = slots_[(size_t)slot]; + if (s.cur_pos > (int)s.prompt.size() || + n_tokens > (int)s.prompt.size() - s.cur_pos) { + return out; + } + + PagedKvAppendResult app = pool_.append(s.handle, (uint32_t)n_tokens); + if (!app) { + // Admission reserved the whole prompt. Treat exhaustion here as a + // broken invariant, not a retryable condition: retrying a batch of + // all-prefill slots without any decoder able to retire would livelock. + if (app.status == PagedKvStatus::BlocksExhausted) { + std::fprintf(stderr, + "[parallel] reserved prefill capacity missing for slot %d\n", + slot); + } + out.busy = false; + return out; + } + + out.rows.reserve(app.write_slots.size()); + for (const PagedKvWriteSlot & write : app.write_slots) { + out.rows.push_back((int64_t)write.physical_token_index); + if (write.block_offset == 0) { + if (out.first_new_block < 0) { + out.first_new_block = + (int)(write.logical_position / pool_.block_size()); + } + out.new_blocks.push_back((int32_t)write.physical_block); + } + } + s.cur_pos += n_tokens; + out.ok = true; + return out; +} + +void SeqSlotManager::commit_prefill(int slot) { + if (!is_prefilling(slot)) return; + SeqSlot & s = slots_[(size_t)slot]; + if (s.cur_pos != (int)s.prompt.size()) return; + s.phase = SeqSlotPhase::decode; +} + +SeqSlotManager::StepAppend SeqSlotManager::append_token(int slot, + int32_t fed_token) { + StepAppend out; + if (!is_active(slot) || !slots_[(size_t)slot].decoding()) return out; + SeqSlot & s = slots_[(size_t)slot]; + if (s.cur_pos >= max_ctx_) { + // No context left; the scheduler should have stopped this slot. + return out; + } + PagedKvAppendResult app = pool_.append( + s.handle, 1, /*only_first_last_slots=*/true); + if (!app || app.token_count != 1 || + app.last.logical_position != (uint32_t)s.cur_pos) { + out.busy = app.status == PagedKvStatus::BlocksExhausted; + return out; + } + s.sample_history.push_back(fed_token); + + out.ok = true; + out.physical_row = (int64_t)app.last.physical_token_index; + out.position = s.cur_pos; + if ((uint32_t)s.cur_pos % pool_.block_size() == 0) { + out.new_block = (int32_t)app.last.physical_block; + out.new_block_index = s.cur_pos / (int)pool_.block_size(); + } + return out; +} + +void SeqSlotManager::commit_step(int slot) { + if (!is_active(slot)) return; + slots_[(size_t)slot].cur_pos += 1; +} + +void SeqSlotManager::retire(int slot) { + if (slot < 0 || slot >= (int)slots_.size()) return; + SeqSlot & s = slots_[(size_t)slot]; + if (!s.active()) return; + const PagedKvStatus status = pool_.release(s.handle); + if (status != PagedKvStatus::Ok && status != PagedKvStatus::StaleHandle) { + std::fprintf(stderr, "[parallel] slot %d release failed: %s\n", + slot, paged_kv_status_string(status)); + } + s = SeqSlot{}; +} + +} // namespace dflash::common diff --git a/server/src/common/concurrency/seq_slot_manager.h b/server/src/common/concurrency/seq_slot_manager.h new file mode 100644 index 000000000..2699ebade --- /dev/null +++ b/server/src/common/concurrency/seq_slot_manager.h @@ -0,0 +1,131 @@ +// SeqSlotManager — complete host-side state for each concurrent serving slot. +// +// Companion of PagedKvPool: the pool hands out sequence handles and physical +// blocks; this class owns everything else a slot needs between admission and +// retirement — the pool-handle lifecycle (including every error path), the +// admission arithmetic (context clamp, prompt reservation, and rolling decode +// headroom), on-demand block allocation, per-slot sampler/RNG/penalty-history +// state, and the position counters. +// +// It deliberately owns NO device state. Prefill/decode allocation returns +// physical rows and block-table deltas as plain vectors. Prompt, KV ownership, +// sampler, and progress live together here; the scheduler keeps +// only its coarse request phase. +// +// Not thread-safe; the single scheduler thread is the only caller. + +#pragma once + +#include "common/paged_kv_pool.h" +#include "common/sampler.h" +#include "common/concurrency/seq_engine.h" + +#include +#include +#include +#include + +namespace dflash::common { + +enum class SeqSlotPhase { + free, + prefill, + decode, +}; + +struct SeqSlot { + SeqSlotPhase phase = SeqSlotPhase::free; + PagedKvSequenceHandle handle; + std::vector prompt; + int cur_pos = 0; + SamplerCfg sampler; + std::mt19937_64 rng{0x9E3779B97F4A7C15ull}; + // Penalty history is recorded as fed rather than sampled: the scheduler + // may override a sample before the model consumes it. + std::vector sample_history; + + bool active() const { return phase != SeqSlotPhase::free; } + bool prefilling() const { return phase == SeqSlotPhase::prefill; } + bool decoding() const { return phase == SeqSlotPhase::decode; } +}; + +class SeqSlotManager { +public: + // `max_ctx` is the per-sequence logical bound; slot count comes from the + // pool's max_sequences. The pool must outlive the manager. + SeqSlotManager(PagedKvPool & pool, int max_ctx); + + // Claim a free slot and atomically reserve all K/V blocks needed by the + // known prompt plus its next logical decode page when that page can exist + // in both max_ctx and the physical pool. Existing decoders are topped up + // first, so a younger admission cannot steal their next-page headroom. + // Prompts larger than the whole pool hard-fail; temporary capacity pressure + // reports busy. Seeds the slot RNG from sampler.seed only when the sampler + // actually draws, else nondeterministically. + SeqEngine::AdmitResult admit(uint64_t request_id, + const std::vector & prompt, + const SamplerCfg & sampler); + + struct PrefillChunk { + bool ok = false; + // The pool is temporarily out of blocks; retrying after another slot + // retires can succeed. BlocksExhausted leaves the pool unchanged. + bool busy = false; + std::vector rows; + // Delta to patch into the slot's device block-table column. + std::vector new_blocks; + int first_new_block = -1; + }; + + // Append `n_tokens` more prompt rows for a prefilling slot. Physical block + // ids come from the slot's admission reservation, so any append within the + // admitted prompt is guaranteed not to wait on another sequence. + PrefillChunk append_prefill(int slot, int n_tokens); + + // Record a finished prefill and expose the slot to decode. + void commit_prefill(int slot); + + struct StepAppend { + bool ok = false; + bool busy = false; // no physical block available right now + int64_t physical_row = -1; + int position = -1; // logical position the fed token is written at + int32_t new_block = -1; + int new_block_index = -1; + }; + + // Allocate the next decode token's cache row, report any new block-table + // entry, and log it to sample_history. cur_pos waits for commit_step(). + StepAppend append_token(int slot, int32_t fed_token); + + // The batched step's compute succeeded: cur_pos++. + void commit_step(int slot); + + // Release the slot's blocks and clear its state. Safe on inactive slots + // and after a failed admission/prefill. + void retire(int slot); + + int slot_count() const { return (int)slots_.size(); } + int max_context() const { return max_ctx_; } + int decoding_count() const; + bool is_active(int slot) const; + bool is_prefilling(int slot) const; + SeqSlot & slot(int i) { return slots_[(size_t)i]; } + const SeqSlot & slot(int i) const { return slots_[(size_t)i]; } + +private: + // Logical extent whose block count includes the sequence's current pages + // plus one future page, capped at max_ctx. + uint32_t decode_headroom_capacity(int logical_tokens) const; + bool capacity_fits_pool(uint32_t token_capacity) const; + + // Atomically preflight and top up every decoding slot as one cohort before + // a younger sequence may reserve capacity. + PagedKvStatus protect_decode_headroom(); + + PagedKvPool & pool_; + int max_ctx_ = 0; + std::vector slots_; +}; + +} // namespace dflash::common diff --git a/server/src/deepseek4/deepseek4_backend.cpp b/server/src/deepseek4/deepseek4_backend.cpp index c8750bb28..f0a216e10 100644 --- a/server/src/deepseek4/deepseek4_backend.cpp +++ b/server/src/deepseek4/deepseek4_backend.cpp @@ -4,6 +4,7 @@ #include "deepseek4_backend.h" #include "deepseek4_budget_hook.h" #include "deepseek4_internal.h" +#include "deepseek4_page_layout.h" #include "common/dynamic_backend.h" #include "common/peer_access.h" #include "common/platform_env.h" @@ -149,6 +150,17 @@ static bool env_int_in_range(const char * name, int fallback, return true; } +static bool is_gfx1151_device(int gpu) { +#if defined(DFLASH27B_BACKEND_HIP) || defined(GGML_USE_HIP) + cudaDeviceProp prop{}; + return cudaGetDeviceProperties(&prop, gpu) == cudaSuccess && + std::strncmp(prop.gcnArchName, "gfx1151", 7) == 0; +#else + (void) gpu; + return false; +#endif +} + static bool configure_dspark_mmvq_defaults(int gpu) { if (env_flag_enabled("DFLASH_DS4_Q6_VERIFY")) { std::fprintf(stderr, @@ -503,20 +515,19 @@ static uint64_t estimate_ds4_cache_bytes(const DeepSeek4Weights & w, int max_ctx const size_t comp_cap = (size_t) (max_ctx / (int) ratio) + 16; total_bytes += comp_cap * head_dim * sizeof(uint16_t); - const size_t window = (ratio == 4) ? 8 : ratio; - total_bytes += window * head_dim * sizeof(float) * 2; + const size_t state_rows = (ratio == 4) ? 8 : ratio; + const size_t comp_width = head_dim * (ratio == 4 ? 2 : 1); + total_bytes += state_rows * comp_width * sizeof(float) * 2; if (ratio == 4) { - // index_comp_kv is per-head: ne0 = n_indexer_head_dim (see - // deepseek4_graph.cpp). The full 64-head width lives only in the - // fixed-size state_kv/state_score scratch (index_state_rows rows), - // which does not scale with context. The old estimate multiplied - // by n_indexer_head here, overcounting 256K context by ~13x and - // falsely rejecting large contexts in hybrid placement. - total_bytes += comp_cap * (size_t) w.n_indexer_head_dim * sizeof(uint16_t); - total_bytes += window * (size_t) w.n_indexer_head_dim * sizeof(float) * 2; - total_bytes += (size_t) 2 * 2 * ratio * (size_t) w.n_indexer_head * - (size_t) w.n_indexer_head_dim * sizeof(float); + // index_comp_kv is per-head. The full multi-head width lives + // only in fixed-size state scratch and does not scale with context. + const size_t index_dim = (size_t) w.n_indexer_head_dim; + total_bytes += comp_cap * index_dim * sizeof(uint16_t); + total_bytes += state_rows * index_dim * sizeof(float) * 2; + total_bytes += (size_t) 2 * 2 * ratio * + (size_t) w.n_indexer_head * index_dim * + sizeof(float); } } @@ -748,7 +759,8 @@ DeepSeek4Backend::~DeepSeek4Backend() { } bool DeepSeek4Backend::requires_monolithic_model() const { - return cfg_.fused_decode || cfg_.fused_verify_f16_kv || + return cfg_.paged_attention || cfg_.fused_decode || + cfg_.fused_verify_f16_kv || prefill_attention_mode_is_approximate(cfg_.prefill_mode); } @@ -781,8 +793,9 @@ bool DeepSeek4Backend::load_model() { ? compiled_placement_backend() : cfg_.device.backend; - // Fused decode and layer-major prefill normally require monolithic expert - // residency. Heterogeneous TP is the exception: its fused graph owns the + // Paged concurrency, fused decode, and layer-major prefill require + // monolithic expert residency. Heterogeneous TP is the exception for + // non-paged modes: its fused graph owns the // routed experts across two local GPU backends, so forcing a full load would // disable the requested split before the TP runtime can initialize. const bool force_full = env_flag_enabled("DFLASH_DS4_FORCE_FULL_LOAD"); @@ -793,9 +806,10 @@ bool DeepSeek4Backend::load_model() { (force_full || need_monolithic)) { std::fprintf(stderr, "[deepseek4] monolithic execution requested " - "(forced=%s, fused_decode=%s, " + "(forced=%s, paged=%s, fused_decode=%s, " "fused_verify_f16_kv=%s, prefill=%s)\n", force_full ? "yes" : "no", + cfg_.paged_attention ? "on" : "off", cfg_.fused_decode ? "on" : "off", cfg_.fused_verify_f16_kv ? "on" : "off", prefill_attention_mode_name(cfg_.prefill_mode)); @@ -1028,6 +1042,25 @@ bool DeepSeek4Backend::supports_batched_spec_feature_capture( } bool DeepSeek4Backend::init() { + if (cfg_.paged_attention) { + const PlacementBackend target_backend = + cfg_.device.backend == PlacementBackend::Auto + ? compiled_placement_backend() : cfg_.device.backend; + if (target_backend != PlacementBackend::Hip || + !is_gfx1151_device(cfg_.device.gpu)) { + std::fprintf(stderr, + "[deepseek4] paged concurrency currently requires one " + "local Strix Halo (gfx1151) HIP target\n"); + return false; + } + if (env_flag_enabled("DFLASH_DS4_MOE_TP")) { + std::fprintf(stderr, + "[deepseek4] paged concurrency keeps all experts resident " + "on Strix Halo and cannot use DFLASH_DS4_MOE_TP\n"); + return false; + } + } + // The shared MMVQ/MMQ crossover defaults to q=3 for NVIDIA. On gfx1151, // DSpark q=4 is faster through MMVQ. Keep AR and other devices unchanged, // and preserve LUCE_MMVQ_MAX_NCOLS as an explicit override. @@ -1036,6 +1069,18 @@ bool DeepSeek4Backend::init() { } configure_gfx1201_hybrid_sub_batch_default(cfg_.device.gpu); + if (cfg_.paged_attention && + (cfg_.max_concurrency < 1 || cfg_.max_concurrency > 16 || + cfg_.device.is_layer_split() || + cfg_.prefill_mode != PrefillAttentionMode::Exact || + cfg_.fused_decode || env_flag_enabled("DFLASH_DS4_FUSED_DECODE") || + env_flag_enabled("DFLASH_DS4_SPEC"))) { + std::fprintf(stderr, + "[deepseek4] paged serving requires 1..16 local slots, exact " + "prefill, and autoregressive non-fused decode\n"); + return false; + } + backend_ = ggml_backend_cuda_init(cfg_.device.gpu); if (!backend_) { std::fprintf(stderr, "[deepseek4] failed to create CUDA backend (gpu=%d)\n", @@ -1059,15 +1104,48 @@ bool DeepSeek4Backend::init() { } const int max_ctx = cfg_.max_ctx > 0 ? cfg_.max_ctx : 8192; - if (!create_deepseek4_cache(backend_, w_, max_ctx, cache_)) { - std::fprintf(stderr, "[deepseek4] failed to allocate KV cache (ctx=%d)\n", max_ctx); - return false; + if (cfg_.paged_attention) { + uint64_t requested = cfg_.kv_pool_tokens > 0 + ? (uint64_t)cfg_.kv_pool_tokens + : (uint64_t)max_ctx * (uint64_t)cfg_.max_concurrency; + requested = std::max(requested, (uint64_t)max_ctx); + const uint64_t blocks64 = + (requested + DS4_PAGE_TOKENS - 1) / DS4_PAGE_TOKENS; + if (blocks64 == 0 || blocks64 > UINT32_MAX || + !create_deepseek4_paged_cache( + backend_, w_, (uint32_t)cfg_.max_concurrency, + (uint32_t)max_ctx, (uint32_t)blocks64, paged_cache_)) { + std::fprintf(stderr, + "[deepseek4] paged cache allocation failed (ctx=%d slots=%d " + "requested_pool_tokens=%llu); reduce --max-ctx/--max-concurrency " + "or set --kv-pool-tokens\n", max_ctx, cfg_.max_concurrency, + (unsigned long long)requested); + return false; + } + } else { + if (!create_deepseek4_cache(backend_, w_, max_ctx, cache_)) { + std::fprintf(stderr, "[deepseek4] failed to allocate KV cache (ctx=%d)\n", max_ctx); + return false; + } + cache_.prefill_mode = cfg_.prefill_mode; } - cache_.prefill_mode = cfg_.prefill_mode; if (env_flag_enabled("DFLASH_DS4_MOE_TP") && !init_moe_tensor_parallel()) { return false; } + if (cfg_.paged_attention && expert_runtime_.compute) { + std::fprintf(stderr, + "[deepseek4] paged serving cannot use the out-of-process expert " + "compute callback; select in-process DFLASH_DS4_MOE_TP or disable paged attention\n"); + return false; + } + if (cfg_.paged_attention && moe_hybrid_ && + !moe_hybrid_->materialized_cold_experts) { + std::fprintf(stderr, + "[deepseek4] paged serving requires statically materialized " + "expert ownership; enable in-process DFLASH_DS4_MOE_TP\n"); + return false; + } if (const char * stats_path = std::getenv("DFLASH_DS4_ROUTING_STATS_OUT")) { if (*stats_path) { @@ -1092,6 +1170,16 @@ bool DeepSeek4Backend::init() { std::fprintf(stderr, "[deepseek4-moe-tp] in-memory routing stats enabled\n"); } + if (cfg_.paged_attention) { + seq_engine_ = std::make_unique( + *this, *paged_cache_.pool, max_ctx, + paged_cache_.plan.max_blocks_per_sequence); + std::fprintf(stderr, + "[deepseek4-parallel] enabled %d slots, %u x %d-token physical " + "blocks; prefill is exact reference mode at one prompt token per slot per scheduler iteration\n", + cfg_.max_concurrency, paged_cache_.plan.physical_blocks, + DS4_PAGE_TOKENS); + } const int active_experts = w_.routed_expert_top_k > 0 ? w_.routed_expert_top_k : w_.n_expert_used; std::fprintf(stderr, @@ -1102,7 +1190,7 @@ bool DeepSeek4Backend::init() { prefill_attention_mode_name(cfg_.prefill_mode), moe_hybrid_ ? " [hybrid]" : ""); - if (env_flag_enabled("DFLASH_DS4_SPEC")) { + if (!cfg_.paged_attention && env_flag_enabled("DFLASH_DS4_SPEC")) { const char * dp = std::getenv("DFLASH_DS4_DRAFT"); if (dp && *dp) { spec_draft_path_ = dp; @@ -1670,7 +1758,10 @@ bool DeepSeek4Backend::init_hybrid_model() { void DeepSeek4Backend::print_ready_banner() const { std::printf("[deepseek4-daemon] ready layers=%d ctx=%d experts=%d/%d\n", - w_.n_layer, cache_.max_ctx, w_.n_expert_used, w_.n_expert); + w_.n_layer, + cfg_.paged_attention ? (int)paged_cache_.plan.max_ctx + : cache_.max_ctx, + w_.n_expert_used, w_.n_expert); std::fflush(stdout); } @@ -1684,6 +1775,12 @@ bool DeepSeek4Backend::park(ParkTarget target) { std::fflush(stdout); } if (!want_target_model || parked_) return true; + if (cfg_.paged_attention) { + std::fprintf(stderr, + "[deepseek4] target park is unavailable while paged serving owns " + "live graph and slot state\n"); + return false; + } maybe_save_routing_stats(); for (int i = 0; i < PREFIX_SLOTS; ++i) { @@ -2627,6 +2724,8 @@ void DeepSeek4Backend::shutdown() { for (int i = 0; i < PREFIX_SLOTS; i++) { snapshot_free(i); } + seq_engine_.reset(); + free_deepseek4_paged_cache(paged_cache_); free_deepseek4_cache(cache_); expert_runtime_.reset(); stream_engine_.destroy(); diff --git a/server/src/deepseek4/deepseek4_backend.h b/server/src/deepseek4/deepseek4_backend.h index 583132042..ab8e1b64b 100644 --- a/server/src/deepseek4/deepseek4_backend.h +++ b/server/src/deepseek4/deepseek4_backend.h @@ -15,6 +15,7 @@ #include "../common/moe_hybrid_stream.h" #include "deepseek4_internal.h" #include "deepseek4_dspark.h" +#include "deepseek4_seq_engine.h" #include "ggml.h" #include "ggml-backend.h" @@ -75,6 +76,7 @@ class DeepSeek4Backend : public ModelBackend { void free_drafter() override; void shutdown() override; + SeqEngine * seq_engine() override { return seq_engine_.get(); } const MoeHybridRoutingStats * get_routing_stats() const override { return routing_stats_.get(); @@ -87,6 +89,8 @@ class DeepSeek4Backend : public ModelBackend { ggml_backend_t expert_backend_ = nullptr; DeepSeek4Weights w_; DeepSeek4Cache cache_; + DeepSeek4PagedCache paged_cache_; + std::unique_ptr seq_engine_; bool parked_ = false; // Sampler @@ -178,6 +182,7 @@ class DeepSeek4Backend : public ModelBackend { MoeExpertComputeRuntime expert_runtime_; std::shared_ptr routing_stats_; std::string routing_stats_out_path_; + friend class DeepSeek4SeqEngine; }; } // namespace dflash::common diff --git a/server/src/deepseek4/deepseek4_fused_verify.inc b/server/src/deepseek4/deepseek4_fused_verify.inc index ee270cc7d..effa4dac2 100644 --- a/server/src/deepseek4/deepseek4_fused_verify.inc +++ b/server/src/deepseek4/deepseek4_fused_verify.inc @@ -237,7 +237,8 @@ static bool ds4_fused_read_route_matrix( static void ds4_fused_consume_route_diagnostics( DeepSeek4FusedDecodeGraph & fg, const MoeHybridStorage * hybrid, - MoeHybridRoutingStats * routing_stats) { + MoeHybridRoutingStats * routing_stats, + const int32_t * active_slots = nullptr) { const bool cache_audit = ds4_env_flag("DFLASH_DS4_TP_CACHE_AUDIT"); if (!routing_stats && !cache_audit) return; @@ -259,6 +260,7 @@ static void ds4_fused_consume_route_diagnostics( continue; } for (int token = 0; token < route.n_tokens; ++token) { + if (active_slots && active_slots[route.lane_start + token] < 0) continue; const int32_t * token_ids = ids.data() + (size_t) token * route.width; const float * token_weights = weights.data() + @@ -373,7 +375,10 @@ static bool ds4_build_fused_verify_graph( bool have_token_ids, const std::vector & capture_ids, MoeHybridStorage * hybrid, - std::vector && shape_key) { + std::vector && shape_key, + DeepSeek4PagedCache * paged_cache = nullptr, + const std::vector> * paged_rows = nullptr) { + const bool paged_mode = paged_cache && paged_rows; if (fg.sched) { ggml_backend_sched_free(fg.sched); fg.sched = nullptr; @@ -401,7 +406,11 @@ static bool ds4_build_fused_verify_graph( fg.sg.ctx = ggml_init(params); if (!fg.sg.ctx) return false; ggml_context * ctx = fg.sg.ctx; - constexpr size_t graph_capacity = 65536u; + // Gathered paged attention builds one q=1 MLA lane per concurrent + // sequence. Above eight lanes the resulting whole-model graph exceeds + // the verifier-era 64K scheduler hash table even though the metadata + // arena still has ample room. + const size_t graph_capacity = q > 8 ? 131072u : 65536u; fg.sg.gf = ggml_new_graph_custom(ctx, graph_capacity, false); ggml_cgraph * gf = fg.sg.gf; @@ -441,7 +450,8 @@ static bool ds4_build_fused_verify_graph( const int preserved_rows = q > 1 ? q : 0; mask_total += (int64_t) (w.n_swa + padded + preserved_rows) * q; } - fg.mask_bundle = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, mask_total); + fg.mask_bundle = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, + std::max(mask_total, 1)); ggml_set_input(fg.mask_bundle); int64_t mask_off = 0; @@ -481,7 +491,115 @@ static bool ds4_build_fused_verify_graph( // the strided working view once, replacing q kernels and q-1 concats. ggml_tensor * attn_in = ggml_cont(ctx, attn_working); - // ── Batched attention ── + // ── Attention ── + ggml_tensor * attn_out = nullptr; + if (paged_mode) { + // q denotes independent decode lanes here, not a causal sequence. + // Gather each lane's immutable chronological history and run the + // established MLA lane core at q=1; all surrounding HC/MoE/output + // machinery remains q-wide and unchanged. + if (q < 1 || q > 16 || paged_rows->size() != (size_t) w.n_layer || + (*paged_rows)[(size_t) il].size() != (size_t) q) return false; + DeepSeek4PagedLayerCache & plc = paged_cache->layers[(size_t) il]; + ggml_tensor * raw_flat = ggml_reshape_2d( + ctx, plc.raw_kv, w.head_dim, + (int64_t) DS4_PAGE_TOKENS * paged_cache->plan.slots); + for (int t = 0; t < q; ++t) { + const auto & rows = (*paged_rows)[(size_t) il][(size_t) t]; + auto & px = ex.paged.emplace_back(); + px.pos = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, 1); ggml_set_input(px.pos); + px.neg_pos = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, 1); ggml_set_input(px.neg_pos); + px.raw_gather = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, + std::max(rows.raw_history.size(), 1)); ggml_set_input(px.raw_gather); + ggml_tensor * raw_history = rows.raw_history.empty() ? nullptr + : ggml_get_rows(ctx, raw_flat, px.raw_gather); + ggml_tensor * comp_history = nullptr; + ggml_tensor * index_history = nullptr; + if (ratio > 0) { + px.comp_gather = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, + std::max(rows.compressed_history.size(), 1)); + ggml_set_input(px.comp_gather); + if (!rows.compressed_history.empty()) + comp_history = ggml_get_rows(ctx, plc.comp_kv, px.comp_gather); + if (ratio == 4) { + px.index_gather = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, + std::max(rows.compressed_history.size(), 1)); + ggml_set_input(px.index_gather); + if (!rows.compressed_history.empty()) + index_history = ggml_get_rows(ctx, plc.index_comp_kv, px.index_gather); + } + } + px.raw_write = ggml_new_tensor_1d(ctx, GGML_TYPE_I64, 1); ggml_set_input(px.raw_write); + if (ratio > 0) { + px.comp_write = ggml_new_tensor_1d(ctx, GGML_TYPE_I64, 1); ggml_set_input(px.comp_write); + px.comp_read = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, 1); ggml_set_input(px.comp_read); + px.ape = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, 1); ggml_set_input(px.ape); + px.state_row = ggml_new_tensor_1d(ctx, GGML_TYPE_I64, 1); ggml_set_input(px.state_row); + px.comp_pos = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, 1); ggml_set_input(px.comp_pos); + } + DeepSeek4CompressorState attn_state{}, index_state{}; + if (ratio > 0 && rows.slot >= 0) { + const size_t off = (size_t) rows.slot * plc.attn_compressor.state_kv->nb[2]; + attn_state.state_kv = ggml_view_2d(ctx, plc.attn_compressor.state_kv, + plc.attn_compressor.state_kv->ne[0], plc.attn_compressor.state_kv->ne[1], + plc.attn_compressor.state_kv->nb[1], off); + attn_state.state_score = ggml_view_2d(ctx, plc.attn_compressor.state_score, + plc.attn_compressor.state_score->ne[0], plc.attn_compressor.state_score->ne[1], + plc.attn_compressor.state_score->nb[1], off); + if (ratio == 4) { + const size_t io = (size_t) rows.slot * plc.indexer_compressor.state_kv->nb[2]; + index_state.state_kv = ggml_view_2d(ctx, plc.indexer_compressor.state_kv, + plc.indexer_compressor.state_kv->ne[0], plc.indexer_compressor.state_kv->ne[1], + plc.indexer_compressor.state_kv->nb[1], io); + index_state.state_score = ggml_view_2d(ctx, plc.indexer_compressor.state_score, + plc.indexer_compressor.state_score->ne[0], plc.indexer_compressor.state_score->ne[1], + plc.indexer_compressor.state_score->nb[1], io); + } + } + DeepSeek4MlaLaneBindings lane{}; + lane.history_mode = DeepSeek4MlaLaneBindings::HistoryMode::ChronologicalGathered; + lane.raw_history = raw_history; lane.n_raw_history = (int) rows.raw_history.size(); + lane.comp_history = comp_history; lane.n_comp_history = (int) rows.compressed_history.size(); + lane.index_comp_history = index_history; + lane.n_index_comp_history = (int) rows.compressed_history.size(); + // Writes use the same flattened physical-row geometry as the + // gather indices (not the slot-local 128-row second axis). + lane.raw_kv = raw_flat; lane.comp_kv = plc.comp_kv; + lane.index_comp_kv = plc.index_comp_kv; + lane.raw_write_rows = px.raw_write; lane.comp_write_rows = px.comp_write; + lane.index_comp_write_rows = px.comp_write; + lane.comp_read_rows = px.comp_read; + lane.index_comp_read_rows = px.comp_read; + lane.write_enabled = rows.slot >= 0; + lane.attn_compressor = &attn_state; lane.indexer_compressor = &index_state; + DeepSeek4AttentionGraphInputs ci{}; + ci.rope_pos = px.pos; ci.neg_pos = px.neg_pos; + ci.attn_ape_row = px.ape; ci.attn_state_rows = px.state_row; + ci.attn_comp_pos = px.comp_pos; ci.index_ape_row = px.ape; + ci.index_state_rows = px.state_row; ci.index_comp_pos = px.comp_pos; + std::vector ib; + std::vector iab; + std::vector lab; + ggml_tensor * col = ggml_view_2d(ctx, attn_in, n_embd, 1, + attn_in->nb[1], (size_t) t * attn_in->nb[1]); + ggml_tensor * one = build_mla_attention_lane_core( + ctx, gf, build_rms_norm(ctx, col, L.attn_norm, w.rms_eps), + w, L, lane, il, (int) rows.position, 1, &ci, ib, iab, lab, + nullptr, DeepSeek4AttentionImpl::Explicit); + if (!one || !ib.empty() || !iab.empty() || !lab.empty()) { + std::fprintf(stderr, + "[deepseek4-paged] layer %d lane %d attention build " + "failed (graph=%d i32=%zu arrays=%zu i64=%zu, " + "first_array=%d)\n", + il, t, one != nullptr, ib.size(), iab.size(), lab.size(), + iab.empty() || iab[0].values.empty() + ? INT32_MIN : iab[0].values[0]); + return false; + } + attn_out = attn_out ? ggml_concat(ctx, attn_out, one, 1) : one; + } + } else { + // ── Batched speculative attention ── DeepSeek4AttentionGraphInputs ain{}; ain.rope_pos = ex.pos_q; ain.neg_pos = ex.neg_q; @@ -535,7 +653,7 @@ static bool ds4_build_fused_verify_graph( std::vector i32ab; std::vector i64ab; ggml_tensor * normed = build_rms_norm(ctx, attn_in, L.attn_norm, w.rms_eps); - ggml_tensor * attn_out = build_mla_attention(ctx, gf, normed, w, L, lc, il, + attn_out = build_mla_attention(ctx, gf, normed, w, L, lc, il, lane_kv_start, lane_q, &ain, i32b, i32ab, i64ab); if (!attn_out) return false; @@ -543,6 +661,7 @@ static bool ds4_build_fused_verify_graph( std::fprintf(stderr, "[ds4-fused-verify] layer %d dynamic bindings; cannot fuse\n", il); return false; } + } // ── Batched HC post (attention) + HC pre (FFN) ── ggml_tensor * attn_batch = ggml_is_contiguous(attn_out) @@ -749,7 +868,11 @@ static bool ds4_build_fused_verify_graph( } } } - if (!ffn_out) return false; + if (!ffn_out) { + std::fprintf(stderr, + "[deepseek4-paged] layer %d FFN graph build failed\n", il); + return false; + } // ── Batched HC post (FFN); capture at drafter layers ── if (fused_hc_join_inputs) { @@ -788,7 +911,7 @@ static bool ds4_build_fused_verify_graph( fg.logits = ggml_mul_mat(ctx, w.output, out_normed); // [n_vocab, q] ggml_set_output(fg.logits); ggml_build_forward_expand(gf, fg.logits); - if (ds4_env_flag("DFLASH_DS4_GPU_ARGMAX_VERIFY")) { + if (paged_mode || ds4_env_flag("DFLASH_DS4_GPU_ARGMAX_VERIFY")) { ex.argmax = ggml_argmax(ctx, fg.logits); ggml_set_output(ex.argmax); ggml_build_forward_expand(gf, ex.argmax); @@ -914,6 +1037,12 @@ static bool ds4_build_fused_verify_graph( pin_main(fg.i32_bundle); pin_main(fg.i64_bundle); pin_main(fg.mask_bundle); + for (const auto & px : ex.paged) { + pin_main(px.pos); pin_main(px.neg_pos); pin_main(px.raw_gather); pin_main(px.comp_gather); + pin_main(px.index_gather); + pin_main(px.raw_write); pin_main(px.comp_write); pin_main(px.comp_read); pin_main(px.ape); + pin_main(px.state_row); pin_main(px.comp_pos); + } for (ggml_tensor * hids : fg.hash_ids) pin_main(hids); for (const MoeHybridGraphInputs & inputs : fg.hybrid_inputs) { if (mixed_policy.pin_route_weights) { diff --git a/server/src/deepseek4/deepseek4_graph.cpp b/server/src/deepseek4/deepseek4_graph.cpp index 8986328b1..ec3507045 100644 --- a/server/src/deepseek4/deepseek4_graph.cpp +++ b/server/src/deepseek4/deepseek4_graph.cpp @@ -11,6 +11,7 @@ #include "deepseek4_internal.h" #include "deepseek4_hc_cuda.h" #include "deepseek4_roctx.h" +#include "deepseek4_page_layout.h" #include "internal.h" #include "../common/step_graph.h" #include "../common/cuda_graph_overrides.h" @@ -999,7 +1000,9 @@ static void build_compressor_step( ggml_tensor * cur_all = nullptr, int n_tokens_all = 1, int kv_start_all = -1, - bool indexer_qat = false) { + bool indexer_qat = false, + ggml_tensor ** current_comp_out = nullptr, + bool paged_physical_row = false) { if (!gf || !cur_last || !ape || !kv_proj || !gate_proj || !norm_weight || !state.state_kv || !state.state_score || !comp_cache || ratio <= 0) { return; @@ -1210,10 +1213,14 @@ static void build_compressor_step( if (indexer_qat) { pooled = ggml_ds4_indexer_qat(ctx, ggml_cont(ctx, pooled)); } + if (current_comp_out) { + *current_comp_out = pooled; + } ggml_tensor * pooled_f16 = ggml_cast(ctx, pooled, GGML_TYPE_F16); const int comp_row = token_pos / ratio; - if (comp_row >= (int) comp_cache->ne[1]) { + if ((!comp_rows_inp || !paged_physical_row) && + comp_row >= (int) comp_cache->ne[1]) { return; } @@ -1418,7 +1425,8 @@ static void build_indexer_compressor_step( ggml_tensor * cur_last, const DeepSeek4Weights & w, const DeepSeek4Layer & L, - DeepSeek4LayerCache & lc, + DeepSeek4CompressorState & indexer_compressor, + ggml_tensor * index_comp_kv, int token_pos, ggml_tensor * ape_row_inp, ggml_tensor * state_rows_inp, @@ -1431,14 +1439,16 @@ static void build_indexer_compressor_step( ggml_tensor * cur_all = nullptr, int n_tokens_all = 1, int kv_start_all = -1, - bool indexer_qat = false) { + bool indexer_qat = false, + ggml_tensor ** current_comp_out = nullptr, + bool paged_physical_row = false) { build_compressor_step(ctx, gf, cur_last, L.indexer_compressor_ape, L.indexer_compressor_kv, L.indexer_compressor_gate, L.indexer_compressor_norm, - lc.indexer_compressor, - lc.index_comp_kv, + indexer_compressor, + index_comp_kv, 4, w.n_indexer_head_dim, // indexer head_dim = 128 token_pos, @@ -1460,7 +1470,9 @@ static void build_indexer_compressor_step( cur_all, n_tokens_all, kv_start_all, - indexer_qat); + indexer_qat, + current_comp_out, + paged_physical_row); } static int ds4_comp_rows_used(const ggml_tensor * comp_cache, int n_cached, int ratio, int token_pos) { @@ -1613,13 +1625,86 @@ static ggml_tensor * build_indexer_topk( // ─── MLA Attention Block ──────────────────────────────────────────────── -static ggml_tensor * build_mla_attention( +// All persistent and live-state bindings consumed by one MLA lane. Keeping +// this internal seam tensor-based is intentional: a paged adapter can later +// supply gathered history and slot-specific compressor state without the +// graph builder consulting DeepSeek4LayerCache or host cache counters. +struct DeepSeek4MlaLaneBindings { + enum class HistoryMode { + ContiguousRing, + ChronologicalGathered, + }; + + HistoryMode history_mode = HistoryMode::ContiguousRing; + // In gathered mode these are immutable, chronological attention inputs. + // Counts are explicit so adapters may bind capacity-padded tensors. + ggml_tensor * raw_history = nullptr; + int n_raw_history = 0; + ggml_tensor * comp_history = nullptr; + int n_comp_history = 0; + ggml_tensor * index_comp_history = nullptr; + int n_index_comp_history = 0; + + // Persistent mutation targets are deliberately independent of history. + ggml_tensor * raw_kv = nullptr; + ggml_tensor * comp_kv = nullptr; + ggml_tensor * index_comp_kv = nullptr; + ggml_tensor * raw_write_rows = nullptr; + ggml_tensor * comp_write_rows = nullptr; + ggml_tensor * index_comp_write_rows = nullptr; + ggml_tensor * comp_read_rows = nullptr; // GET_ROWS requires I32 + ggml_tensor * index_comp_read_rows = nullptr; + + // Optional passive outputs let a future adapter scatter current products. + ggml_tensor ** current_raw_out = nullptr; + ggml_tensor ** current_comp_out = nullptr; + ggml_tensor ** current_index_comp_out = nullptr; + // False is the padding/inactive-lane contract: build attention against the + // supplied padded history, but emit no persistent current-row mutations. + bool write_enabled = true; + DeepSeek4CompressorState * attn_compressor = nullptr; + DeepSeek4CompressorState * indexer_compressor = nullptr; + int n_comp_live = 0; + int n_index_comp_live = 0; + int n_comp_committed = 0; +}; + +// Projection/RoPE products handed to the history/update portion of a lane. +// This is deliberately a passive bundle: introducing graph operations in a +// separate builder would risk changing decode graph ordering. +struct DeepSeek4PreparedProjectedLane { + ggml_tensor * normalized_q_lora = nullptr; + ggml_tensor * q = nullptr; + ggml_tensor * kv = nullptr; + ggml_tensor * rope_pos = nullptr; +}; + +static DeepSeek4MlaLaneBindings deepseek4_contiguous_lane_bindings( + DeepSeek4LayerCache & lc, + int ratio, + int token_pos) { + DeepSeek4MlaLaneBindings lane; + lane.history_mode = DeepSeek4MlaLaneBindings::HistoryMode::ContiguousRing; + lane.raw_kv = lc.raw_kv; + lane.comp_kv = lc.comp_kv; + lane.index_comp_kv = lc.index_comp_kv; + lane.attn_compressor = &lc.attn_compressor; + lane.indexer_compressor = &lc.indexer_compressor; + lane.n_comp_live = ratio > 0 + ? ds4_comp_rows_used(lc.comp_kv, lc.n_comp, ratio, token_pos) : 0; + lane.n_index_comp_live = ratio == 4 + ? ds4_comp_rows_used(lc.index_comp_kv, lc.n_index_comp, 4, token_pos) : 0; + lane.n_comp_committed = lc.n_comp; + return lane; +} + +static ggml_tensor * build_mla_attention_lane_core( ggml_context * ctx, ggml_cgraph * gf, ggml_tensor * cur, // [n_embd, n_tokens] const DeepSeek4Weights & w, const DeepSeek4Layer & L, - DeepSeek4LayerCache & lc, + const DeepSeek4MlaLaneBindings & lane, int layer_idx, int kv_start, int n_tokens, @@ -1637,6 +1722,8 @@ static ggml_tensor * build_mla_attention( const int n_out_group = w.n_out_group; const int n_lora_o = w.n_lora_o; const int ratio = w.compress_ratios[layer_idx]; + const bool gathered_history = lane.history_mode == + DeepSeek4MlaLaneBindings::HistoryMode::ChronologicalGathered; // ── Q path: cur → q_a → norm → q_b → per-head norm ───────────── // q_a: [n_embd, n_tokens] → [n_lora_q, n_tokens] @@ -1694,6 +1781,11 @@ static ggml_tensor * build_mla_attention( rope_freq, rope_scale, rope_ext, rope_attn, w.rope_yarn_beta_fast, w.rope_yarn_beta_slow, rope_n_ctx_orig); + const DeepSeek4PreparedProjectedLane projected = {qr, q, kv, rope_pos}; + // Keep the established local names below to make the no-topology-change + // property obvious; the bundle is the handoff seam for a future adapter. + (void) projected; + // ── Causal batched step (exact multi-token target semantics) ─── // The target model is causal: token i must not attend to batch tokens // j > i, must see the compressed-row count as of its own position, and — @@ -1709,15 +1801,15 @@ static ggml_tensor * build_mla_attention( ggml_tensor * old_rows_scratch_f16 = nullptr; int n_old_rows = 0; ggml_tensor * prior_rows_scratch = nullptr; - int n_prior_rows = 0; + int n_prior_rows = gathered_history ? lane.n_raw_history : 0; const bool fused_causal = cached_inputs && cached_inputs->attn_row_mask && n_tokens > 1; - if (fused_causal) { + if (!gathered_history && fused_causal) { // Fused verify: ALWAYS q preserved rows so the topology is stable; // unwrapped/garbage rows are masked by the host-filled mask values. for (int ti = 0; ti < n_tokens; ti++) { ggml_tensor * slot = ggml_view_2d( - ctx, lc.raw_kv, head_dim, 1, lc.raw_kv->nb[1], - (size_t)((kv_start + ti) % w.n_swa) * lc.raw_kv->nb[1]); + ctx, lane.raw_kv, head_dim, 1, lane.raw_kv->nb[1], + (size_t)((kv_start + ti) % w.n_swa) * lane.raw_kv->nb[1]); ggml_tensor * saved = ggml_cont(ctx, slot); ggml_build_forward_expand(gf, saved); old_rows_scratch = old_rows_scratch @@ -1726,14 +1818,14 @@ static ggml_tensor * build_mla_attention( } old_rows_scratch_f16 = old_rows_scratch; old_rows_scratch = ds4_cast_if_needed(ctx, old_rows_scratch, GGML_TYPE_F32); - } else if (causal_batch && !layer_major_batch) { + } else if (!gathered_history && causal_batch && !layer_major_batch) { // Copy the to-be-overwritten rows FIRST; same-stream build order runs // these before the ring writes below. for (int ti = 0; ti < n_tokens; ti++) { if (kv_start + ti < w.n_swa) continue; // slot never held an older pos ggml_tensor * slot = ggml_view_2d( - ctx, lc.raw_kv, head_dim, 1, lc.raw_kv->nb[1], - (size_t)((kv_start + ti) % w.n_swa) * lc.raw_kv->nb[1]); + ctx, lane.raw_kv, head_dim, 1, lane.raw_kv->nb[1], + (size_t)((kv_start + ti) % w.n_swa) * lane.raw_kv->nb[1]); ggml_tensor * saved = ggml_cont(ctx, slot); ggml_build_forward_expand(gf, saved); old_rows_scratch = old_rows_scratch @@ -1743,7 +1835,7 @@ static ggml_tensor * build_mla_attention( if (old_rows_scratch) { old_rows_scratch = ds4_cast_if_needed(ctx, old_rows_scratch, GGML_TYPE_F32); } - } else if (layer_major_batch) { + } else if (!gathered_history && layer_major_batch) { // Snapshot the chronological pre-chunk window before any ring writes. // Attention then consumes [prior F16 rows | current F32 rows], matching // the single-token path and avoiding an F16 round-trip for this chunk. @@ -1753,8 +1845,8 @@ static ggml_tensor * build_mla_attention( const int tail = std::min(n_prior_rows, w.n_swa - first); auto snapshot_span = [&](int row, int count) { ggml_tensor * span = ggml_view_2d( - ctx, lc.raw_kv, head_dim, count, lc.raw_kv->nb[1], - (size_t) row * lc.raw_kv->nb[1]); + ctx, lane.raw_kv, head_dim, count, lane.raw_kv->nb[1], + (size_t) row * lane.raw_kv->nb[1]); return ggml_cont(ctx, span); }; prior_rows_scratch = snapshot_span(first, tail); @@ -1772,13 +1864,18 @@ static ggml_tensor * build_mla_attention( // ── Store ALL KV rows in the raw SWA ring ───────────────────── // For decode (n_tokens=1): write single row. For prefill: write all rows. - ggml_tensor * raw_kv_source = lc.raw_kv; - ggml_tensor * raw_kv_rows = cached_inputs - ? cached_inputs->raw_kv_rows - : nullptr; - if (raw_kv_rows) { + ggml_tensor * raw_kv_source = lane.raw_kv; + ggml_tensor * raw_kv_rows = lane.raw_write_rows + ? lane.raw_write_rows + : (cached_inputs ? cached_inputs->raw_kv_rows : nullptr); + if (lane.current_raw_out) { + *lane.current_raw_out = kv; + } + if (!lane.write_enabled) { + // Inactive/padding lanes intentionally have no cache mutation. + } else if (raw_kv_rows) { ggml_tensor * kv_f32 = ggml_is_contiguous(kv) ? kv : ggml_cont(ctx, kv); - raw_kv_source = ggml_set_rows(ctx, lc.raw_kv, kv_f32, raw_kv_rows); + raw_kv_source = ggml_set_rows(ctx, lane.raw_kv, kv_f32, raw_kv_rows); ggml_build_forward_expand(gf, raw_kv_source); } else { // The attention graph consumes the whole current ubatch directly. @@ -1790,8 +1887,8 @@ static ggml_tensor * build_mla_attention( ggml_tensor * kv_row = ggml_view_2d( ctx, kv, head_dim, 1, kv->nb[1], (size_t)ti * kv->nb[1]); ggml_tensor * kv_slot = ggml_view_2d( - ctx, lc.raw_kv, head_dim, 1, lc.raw_kv->nb[1], - (size_t)(pos_ti % w.n_swa) * lc.raw_kv->nb[1]); + ctx, lane.raw_kv, head_dim, 1, lane.raw_kv->nb[1], + (size_t)(pos_ti % w.n_swa) * lane.raw_kv->nb[1]); ggml_build_forward_expand(gf, ggml_cpy(ctx, ggml_cast(ctx, kv_row, GGML_TYPE_F16), kv_slot)); } } @@ -1800,15 +1897,15 @@ static ggml_tensor * build_mla_attention( // ── Learned compression update ────────────────────────────────── ggml_tensor * cur_last = ggml_view_2d( ctx, cur, n_embd, 1, cur->nb[1], (size_t)(n_tokens - 1) * cur->nb[1]); - ggml_tensor * comp_kv_source = lc.comp_kv; - if (ratio > 0 && L.attn_compressor_kv) { + ggml_tensor * comp_kv_source = lane.comp_kv; + if (lane.write_enabled && ratio > 0 && L.attn_compressor_kv) { build_compressor_step(ctx, gf, cur_last, L.attn_compressor_ape, L.attn_compressor_kv, L.attn_compressor_gate, L.attn_compressor_norm, - lc.attn_compressor, - lc.comp_kv, + *lane.attn_compressor, + lane.comp_kv, ratio, head_dim, token_pos, @@ -1821,7 +1918,8 @@ static ggml_tensor * build_mla_attention( (int)w.rope_orig_ctx, cached_inputs ? cached_inputs->attn_ape_row : nullptr, cached_inputs ? cached_inputs->attn_state_rows : nullptr, - cached_inputs ? cached_inputs->attn_comp_rows : nullptr, + lane.comp_write_rows ? lane.comp_write_rows : + (cached_inputs ? cached_inputs->attn_comp_rows : nullptr), cached_inputs ? cached_inputs->attn_comp_pos : nullptr, i64_array_inputs, i32_array_inputs, @@ -1829,15 +1927,20 @@ static ggml_tensor * build_mla_attention( cached_inputs ? cached_inputs->flush_rows : nullptr, (causal_batch || fused_causal) ? cur : nullptr, n_tokens, - kv_start); + kv_start, + false, + lane.current_comp_out, + gathered_history); } - ggml_tensor * index_comp_kv_source = lc.index_comp_kv; - if (ratio == 4 && L.indexer_compressor_kv) { - build_indexer_compressor_step(ctx, gf, cur_last, w, L, lc, token_pos, + ggml_tensor * index_comp_kv_source = lane.index_comp_kv; + if (lane.write_enabled && ratio == 4 && L.indexer_compressor_kv) { + build_indexer_compressor_step(ctx, gf, cur_last, w, L, + *lane.indexer_compressor, lane.index_comp_kv, token_pos, cached_inputs ? cached_inputs->index_ape_row : nullptr, cached_inputs ? cached_inputs->index_state_rows : nullptr, - cached_inputs ? cached_inputs->index_comp_rows : nullptr, + lane.index_comp_write_rows ? lane.index_comp_write_rows : + (cached_inputs ? cached_inputs->index_comp_rows : nullptr), cached_inputs ? cached_inputs->index_comp_pos : nullptr, i64_array_inputs, i32_array_inputs, @@ -1847,7 +1950,9 @@ static ggml_tensor * build_mla_attention( n_tokens, kv_start, attention_impl == - DeepSeek4AttentionImpl::SparseFlash); + DeepSeek4AttentionImpl::SparseFlash, + lane.current_index_comp_out, + gathered_history); } // ── MLA Dot-Product Attention (SWA + compressed KV) ──────────── @@ -1856,43 +1961,67 @@ static ggml_tensor * build_mla_attention( // comp_kv: [head_dim, comp_cap] F16 compressed rows. // n_raw = min(kv_start + n_tokens, n_swa) const bool masked_kv = cached_inputs && cached_inputs->attn_row_mask; - const int n_comp_live = (ratio > 0) ? ds4_comp_rows_used(lc.comp_kv, lc.n_comp, ratio, token_pos) : 0; + const bool gathered_emits_comp = gathered_history && lane.write_enabled && + ratio > 0 && ((token_pos + 1) % ratio) == 0; + const int n_comp_live = gathered_history + ? lane.n_comp_history + (gathered_emits_comp ? 1 : 0) : lane.n_comp_live; + ggml_tensor * comp_history_source = gathered_history + ? lane.comp_history : comp_kv_source; + ggml_tensor * index_comp_history_source = gathered_history + ? lane.index_comp_history : index_comp_kv_source; + if (gathered_emits_comp) { + // Gather through the post-update source to make the compressor write a + // graph dependency. Reading the F16 cache row preserves ordinary q=1 + // rounding at a boundary instead of feeding the transient F32 pool. + ggml_tensor * emitted = ggml_get_rows( + ctx, comp_kv_source, lane.comp_read_rows); + comp_history_source = lane.comp_history + ? ggml_concat(ctx, lane.comp_history, emitted, 1) : emitted; + if (ratio == 4) { + ggml_tensor * index_emitted = ggml_get_rows( + ctx, index_comp_kv_source, lane.index_comp_read_rows); + index_comp_history_source = lane.index_comp_history + ? ggml_concat(ctx, lane.index_comp_history, index_emitted, 1) + : index_emitted; + } + } ggml_tensor * indexer_topk = nullptr; if (attention_impl == DeepSeek4AttentionImpl::SparseFlash && ratio == 4 && f32_array_inputs) { - const int n_index_comp_live = ds4_comp_rows_used( - lc.index_comp_kv, lc.n_index_comp, 4, token_pos); - // Attention and index compression advance together at ratio 4. Reusing - // the attention mask is safe only while that invariant and the index - // buffer capacity hold; fail at graph construction if state diverges. - GGML_ASSERT(lc.index_comp_kv && index_comp_kv_source); - GGML_ASSERT(n_index_comp_live == n_comp_live); - GGML_ASSERT(!masked_kv || - cached_inputs->padded_comp <= lc.index_comp_kv->ne[1]); - // Use the same padded span as attention in a replayable decode graph. - // The dynamic compressed portion of attn_row_mask is added to the - // indexer scores, so padding stays invisible while live rows can grow - // within the fixed graph shape. - const int n_index_comp = masked_kv - ? cached_inputs->padded_comp - : n_index_comp_live; + int n_index_comp = 0; ggml_tensor * index_visibility_mask = nullptr; - if (masked_kv && n_index_comp > 0) { - index_visibility_mask = ggml_view_2d( - ctx, cached_inputs->attn_row_mask, - n_index_comp, 1, - (size_t) n_index_comp * sizeof(float), - (size_t) w.n_swa * sizeof(float)); + if (gathered_history) { + n_index_comp = lane.n_index_comp_history + + (gathered_emits_comp ? 1 : 0); + } else { + const int n_index_comp_live = ds4_comp_rows_used( + lc.index_comp_kv, lc.n_index_comp, 4, token_pos); + // Attention and index compression advance together at ratio 4. + GGML_ASSERT(lc.index_comp_kv && index_comp_kv_source); + GGML_ASSERT(n_index_comp_live == n_comp_live); + GGML_ASSERT(!masked_kv || + cached_inputs->padded_comp <= lc.index_comp_kv->ne[1]); + n_index_comp = masked_kv + ? cached_inputs->padded_comp + : n_index_comp_live; + if (masked_kv && n_index_comp > 0) { + index_visibility_mask = ggml_view_2d( + ctx, cached_inputs->attn_row_mask, + n_index_comp, 1, + (size_t) n_index_comp * sizeof(float), + (size_t) w.n_swa * sizeof(float)); + } } indexer_topk = build_indexer_topk( - ctx, qr, cur, w, L, index_comp_kv_source, + ctx, qr, cur, w, L, index_comp_history_source, n_index_comp, kv_start, n_tokens, rope_pos, index_visibility_mask, i32_array_inputs); } // Stable path reads the full physical ring (masking not-yet-written slots) // and a padded compressed-row span; the plain path reads only valid rows. - const int n_raw = masked_kv ? w.n_swa + const int n_raw = gathered_history ? lane.n_raw_history + n_tokens + : masked_kv ? w.n_swa : layer_major_batch ? n_prior_rows + n_tokens : std::min(kv_start + n_tokens, w.n_swa); const int n_comp_attn = masked_kv ? cached_inputs->padded_comp : n_comp_live; @@ -1904,13 +2033,24 @@ static ggml_tensor * build_mla_attention( // write and see the previous contents of the raw KV slot. auto raw_kv_view = [&](int row, int count) -> ggml_tensor * { ggml_tensor * view = ggml_view_2d( - ctx, lc.raw_kv, head_dim, count, lc.raw_kv->nb[1], - (size_t)row * lc.raw_kv->nb[1]); + ctx, lane.raw_kv, head_dim, count, lane.raw_kv->nb[1], + (size_t)row * lane.raw_kv->nb[1]); return ds4_cast_if_needed(ctx, view, GGML_TYPE_F32); }; ggml_tensor * kv_attn = nullptr; - if (masked_kv) { + if (gathered_history) { + ggml_tensor * current = ds4_cast_if_needed(ctx, kv, GGML_TYPE_F32); + if (lane.n_raw_history > 0 && lane.raw_history) { + ggml_tensor * history = ggml_view_2d( + ctx, lane.raw_history, head_dim, lane.n_raw_history, + lane.raw_history->nb[1], 0); + history = ds4_cast_if_needed(ctx, history, GGML_TYPE_F32); + kv_attn = ggml_concat(ctx, history, current, 1); + } else { + kv_attn = current; + } + } else if (masked_kv) { // Fused stable-KV path: read the full physical ring; rows not yet // written are masked to -1e30 in the score matrix (exact 0 after // softmax). Only the fused decode graph sets attn_row_mask. Read @@ -1934,7 +2074,7 @@ static ggml_tensor * build_mla_attention( // KV at its runtime row in an F32 snapshot instead. The tokenwise // prefill helper takes the same branch and row ordering. ggml_tensor * ring = ggml_view_2d( - ctx, lc.raw_kv, head_dim, w.n_swa, lc.raw_kv->nb[1], 0); + ctx, lane.raw_kv, head_dim, w.n_swa, lane.raw_kv->nb[1], 0); ring = ds4_cast_if_needed(ctx, ring, GGML_TYPE_F32); kv_attn = ggml_set_rows(ctx, ring, cur_kv, raw_kv_rows); ggml_build_forward_expand(gf, kv_attn); @@ -1959,21 +2099,20 @@ static ggml_tensor * build_mla_attention( attention_impl == DeepSeek4AttentionImpl::Explicit && kv_attn->type == GGML_TYPE_F32 && raw_kv_source->type == GGML_TYPE_F16 && - (!comp_kv_source || comp_kv_source->type == GGML_TYPE_F16) && + (!comp_history_source || + comp_history_source->type == GGML_TYPE_F16) && (!old_rows_scratch_f16 || old_rows_scratch_f16->type == GGML_TYPE_F16); if (fused_explicit_f16_kv) { // DS4's persistent MLA caches are already F16. Feed those tensors // directly to the established explicit attention matmuls instead of // casting the entire long-context cache to F32 on every verifier step. - // Current writes are consumed through their set_rows results, while - // preserved overwritten rows retain the same cached F16 values. kv_attn = ggml_view_2d( ctx, raw_kv_source, head_dim, n_raw, raw_kv_source->nb[1], 0); - if (n_comp_attn > 0 && comp_kv_source) { + if (n_comp_attn > 0 && comp_history_source) { ggml_tensor * comp = ggml_view_2d( - ctx, comp_kv_source, head_dim, n_comp_attn, - comp_kv_source->nb[1], 0); + ctx, comp_history_source, head_dim, n_comp_attn, + comp_history_source->nb[1], 0); kv_attn = ggml_concat(ctx, kv_attn, comp, 1); } if (old_rows_scratch_f16) { @@ -1986,13 +2125,12 @@ static ggml_tensor * build_mla_attention( "[deepseek4] fused explicit F16 K/V active: tokens=%d " "compressed=%d\n", n_tokens, n_comp_attn); - explicit_f16_kv_logged = true; } } else { - if (n_comp_attn > 0 && comp_kv_source) { + if (n_comp_attn > 0 && comp_history_source) { ggml_tensor * comp = ggml_view_2d( - ctx, comp_kv_source, head_dim, n_comp_attn, - comp_kv_source->nb[1], 0); + ctx, comp_history_source, head_dim, n_comp_attn, + comp_history_source->nb[1], 0); comp = ds4_cast_if_needed(ctx, comp, GGML_TYPE_F32); kv_attn = ggml_concat(ctx, kv_attn, comp, 1); } @@ -2039,7 +2177,9 @@ static ggml_tensor * build_mla_attention( } } if (n_comp_attn > 0) { - const int vis = ds4_comp_rows_used(lc.comp_kv, lc.n_comp, ratio, pos_i); + const int vis = gathered_history ? n_comp_attn + : ds4_comp_rows_used( + lane.comp_kv, lane.n_comp_committed, ratio, pos_i); for (int c = vis; c < n_comp_attn; c++) col[n_raw + c] = -1e30f; } } @@ -2063,8 +2203,9 @@ static ggml_tensor * build_mla_attention( if (pos_r > pos_i) col[r] = -1e30f; } if (n_comp_attn > 0) { - const int visible = ds4_comp_rows_used( - lc.comp_kv, lc.n_comp, ratio, pos_i); + const int visible = gathered_history ? n_comp_attn + : ds4_comp_rows_used( + lane.comp_kv, lane.n_comp_committed, ratio, pos_i); for (int c = visible; c < n_comp_attn; ++c) { col[n_raw + c] = -1e30f; } @@ -2116,7 +2257,7 @@ static ggml_tensor * build_mla_attention( const int first_count = DS4_NUMERICAL_PREFILL_BAND; const int second_count = n_tokens - first_count; const int first_comp = ratio > 0 - ? ds4_comp_rows_used(lc.comp_kv, lc.n_comp, ratio, + ? ds4_comp_rows_used(lane.comp_kv, lane.n_comp_committed, ratio, kv_start + first_count - 1) : 0; const int second_comp = n_comp_live; @@ -2160,7 +2301,7 @@ static ggml_tensor * build_mla_attention( } if (comp_count > 0) { const int visible = ds4_comp_rows_used( - lc.comp_kv, lc.n_comp, ratio, pos_i); + lane.comp_kv, lane.n_comp_committed, ratio, pos_i); for (int c = visible; c < comp_count; ++c) { col[raw_count + c] = -1e30f; } @@ -2403,6 +2544,34 @@ static ggml_tensor * build_mla_attention( return out; } +// Legacy contiguous-cache adapter. Both decode and the consecutive q>1 +// verifier/prefill path enter through here, so their graph construction order +// remains exactly the order in build_mla_attention_lane_core. +static ggml_tensor * build_mla_attention( + ggml_context * ctx, + ggml_cgraph * gf, + ggml_tensor * cur, + const DeepSeek4Weights & w, + const DeepSeek4Layer & L, + DeepSeek4LayerCache & lc, + int layer_idx, + int kv_start, + int n_tokens, + const DeepSeek4AttentionGraphInputs * cached_inputs, + std::vector & i32_inputs, + std::vector & i32_array_inputs, + std::vector & i64_array_inputs, + std::vector * f32_array_inputs = nullptr, + DeepSeek4AttentionImpl attention_impl = DeepSeek4AttentionImpl::Explicit) { + const int ratio = w.compress_ratios[layer_idx]; + DeepSeek4MlaLaneBindings lane = deepseek4_contiguous_lane_bindings( + lc, ratio, kv_start + n_tokens - 1); + return build_mla_attention_lane_core( + ctx, gf, cur, w, L, lane, layer_idx, kv_start, n_tokens, + cached_inputs, i32_inputs, i32_array_inputs, i64_array_inputs, + f32_array_inputs, attention_impl); +} + struct DeepSeek4CachedDecodeHcPreGraph { const ggml_context * owner_ctx = nullptr; ggml_backend_t backend = nullptr; @@ -4810,6 +4979,19 @@ struct Ds4FusedVerifyCache { std::array slots; struct Extra { + struct PagedLane { + ggml_tensor * pos = nullptr; + ggml_tensor * neg_pos = nullptr; + ggml_tensor * raw_gather = nullptr; + ggml_tensor * comp_gather = nullptr; + ggml_tensor * index_gather = nullptr; + ggml_tensor * raw_write = nullptr; + ggml_tensor * comp_write = nullptr; + ggml_tensor * comp_read = nullptr; + ggml_tensor * ape = nullptr; + ggml_tensor * state_row = nullptr; + ggml_tensor * comp_pos = nullptr; + }; ggml_tensor * pos_q = nullptr; // i32 [q] ggml_tensor * neg_q = nullptr; // i32 [q] ggml_tensor * rawrows = nullptr; // i64 [1,q] @@ -4822,6 +5004,7 @@ struct Ds4FusedVerifyCache { // Reused host staging for the context-sized additive attention mask. // Keeping it per slot removes one allocation from every verify step. std::vector mask_values; + std::vector paged; // [layer*q], paged mode only int q = 0; void reset() { *this = Extra{}; } @@ -6854,6 +7037,222 @@ static bool initialize_layer_range_cache( runtime.owns_output = owns_output; return true; } + +struct Ds4PagedGatheredRuntime { + DeepSeek4LayerRangeCache model; + const MoeHybridStorage * hybrid_identity = nullptr; + ggml_backend_t hybrid_cpu_backend = nullptr; + ggml_backend_t hybrid_cold_backend = nullptr; +}; + +void deepseek4_release_paged_gathered_runtime(DeepSeek4PagedCache & cache) { + delete static_cast(cache.gathered_runtime); + cache.gathered_runtime = nullptr; +} + +bool deepseek4_paged_gathered_step( + ggml_backend_t backend, int device, const DeepSeek4Weights & w, + DeepSeek4PagedCache & cache, const float * embeddings, + const int32_t * token_ids, const int64_t * positions, + const int32_t * slots, uint32_t lanes, const int32_t * block_tables, + uint32_t block_table_stride, std::vector & out_logits, + std::vector & out_argmax, MoeHybridStorage * hybrid, + MoeHybridRoutingStats * routing_stats) { + if (!backend || !embeddings || !positions || !slots || !block_tables || + lanes < 1 || lanes > 16 || cache.layers.size() != (size_t) w.n_layer || + block_table_stride < cache.plan.max_blocks_per_sequence) return false; + for (uint32_t lane = 0; lane < lanes; ++lane) { + if (slots[lane] < 0) continue; + if ((uint32_t) slots[lane] >= cache.plan.slots || positions[lane] < 0 || + (uint64_t) positions[lane] >= cache.plan.max_ctx || + positions[lane] > INT32_MAX) return false; + for (uint32_t prior = 0; prior < lane; ++prior) + if (slots[prior] == slots[lane]) return false; + } + // Active logical pages must have valid, exclusive physical ownership. + // Aliasing would make one lane's compressor write mutate another lane's + // chronological history and is therefore malformed addressing. + std::vector physical_owner(cache.plan.physical_blocks, -1); + for (uint32_t lane = 0; lane < lanes; ++lane) { + if (slots[lane] < 0) continue; + const uint64_t last_block = (uint64_t) positions[lane] / DS4_PAGE_TOKENS; + if (last_block >= block_table_stride) return false; + for (uint64_t logical = 0; logical <= last_block; ++logical) { + const int32_t physical = block_tables[(size_t) lane * block_table_stride + logical]; + if (physical < 0 || (uint32_t) physical >= cache.plan.physical_blocks || + physical_owner[(size_t) physical] >= 0) return false; + physical_owner[(size_t) physical] = (int32_t) lane; + } + } + if (hybrid) { + for (size_t il = 0; il < hybrid->layers.size(); ++il) { + if (hybrid->layers[il].cache_slots > 0) { + std::fprintf(stderr, + "[deepseek4-paged] layer %zu uses mutable expert-cache " + "placement, which gathered serving cannot capture\n", il); + return false; + } + } + } + auto * rt = static_cast(cache.gathered_runtime); + if (!rt) { + rt = new (std::nothrow) Ds4PagedGatheredRuntime; + if (!rt) return false; + cache.gathered_runtime = rt; + } + if (rt->hybrid_identity != hybrid || + rt->hybrid_cpu_backend != (hybrid ? hybrid->cpu_backend : nullptr) || + rt->hybrid_cold_backend != (hybrid ? hybrid->cold_backend : nullptr)) { + rt->model.fused_verify_graph_cache.destroy(); + rt->hybrid_identity = hybrid; + rt->hybrid_cpu_backend = hybrid ? hybrid->cpu_backend : nullptr; + rt->hybrid_cold_backend = hybrid ? hybrid->cold_backend : nullptr; + } + if (!rt->model.matches(w, backend, device, 0, w.n_layer, true) && + !initialize_layer_range_cache(rt->model, backend, device, w, + 0, w.n_layer, true)) { + std::fprintf(stderr, + "[deepseek4-paged] failed to initialize whole-model graph cache\n"); + return false; + } + + std::vector> prepared((size_t) w.n_layer); + std::vector key = {0x5041474544LL, (int64_t) lanes, + token_ids ? 1 : 0, hybrid ? 1 : 0}; + for (uint32_t lane = 0; lane < lanes; ++lane) key.push_back(slots[lane]); + for (int il = 0; il < w.n_layer; ++il) { + const uint32_t ratio = cache.layers[(size_t) il].ratio; + if (!prepare_deepseek4_gathered_lane_rows( + slots, positions, lanes, block_tables, block_table_stride, + cache.plan.physical_blocks, ratio, prepared[(size_t) il])) return false; + for (const auto & row : prepared[(size_t) il]) { + key.push_back((int64_t) row.raw_history.size()); + key.push_back((int64_t) row.compressed_history.size()); + key.push_back(row.slot < 0 ? -1 : + (ratio ? row.position % ratio : row.position % DS4_PAGE_TOKENS)); + } + } + + auto & vc = rt->model.fused_verify_graph_cache; + auto & mc = rt->model.fused_decode_graph_cache; + if (vc.owner_ctx != w.ctx || vc.backend != backend || + vc.peer_backend != (hybrid ? hybrid->cold_backend : nullptr)) { + vc.destroy(); vc.owner_ctx = w.ctx; vc.backend = backend; + vc.peer_backend = hybrid ? hybrid->cold_backend : nullptr; + } + if (mc.owner_ctx != w.ctx || mc.backend != backend) { + mc.destroy(); mc.owner_ctx = w.ctx; mc.backend = backend; + } + if (!ds4_fused_ensure_fn_mirrors(mc, backend, w, + rt->model.hc_layer_weights, rt->model.hc_output_weights)) return false; + vc.counter++; + DeepSeek4FusedDecodeGraph * fg = nullptr; + Ds4FusedVerifyCache::Extra * ex = nullptr; + const size_t slot_limit = hybrid ? ds4_fused_verify_hybrid_slot_limit() + : vc.slots.size(); + for (size_t i = 0; i < slot_limit; ++i) { + if (vc.slots[i].built() && vc.slots[i].shape_key == key) { + fg = &vc.slots[i]; ex = &vc.extra[i]; break; + } + } + if (!fg) { + size_t pick = 0; + for (size_t i = 0; i < slot_limit; ++i) { + if (!vc.slots[i].built()) { pick = i; break; } + if (vc.slots[i].last_use < vc.slots[pick].last_use) pick = i; + } + fg = &vc.slots[pick]; ex = &vc.extra[pick]; + fg->destroy(vc.backend, vc.peer_backend); ex->reset(); + if (!ds4_build_fused_verify_graph( + mc, *fg, *ex, backend, w, cache.prefill_staging, + rt->model.hc_layer_weights, rt->model.hc_output_weights, + rt->model.hash_routing_tables, 0, (int) lanes, + token_ids != nullptr, {}, hybrid, std::move(key), + &cache, &prepared)) { + std::fprintf(stderr, + "[deepseek4-paged] failed to build gathered graph " + "(lanes=%u)\n", lanes); + return false; + } + } + fg->last_use = vc.counter; + ds4_fv_set(fg->inp_embed, embeddings, + sizeof(float) * (size_t) w.n_embd * lanes); + size_t pi = 0; + for (int il = 0; il < w.n_layer; ++il) { + const int ratio = (int) cache.layers[(size_t) il].ratio; + for (uint32_t lane = 0; lane < lanes; ++lane, ++pi) { + const auto & row = prepared[(size_t) il][lane]; + const auto & px = ex->paged[pi]; + const int32_t pos = (int32_t) row.position; + const int32_t neg_pos = -pos; + ds4_fv_set(px.pos, &pos, sizeof(pos)); + ds4_fv_set(px.neg_pos, &neg_pos, sizeof(neg_pos)); + std::vector idx(std::max(row.raw_history.size(), 1), 0); + for (size_t i = 0; i < row.raw_history.size(); ++i) idx[i] = (int32_t) row.raw_history[i]; + ds4_fv_set(px.raw_gather, idx.data(), idx.size() * sizeof(int32_t)); + const int64_t raw_write = std::max(row.raw_scatter, 0); + ds4_fv_set(px.raw_write, &raw_write, sizeof(raw_write)); + if (ratio > 0) { + idx.assign(std::max(row.compressed_history.size(), 1), 0); + for (size_t i = 0; i < row.compressed_history.size(); ++i) + idx[i] = (int32_t) row.compressed_history[i]; + ds4_fv_set(px.comp_gather, idx.data(), idx.size() * sizeof(int32_t)); + if (px.index_gather) + ds4_fv_set(px.index_gather, idx.data(), idx.size() * sizeof(int32_t)); + const int64_t cw = std::max(row.compressed_scatter, 0); + const int32_t cr = (int32_t) cw; + const int32_t ape = pos % ratio; + const int64_t state = ratio == 4 ? 4 + ape : ape; + const int32_t comp_pos = pos + 1 - ratio; + ds4_fv_set(px.comp_write, &cw, sizeof(cw)); + ds4_fv_set(px.comp_read, &cr, sizeof(cr)); + ds4_fv_set(px.ape, &ape, sizeof(ape)); + ds4_fv_set(px.state_row, &state, sizeof(state)); + ds4_fv_set(px.comp_pos, &comp_pos, sizeof(comp_pos)); + } + } + } + if (token_ids) { + for (int il = 0; il < w.n_layer; ++il) { + ggml_tensor * ids = fg->hash_ids[(size_t) il]; if (!ids) continue; + std::vector values((size_t) ids->ne[0] * lanes); + for (uint32_t lane = 0; lane < lanes; ++lane) { + const int32_t * src = hash_routing_row(rt->model.hash_routing_tables[(size_t) il], + slots[lane] < 0 ? 0 : token_ids[lane], + w.n_expert_used); + if (!src) return false; + std::memcpy(values.data() + lane * ids->ne[0], src, + (size_t) ids->ne[0] * sizeof(int32_t)); + } + ds4_fv_set(ids, values.data(), values.size() * sizeof(int32_t)); + } + } + const enum ggml_status status = fg->sched + ? ggml_backend_sched_graph_compute(fg->sched, fg->sg.gf) + : ggml_backend_graph_compute(backend, fg->sg.gf); + if (status != GGML_STATUS_SUCCESS) { + std::fprintf(stderr, + "[deepseek4-paged] gathered graph compute failed: status=%d\n", + (int) status); + return false; + } + ds4_fused_consume_route_diagnostics(*fg, hybrid, routing_stats, slots); + out_logits.resize((size_t) w.n_vocab * lanes); + out_argmax.resize(lanes); + ggml_backend_tensor_get(fg->logits, out_logits.data(), 0, + out_logits.size() * sizeof(float)); + ggml_backend_tensor_get(ex->argmax, out_argmax.data(), 0, + out_argmax.size() * sizeof(int32_t)); + for (uint32_t lane = 0; lane < lanes; ++lane) { + if (slots[lane] >= 0) continue; + std::fill_n(out_logits.data() + (size_t) lane * w.n_vocab, + w.n_vocab, 0.0f); + out_argmax[lane] = -1; + } + return true; +} + bool deepseek4_step_layer_range( ggml_backend_t backend, int device, diff --git a/server/src/deepseek4/deepseek4_internal.h b/server/src/deepseek4/deepseek4_internal.h index 66d80e417..4f587925a 100644 --- a/server/src/deepseek4/deepseek4_internal.h +++ b/server/src/deepseek4/deepseek4_internal.h @@ -15,6 +15,7 @@ #pragma once #include +#include #include #include @@ -25,6 +26,8 @@ #include "internal.h" #include "common/layer_split_utils.h" #include "common/prefill_attention_mode.h" +#include "common/paged_kv_pool.h" +#include "deepseek4_paged_cache.h" namespace dflash::common { @@ -296,6 +299,26 @@ struct DeepSeek4Cache { ggml_backend_buffer_t buf = nullptr; }; +struct DeepSeek4PagedLayerCache : DeepSeek4LayerCache { + uint32_t ratio = 0; + uint64_t physical_rows = 0; +}; + +struct DeepSeek4PagedCache { + std::unique_ptr pool; + DeepSeek4PagedCachePlan plan; + ggml_tensor * block_table = nullptr; + ggml_tensor * sequence_lengths = nullptr; + ggml_tensor * active_slot_ids = nullptr; + std::vector layers; + DeepSeek4Cache prefill_staging; + ggml_context * ctx = nullptr; + ggml_backend_buffer_t buf = nullptr; + // Dedicated bounded gathered-reference graph cache (opaque here because + // its implementation shares the fused verifier's private machinery). + void * gathered_runtime = nullptr; +}; + struct DeepSeek4Snapshot; struct DeepSeek4RawRingSpan { @@ -315,6 +338,9 @@ struct DeepSeek4BackendConfig { int expert_top_k = 0; // 0 = use all model-routed experts bool fused_decode = false; // single-graph GPU decode bool fused_verify_f16_kv = false; // F16 KV in batched verifier attention + bool paged_attention = false; + int max_concurrency = 1; + long long kv_pool_tokens = 0; }; // ─── Function declarations ────────────────────────────────────────────── @@ -340,6 +366,26 @@ bool create_deepseek4_cache(ggml_backend_t backend, DeepSeek4Cache & out); void free_deepseek4_cache(DeepSeek4Cache & c); +bool create_deepseek4_paged_cache(ggml_backend_t backend, + const DeepSeek4Weights & w, + uint32_t slots, uint32_t max_ctx, + uint32_t physical_blocks, + DeepSeek4PagedCache & out); +void reset_deepseek4_paged_slot(DeepSeek4PagedCache & c, uint32_t slot); +void free_deepseek4_paged_cache(DeepSeek4PagedCache & c); +// Exact gathered-reference decode for 1..16 independent lanes. Inputs are +// lane-major; negative slots are inactive padding lanes. `out_logits` is +// [n_vocab, lanes] and `out_argmax` is [lanes]. +bool deepseek4_paged_gathered_step( + ggml_backend_t backend, int device, const DeepSeek4Weights & w, + DeepSeek4PagedCache & cache, const float * embeddings, + const int32_t * token_ids, const int64_t * positions, + const int32_t * slots, uint32_t lanes, const int32_t * block_tables, + uint32_t block_table_stride, std::vector & out_logits, + std::vector & out_argmax, + MoeHybridStorage * moe_hybrid = nullptr, + MoeHybridRoutingStats * routing_stats = nullptr); +void deepseek4_release_paged_gathered_runtime(DeepSeek4PagedCache & cache); void reset_deepseek4_cache(DeepSeek4Cache & c); // Release only reproducible large-batch graph arenas after prefill. KV/model // state and the DSpark feature tail remain live for the following decode. diff --git a/server/src/deepseek4/deepseek4_page_layout.h b/server/src/deepseek4/deepseek4_page_layout.h new file mode 100644 index 000000000..b2ee71631 --- /dev/null +++ b/server/src/deepseek4/deepseek4_page_layout.h @@ -0,0 +1,55 @@ +// Host-side address geometry for DeepSeek V4's paged raw and compressed KV. +#pragma once + +#include +#include + +namespace dflash::common { + +inline constexpr uint32_t DS4_PAGE_TOKENS = 128; + +// Raw KV remains slot-indexed: every sequence reuses this 128-row ring. +inline constexpr uint32_t ds4_raw_ring_row(uint64_t logical_token) { + return static_cast(logical_token % DS4_PAGE_TOKENS); +} + +// Number of physically paged compressed rows. Rejects unsupported ratios and +// arithmetic that cannot be represented by the row-index type. +inline bool ds4_compressed_page_capacity(uint64_t physical_blocks, + uint32_t ratio, + uint64_t & rows) { + if (ratio != 4 && ratio != 128) return false; + const uint64_t rows_per_block = DS4_PAGE_TOKENS / ratio; + if (physical_blocks > + std::numeric_limits::max() / rows_per_block) { + return false; + } + rows = physical_blocks * rows_per_block; + return true; +} + +// Computes the destination for a completed compression group. `emitted` is +// false between group boundaries and `row` is left unchanged in that case. +// Physical block IDs need not be contiguous. +inline bool ds4_compressed_page_row(uint64_t logical_token, + uint64_t physical_block, + uint32_t ratio, + uint64_t & row, + bool & emitted) { + if (ratio != 4 && ratio != 128) return false; + emitted = logical_token % ratio == ratio - 1; + if (!emitted) return true; + + const uint64_t rows_per_block = DS4_PAGE_TOKENS / ratio; + if (physical_block > + std::numeric_limits::max() / rows_per_block) { + return false; + } + const uint64_t base = physical_block * rows_per_block; + const uint64_t offset = (logical_token % DS4_PAGE_TOKENS) / ratio; + if (base > std::numeric_limits::max() - offset) return false; + row = base + offset; + return true; +} + +} // namespace dflash::common diff --git a/server/src/deepseek4/deepseek4_paged_cache.cpp b/server/src/deepseek4/deepseek4_paged_cache.cpp new file mode 100644 index 000000000..cdb395dbb --- /dev/null +++ b/server/src/deepseek4/deepseek4_paged_cache.cpp @@ -0,0 +1,219 @@ +#include "deepseek4_paged_cache.h" + +#include "deepseek4_page_layout.h" + +#ifndef DFLASH_DS4_PLAN_ONLY +#include "deepseek4_internal.h" +#endif + +#include +#include +#include + +namespace dflash::common { +namespace { +bool add_mul(uint64_t & dst, uint64_t a, uint64_t b) { + if (a && b > std::numeric_limits::max() / a) return false; + const uint64_t v = a * b; + if (dst > std::numeric_limits::max() - v) return false; + dst += v; + return true; +} +} + +bool prepare_deepseek4_gathered_lane_rows( + const int32_t * slots, const int64_t * positions, uint32_t lanes, + const int32_t * block_tables, uint32_t block_table_stride, + uint32_t physical_blocks, uint32_t ratio, + std::vector & out) { + if (!slots || !positions || !block_tables || !block_table_stride || + !physical_blocks || (ratio != 0 && ratio != 4 && ratio != 128)) { + return false; + } + std::vector prepared(lanes); + for (uint32_t lane = 0; lane < lanes; ++lane) { + auto & rows = prepared[lane]; + rows.slot = slots[lane]; + rows.position = positions[lane]; + if (rows.slot < 0) continue; // Padding must remain entirely passive. + if (rows.position < 0) return false; + const uint64_t pos = static_cast(rows.position); + // The current row is appended in-graph, so retain at most the 127 + // preceding rows that can coexist with it in the 128-row SWA window. + const uint64_t first_raw = pos >= DS4_PAGE_TOKENS + ? pos - DS4_PAGE_TOKENS + 1 : 0; + rows.raw_history.reserve(static_cast(pos - first_raw)); + for (uint64_t p = first_raw; p < pos; ++p) { + rows.raw_history.push_back( + int64_t(rows.slot) * DS4_PAGE_TOKENS + ds4_raw_ring_row(p)); + } + rows.raw_scatter = int64_t(rows.slot) * DS4_PAGE_TOKENS + + ds4_raw_ring_row(pos); + if (!ratio) continue; + + // Every completed group before the current token contributes one + // chronological row. Looking up each logical page (rather than + // assuming contiguous physical pages) is the reference behaviour. + const uint64_t completed = pos / ratio; + rows.compressed_history.reserve(static_cast(completed)); + for (uint64_t group = 0; group < completed; ++group) { + const uint64_t end_token = group * ratio + ratio - 1; + const uint64_t logical_block = end_token / DS4_PAGE_TOKENS; + if (logical_block >= block_table_stride) return false; + const int32_t physical = + block_tables[size_t(lane) * block_table_stride + logical_block]; + if (physical < 0 || uint32_t(physical) >= physical_blocks) return false; + uint64_t row = 0; bool emitted = false; + if (!ds4_compressed_page_row(end_token, uint32_t(physical), ratio, + row, emitted) || !emitted || + row > uint64_t(INT64_MAX)) return false; + rows.compressed_history.push_back(static_cast(row)); + } + const uint64_t logical_block = pos / DS4_PAGE_TOKENS; + if (logical_block >= block_table_stride) return false; + const int32_t physical = + block_tables[size_t(lane) * block_table_stride + logical_block]; + if (physical < 0 || uint32_t(physical) >= physical_blocks) return false; + uint64_t scatter = 0; + if (!ds4_compressed_page_row(pos, uint32_t(physical), ratio, scatter, + rows.compressed_emitted) || + scatter > uint64_t(INT64_MAX)) return false; + if (rows.compressed_emitted) rows.compressed_scatter = int64_t(scatter); + } + out = std::move(prepared); + return true; +} + +bool plan_deepseek4_paged_cache(uint32_t head_dim, uint32_t indexer_head_dim, + uint32_t slots, uint32_t max_ctx, + uint32_t physical_blocks, + const std::vector & ratios, + DeepSeek4PagedCachePlan & out) { + DeepSeek4PagedCachePlan p; + if (!head_dim || !indexer_head_dim || !slots || !max_ctx || + !physical_blocks || ratios.empty() || + physical_blocks > UINT32_MAX / DS4_PAGE_TOKENS) return false; + p.slots = slots; p.max_ctx = max_ctx; p.physical_blocks = physical_blocks; + p.max_blocks_per_sequence = 1 + (max_ctx - 1) / DS4_PAGE_TOKENS; + p.ratios = ratios; + p.physical_rows.resize(ratios.size()); + // block table, lengths, and active IDs, all I32. + if (!add_mul(p.metadata_bytes, p.max_blocks_per_sequence, uint64_t(slots) * 4) || + !add_mul(p.metadata_bytes, slots, 8)) return false; + for (size_t i = 0; i < ratios.size(); ++i) { + const uint32_t r = ratios[i]; + if (r != 0 && r != 4 && r != 128) return false; + if (!add_mul(p.raw_bytes, uint64_t(head_dim) * DS4_PAGE_TOKENS * 2, slots)) return false; + if (!r) continue; + uint64_t rows = 0; + if (!ds4_compressed_page_capacity(physical_blocks, r, rows)) return false; + p.physical_rows[i] = rows; + if (!add_mul(p.compressed_bytes, uint64_t(head_dim) * 2, rows)) return false; + const uint64_t width = uint64_t(head_dim) * (r == 4 ? 2 : 1); + const uint64_t state_rows = r == 4 ? 8 : 128; + if (!add_mul(p.state_bytes, width * state_rows * 8, slots)) return false; // KV + score F32 + if (r == 4) { + if (!add_mul(p.compressed_bytes, uint64_t(indexer_head_dim) * 2, rows)) return false; + if (!add_mul(p.state_bytes, uint64_t(indexer_head_dim) * 2 * 8 * 8, slots)) return false; + } + } + p.total_persistent_bytes = p.metadata_bytes; + if (p.total_persistent_bytes > UINT64_MAX - p.raw_bytes) return false; + p.total_persistent_bytes += p.raw_bytes; + if (p.total_persistent_bytes > UINT64_MAX - p.compressed_bytes) return false; + p.total_persistent_bytes += p.compressed_bytes; + if (p.total_persistent_bytes > UINT64_MAX - p.state_bytes) return false; + p.total_persistent_bytes += p.state_bytes; + out = std::move(p); + return true; +} + +#ifndef DFLASH_DS4_PLAN_ONLY +bool create_deepseek4_paged_cache(ggml_backend_t backend, + const DeepSeek4Weights & w, uint32_t slots, + uint32_t max_ctx, uint32_t physical_blocks, + DeepSeek4PagedCache & out) { + free_deepseek4_paged_cache(out); + DeepSeek4PagedCachePlan plan; + if (!backend || w.n_layer <= 0 || w.compress_ratios.size() != size_t(w.n_layer) || + !plan_deepseek4_paged_cache(w.head_dim, w.n_indexer_head_dim, slots, + max_ctx, physical_blocks, w.compress_ratios, plan)) return false; + try { out.pool = std::make_unique(physical_blocks, slots, DS4_PAGE_TOKENS); } + catch (...) { free_deepseek4_paged_cache(out); return false; } + out.plan = plan; + out.layers.resize(w.n_layer); + ggml_init_params ip{ggml_tensor_overhead() * size_t(w.n_layer * 9 + 8) + 4096, nullptr, true}; + out.ctx = ggml_init(ip); + if (!out.ctx) { free_deepseek4_paged_cache(out); return false; } + out.block_table = ggml_new_tensor_2d(out.ctx, GGML_TYPE_I32, plan.max_blocks_per_sequence, slots); + out.sequence_lengths = ggml_new_tensor_1d(out.ctx, GGML_TYPE_I32, slots); + out.active_slot_ids = ggml_new_tensor_1d(out.ctx, GGML_TYPE_I32, slots); + for (int il = 0; il < w.n_layer; ++il) { + auto & l = out.layers[il]; const uint32_t r = plan.ratios[il]; + l.ratio = r; l.physical_rows = plan.physical_rows[il]; + l.raw_kv = ggml_new_tensor_3d(out.ctx, GGML_TYPE_F16, w.head_dim, DS4_PAGE_TOKENS, slots); + if (!r) continue; + l.comp_kv = ggml_new_tensor_2d(out.ctx, GGML_TYPE_F16, w.head_dim, l.physical_rows); + const int64_t width = int64_t(w.head_dim) * (r == 4 ? 2 : 1), sr = r == 4 ? 8 : 128; + l.attn_compressor.state_kv = ggml_new_tensor_3d(out.ctx, GGML_TYPE_F32, width, sr, slots); + l.attn_compressor.state_score = ggml_new_tensor_3d(out.ctx, GGML_TYPE_F32, width, sr, slots); + if (r == 4) { + l.index_comp_kv = ggml_new_tensor_2d(out.ctx, GGML_TYPE_F16, w.n_indexer_head_dim, l.physical_rows); + const int64_t iw = int64_t(w.n_indexer_head_dim) * 2; + l.indexer_compressor.state_kv = ggml_new_tensor_3d(out.ctx, GGML_TYPE_F32, iw, 8, slots); + l.indexer_compressor.state_score = ggml_new_tensor_3d(out.ctx, GGML_TYPE_F32, iw, 8, slots); + } + } + out.buf = ggml_backend_alloc_ctx_tensors(out.ctx, backend); + if (!out.buf) { free_deepseek4_paged_cache(out); return false; } + ggml_backend_buffer_clear(out.buf, 0); + // One shared, contiguous legacy cache is intentionally retained for prefill. + if (!create_deepseek4_cache(backend, w, int(max_ctx), out.prefill_staging)) { + free_deepseek4_paged_cache(out); return false; + } + return true; +} + +void reset_deepseek4_paged_slot(DeepSeek4PagedCache & c, uint32_t slot) { + if (!c.buf || slot >= c.plan.slots) return; + auto clear_slot = [slot](ggml_tensor * tensor) { + if (!tensor || tensor->ne[2] <= (int64_t) slot) return; + const size_t bytes = tensor->nb[2]; + std::vector zeros(bytes, 0); + ggml_backend_tensor_set(tensor, zeros.data(), (size_t) slot * bytes, + bytes); + }; + for (DeepSeek4PagedLayerCache & layer : c.layers) { + clear_slot(layer.attn_compressor.state_kv); + clear_slot(layer.attn_compressor.state_score); + clear_slot(layer.indexer_compressor.state_kv); + clear_slot(layer.indexer_compressor.state_score); + } + if (c.block_table) { + std::vector empty(c.plan.max_blocks_per_sequence, -1); + ggml_backend_tensor_set(c.block_table, empty.data(), + (size_t) slot * c.block_table->nb[1], + empty.size() * sizeof(int32_t)); + } + const int32_t zero = 0; + const int32_t inactive = -1; + if (c.sequence_lengths) { + ggml_backend_tensor_set(c.sequence_lengths, &zero, + (size_t) slot * sizeof(int32_t), sizeof(zero)); + } + if (c.active_slot_ids) { + ggml_backend_tensor_set(c.active_slot_ids, &inactive, + (size_t) slot * sizeof(int32_t), sizeof(inactive)); + } +} + +void free_deepseek4_paged_cache(DeepSeek4PagedCache & c) { + deepseek4_release_paged_gathered_runtime(c); + free_deepseek4_cache(c.prefill_staging); + if (c.buf) { ggml_backend_buffer_free(c.buf); c.buf = nullptr; } + if (c.ctx) { ggml_free(c.ctx); c.ctx = nullptr; } + c.pool.reset(); c.layers.clear(); c.block_table = nullptr; + c.sequence_lengths = nullptr; c.active_slot_ids = nullptr; c.plan = {}; +} +#endif +} // namespace dflash::common diff --git a/server/src/deepseek4/deepseek4_paged_cache.h b/server/src/deepseek4/deepseek4_paged_cache.h new file mode 100644 index 000000000..d8bb3cc83 --- /dev/null +++ b/server/src/deepseek4/deepseek4_paged_cache.h @@ -0,0 +1,63 @@ +#pragma once + +#include +#include +#include + +namespace dflash::common { + +// Pure host-side allocation plan. Byte counts describe tensor payloads (ggml +// alignment/padding is deliberately excluded). +struct DeepSeek4PagedCachePlan { + uint32_t slots = 0; + uint32_t max_ctx = 0; + uint32_t physical_blocks = 0; + uint32_t max_blocks_per_sequence = 0; + uint64_t metadata_bytes = 0; + uint64_t raw_bytes = 0; + uint64_t compressed_bytes = 0; + uint64_t state_bytes = 0; + uint64_t total_persistent_bytes = 0; + std::vector ratios; + std::vector physical_rows; +}; + +// Host metadata for the gathered-reference decode graph. Rows are expressed +// in the flattened persistent tensors: raw rows are [slot, ring-row], while +// compressed rows use the physical page geometry from deepseek4_page_layout.h. +// A negative slot denotes a padding lane and consequently has no scatter rows. +struct DeepSeek4GatheredLaneRows { + int32_t slot = -1; + int64_t position = 0; + std::vector raw_history; + std::vector compressed_history; + int64_t raw_scatter = -1; + int64_t compressed_scatter = -1; + bool compressed_emitted = false; +}; + +// block_tables is lane-major with block_table_stride entries per lane. +// Physical block IDs may be fragmented and are validated against +// physical_blocks. History excludes the current token; compressed history is +// in chronological group order. Returns false for malformed active lanes. +bool prepare_deepseek4_gathered_lane_rows( + const int32_t * slots, + const int64_t * positions, + uint32_t lanes, + const int32_t * block_tables, + uint32_t block_table_stride, + uint32_t physical_blocks, + uint32_t ratio, + std::vector & out); + +// Ratios must contain only 0, 4, or 128. A ratio-zero layer has a raw ring +// but no compressed storage or compressor state. +bool plan_deepseek4_paged_cache(uint32_t head_dim, + uint32_t indexer_head_dim, + uint32_t slots, + uint32_t max_ctx, + uint32_t physical_blocks, + const std::vector & ratios, + DeepSeek4PagedCachePlan & out); + +} // namespace dflash::common diff --git a/server/src/deepseek4/deepseek4_seq_engine.cpp b/server/src/deepseek4/deepseek4_seq_engine.cpp new file mode 100644 index 000000000..745046783 --- /dev/null +++ b/server/src/deepseek4/deepseek4_seq_engine.cpp @@ -0,0 +1,272 @@ +#include "deepseek4_seq_engine.h" + +#include "deepseek4_backend.h" +#include "common/sampler.h" + +#include +#include + +namespace dflash::common { + +DeepSeek4SeqEngine::DeepSeek4SeqEngine( + DeepSeek4Backend & backend, PagedKvPool & pool, int max_ctx, + uint32_t table_stride) + : b_(backend), slots_(pool, max_ctx), stride_(table_stride), + host_tables_((size_t)pool.max_sequences() * table_stride, -1) {} + +bool DeepSeek4SeqEngine::token_is_eos(int32_t token) const { + return deepseek4_is_eos_tok(token, b_.w_); +} + +StepPlanLimits DeepSeek4SeqEngine::step_plan_limits( + int decode_rows) const { + // The gathered graph accepts at most sixteen independent lanes and does + // not permit two rows from the same sequence. A prompt therefore advances + // by one token while every live decoder still shares the same weight pass. + const int available = std::max(0, 16 - decode_rows); + return {available, 1, available, 1}; +} + +SeqEngine::AdmitResult DeepSeek4SeqEngine::admit( + uint64_t request_id, const std::vector & prompt, + const SamplerCfg & sampler) { + using AdmitStatus = AdmitResult::Status; + AdmitResult result = slots_.admit( + request_id, prompt, sampler); + if (result.status != AdmitStatus::admitted) return result; + if (result.slot < 0 || result.slot >= slots_.slot_count()) { + result.status = AdmitStatus::failed; + result.error = "invalid DeepSeek4 serving slot"; + return result; + } + std::fill_n(host_tables_.data() + (size_t)result.slot * stride_, + stride_, -1); + reset_deepseek4_paged_slot(b_.paged_cache_, (uint32_t)result.slot); + return result; +} + +bool DeepSeek4SeqEngine::set_block(int slot, int logical, int32_t physical) { + if (slot < 0 || slot >= slots_.slot_count() || logical < 0 || + (uint32_t)logical >= stride_ || !b_.paged_cache_.block_table) { + return false; + } + host_tables_[(size_t)slot * stride_ + (size_t)logical] = physical; + ggml_tensor * table = b_.paged_cache_.block_table; + ggml_backend_tensor_set( + table, &physical, + (size_t)slot * table->nb[1] + + (size_t)logical * sizeof(int32_t), + sizeof(physical)); + return true; +} + +void DeepSeek4SeqEngine::fail_prefill( + int slot, std::vector & outputs, + const std::string & error) { + std::fprintf(stderr, "[deepseek4-parallel] prefill slot %d: %s\n", + slot, error.c_str()); + PrefillOutput out; + out.slot = slot; + out.status = PrefillOutput::Status::failed; + out.error = error; + outputs.push_back(std::move(out)); +} + +SeqEngine::StepResult DeepSeek4SeqEngine::step(const StepPlan & plan) { + StepResult result; + const std::vector & inputs = plan.decode; + const int n_slots = slots_.slot_count(); + + auto fail_step = [&](const std::string & error) { + result.decode.clear(); + result.prefills.clear(); + result.error = error; + return std::move(result); + }; + + if ((int)inputs.size() != slots_.decoding_count()) { + return fail_step("decode plan does not cover every live DeepSeek4 slot"); + } + std::vector decode_seen((size_t)n_slots, 0); + for (const StepInput & input : inputs) { + if (input.slot < 0 || input.slot >= n_slots || input.token < 0 || + decode_seen[(size_t)input.slot] || + !slots_.is_active(input.slot) || + slots_.is_prefilling(input.slot)) { + return fail_step("invalid or duplicate DeepSeek4 decode row"); + } + decode_seen[(size_t)input.slot] = 1; + } + + const StepPlanLimits limits = step_plan_limits((int)inputs.size()); + if ((int)plan.prefills.size() > limits.max_prefill_sequences) { + return fail_step("DeepSeek4 step exceeds the sixteen-lane graph"); + } + std::vector prefill_seen((size_t)n_slots, 0); + for (const PrefillSlice & slice : plan.prefills) { + if (slice.slot < 0 || slice.slot >= n_slots || + slice.max_tokens != 1 || prefill_seen[(size_t)slice.slot] || + decode_seen[(size_t)slice.slot] || + !slots_.is_prefilling(slice.slot)) { + return fail_step("invalid or duplicate DeepSeek4 prefill row"); + } + prefill_seen[(size_t)slice.slot] = 1; + } + if (inputs.empty() && plan.prefills.empty()) return result; + + std::vector lane_tokens; + std::vector lane_positions; + std::vector lane_slots; + std::vector decode_lanes; + lane_tokens.reserve(inputs.size() + plan.prefills.size()); + lane_positions.reserve(inputs.size() + plan.prefills.size()); + lane_slots.reserve(inputs.size() + plan.prefills.size()); + decode_lanes.reserve(inputs.size()); + result.decode.reserve(inputs.size()); + result.prefills.reserve(plan.prefills.size()); + + for (const StepInput & input : inputs) { + DecodeOutput out; + out.slot = input.slot; + const SeqSlotManager::StepAppend append = + slots_.append_token(input.slot, input.token); + if (!append.ok) { + out.failed = true; + out.error = append.busy + ? "paged KV pool exhausted during DeepSeek4 decode; raise " + "--kv-pool-tokens or lower --max-ctx/--max-concurrency" + : "DeepSeek4 decode K/V append failed"; + decode_lanes.push_back(-1); + result.decode.push_back(std::move(out)); + continue; + } + if (append.new_block >= 0 && + !set_block(input.slot, append.new_block_index, + append.new_block)) { + out.failed = true; + out.error = "DeepSeek4 decode block-table update failed"; + decode_lanes.push_back(-1); + result.decode.push_back(std::move(out)); + continue; + } + decode_lanes.push_back((int)lane_tokens.size()); + lane_tokens.push_back(input.token); + lane_positions.push_back(append.position); + lane_slots.push_back(input.slot); + result.decode.push_back(std::move(out)); + } + + struct PrefillLane { + int slot = -1; + int lane = -1; + bool commit = false; + }; + std::vector prefill_lanes; + prefill_lanes.reserve(plan.prefills.size()); + for (const PrefillSlice & slice : plan.prefills) { + SeqSlotManager::PrefillChunk chunk = + slots_.append_prefill(slice.slot, 1); + if (!chunk.ok || chunk.rows.size() != 1) { + fail_prefill(slice.slot, result.prefills, + "DeepSeek4 prefill K/V append failed"); + continue; + } + if (!chunk.new_blocks.empty() && + !set_block(slice.slot, chunk.first_new_block, + chunk.new_blocks.front())) { + fail_prefill(slice.slot, result.prefills, + "DeepSeek4 prefill block-table update failed"); + continue; + } + const SeqSlot & slot = slots_.slot(slice.slot); + const bool commit = slot.cur_pos == (int)slot.prompt.size(); + prefill_lanes.push_back( + {slice.slot, (int)lane_tokens.size(), commit}); + lane_tokens.push_back(slot.prompt[(size_t)slot.cur_pos - 1]); + lane_positions.push_back(slot.cur_pos - 1); + lane_slots.push_back(slice.slot); + } + + if (lane_tokens.empty()) return result; + if (lane_tokens.size() > 16) { + return fail_step("DeepSeek4 gathered step exceeds sixteen lanes"); + } + + std::vector embeddings( + (size_t)b_.w_.n_embd * lane_tokens.size()); + if (!b_.w_.embedder.embed(lane_tokens.data(), (int)lane_tokens.size(), + embeddings.data())) { + return fail_step("DeepSeek4 token embedding failed"); + } + + std::vector compact_tables( + lane_tokens.size() * stride_, -1); + for (size_t lane = 0; lane < lane_slots.size(); ++lane) { + std::copy_n( + host_tables_.data() + (size_t)lane_slots[lane] * stride_, + stride_, compact_tables.data() + lane * stride_); + } + + std::vector lengths((size_t)n_slots, 0); + std::vector active((size_t)n_slots, -1); + for (size_t lane = 0; lane < lane_slots.size(); ++lane) { + lengths[(size_t)lane_slots[lane]] = + (int32_t)lane_positions[lane] + 1; + active[lane] = lane_slots[lane]; + } + ggml_backend_tensor_set( + b_.paged_cache_.sequence_lengths, lengths.data(), 0, + lengths.size() * sizeof(int32_t)); + ggml_backend_tensor_set( + b_.paged_cache_.active_slot_ids, active.data(), 0, + active.size() * sizeof(int32_t)); + + std::vector logits; + std::vector argmax; + if (!deepseek4_paged_gathered_step( + b_.backend_, b_.cfg_.device.gpu, b_.w_, b_.paged_cache_, + embeddings.data(), lane_tokens.data(), lane_positions.data(), + lane_slots.data(), (uint32_t)lane_tokens.size(), + compact_tables.data(), stride_, logits, argmax, + b_.moe_hybrid_.get(), b_.routing_stats_.get())) { + return fail_step("DeepSeek4 gathered paged graph failed"); + } + + auto sample_lane = [&](int slot_id, int lane) { + SeqSlot & slot = slots_.slot(slot_id); + if (!slot.sampler.needs_logit_processing()) { + return argmax[(size_t)lane]; + } + return sample_logits( + logits.data() + (size_t)lane * b_.w_.n_vocab, + b_.w_.n_vocab, slot.sampler, slot.sample_history, slot.rng); + }; + + for (size_t i = 0; i < inputs.size(); ++i) { + const int lane = decode_lanes[i]; + if (lane < 0) continue; + DecodeOutput & out = result.decode[i]; + slots_.commit_step(out.slot); + out.token = sample_lane(out.slot, lane); + } + for (const PrefillLane & prefill : prefill_lanes) { + PrefillOutput out; + out.slot = prefill.slot; + if (prefill.commit) { + out.status = PrefillOutput::Status::completed; + out.token = sample_lane(prefill.slot, prefill.lane); + slots_.commit_prefill(prefill.slot); + } + result.prefills.push_back(std::move(out)); + } + return result; +} + +void DeepSeek4SeqEngine::retire(int slot) { + if (!slots_.is_active(slot)) return; + slots_.retire(slot); + reset_deepseek4_paged_slot(b_.paged_cache_, (uint32_t)slot); + std::fill_n(host_tables_.data() + (size_t)slot * stride_, stride_, -1); +} + +} // namespace dflash::common diff --git a/server/src/deepseek4/deepseek4_seq_engine.h b/server/src/deepseek4/deepseek4_seq_engine.h new file mode 100644 index 000000000..9eeadfd81 --- /dev/null +++ b/server/src/deepseek4/deepseek4_seq_engine.h @@ -0,0 +1,41 @@ +#pragma once + +#include "common/concurrency/seq_engine.h" +#include "common/concurrency/seq_slot_manager.h" + +#include +#include + +namespace dflash::common { + +class DeepSeek4Backend; + +// Exact concurrent serving path for DeepSeek4. Model state remains in +// DeepSeek4PagedCache; this class owns only scheduler-facing slot state and +// the host mirror of the model's block table. +class DeepSeek4SeqEngine final : public SeqEngine { +public: + DeepSeek4SeqEngine(DeepSeek4Backend & backend, PagedKvPool & pool, + int max_ctx, uint32_t table_stride); + + int slot_count() const override { return slots_.slot_count(); } + int max_context() const override { return slots_.max_context(); } + AdmitResult admit(uint64_t request_id, const std::vector & prompt, + const SamplerCfg & sampler) override; + StepResult step(const StepPlan & plan) override; + StepPlanLimits step_plan_limits(int decode_rows) const override; + void retire(int slot) override; + bool token_is_eos(int32_t token) const override; + +private: + bool set_block(int slot, int logical, int32_t physical); + void fail_prefill(int slot, std::vector & outputs, + const std::string & error); + + DeepSeek4Backend & b_; + SeqSlotManager slots_; + uint32_t stride_ = 0; + std::vector host_tables_; +}; + +} // namespace dflash::common diff --git a/server/test/test_deepseek4_page_layout.cpp b/server/test/test_deepseek4_page_layout.cpp new file mode 100644 index 000000000..5b98665cf --- /dev/null +++ b/server/test/test_deepseek4_page_layout.cpp @@ -0,0 +1,53 @@ +#include "deepseek4/deepseek4_page_layout.h" +#include "host_check.h" + +#include +#include +#include + +using namespace dflash::common; + +static int g_checks = 0; + +int main() { + CHECK(ds4_raw_ring_row(0) == 0); + CHECK(ds4_raw_ring_row(127) == 127); + CHECK(ds4_raw_ring_row(128) == 0); + CHECK(ds4_raw_ring_row(255) == 127); + + uint64_t row = 999; + bool emitted = true; + CHECK(ds4_compressed_page_row(3, 7, 4, row, emitted)); + CHECK(emitted && row == 7 * 32); + CHECK(ds4_compressed_page_row(4, 7, 4, row, emitted)); + CHECK(!emitted && row == 7 * 32); + CHECK(ds4_compressed_page_row(127, 7, 4, row, emitted)); + CHECK(emitted && row == 7 * 32 + 31); + CHECK(ds4_compressed_page_row(128, 42, 4, row, emitted)); + CHECK(!emitted); + CHECK(ds4_compressed_page_row(131, 42, 4, row, emitted)); + CHECK(emitted && row == 42 * 32); + CHECK(ds4_compressed_page_row(255, 3, 4, row, emitted)); + CHECK(emitted && row == 3 * 32 + 31); + + CHECK(ds4_compressed_page_row(127, 91, 128, row, emitted)); + CHECK(emitted && row == 91); + CHECK(ds4_compressed_page_row(128, 2, 128, row, emitted)); + CHECK(!emitted); + CHECK(ds4_compressed_page_row(255, 2, 128, row, emitted)); + CHECK(emitted && row == 2); + + uint64_t capacity = 0; + CHECK(ds4_compressed_page_capacity(5, 4, capacity) && capacity == 160); + CHECK(ds4_compressed_page_capacity(5, 128, capacity) && capacity == 5); + CHECK(!ds4_compressed_page_capacity(5, 0, capacity)); + CHECK(!ds4_compressed_page_capacity(5, 16, capacity)); + CHECK(!ds4_compressed_page_capacity( + std::numeric_limits::max(), 4, capacity)); + CHECK(!ds4_compressed_page_row(3, + std::numeric_limits::max(), 4, row, emitted)); + CHECK(!ds4_compressed_page_row(3, 0, 16, row, emitted)); + + std::printf("OK test_deepseek4_page_layout (%d checks)\n", g_checks); + return 0; +} diff --git a/server/test/test_deepseek4_paged_cache.cpp b/server/test/test_deepseek4_paged_cache.cpp new file mode 100644 index 000000000..dd9681bf0 --- /dev/null +++ b/server/test/test_deepseek4_paged_cache.cpp @@ -0,0 +1,86 @@ +#include "deepseek4/deepseek4_paged_cache.h" +#include "host_check.h" +#include +#include +using namespace dflash::common; +static int g_checks = 0; +int main() { + DeepSeek4PagedCachePlan p, twice; + CHECK(plan_deepseek4_paged_cache(512, 128, 3, 4096, 40, {0, 4, 128}, p)); + CHECK(p.max_blocks_per_sequence == 32 && p.physical_rows[0] == 0); + CHECK(p.physical_rows[1] == 1280 && p.physical_rows[2] == 40); + CHECK(p.raw_bytes == uint64_t(3) * 512 * 128 * 3 * 2); + CHECK(p.metadata_bytes == uint64_t(32 * 3 + 3 + 3) * 4); + CHECK(plan_deepseek4_paged_cache(512, 128, 6, 4096, 40, {0, 4, 128}, twice)); + // Paged rows are shared; only raw rings, metadata, and compressor state scale by slots. + CHECK(twice.compressed_bytes == p.compressed_bytes); + CHECK(twice.raw_bytes == p.raw_bytes * 2 && twice.state_bytes == p.state_bytes * 2); + DeepSeek4PagedCachePlan sixteen; + CHECK(plan_deepseek4_paged_cache(512, 128, 16, 4096, 40, + {0, 4, 128}, sixteen)); + CHECK(sixteen.slots == 16 && sixteen.max_blocks_per_sequence == 32); + CHECK(sixteen.compressed_bytes == p.compressed_bytes); + CHECK(sixteen.raw_bytes == p.raw_bytes / 3 * 16); + CHECK(!plan_deepseek4_paged_cache(512, 128, 1, 4096, 40, {4, 16}, twice)); + CHECK(!plan_deepseek4_paged_cache(512, 128, 1, 4096, + std::numeric_limits::max(), {4}, twice)); + + const int32_t slots[] = {2, 5, -1}; + const int64_t positions[] = {3, 259, 999}; + const int32_t tables[] = {4, 3, 2, 6, 1, 7, 0, 0, 0}; + std::vector rows; + CHECK(prepare_deepseek4_gathered_lane_rows( + slots, positions, 3, tables, 3, 8, 4, rows)); + CHECK(rows.size() == 3); + CHECK(rows[0].raw_history == std::vector({256, 257, 258})); + CHECK(rows[0].raw_scatter == 259); + CHECK(rows[0].compressed_emitted && rows[0].compressed_scatter == 4 * 32); + CHECK(rows[0].compressed_history.empty()); + CHECK(rows[1].raw_history.size() == 127); + CHECK(rows[1].raw_history.front() == 5 * 128 + 4); + CHECK(rows[1].raw_history.back() == 5 * 128 + 2); + CHECK(rows[1].compressed_history.size() == 64); + CHECK(rows[1].compressed_history.front() == 6 * 32); + CHECK(rows[1].compressed_history[31] == 6 * 32 + 31); + CHECK(rows[1].compressed_history[32] == 1 * 32); + CHECK(rows[1].compressed_history.back() == 1 * 32 + 31); + CHECK(rows[1].compressed_emitted && rows[1].compressed_scatter == 7 * 32); + CHECK(rows[2].raw_history.empty() && rows[2].compressed_history.empty()); + CHECK(rows[2].raw_scatter == -1 && rows[2].compressed_scatter == -1); + + std::vector sixteen_slots(16); + std::vector sixteen_positions(16, 0); + std::vector sixteen_tables(16); + for (int i = 0; i < 16; ++i) { + sixteen_slots[(size_t) i] = i; + sixteen_tables[(size_t) i] = i; + } + CHECK(prepare_deepseek4_gathered_lane_rows( + sixteen_slots.data(), sixteen_positions.data(), 16, + sixteen_tables.data(), 1, 16, 4, rows)); + CHECK(rows.size() == 16); + for (int i = 0; i < 16; ++i) { + CHECK(rows[(size_t) i].slot == i); + CHECK(rows[(size_t) i].raw_history.empty()); + CHECK(rows[(size_t) i].raw_scatter == int64_t(i * 128)); + } + + const int32_t boundary_slot[] = {1}; + const int32_t boundary_table[] = {0, 1}; + for (int64_t pos : {127LL, 128LL, 129LL}) { + CHECK(prepare_deepseek4_gathered_lane_rows( + boundary_slot, &pos, 1, boundary_table, 2, 2, 0, rows)); + CHECK(rows[0].raw_history.size() == (pos == 127 ? 127u : 127u)); + CHECK(rows[0].raw_history.front() == 128 + (pos == 127 ? 0 : pos - 127)); + CHECK(rows[0].raw_history.back() == 128 + ((pos - 1) % 128)); + } + + const int64_t ratio128_pos[] = {255}; + CHECK(prepare_deepseek4_gathered_lane_rows( + slots, ratio128_pos, 1, tables, 3, 8, 128, rows)); + CHECK(rows[0].compressed_history.size() == 1); + CHECK(rows[0].compressed_history[0] == 4); + CHECK(rows[0].compressed_emitted && rows[0].compressed_scatter == 3); + std::printf("OK test_deepseek4_paged_cache (%d checks)\n", g_checks); + return 0; +} From 4434880b1d0ad7ed37a9773541b637f50cfafe67 Mon Sep 17 00:00:00 2001 From: Graffioh Date: Wed, 12 Aug 2026 07:52:46 +0000 Subject: [PATCH 2/5] perf(ds4): batch gathered concurrency work --- .../src/deepseek4/deepseek4_fused_verify.inc | 176 +++++++-- server/src/deepseek4/deepseek4_graph.cpp | 373 +++++++++++++----- .../src/deepseek4/deepseek4_paged_cache.cpp | 5 +- server/test/test_deepseek4_paged_cache.cpp | 1 + 4 files changed, 405 insertions(+), 150 deletions(-) diff --git a/server/src/deepseek4/deepseek4_fused_verify.inc b/server/src/deepseek4/deepseek4_fused_verify.inc index effa4dac2..0b5b594f9 100644 --- a/server/src/deepseek4/deepseek4_fused_verify.inc +++ b/server/src/deepseek4/deepseek4_fused_verify.inc @@ -379,9 +379,27 @@ static bool ds4_build_fused_verify_graph( DeepSeek4PagedCache * paged_cache = nullptr, const std::vector> * paged_rows = nullptr) { const bool paged_mode = paged_cache && paged_rows; - if (fg.sched) { + if (paged_mode) { + if (q < 1 || q > 16 || + paged_rows->size() != (size_t) w.n_layer) { + return false; + } + for (const auto & layer_rows : *paged_rows) { + if (layer_rows.size() != (size_t) q) return false; + } + } + const size_t graph_capacity = q > 8 ? 131072u : 65536u; + std::array sched_backends{}; + if (hybrid) { + sched_backends = {backend, hybrid->cold_backend, hybrid->cpu_backend}; + } + const bool reuse_sched = + hybrid && fg.sched_reusable(sched_backends, graph_capacity); + if (fg.sched && !reuse_sched) { ggml_backend_sched_free(fg.sched); fg.sched = nullptr; + fg.sched_capacity = 0; + fg.sched_backends = {}; } step_graph_free(fg.sg); fg.reset_nodes(); @@ -410,7 +428,6 @@ static bool ds4_build_fused_verify_graph( // sequence. Above eight lanes the resulting whole-model graph exceeds // the verifier-era 64K scheduler hash table even though the metadata // arena still has ample room. - const size_t graph_capacity = q > 8 ? 131072u : 65536u; fg.sg.gf = ggml_new_graph_custom(ctx, graph_capacity, false); ggml_cgraph * gf = fg.sg.gf; @@ -455,10 +472,39 @@ static bool ds4_build_fused_verify_graph( ggml_set_input(fg.mask_bundle); int64_t mask_off = 0; - // Keep every speculative lane in one HC tensor. The previous graph built - // five independent HC ops and progressively concatenated their outputs at - // every pre/post boundary. HC kernels already support a token dimension, - // so preserve it throughout the verifier and issue one batched op instead. + // Back every gathered lane's scalar and index inputs with shared + // tensors. Their sizes are fixed by the prepared history shape key. + int64_t paged_gather_total = 0; + if (paged_mode) { + for (int il = 0; il < w.n_layer; ++il) { + const int layer_ratio = (int) w.compress_ratios[il]; + for (int t = 0; t < q; ++t) { + const auto & rows = (*paged_rows)[(size_t) il][(size_t) t]; + paged_gather_total += + (int64_t) std::max(rows.raw_history.size(), 1); + if (layer_ratio > 0) { + paged_gather_total += (int64_t) std::max( + rows.compressed_history.size(), 1); + } + } + } + ex.paged_i32_n = (int64_t) 5 * w.n_layer * q; + ex.paged_i64_n = (int64_t) 3 * w.n_layer * q; + ex.paged_gather_n = std::max(paged_gather_total, 1); + ex.paged_i32 = ggml_new_tensor_1d( + ctx, GGML_TYPE_I32, ex.paged_i32_n); + ggml_set_input(ex.paged_i32); + ex.paged_i64 = ggml_new_tensor_1d( + ctx, GGML_TYPE_I64, ex.paged_i64_n); + ggml_set_input(ex.paged_i64); + ex.paged_gather = ggml_new_tensor_1d( + ctx, GGML_TYPE_I32, ex.paged_gather_n); + ggml_set_input(ex.paged_gather); + } + int64_t paged_gather_off = 0; + + // Keep every speculative or gathered lane in one HC tensor. HC kernels + // already support a token dimension, so issue one batched op per boundary. ggml_tensor * embed_3d = ggml_reshape_3d( ctx, fg.inp_embed, n_embd, 1, q); ggml_tensor * hc_repeated = ggml_repeat_4d( @@ -498,44 +544,71 @@ static bool ds4_build_fused_verify_graph( // Gather each lane's immutable chronological history and run the // established MLA lane core at q=1; all surrounding HC/MoE/output // machinery remains q-wide and unchanged. - if (q < 1 || q > 16 || paged_rows->size() != (size_t) w.n_layer || - (*paged_rows)[(size_t) il].size() != (size_t) q) return false; DeepSeek4PagedLayerCache & plc = paged_cache->layers[(size_t) il]; ggml_tensor * raw_flat = ggml_reshape_2d( ctx, plc.raw_kv, w.head_dim, (int64_t) DS4_PAGE_TOKENS * paged_cache->plan.slots); + ggml_tensor * attn_normed = + build_rms_norm(ctx, attn_in, L.attn_norm, w.rms_eps); + DeepSeek4PreparedProjectedLane batched_proj = + build_mla_qkv_projection(ctx, attn_normed, w, L, q); + build_mla_qkv_rope( + ctx, batched_proj, w, ds4_rope_params(w, ratio), q, + ex.pos_q, /*fuse_q_rope=*/false); for (int t = 0; t < q; ++t) { const auto & rows = (*paged_rows)[(size_t) il][(size_t) t]; auto & px = ex.paged.emplace_back(); - px.pos = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, 1); ggml_set_input(px.pos); - px.neg_pos = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, 1); ggml_set_input(px.neg_pos); - px.raw_gather = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, - std::max(rows.raw_history.size(), 1)); ggml_set_input(px.raw_gather); + const int64_t lane_idx = (int64_t) il * q + t; + px.i32_base = lane_idx * 5; + px.i64_base = lane_idx * 3; + auto scalar_i32 = [&](int slot) { + return ggml_view_1d( + ctx, ex.paged_i32, 1, + (size_t) (px.i32_base + slot) * sizeof(int32_t)); + }; + auto scalar_i64 = [&](int slot) { + return ggml_view_1d( + ctx, ex.paged_i64, 1, + (size_t) (px.i64_base + slot) * sizeof(int64_t)); + }; + px.pos = scalar_i32(0); + px.neg_pos = scalar_i32(1); + px.raw_n = + (int64_t) std::max(rows.raw_history.size(), 1); + px.raw_off = paged_gather_off; + px.raw_gather = ggml_view_1d( + ctx, ex.paged_gather, px.raw_n, + (size_t) px.raw_off * sizeof(int32_t)); + paged_gather_off += px.raw_n; ggml_tensor * raw_history = rows.raw_history.empty() ? nullptr : ggml_get_rows(ctx, raw_flat, px.raw_gather); ggml_tensor * comp_history = nullptr; ggml_tensor * index_history = nullptr; if (ratio > 0) { - px.comp_gather = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, - std::max(rows.compressed_history.size(), 1)); - ggml_set_input(px.comp_gather); + px.comp_n = (int64_t) std::max( + rows.compressed_history.size(), 1); + px.comp_off = paged_gather_off; + px.comp_gather = ggml_view_1d( + ctx, ex.paged_gather, px.comp_n, + (size_t) px.comp_off * sizeof(int32_t)); + paged_gather_off += px.comp_n; if (!rows.compressed_history.empty()) comp_history = ggml_get_rows(ctx, plc.comp_kv, px.comp_gather); if (ratio == 4) { - px.index_gather = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, - std::max(rows.compressed_history.size(), 1)); - ggml_set_input(px.index_gather); - if (!rows.compressed_history.empty()) - index_history = ggml_get_rows(ctx, plc.index_comp_kv, px.index_gather); + // The indexer and attention compressors address the + // same chronological compressed rows. Explicit mode + // has no indexer consumer, so share the gather and do + // not emit the otherwise-dead index-history read. + px.index_gather = px.comp_gather; } } - px.raw_write = ggml_new_tensor_1d(ctx, GGML_TYPE_I64, 1); ggml_set_input(px.raw_write); + px.raw_write = scalar_i64(0); if (ratio > 0) { - px.comp_write = ggml_new_tensor_1d(ctx, GGML_TYPE_I64, 1); ggml_set_input(px.comp_write); - px.comp_read = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, 1); ggml_set_input(px.comp_read); - px.ape = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, 1); ggml_set_input(px.ape); - px.state_row = ggml_new_tensor_1d(ctx, GGML_TYPE_I64, 1); ggml_set_input(px.state_row); - px.comp_pos = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, 1); ggml_set_input(px.comp_pos); + px.comp_write = scalar_i64(1); + px.comp_read = scalar_i32(2); + px.ape = scalar_i32(3); + px.state_row = scalar_i64(2); + px.comp_pos = scalar_i32(4); } DeepSeek4CompressorState attn_state{}, index_state{}; if (ratio > 0 && rows.slot >= 0) { @@ -580,13 +653,19 @@ static bool ds4_build_fused_verify_graph( std::vector ib; std::vector iab; std::vector lab; - ggml_tensor * col = ggml_view_2d(ctx, attn_in, n_embd, 1, - attn_in->nb[1], (size_t) t * attn_in->nb[1]); + ggml_tensor * col = ggml_view_2d( + ctx, attn_normed, n_embd, 1, attn_normed->nb[1], + (size_t) t * attn_normed->nb[1]); + const DeepSeek4PreparedProjectedLane lane_proj = + ds4_slice_projected_lane(ctx, batched_proj, w, t, px.pos); + ggml_tensor * lane_context = nullptr; ggml_tensor * one = build_mla_attention_lane_core( - ctx, gf, build_rms_norm(ctx, col, L.attn_norm, w.rms_eps), - w, L, lane, il, (int) rows.position, 1, &ci, ib, iab, lab, - nullptr, DeepSeek4AttentionImpl::Explicit); - if (!one || !ib.empty() || !iab.empty() || !lab.empty()) { + ctx, gf, col, w, L, lane, il, (int) rows.position, 1, + &ci, ib, iab, lab, nullptr, + DeepSeek4AttentionImpl::Explicit, &lane_proj, + &lane_context); + if (!one || !lane_context || !ib.empty() || !iab.empty() || + !lab.empty()) { std::fprintf(stderr, "[deepseek4-paged] layer %d lane %d attention build " "failed (graph=%d i32=%zu arrays=%zu i64=%zu, " @@ -596,8 +675,12 @@ static bool ds4_build_fused_verify_graph( ? INT32_MIN : iab[0].values[0]); return false; } - attn_out = attn_out ? ggml_concat(ctx, attn_out, one, 1) : one; + attn_out = attn_out + ? ggml_concat(ctx, attn_out, lane_context, 1) + : lane_context; } + attn_out = build_mla_output_projection( + ctx, attn_out, w, L, q, /*allow_grouped=*/false); } else { // ── Batched speculative attention ── DeepSeek4AttentionGraphInputs ain{}; @@ -989,14 +1072,22 @@ static bool ds4_build_fused_verify_graph( warned_cross_vendor_join = true; } } - ggml_backend_t backends[3] = { - backend, peer, hybrid->cpu_backend}; - fg.sched = ggml_backend_sched_new( - backends, nullptr, 3, graph_capacity, false, true); - if (!fg.sched) { - std::fprintf(stderr, - "[ds4-fused-verify] scheduler creation failed\n"); - return false; + if (reuse_sched) { + // Drop prior tensor assignments and deferred events while keeping + // the scheduler-owned pinned staging allocations alive. + ggml_backend_sched_reset(fg.sched); + } else { + ggml_backend_t backends[3] = { + backend, peer, hybrid->cpu_backend}; + fg.sched = ggml_backend_sched_new( + backends, nullptr, 3, graph_capacity, false, true); + if (!fg.sched) { + std::fprintf(stderr, + "[ds4-fused-verify] scheduler creation failed\n"); + return false; + } + fg.sched_capacity = graph_capacity; + fg.sched_backends = sched_backends; } const bool late_join_split = mixed_policy.late_join_split; const MoeHybridGraphPolicy & moe_policy = @@ -1037,6 +1128,9 @@ static bool ds4_build_fused_verify_graph( pin_main(fg.i32_bundle); pin_main(fg.i64_bundle); pin_main(fg.mask_bundle); + pin_main(ex.paged_i32); + pin_main(ex.paged_i64); + pin_main(ex.paged_gather); for (const auto & px : ex.paged) { pin_main(px.pos); pin_main(px.neg_pos); pin_main(px.raw_gather); pin_main(px.comp_gather); pin_main(px.index_gather); diff --git a/server/src/deepseek4/deepseek4_graph.cpp b/server/src/deepseek4/deepseek4_graph.cpp index ec3507045..a1fbe24da 100644 --- a/server/src/deepseek4/deepseek4_graph.cpp +++ b/server/src/deepseek4/deepseek4_graph.cpp @@ -1679,6 +1679,130 @@ struct DeepSeek4PreparedProjectedLane { ggml_tensor * rope_pos = nullptr; }; +// Per-layer RoPE parameters. Compressed layers use YaRN scaling, and +// attn_factor cancels the magnitude scaling rope_yarn applies. +struct Ds4RopeParams { + float freq = 0.0f; + float scale = 1.0f; + float ext = 0.0f; + float attn = 1.0f; + int n_ctx_orig = 0; +}; + +static Ds4RopeParams ds4_rope_params(const DeepSeek4Weights & w, int ratio) { + const bool compressed = ratio > 0; + Ds4RopeParams p; + p.freq = compressed ? w.compress_rope_freq_base : w.rope_freq_base; + p.scale = compressed ? (1.0f / w.rope_scale_factor) : 1.0f; + p.ext = compressed ? 1.0f : 0.0f; + if (p.ext != 0.0f && p.scale > 0.0f) { + p.attn /= (1.0f + 0.1f * logf(1.0f / p.scale)); + } + p.n_ctx_orig = (int) w.rope_orig_ctx; + return p; +} + +// Q/KV projections and their tail RoPE are independent per token. A paged +// caller can evaluate them once at width q and hand each lane a column view, +// avoiding one reread of all three projection weights per active lane. +static DeepSeek4PreparedProjectedLane build_mla_qkv_projection( + ggml_context * ctx, + ggml_tensor * cur, + const DeepSeek4Weights & w, + const DeepSeek4Layer & L, + int n_tokens) { + DeepSeek4PreparedProjectedLane out; + ggml_tensor * qr = ggml_mul_mat(ctx, L.attn_q_a, cur); + qr = build_rms_norm(ctx, qr, L.attn_q_a_norm, w.rms_eps); + ggml_tensor * q = ggml_mul_mat(ctx, L.attn_q_b, qr); + q = ggml_reshape_3d(ctx, q, w.head_dim, w.n_head, n_tokens); + q = ggml_rms_norm(ctx, q, w.rms_eps); + + ggml_tensor * kv = ggml_mul_mat(ctx, L.attn_kv, cur); + kv = build_rms_norm(ctx, kv, L.attn_kv_a_norm, w.rms_eps); + + out.normalized_q_lora = qr; + out.q = q; + out.kv = kv; + return out; +} + +static void build_mla_qkv_rope( + ggml_context * ctx, + DeepSeek4PreparedProjectedLane & p, + const DeepSeek4Weights & w, + const Ds4RopeParams & rope, + int n_tokens, + ggml_tensor * rope_pos, + bool fuse_q_rope) { + if (!fuse_q_rope) { + p.q = build_tail_rope_3d(ctx, p.q, rope_pos, w.n_rot, w.head_dim, + w.n_head, n_tokens, rope.freq, rope.scale, + rope.ext, rope.attn, w.rope_yarn_beta_fast, + w.rope_yarn_beta_slow, rope.n_ctx_orig); + } + p.kv = build_tail_rope_2d(ctx, p.kv, rope_pos, w.n_rot, w.head_dim, + n_tokens, rope.freq, rope.scale, rope.ext, + rope.attn, w.rope_yarn_beta_fast, + w.rope_yarn_beta_slow, rope.n_ctx_orig); + p.rope_pos = rope_pos; +} + +// Grouped low-rank output projection. Several independent gathered lanes can +// concatenate their pre-projection contexts and share one q-wide evaluation. +static ggml_tensor * build_mla_output_projection( + ggml_context * ctx, + ggml_tensor * attn_out, + const DeepSeek4Weights & w, + const DeepSeek4Layer & L, + int n_tokens, + bool allow_grouped) { + const int group_dim = w.head_dim * (w.n_head / w.n_out_group); + attn_out = ggml_reshape_3d( + ctx, attn_out, group_dim, w.n_out_group, n_tokens); + attn_out = ggml_permute(ctx, attn_out, 0, 2, 1, 3); + if (n_tokens == 1) { + attn_out = ggml_cont(ctx, attn_out); + } + ggml_tensor * out_a_3d = ggml_reshape_3d( + ctx, L.attn_output_a, group_dim, w.n_lora_o, w.n_out_group); + ggml_tensor * attn_low = ggml_mul_mat(ctx, out_a_3d, attn_out); + + const bool grouped_output_projection = + allow_grouped && n_tokens > 1 && + !ds4_env_flag("DFLASH_DS4_DISABLE_GROUPED_OUTPUT_PROJECTION"); + if (grouped_output_projection) { + return ggml_mul_mat_grouped_src(ctx, L.attn_output_b, attn_low); + } + attn_low = ggml_cont(ctx, ggml_permute(ctx, attn_low, 0, 2, 1, 3)); + attn_low = ggml_reshape_2d( + ctx, attn_low, (int64_t) w.n_lora_o * w.n_out_group, n_tokens); + return ggml_mul_mat(ctx, L.attn_output_b, attn_low); +} + +// One lane's column of a batched prologue. These are views only. +static DeepSeek4PreparedProjectedLane ds4_slice_projected_lane( + ggml_context * ctx, + const DeepSeek4PreparedProjectedLane & batched, + const DeepSeek4Weights & w, + int lane, + ggml_tensor * lane_rope_pos) { + DeepSeek4PreparedProjectedLane out; + out.normalized_q_lora = ggml_view_2d( + ctx, batched.normalized_q_lora, batched.normalized_q_lora->ne[0], 1, + batched.normalized_q_lora->nb[1], + (size_t) lane * batched.normalized_q_lora->nb[1]); + out.q = ggml_view_3d( + ctx, batched.q, w.head_dim, w.n_head, 1, + batched.q->nb[1], batched.q->nb[2], + (size_t) lane * batched.q->nb[2]); + out.kv = ggml_view_2d( + ctx, batched.kv, batched.kv->ne[0], 1, batched.kv->nb[1], + (size_t) lane * batched.kv->nb[1]); + out.rope_pos = lane_rope_pos; + return out; +} + static DeepSeek4MlaLaneBindings deepseek4_contiguous_lane_bindings( DeepSeek4LayerCache & lc, int ratio, @@ -1713,49 +1837,36 @@ static ggml_tensor * build_mla_attention_lane_core( std::vector & i32_array_inputs, std::vector & i64_array_inputs, std::vector * f32_array_inputs = nullptr, - DeepSeek4AttentionImpl attention_impl = DeepSeek4AttentionImpl::Explicit) { + DeepSeek4AttentionImpl attention_impl = DeepSeek4AttentionImpl::Explicit, + const DeepSeek4PreparedProjectedLane * prepared = nullptr, + ggml_tensor ** out_attn_context = nullptr) { const int n_embd = w.n_embd; const int head_dim = w.head_dim; const int n_head = w.n_head; const int n_rot = w.n_rot; - const int n_out_group = w.n_out_group; - const int n_lora_o = w.n_lora_o; const int ratio = w.compress_ratios[layer_idx]; const bool gathered_history = lane.history_mode == DeepSeek4MlaLaneBindings::HistoryMode::ChronologicalGathered; - // ── Q path: cur → q_a → norm → q_b → per-head norm ───────────── - // q_a: [n_embd, n_tokens] → [n_lora_q, n_tokens] - ggml_tensor * qr = ggml_mul_mat(ctx, L.attn_q_a, cur); - // qr_norm is reused by the ratio-4 indexer before the main q_b projection. - qr = build_rms_norm(ctx, qr, L.attn_q_a_norm, w.rms_eps); - // q_b: [n_lora_q, n_tokens] → [n_head * head_dim, n_tokens] - ggml_tensor * q = ggml_mul_mat(ctx, L.attn_q_b, qr); - // Reshape to [head_dim, n_head, n_tokens] for per-head ops - q = ggml_reshape_3d(ctx, q, head_dim, n_head, n_tokens); - // Reference DS4 applies unweighted RMSNorm independently to every Q head. - q = ggml_rms_norm(ctx, q, w.rms_eps); - - // ── KV path: cur → kv → norm ─────────────────────────────────── - // kv: [n_embd, n_tokens] → [head_dim, n_tokens] - ggml_tensor * kv = ggml_mul_mat(ctx, L.attn_kv, cur); - kv = build_rms_norm(ctx, kv, L.attn_kv_a_norm, w.rms_eps); + // Existing callers leave prepared null and emit the original prologue in + // place. Only gathered paged concurrency supplies a q-wide projection. + DeepSeek4PreparedProjectedLane projected; + if (!prepared) { + projected = build_mla_qkv_projection(ctx, cur, w, L, n_tokens); + } // ── RoPE on Q and KV (tail rotation on last n_rot dims) ──────── - // DS4 uses per-layer RoPE params: compressed layers get YaRN scaling. - const bool compressed = (ratio > 0); - const float rope_freq = compressed ? w.compress_rope_freq_base : w.rope_freq_base; - const float rope_scale = compressed ? (1.0f / w.rope_scale_factor) : 1.0f; - const float rope_ext = compressed ? 1.0f : 0.0f; - // For YaRN: attn_factor cancels the magnitude scaling in rope_yarn - float rope_attn = 1.0f; - if (rope_ext != 0.0f && rope_scale > 0.0f) { - rope_attn /= (1.0f + 0.1f * logf(1.0f / rope_scale)); - } + const Ds4RopeParams rope = ds4_rope_params(w, ratio); + const float rope_freq = rope.freq; + const float rope_scale = rope.scale; + const float rope_ext = rope.ext; + const float rope_attn = rope.attn; + const int rope_n_ctx_orig = rope.n_ctx_orig; // Position tensor for this token batch - ggml_tensor * rope_pos = cached_inputs ? cached_inputs->rope_pos : nullptr; + ggml_tensor * rope_pos = prepared ? prepared->rope_pos + : (cached_inputs ? cached_inputs->rope_pos : nullptr); if (!rope_pos) { rope_pos = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, n_tokens); ggml_set_input(rope_pos); @@ -1764,27 +1875,20 @@ static ggml_tensor * build_mla_attention_lane_core( i32_array_inputs.push_back({rope_pos, std::move(pos_vals)}); } - // n_ctx_orig is critical for YaRN correction on compressed layers - const int rope_n_ctx_orig = (int)w.rope_orig_ctx; // 65536 - // D=512 flash prefill can rotate Q's 64-d tail inside the exact attention // kernel. This avoids materializing cont(nope), cont(tail), rope(tail), // and concat(nope, tail) while retaining the same F32 rounding boundary. const bool fuse_q_rope = attention_impl != DeepSeek4AttentionImpl::Explicit && n_tokens > 1 && head_dim == 512 && n_rot == 64; - if (!fuse_q_rope) { - q = build_tail_rope_3d(ctx, q, rope_pos, n_rot, head_dim, n_head, n_tokens, - rope_freq, rope_scale, rope_ext, rope_attn, - w.rope_yarn_beta_fast, w.rope_yarn_beta_slow, rope_n_ctx_orig); + if (prepared) { + projected = *prepared; + } else { + build_mla_qkv_rope( + ctx, projected, w, rope, n_tokens, rope_pos, fuse_q_rope); } - kv = build_tail_rope_2d(ctx, kv, rope_pos, n_rot, head_dim, n_tokens, - rope_freq, rope_scale, rope_ext, rope_attn, - w.rope_yarn_beta_fast, w.rope_yarn_beta_slow, rope_n_ctx_orig); - - const DeepSeek4PreparedProjectedLane projected = {qr, q, kv, rope_pos}; - // Keep the established local names below to make the no-topology-change - // property obvious; the bundle is the handoff seam for a future adapter. - (void) projected; + ggml_tensor * qr = projected.normalized_q_lora; + ggml_tensor * q = projected.q; + ggml_tensor * kv = projected.kv; // ── Causal batched step (exact multi-token target semantics) ─── // The target model is causal: token i must not attend to batch tokens @@ -1934,7 +2038,13 @@ static ggml_tensor * build_mla_attention_lane_core( } ggml_tensor * index_comp_kv_source = lane.index_comp_kv; - if (lane.write_enabled && ratio == 4 && L.indexer_compressor_kv) { + // Gathered paged concurrency always uses Explicit attention, whose + // build_indexer_topk path is disabled. In that mode the indexer compressor + // only writes state that no graph node reads, so omit the dead subgraph. + const bool indexer_compressor_is_dead = + gathered_history && attention_impl != DeepSeek4AttentionImpl::SparseFlash; + if (lane.write_enabled && ratio == 4 && L.indexer_compressor_kv && + !indexer_compressor_is_dead) { build_indexer_compressor_step(ctx, gf, cur_last, w, L, *lane.indexer_compressor, lane.index_comp_kv, token_pos, cached_inputs ? cached_inputs->index_ape_row : nullptr, @@ -2502,46 +2612,13 @@ static ggml_tensor * build_mla_attention_lane_core( // Flatten to [head_dim*n_head, n_tokens] for output projection ggml_tensor * attn_out = ggml_reshape_2d(ctx, context, head_dim * n_head, n_tokens); - // ── Grouped output projection ────────────────────────────────── - // DS4 output uses grouped low-rank projection: - // attn_out: [head_dim*n_head, n_tokens] → reshape [group_dim, n_tokens, n_groups] - // out_a: [group_dim, n_groups*n_lora_o] → reshape [group_dim, n_lora_o, n_groups] - // batched matmul over n_groups: → [n_lora_o, n_tokens, n_groups] - // → reshape [n_lora_o*n_groups, n_tokens] - // out_b: [n_lora_o*n_groups, n_embd] → final: [n_embd, n_tokens] - const int group_dim = head_dim * (n_head / n_out_group); // 512 * 8 = 4096 - // Reshape attn_out: [32768, n_tokens] → [4096, 8, n_tokens] → permute to [4096, n_tokens, 8] - attn_out = ggml_reshape_3d(ctx, attn_out, group_dim, n_out_group, n_tokens); - attn_out = ggml_permute(ctx, attn_out, 0, 2, 1, 3); - if (n_tokens == 1) { - attn_out = ggml_cont(ctx, attn_out); - } - // attn_out is now [group_dim, n_tokens, n_out_group] - ggml_tensor * out_a_3d = ggml_reshape_3d(ctx, L.attn_output_a, group_dim, n_lora_o, n_out_group); - // out_a_3d: [group_dim, n_lora_o, n_out_group] — ne[2] matches - ggml_tensor * attn_low = ggml_mul_mat(ctx, out_a_3d, attn_out); - // attn_low: [n_lora_o, n_tokens, n_out_group] - ggml_tensor * out = nullptr; - const bool grouped_output_projection = - n_tokens > 1 && - !ds4_env_flag("DFLASH_DS4_DISABLE_GROUPED_OUTPUT_PROJECTION"); - if (grouped_output_projection) { - // Batched ROCmFPX MMQ consumes src1's channel stride directly. This - // avoids materializing both permutations (~256 MiB/layer at 2K). - out = ggml_mul_mat_grouped_src(ctx, L.attn_output_b, attn_low); - } else { - // Preserve the established single-token graph and provide an exact - // fallback for heterogeneous runtimes that cannot retain grouped-view - // metadata across a scheduler copy. At verifier widths (q <= 4), this - // materializes at most 128 KiB per layer rather than the long-prefill - // volume avoided by the grouped path. - attn_low = ggml_cont(ctx, ggml_permute(ctx, attn_low, 0, 2, 1, 3)); - attn_low = ggml_reshape_2d( - ctx, attn_low, n_lora_o * n_out_group, n_tokens); - out = ggml_mul_mat(ctx, L.attn_output_b, attn_low); + if (out_attn_context) { + *out_attn_context = attn_out; + return attn_out; } - return out; + return build_mla_output_projection(ctx, attn_out, w, L, n_tokens, + /*allow_grouped=*/true); } // Legacy contiguous-cache adapter. Both decode and the consecutive q>1 @@ -4878,6 +4955,16 @@ struct DeepSeek4FusedDecodeGraph { std::vector authoritative_routes; ggml_tensor * logits = nullptr; ggml_backend_sched_t sched = nullptr; + // The scheduler owns large pinned cross-backend staging buffers. Retain it + // across gathered-paged shape rebuilds when its backend set and capacity + // still match. + size_t sched_capacity = 0; + std::array sched_backends{}; + + bool sched_reusable(const std::array & backends, + size_t capacity) const { + return sched && sched_capacity >= capacity && sched_backends == backends; + } void reset_nodes() { inp_embed = nullptr; @@ -4913,6 +5000,22 @@ struct DeepSeek4FusedDecodeGraph { } } + // Retain shape-independent resources, but first retire native graph + // executables whose keys point into this metadata arena. The allocator and + // scheduler remain alive; the builder resets their per-graph state. + void release_for_rebuild(ggml_backend_t main_backend, + ggml_backend_t peer_backend = nullptr) { + invalidate_native_graphs(main_backend, peer_backend); + // Clear scheduler registrations while their tensor metadata is still + // valid. The builder may reset again after deciding to reuse it; that + // second reset is a no-op but keeps the builder self-contained. + if (sched) { + ggml_backend_sched_reset(sched); + } + step_graph_free(sg); + reset_nodes(); + } + void destroy(ggml_backend_t main_backend, ggml_backend_t peer_backend = nullptr) { // Native graph executables outlive ggml graph metadata in the backend @@ -4923,6 +5026,8 @@ struct DeepSeek4FusedDecodeGraph { ggml_backend_sched_free(sched); sched = nullptr; } + sched_capacity = 0; + sched_backends = {}; step_graph_destroy(sg); reset_nodes(); } @@ -4991,6 +5096,15 @@ struct Ds4FusedVerifyCache { ggml_tensor * ape = nullptr; ggml_tensor * state_row = nullptr; ggml_tensor * comp_pos = nullptr; + // Element offsets into the shared per-dtype upload bundles. + int64_t i32_base = -1; // pos, neg_pos, comp_read, ape, comp_pos + int64_t i64_base = -1; // raw_write, comp_write, state_row + int64_t raw_off = -1; + int64_t raw_n = 0; + int64_t comp_off = -1; + int64_t comp_n = 0; + int64_t index_off = -1; + int64_t index_n = 0; }; ggml_tensor * pos_q = nullptr; // i32 [q] ggml_tensor * neg_q = nullptr; // i32 [q] @@ -5005,6 +5119,12 @@ struct Ds4FusedVerifyCache { // Keeping it per slot removes one allocation from every verify step. std::vector mask_values; std::vector paged; // [layer*q], paged mode only + ggml_tensor * paged_i32 = nullptr; + ggml_tensor * paged_i64 = nullptr; + ggml_tensor * paged_gather = nullptr; + int64_t paged_i32_n = 0; + int64_t paged_i64_n = 0; + int64_t paged_gather_n = 0; int q = 0; void reset() { *this = Extra{}; } @@ -7162,7 +7282,8 @@ bool deepseek4_paged_gathered_step( if (vc.slots[i].last_use < vc.slots[pick].last_use) pick = i; } fg = &vc.slots[pick]; ex = &vc.extra[pick]; - fg->destroy(vc.backend, vc.peer_backend); ex->reset(); + fg->release_for_rebuild(vc.backend, vc.peer_backend); + ex->reset(); if (!ds4_build_fused_verify_graph( mc, *fg, *ex, backend, w, cache.prefill_staging, rt->model.hc_layer_weights, rt->model.hc_output_weights, @@ -7172,47 +7293,83 @@ bool deepseek4_paged_gathered_step( std::fprintf(stderr, "[deepseek4-paged] failed to build gathered graph " "(lanes=%u)\n", lanes); + fg->destroy(vc.backend, vc.peer_backend); + ex->reset(); return false; } } fg->last_use = vc.counter; ds4_fv_set(fg->inp_embed, embeddings, sizeof(float) * (size_t) w.n_embd * lanes); + // The shared Q/KV prologue rotates all gathered lanes at once. Padding + // lanes use position zero, matching their passive prepared row record. + { + std::vector pos_batch(lanes, 0); + std::vector neg_batch(lanes, 0); + for (uint32_t lane = 0; lane < lanes; ++lane) { + if (slots[lane] < 0) continue; + pos_batch[lane] = (int32_t) positions[lane]; + neg_batch[lane] = -(int32_t) positions[lane]; + } + ds4_fv_set(ex->pos_q, pos_batch.data(), sizeof(int32_t) * lanes); + ds4_fv_set(ex->neg_q, neg_batch.data(), sizeof(int32_t) * lanes); + } + + std::vector bundle_i32( + (size_t) std::max(ex->paged_i32_n, 0), 0); + std::vector bundle_i64( + (size_t) std::max(ex->paged_i64_n, 0), 0); + std::vector bundle_gather( + (size_t) std::max(ex->paged_gather_n, 0), 0); size_t pi = 0; for (int il = 0; il < w.n_layer; ++il) { const int ratio = (int) cache.layers[(size_t) il].ratio; for (uint32_t lane = 0; lane < lanes; ++lane, ++pi) { const auto & row = prepared[(size_t) il][lane]; const auto & px = ex->paged[pi]; + if (px.i32_base < 0 || px.i64_base < 0) return false; const int32_t pos = (int32_t) row.position; - const int32_t neg_pos = -pos; - ds4_fv_set(px.pos, &pos, sizeof(pos)); - ds4_fv_set(px.neg_pos, &neg_pos, sizeof(neg_pos)); - std::vector idx(std::max(row.raw_history.size(), 1), 0); - for (size_t i = 0; i < row.raw_history.size(); ++i) idx[i] = (int32_t) row.raw_history[i]; - ds4_fv_set(px.raw_gather, idx.data(), idx.size() * sizeof(int32_t)); - const int64_t raw_write = std::max(row.raw_scatter, 0); - ds4_fv_set(px.raw_write, &raw_write, sizeof(raw_write)); - if (ratio > 0) { - idx.assign(std::max(row.compressed_history.size(), 1), 0); - for (size_t i = 0; i < row.compressed_history.size(); ++i) - idx[i] = (int32_t) row.compressed_history[i]; - ds4_fv_set(px.comp_gather, idx.data(), idx.size() * sizeof(int32_t)); - if (px.index_gather) - ds4_fv_set(px.index_gather, idx.data(), idx.size() * sizeof(int32_t)); + bundle_i32[(size_t) px.i32_base + 0] = pos; + bundle_i32[(size_t) px.i32_base + 1] = -pos; + if (px.raw_off < 0 || + px.raw_off + px.raw_n > ex->paged_gather_n || + (int64_t) row.raw_history.size() > px.raw_n) return false; + for (size_t i = 0; i < row.raw_history.size(); ++i) { + bundle_gather[(size_t) px.raw_off + i] = + (int32_t) row.raw_history[i]; + } + bundle_i64[(size_t) px.i64_base + 0] = + std::max(row.raw_scatter, 0); + if (ratio > 0 && px.comp_off >= 0) { + if (px.comp_off + px.comp_n > ex->paged_gather_n || + (int64_t) row.compressed_history.size() > px.comp_n) { + return false; + } + for (size_t i = 0; i < row.compressed_history.size(); ++i) { + const int32_t value = + (int32_t) row.compressed_history[i]; + bundle_gather[(size_t) px.comp_off + i] = value; + if (px.index_off >= 0) { + bundle_gather[(size_t) px.index_off + i] = value; + } + } const int64_t cw = std::max(row.compressed_scatter, 0); - const int32_t cr = (int32_t) cw; const int32_t ape = pos % ratio; - const int64_t state = ratio == 4 ? 4 + ape : ape; - const int32_t comp_pos = pos + 1 - ratio; - ds4_fv_set(px.comp_write, &cw, sizeof(cw)); - ds4_fv_set(px.comp_read, &cr, sizeof(cr)); - ds4_fv_set(px.ape, &ape, sizeof(ape)); - ds4_fv_set(px.state_row, &state, sizeof(state)); - ds4_fv_set(px.comp_pos, &comp_pos, sizeof(comp_pos)); + bundle_i64[(size_t) px.i64_base + 1] = cw; + bundle_i64[(size_t) px.i64_base + 2] = + ratio == 4 ? 4 + ape : ape; + bundle_i32[(size_t) px.i32_base + 2] = (int32_t) cw; + bundle_i32[(size_t) px.i32_base + 3] = ape; + bundle_i32[(size_t) px.i32_base + 4] = pos + 1 - ratio; } } } + ds4_fv_set(ex->paged_i32, bundle_i32.data(), + bundle_i32.size() * sizeof(int32_t)); + ds4_fv_set(ex->paged_i64, bundle_i64.data(), + bundle_i64.size() * sizeof(int64_t)); + ds4_fv_set(ex->paged_gather, bundle_gather.data(), + bundle_gather.size() * sizeof(int32_t)); if (token_ids) { for (int il = 0; il < w.n_layer; ++il) { ggml_tensor * ids = fg->hash_ids[(size_t) il]; if (!ids) continue; diff --git a/server/src/deepseek4/deepseek4_paged_cache.cpp b/server/src/deepseek4/deepseek4_paged_cache.cpp index cdb395dbb..caca04324 100644 --- a/server/src/deepseek4/deepseek4_paged_cache.cpp +++ b/server/src/deepseek4/deepseek4_paged_cache.cpp @@ -34,8 +34,11 @@ bool prepare_deepseek4_gathered_lane_rows( for (uint32_t lane = 0; lane < lanes; ++lane) { auto & rows = prepared[lane]; rows.slot = slots[lane]; - rows.position = positions[lane]; if (rows.slot < 0) continue; // Padding must remain entirely passive. + // Padding lanes do not have a validated position. Keep the default + // zero so inverse RoPE and compressor bookkeeping cannot inherit an + // arbitrary positions[] value once callers use constant-width steps. + rows.position = positions[lane]; if (rows.position < 0) return false; const uint64_t pos = static_cast(rows.position); // The current row is appended in-graph, so retain at most the 127 diff --git a/server/test/test_deepseek4_paged_cache.cpp b/server/test/test_deepseek4_paged_cache.cpp index dd9681bf0..11df40ef3 100644 --- a/server/test/test_deepseek4_paged_cache.cpp +++ b/server/test/test_deepseek4_paged_cache.cpp @@ -47,6 +47,7 @@ int main() { CHECK(rows[1].compressed_emitted && rows[1].compressed_scatter == 7 * 32); CHECK(rows[2].raw_history.empty() && rows[2].compressed_history.empty()); CHECK(rows[2].raw_scatter == -1 && rows[2].compressed_scatter == -1); + CHECK(rows[2].position == 0); std::vector sixteen_slots(16); std::vector sixteen_positions(16, 0); From 17600d85f5952a5c1262c86f7c976a3b179ef473 Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 25 Aug 2026 10:03:29 +0000 Subject: [PATCH 3/5] fix(ds4): integrate concurrency with current main --- .github/workflows/ci.yml | 3 +- README.md | 6 +- server/CMakeLists.txt | 24 +- server/docs/DS4.md | 11 +- server/src/common/backend_args.h | 4 +- server/src/common/backend_factory.cpp | 6 + .../common/concurrency/seq_slot_manager.cpp | 17 +- .../src/common/concurrency/seq_slot_manager.h | 16 +- server/src/common/feature_gate.cpp | 36 ++- server/src/common/model_capabilities.h | 2 +- server/src/deepseek4/deepseek4_backend.cpp | 3 +- server/src/deepseek4/deepseek4_graph.cpp | 7 +- server/src/deepseek4/deepseek4_internal.h | 2 +- server/src/deepseek4/deepseek4_seq_engine.cpp | 5 +- .../concurrency/qwen35_slot_manager.cpp | 277 ------------------ .../qwen35/concurrency/qwen35_slot_manager.h | 136 +-------- server/test/test_feature_gate.cpp | 41 ++- 17 files changed, 137 insertions(+), 459 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 32187e44f..713c4ca16 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,13 +84,14 @@ jobs: test_dflash test_generate test_flash_attn_sparse test_server_unit \ test_deepseek4_unit test_feature_gate test_seq_slot_manager \ test_seq_engine_contract test_seq_batch_plan test_client_send_buffer \ + test_deepseek4_page_layout test_deepseek4_paged_cache \ test_model_smoke test_batched_gdn test_concat_transpose -j$(nproc) - name: Run C++ server unit tests run: | cd server/build ctest --output-on-failure \ - -R "server_unit|deepseek4_unit|feature_gate|seq_slot_manager|seq_engine_contract|seq_batch_plan|client_send_buffer|batched_gdn_cpu" \ + -R "server_unit|deepseek4_unit|feature_gate|seq_slot_manager|seq_engine_contract|seq_batch_plan|client_send_buffer|deepseek4_page_layout|deepseek4_paged_cache|batched_gdn_cpu" \ --no-tests=error - name: Populate venv with cu128 torch + setuptools diff --git a/README.md b/README.md index 2fa12e67d..77b220627 100644 --- a/README.md +++ b/README.md @@ -375,9 +375,9 @@ When compression is on, multi-turn continuations automatically use **FlowKV**: a | `DFLASH_PREFILL_CACHE_SLOTS=N` | `0` | Container-entrypoint equivalent of `--prefill-cache-slots`; the native binary itself uses the CLI flag. | | `--kv-cache-dir ` | — | Persist prefix cache to disk | | `--kv-cache-budget N` | — | On-disk cache size cap | -| `--paged-attention` | off | Exact 16-token block-table attention for Qwen3.6-27B; see [paged attention](optimizations/paged_attention/README.md) | -| `--max-concurrency N` | `1` | Maximum concurrent sequence slots. Values 2–64 enable paged attention automatically. | -| `--kv-pool-tokens N` | `0` (auto) | Shared physical K/V capacity for concurrent paged serving. Requires `--max-concurrency` greater than 1. Zero derives capacity from available device memory; explicit values are rounded to whole 16-token blocks. | +| `--paged-attention` | off | Exact block-table attention for monolithic Qwen3.6-27B (16-token blocks) and DeepSeek4 on Strix Halo (128-token pages); see [paged attention](optimizations/paged_attention/README.md) and [DeepSeek4 concurrent serving](server/docs/DS4.md#strix-halo-concurrent-serving) | +| `--max-concurrency N` | `1` | Maximum concurrent sequence slots. Values above 1 enable paged attention automatically; Qwen supports up to 64 and DeepSeek4 up to 16. | +| `--kv-pool-tokens N` | `0` (auto) | Shared physical K/V capacity for concurrent paged serving. Requires `--max-concurrency` greater than 1. Zero derives capacity from available device memory; explicit values are rounded to the backend's page size. | | `--admission-coalesce-ms N` | `20` | Idle-to-busy batching window for concurrent serving, from 0 to 1000 ms; `0` disables it. | **Bounded KV residency (KVFlash)** diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt index cf1e4ee8b..0098f3a8d 100644 --- a/server/CMakeLists.txt +++ b/server/CMakeLists.txt @@ -432,6 +432,8 @@ add_library(dflash_common STATIC src/deepseek4/deepseek4_loader.cpp src/deepseek4/deepseek4_graph.cpp src/deepseek4/deepseek4_roctx.cpp + src/deepseek4/deepseek4_paged_cache.cpp + src/deepseek4/deepseek4_seq_engine.cpp src/deepseek4/deepseek4_backend.cpp src/deepseek4/deepseek4_daemon.cpp src/deepseek4/deepseek4_layer_split_adapter.cpp @@ -466,7 +468,7 @@ add_library(dflash_common STATIC src/common/dflash_draft_kv.cpp src/common/dflash_spec_decode.cpp src/common/concurrency/paged_kv_pool.cpp - src/qwen35/concurrency/qwen35_slot_manager.cpp + src/common/concurrency/seq_slot_manager.cpp src/common/layer_split_backend.cpp src/common/layer_split_runtime.cpp src/qwen35/graph_builders.cpp @@ -1417,11 +1419,29 @@ if(DFLASH27B_TESTS) ${CMAKE_CURRENT_SOURCE_DIR}/src) list(APPEND _raw_unit_test_targets test_paged_kv_pool) endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/test_deepseek4_page_layout.cpp") + add_executable(test_deepseek4_page_layout + test/test_deepseek4_page_layout.cpp) + target_include_directories(test_deepseek4_page_layout PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/src + ${CMAKE_CURRENT_SOURCE_DIR}/include) + list(APPEND _raw_unit_test_targets test_deepseek4_page_layout) + endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/test_deepseek4_paged_cache.cpp") + add_executable(test_deepseek4_paged_cache + test/test_deepseek4_paged_cache.cpp + src/deepseek4/deepseek4_paged_cache.cpp) + target_compile_definitions(test_deepseek4_paged_cache PRIVATE + DFLASH_DS4_PLAN_ONLY=1) + target_include_directories(test_deepseek4_paged_cache PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/src) + list(APPEND _raw_unit_test_targets test_deepseek4_paged_cache) + endif() if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/test_seq_slot_manager.cpp") # Host-side slot bookkeeping test (concurrent serving): no GPU. add_executable(test_seq_slot_manager test/test_seq_slot_manager.cpp - src/qwen35/concurrency/qwen35_slot_manager.cpp + src/common/concurrency/seq_slot_manager.cpp src/common/concurrency/paged_kv_pool.cpp) target_include_directories(test_seq_slot_manager PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src) diff --git a/server/docs/DS4.md b/server/docs/DS4.md index 086da1683..f8ab267a6 100644 --- a/server/docs/DS4.md +++ b/server/docs/DS4.md @@ -307,13 +307,18 @@ pass; each selected prompt advances by one exact token because the graph must not contain two rows from the same sequence. ```bash -cmake -S . -B build-hip \ +hf download Lucebox/DeepSeek-V4-Flash-0731-ROCmFP3 \ + DeepSeek-V4-Flash-0731-ROCMFPX-MIX-STRIX.gguf \ + --local-dir /path/to/models + +cmake -S server -B server/build-hip \ -DDFLASH27B_GPU_BACKEND=hip \ -DDFLASH27B_HIP_ARCHITECTURES=gfx1151 \ -DDFLASH27B_SERVER=ON -cmake --build build-hip -j +cmake --build server/build-hip -j -./build-hip/dflash_server /path/to/deepseek4-target.gguf \ +./server/build-hip/dflash_server \ + /path/to/models/DeepSeek-V4-Flash-0731-ROCMFPX-MIX-STRIX.gguf \ --target-device hip:0 \ --paged-attention \ --max-concurrency 16 \ diff --git a/server/src/common/backend_args.h b/server/src/common/backend_args.h index 251f1bae2..3bdd945a4 100644 --- a/server/src/common/backend_args.h +++ b/server/src/common/backend_args.h @@ -66,12 +66,12 @@ struct BackendArgs { // Attention and speculative-decode options. Individual backends consume // only the fields they support. int fa_window = 0; // 0 = full attention. qwen3.6 full-attn layers must see the whole context; a finite window drops the system prompt/tools -> breaks tool calls. - bool paged_attention = false; // 16-token paged K/V blocks for AR decode + bool paged_attention = false; // model-specific paged K/V blocks for AR decode // Concurrent decode slots (--max-concurrency). > 1 requires paged_attention; // the backend serves that many sequences through the seq_* slot API. int max_concurrency = 1; // Total paged K/V pool in tokens shared by all slots (--kv-pool-tokens; - // block-rounded). 0 = derive capacity from available device memory. + // model-page-rounded). 0 = derive capacity from available device memory. long long kv_pool_tokens = 0; int kq_stride_pad = 32; int draft_block_size = 0; // 0 = drafter metadata diff --git a/server/src/common/backend_factory.cpp b/server/src/common/backend_factory.cpp index ee9b198bf..51d44ff3e 100644 --- a/server/src/common/backend_factory.cpp +++ b/server/src/common/backend_factory.cpp @@ -103,6 +103,9 @@ DFLASH_CHECK_ARCH("deepseek4", DeepSeek4BackendConfig, DeepSeek4LayerSplitAdapte // that shared struct would fail a check that is really about dispatch. DFLASH_CHECK_ARCH_OPTION("qwen35", Qwen35Config, Qwen35LayerSplitAdapterConfig, has_paged_attention, paged_attn); +DFLASH_CHECK_ARCH_OPTION("deepseek4", DeepSeek4BackendConfig, + DeepSeek4LayerSplitAdapterConfig, + has_paged_attention, paged_attn); DFLASH_CHECK_ARCH_OPTION("qwen35", Qwen35Config, Qwen35LayerSplitAdapterConfig, has_draft_block_size, draft_block_size); @@ -430,6 +433,9 @@ std::unique_ptr create_backend( cfg.fused_decode = args.ds4_fused_decode; cfg.fused_verify_f16_kv = args.ds4_fused_verify_f16_kv; cfg.prefill_mode = args.ds4_prefill_mode; + cfg.paged_attention = args.paged_attention; + cfg.max_concurrency = args.max_concurrency; + cfg.kv_pool_tokens = args.kv_pool_tokens; auto backend = std::make_unique(cfg); if (!backend->init()) { diff --git a/server/src/common/concurrency/seq_slot_manager.cpp b/server/src/common/concurrency/seq_slot_manager.cpp index 1ff91ed8d..0309fd908 100644 --- a/server/src/common/concurrency/seq_slot_manager.cpp +++ b/server/src/common/concurrency/seq_slot_manager.cpp @@ -84,6 +84,14 @@ bool SeqSlotManager::is_prefilling(int slot) const { return is_active(slot) && slots_[(size_t)slot].prefilling(); } +bool SeqSlotManager::has_prefill_prompt_at_least(int tokens) const { + if (tokens <= 0) return true; + return std::any_of(slots_.begin(), slots_.end(), + [tokens](const SeqSlot & slot) { + return slot.prefilling() && slot.prompt_len >= tokens; + }); +} + SeqEngine::AdmitResult SeqSlotManager::admit( uint64_t request_id, const std::vector & prompt, const SamplerCfg & sampler) { @@ -159,7 +167,7 @@ SeqEngine::AdmitResult SeqSlotManager::admit( s.phase = SeqSlotPhase::prefill; s.handle = handle; s.cur_pos = 0; - s.prompt = prompt; + s.prompt_len = prompt_len; s.sampler = sampler; s.sample_history = prompt; // Same predicate the engine uses to pick CPU sampling over GPU argmax: @@ -181,8 +189,8 @@ SeqSlotManager::PrefillChunk SeqSlotManager::append_prefill( if (!is_prefilling(slot) || n_tokens < 1) return out; SeqSlot & s = slots_[(size_t)slot]; - if (s.cur_pos > (int)s.prompt.size() || - n_tokens > (int)s.prompt.size() - s.cur_pos) { + if (s.cur_pos > s.prompt_len || + n_tokens > s.prompt_len - s.cur_pos) { return out; } @@ -196,7 +204,6 @@ SeqSlotManager::PrefillChunk SeqSlotManager::append_prefill( "[parallel] reserved prefill capacity missing for slot %d\n", slot); } - out.busy = false; return out; } @@ -219,7 +226,7 @@ SeqSlotManager::PrefillChunk SeqSlotManager::append_prefill( void SeqSlotManager::commit_prefill(int slot) { if (!is_prefilling(slot)) return; SeqSlot & s = slots_[(size_t)slot]; - if (s.cur_pos != (int)s.prompt.size()) return; + if (s.cur_pos != s.prompt_len) return; s.phase = SeqSlotPhase::decode; } diff --git a/server/src/common/concurrency/seq_slot_manager.h b/server/src/common/concurrency/seq_slot_manager.h index 2699ebade..c89d4f490 100644 --- a/server/src/common/concurrency/seq_slot_manager.h +++ b/server/src/common/concurrency/seq_slot_manager.h @@ -16,7 +16,7 @@ #pragma once -#include "common/paged_kv_pool.h" +#include "common/concurrency/paged_kv_pool.h" #include "common/sampler.h" #include "common/concurrency/seq_engine.h" @@ -36,7 +36,9 @@ enum class SeqSlotPhase { struct SeqSlot { SeqSlotPhase phase = SeqSlotPhase::free; PagedKvSequenceHandle handle; - std::vector prompt; + // Prompt tokens are the immutable prefix of sample_history. Decode tokens + // append to the same allocation, avoiding a second full prompt copy. + int prompt_len = 0; int cur_pos = 0; SamplerCfg sampler; std::mt19937_64 rng{0x9E3779B97F4A7C15ull}; @@ -44,6 +46,12 @@ struct SeqSlot { // may override a sample before the model consumes it. std::vector sample_history; + int generated_tokens() const { + return sample_history.size() > (size_t)prompt_len + ? (int)(sample_history.size() - (size_t)prompt_len) + : 0; + } + bool active() const { return phase != SeqSlotPhase::free; } bool prefilling() const { return phase == SeqSlotPhase::prefill; } bool decoding() const { return phase == SeqSlotPhase::decode; } @@ -68,9 +76,6 @@ class SeqSlotManager { struct PrefillChunk { bool ok = false; - // The pool is temporarily out of blocks; retrying after another slot - // retires can succeed. BlocksExhausted leaves the pool unchanged. - bool busy = false; std::vector rows; // Delta to patch into the slot's device block-table column. std::vector new_blocks; @@ -110,6 +115,7 @@ class SeqSlotManager { int decoding_count() const; bool is_active(int slot) const; bool is_prefilling(int slot) const; + bool has_prefill_prompt_at_least(int tokens) const; SeqSlot & slot(int i) { return slots_[(size_t)i]; } const SeqSlot & slot(int i) const { return slots_[(size_t)i]; } diff --git a/server/src/common/feature_gate.cpp b/server/src/common/feature_gate.cpp index df7b03f90..28ae4c69f 100644 --- a/server/src/common/feature_gate.cpp +++ b/server/src/common/feature_gate.cpp @@ -176,13 +176,14 @@ std::string check_feature_compatibility( // ── --paged-attention × architecture, placement, and decode features // Paged decode swaps the contiguous K/V cache for a block table owned by - // the monolithic qwen35 backend, so every rule below is about reaching - // that one code path. All are errors rather than warnings: running dense + // a monolithic Qwen or DeepSeek backend. All are errors rather than + // warnings: running dense // instead would hide the memory behavior the flag was chosen for. if (args.paged_attention) { if (!arch_supports_paged_attention(arch, /*is_layer_split=*/false)) { - return "--paged-attention requires a Qwen3.5/Qwen3.6 dense target " - "(architecture '" + arch + "' has no paged decode path)"; + return "--paged-attention requires a dense Qwen3.5/Qwen3.6 or " + "DeepSeek4 target (architecture '" + arch + + "' has no paged decode path)"; } // No rule for "requires a CUDA or HIP build": those are the only two // backends this binary can be configured with, and GGML_OP_PAGED_ATTN @@ -206,6 +207,18 @@ std::string check_feature_compatibility( if (features.kvflash_enabled) { return "--paged-attention cannot be combined with KVFlash"; } + if (arch == "deepseek4") { + if (target_backend != PlacementBackend::Hip) { + return "DeepSeek4 paged attention requires a local HIP target"; + } + if (args.ds4_prefill_mode != PrefillAttentionMode::Exact) { + return "DeepSeek4 paged attention requires --ds4-prefill exact"; + } + if (args.ds4_fused_decode || args.ds4_fused_verify_f16_kv) { + return "DeepSeek4 paged attention requires non-fused " + "autoregressive decode"; + } + } // The pool rounds max_ctx up to a whole number of blocks, so the top // of the range is what can be rounded without overflowing int. if (args.device.max_ctx <= 0 || @@ -216,8 +229,8 @@ std::string check_feature_compatibility( } // ── --max-concurrency × paged attention - // Concurrent decode slots are currently implemented only by the paged - // qwen35 backend. The common scheduler does not require a particular + // Concurrent decode slots are implemented by model-specific paged + // backends. The common scheduler does not require a particular // model-state representation; each backend owns whatever per-slot state // its graph needs alongside one block-table column per sequence. // Everything the paged cluster above rejects is transitively rejected, @@ -229,11 +242,12 @@ std::string check_feature_compatibility( if (!args.paged_attention) { return "--max-concurrency requires --paged-attention"; } - // The paged pool addresses tokens with uint32; 64 slots is far above - // any batch the decode kernel has been sized for and keeps the - // fixed-width decode batch bounded. - if (args.max_concurrency > 64) { - return "--max-concurrency must be at most 64"; + // Qwen's graph is qualified through 64 lanes. DeepSeek's gathered + // whole-model graph is intentionally bounded to 16 independent lanes. + const int max_slots = arch == "deepseek4" ? 16 : 64; + if (args.max_concurrency > max_slots) { + return "--max-concurrency must be at most " + + std::to_string(max_slots) + " for " + arch; } // Physical capacity is memory-derived and capped independently of the // logical slot count, so max-concurrency no longer multiplies max_ctx diff --git a/server/src/common/model_capabilities.h b/server/src/common/model_capabilities.h index f62087141..36c5d9d3c 100644 --- a/server/src/common/model_capabilities.h +++ b/server/src/common/model_capabilities.h @@ -76,7 +76,7 @@ inline constexpr ArchCapabilities kArchCapabilities[] = { {"laguna", true, false, false, true, kMono, kMono, kMono, kNever, kNever, kNever, kNever}, {"qwen3", false, false, true, false, kNever, kNever, kNever, kNever, kNever, kNever, kNever}, {"gemma4", true, false, false, false, kMono, kNever, kNever, kNever, kBoth, kNever, kNever}, - {"deepseek4", true, false, false, false, kNever, kNever, kNever, kNever, kNever, kNever, kNever}, + {"deepseek4", true, false, false, false, kNever, kNever, kNever, kNever, kNever, kNever, kMono}, }; inline constexpr std::size_t kArchCount = diff --git a/server/src/deepseek4/deepseek4_backend.cpp b/server/src/deepseek4/deepseek4_backend.cpp index f0a216e10..49e2ddb64 100644 --- a/server/src/deepseek4/deepseek4_backend.cpp +++ b/server/src/deepseek4/deepseek4_backend.cpp @@ -1073,7 +1073,8 @@ bool DeepSeek4Backend::init() { (cfg_.max_concurrency < 1 || cfg_.max_concurrency > 16 || cfg_.device.is_layer_split() || cfg_.prefill_mode != PrefillAttentionMode::Exact || - cfg_.fused_decode || env_flag_enabled("DFLASH_DS4_FUSED_DECODE") || + cfg_.fused_decode || cfg_.fused_verify_f16_kv || + env_flag_enabled("DFLASH_DS4_FUSED_DECODE") || env_flag_enabled("DFLASH_DS4_SPEC"))) { std::fprintf(stderr, "[deepseek4] paged serving requires 1..16 local slots, exact " diff --git a/server/src/deepseek4/deepseek4_graph.cpp b/server/src/deepseek4/deepseek4_graph.cpp index a1fbe24da..a1a2b5a86 100644 --- a/server/src/deepseek4/deepseek4_graph.cpp +++ b/server/src/deepseek4/deepseek4_graph.cpp @@ -2104,13 +2104,12 @@ static ggml_tensor * build_mla_attention_lane_core( n_index_comp = lane.n_index_comp_history + (gathered_emits_comp ? 1 : 0); } else { - const int n_index_comp_live = ds4_comp_rows_used( - lc.index_comp_kv, lc.n_index_comp, 4, token_pos); + const int n_index_comp_live = lane.n_index_comp_live; // Attention and index compression advance together at ratio 4. - GGML_ASSERT(lc.index_comp_kv && index_comp_kv_source); + GGML_ASSERT(lane.index_comp_kv && index_comp_kv_source); GGML_ASSERT(n_index_comp_live == n_comp_live); GGML_ASSERT(!masked_kv || - cached_inputs->padded_comp <= lc.index_comp_kv->ne[1]); + cached_inputs->padded_comp <= lane.index_comp_kv->ne[1]); n_index_comp = masked_kv ? cached_inputs->padded_comp : n_index_comp_live; diff --git a/server/src/deepseek4/deepseek4_internal.h b/server/src/deepseek4/deepseek4_internal.h index 4f587925a..9848775e9 100644 --- a/server/src/deepseek4/deepseek4_internal.h +++ b/server/src/deepseek4/deepseek4_internal.h @@ -26,7 +26,7 @@ #include "internal.h" #include "common/layer_split_utils.h" #include "common/prefill_attention_mode.h" -#include "common/paged_kv_pool.h" +#include "common/concurrency/paged_kv_pool.h" #include "deepseek4_paged_cache.h" namespace dflash::common { diff --git a/server/src/deepseek4/deepseek4_seq_engine.cpp b/server/src/deepseek4/deepseek4_seq_engine.cpp index 745046783..9615a1440 100644 --- a/server/src/deepseek4/deepseek4_seq_engine.cpp +++ b/server/src/deepseek4/deepseek4_seq_engine.cpp @@ -179,10 +179,11 @@ SeqEngine::StepResult DeepSeek4SeqEngine::step(const StepPlan & plan) { continue; } const SeqSlot & slot = slots_.slot(slice.slot); - const bool commit = slot.cur_pos == (int)slot.prompt.size(); + const bool commit = slot.cur_pos == slot.prompt_len; prefill_lanes.push_back( {slice.slot, (int)lane_tokens.size(), commit}); - lane_tokens.push_back(slot.prompt[(size_t)slot.cur_pos - 1]); + lane_tokens.push_back( + slot.sample_history[(size_t)slot.cur_pos - 1]); lane_positions.push_back(slot.cur_pos - 1); lane_slots.push_back(slice.slot); } diff --git a/server/src/qwen35/concurrency/qwen35_slot_manager.cpp b/server/src/qwen35/concurrency/qwen35_slot_manager.cpp index b8dcc36ec..02e07f60b 100644 --- a/server/src/qwen35/concurrency/qwen35_slot_manager.cpp +++ b/server/src/qwen35/concurrency/qwen35_slot_manager.cpp @@ -1,278 +1 @@ #include "qwen35_slot_manager.h" - -#include -#include - -namespace dflash::common { - -Qwen35SlotManager::Qwen35SlotManager(PagedKvPool & pool, int max_ctx) - : pool_(pool), max_ctx_(max_ctx) { - slots_.assign(pool.max_sequences(), Qwen35Slot{}); -} - -int Qwen35SlotManager::decoding_count() const { - int n = 0; - for (const Qwen35Slot & s : slots_) { - n += s.decoding() ? 1 : 0; - } - return n; -} - -uint32_t Qwen35SlotManager::decode_headroom_capacity(int logical_tokens) const { - const uint64_t extended = - static_cast(std::max(0, logical_tokens)) + - pool_.block_size(); - return static_cast(std::min( - static_cast(max_ctx_), extended)); -} - -bool Qwen35SlotManager::capacity_fits_pool(uint32_t token_capacity) const { - const uint64_t blocks = token_capacity == 0 ? 0 : - 1 + (static_cast(token_capacity) - 1) / - pool_.block_size(); - return blocks <= pool_.physical_block_count(); -} - -PagedKvStatus Qwen35SlotManager::protect_decode_headroom() { - struct TopUp { - PagedKvSequenceHandle handle; - uint32_t token_capacity = 0; - }; - - std::vector topups; - topups.reserve(slots_.size()); - uint64_t total_additional = 0; - const uint64_t block_size = pool_.block_size(); - for (const Qwen35Slot & slot : slots_) { - if (!slot.decoding()) continue; - const uint32_t capacity = decode_headroom_capacity(slot.cur_pos); - if (!capacity_fits_pool(capacity)) continue; - - uint32_t owned_blocks = 0; - const PagedKvStatus status = - pool_.owned_block_count(slot.handle, owned_blocks); - if (status != PagedKvStatus::Ok) return status; - const uint64_t target_blocks = capacity == 0 ? 0 : - 1 + (static_cast(capacity) - 1) / block_size; - if (target_blocks <= owned_blocks) continue; - const uint32_t additional = - static_cast(target_blocks - owned_blocks); - total_additional += additional; - topups.push_back({slot.handle, capacity}); - } - - // Preflight the whole cohort before moving a block, so a failed admission - // attempt cannot protect only whichever decoder happened to be visited - // first. - if (total_additional > pool_.free_block_count()) { - return PagedKvStatus::BlocksExhausted; - } - for (const TopUp & topup : topups) { - const PagedKvStatus status = - pool_.reserve_capacity(topup.handle, topup.token_capacity); - if (status != PagedKvStatus::Ok) return status; - } - return PagedKvStatus::Ok; -} - -bool Qwen35SlotManager::is_active(int slot) const { - return slot >= 0 && slot < (int)slots_.size() && - slots_[(size_t)slot].active(); -} - -bool Qwen35SlotManager::is_prefilling(int slot) const { - return is_active(slot) && slots_[(size_t)slot].prefilling(); -} - -bool Qwen35SlotManager::has_prefill_prompt_at_least(int tokens) const { - if (tokens <= 0) return true; - return std::any_of(slots_.begin(), slots_.end(), - [tokens](const Qwen35Slot & slot) { - return slot.prefilling() && slot.prompt_len >= tokens; - }); -} - -SeqEngine::AdmitResult Qwen35SlotManager::admit( - uint64_t request_id, const std::vector & prompt, - const SamplerCfg & sampler) { - using AdmitStatus = SeqEngine::AdmitResult::Status; - SeqEngine::AdmitResult r; - if (prompt.empty()) { - r.error = "empty prompt"; - return r; - } - if (prompt.size() > static_cast(max_ctx_)) { - r.error = "prompt exceeds max_ctx"; - return r; - } - const int prompt_len = static_cast(prompt.size()); - - // A prompt larger than the whole pool can NEVER be admitted; waiting - // for other sequences to drain would stall the queue forever and then - // fail anyway. Hard-fail it up front instead of reporting busy. - const uint64_t pool_capacity = - (uint64_t)pool_.physical_block_count() * pool_.block_size(); - if ((uint64_t)prompt_len > pool_capacity) { - r.error = "prompt needs " + std::to_string(prompt_len) + - " KV tokens but the pool holds " + - std::to_string(pool_capacity) + - "; raise --kv-pool-tokens or shorten the prompt"; - return r; - } - - int slot = -1; - for (int i = 0; i < (int)slots_.size(); i++) { - if (!slots_[(size_t)i].active()) { slot = i; break; } - } - if (slot < 0) { - r.status = AdmitStatus::busy; - r.error = "all decode slots are busy"; - return r; - } - - // A newly freed block belongs to any older decoder missing its rolling - // next-page reserve before it can belong to this admission. - const PagedKvStatus headroom_status = protect_decode_headroom(); - if (headroom_status != PagedKvStatus::Ok) { - r.status = headroom_status == PagedKvStatus::BlocksExhausted - ? AdmitStatus::busy : AdmitStatus::failed; - r.error = r.status == AdmitStatus::busy - ? "existing decoders need the available KV headroom" - : paged_kv_status_string(headroom_status); - return r; - } - - PagedKvSequenceHandle handle; - uint32_t reservation_capacity = - decode_headroom_capacity(prompt_len); - if (!capacity_fits_pool(reservation_capacity)) { - // The prompt itself fits, but this physical pool can never hold its - // following page. Preserve useful prompt-only behavior and report - // decode exhaustion later if the sequence reaches that boundary. - reservation_capacity = static_cast(prompt_len); - } - const PagedKvStatus status = pool_.acquire_reserved( - request_id, reservation_capacity, handle); - if (status != PagedKvStatus::Ok) { - r.status = status == PagedKvStatus::SequenceSlotsExhausted || - status == PagedKvStatus::BlocksExhausted - ? AdmitStatus::busy : AdmitStatus::failed; - r.error = status == PagedKvStatus::BlocksExhausted - ? "not enough unreserved KV blocks for the prompt and decode headroom" - : paged_kv_status_string(status); - return r; - } - - Qwen35Slot & s = slots_[(size_t)slot]; - s.phase = Qwen35SlotPhase::prefill; - s.handle = handle; - s.cur_pos = 0; - s.prompt_len = prompt_len; - s.sampler = sampler; - s.sample_history = prompt; - // Same predicate the engine uses to pick CPU sampling over GPU argmax: - // a seed only means anything when the sampler actually draws. - if (sampler.needs_logit_processing() && sampler.seed != 0) { - s.rng.seed(sampler.seed); - } else { - s.rng.seed(std::random_device{}()); - } - - r.status = AdmitStatus::admitted; - r.slot = slot; - return r; -} - -Qwen35SlotManager::PrefillChunk Qwen35SlotManager::append_prefill( - int slot, int n_tokens) { - PrefillChunk out; - if (!is_prefilling(slot) || n_tokens < 1) return out; - - Qwen35Slot & s = slots_[(size_t)slot]; - if (s.cur_pos > s.prompt_len || - n_tokens > s.prompt_len - s.cur_pos) { - return out; - } - - PagedKvAppendResult app = pool_.append(s.handle, (uint32_t)n_tokens); - if (!app) { - // Admission reserved the whole prompt. Treat exhaustion here as a - // broken invariant, not a retryable condition: retrying a batch of - // all-prefill slots without any decoder able to retire would livelock. - if (app.status == PagedKvStatus::BlocksExhausted) { - std::fprintf(stderr, - "[parallel] reserved prefill capacity missing for slot %d\n", - slot); - } - return out; - } - - out.rows.reserve(app.write_slots.size()); - for (const PagedKvWriteSlot & write : app.write_slots) { - out.rows.push_back((int64_t)write.physical_token_index); - if (write.block_offset == 0) { - if (out.first_new_block < 0) { - out.first_new_block = - (int)(write.logical_position / pool_.block_size()); - } - out.new_blocks.push_back((int32_t)write.physical_block); - } - } - s.cur_pos += n_tokens; - out.ok = true; - return out; -} - -void Qwen35SlotManager::commit_prefill(int slot) { - if (!is_prefilling(slot)) return; - Qwen35Slot & s = slots_[(size_t)slot]; - if (s.cur_pos != s.prompt_len) return; - s.phase = Qwen35SlotPhase::decode; -} - -Qwen35SlotManager::StepAppend Qwen35SlotManager::append_token(int slot, - int32_t fed_token) { - StepAppend out; - if (!is_active(slot) || !slots_[(size_t)slot].decoding()) return out; - Qwen35Slot & s = slots_[(size_t)slot]; - if (s.cur_pos >= max_ctx_) { - // No context left; the scheduler should have stopped this slot. - return out; - } - PagedKvAppendResult app = pool_.append( - s.handle, 1, /*only_first_last_slots=*/true); - if (!app || app.token_count != 1 || - app.last.logical_position != (uint32_t)s.cur_pos) { - out.busy = app.status == PagedKvStatus::BlocksExhausted; - return out; - } - s.sample_history.push_back(fed_token); - - out.ok = true; - out.physical_row = (int64_t)app.last.physical_token_index; - out.position = s.cur_pos; - if ((uint32_t)s.cur_pos % pool_.block_size() == 0) { - out.new_block = (int32_t)app.last.physical_block; - out.new_block_index = s.cur_pos / (int)pool_.block_size(); - } - return out; -} - -void Qwen35SlotManager::commit_step(int slot) { - if (!is_active(slot)) return; - slots_[(size_t)slot].cur_pos += 1; -} - -void Qwen35SlotManager::retire(int slot) { - if (slot < 0 || slot >= (int)slots_.size()) return; - Qwen35Slot & s = slots_[(size_t)slot]; - if (!s.active()) return; - const PagedKvStatus status = pool_.release(s.handle); - if (status != PagedKvStatus::Ok && status != PagedKvStatus::StaleHandle) { - std::fprintf(stderr, "[parallel] slot %d release failed: %s\n", - slot, paged_kv_status_string(status)); - } - s = Qwen35Slot{}; -} - -} // namespace dflash::common diff --git a/server/src/qwen35/concurrency/qwen35_slot_manager.h b/server/src/qwen35/concurrency/qwen35_slot_manager.h index 1da009f69..65dc0dee1 100644 --- a/server/src/qwen35/concurrency/qwen35_slot_manager.h +++ b/server/src/qwen35/concurrency/qwen35_slot_manager.h @@ -1,137 +1,13 @@ -// Qwen35SlotManager — complete host-side state for each Qwen serving slot. -// -// Companion of PagedKvPool: the pool hands out sequence handles and physical -// blocks; this class owns everything else a slot needs between admission and -// retirement — the pool-handle lifecycle (including every error path), the -// admission arithmetic (context clamp, prompt reservation, and rolling decode -// headroom), on-demand block allocation, per-slot sampler/RNG/penalty-history -// state, and the position counters. -// -// It deliberately owns NO device state. Prefill/decode allocation returns -// physical rows and block-table deltas as plain vectors. Prompt, KV ownership, -// sampler, and progress live together here; the scheduler keeps -// only its coarse request phase. -// -// Not thread-safe; the single scheduler thread is the only caller. - #pragma once -#include "common/concurrency/paged_kv_pool.h" -#include "common/sampler.h" -#include "common/concurrency/seq_engine.h" - -#include -#include -#include -#include +#include "common/concurrency/seq_slot_manager.h" namespace dflash::common { -enum class Qwen35SlotPhase { - free, - prefill, - decode, -}; - -struct Qwen35Slot { - Qwen35SlotPhase phase = Qwen35SlotPhase::free; - PagedKvSequenceHandle handle; - // Prompt tokens are the immutable prefix of sample_history. Decode tokens - // append to the same allocation, avoiding a second full prompt copy. - int prompt_len = 0; - int cur_pos = 0; - SamplerCfg sampler; - std::mt19937_64 rng{0x9E3779B97F4A7C15ull}; - // Penalty history is recorded as fed rather than sampled: the scheduler - // may override a sample before the model consumes it. - std::vector sample_history; - - int generated_tokens() const { - return sample_history.size() > (size_t)prompt_len - ? (int)(sample_history.size() - (size_t)prompt_len) - : 0; - } - - bool active() const { return phase != Qwen35SlotPhase::free; } - bool prefilling() const { return phase == Qwen35SlotPhase::prefill; } - bool decoding() const { return phase == Qwen35SlotPhase::decode; } -}; - -class Qwen35SlotManager { -public: - // `max_ctx` is the per-sequence logical bound; slot count comes from the - // pool's max_sequences. The pool must outlive the manager. - Qwen35SlotManager(PagedKvPool & pool, int max_ctx); - - // Claim a free slot and atomically reserve all K/V blocks needed by the - // known prompt plus its next logical decode page when that page can exist - // in both max_ctx and the physical pool. Existing decoders are topped up - // first, so a younger admission cannot steal their next-page headroom. - // Prompts larger than the whole pool hard-fail; temporary capacity pressure - // reports busy. Seeds the slot RNG from sampler.seed only when the sampler - // actually draws, else nondeterministically. - SeqEngine::AdmitResult admit(uint64_t request_id, - const std::vector & prompt, - const SamplerCfg & sampler); - - struct PrefillChunk { - bool ok = false; - std::vector rows; - // Delta to patch into the slot's device block-table column. - std::vector new_blocks; - int first_new_block = -1; - }; - - // Append `n_tokens` more prompt rows for a prefilling slot. Physical block - // ids come from the slot's admission reservation, so any append within the - // admitted prompt is guaranteed not to wait on another sequence. - PrefillChunk append_prefill(int slot, int n_tokens); - - // Record a finished prefill and expose the slot to decode. - void commit_prefill(int slot); - - struct StepAppend { - bool ok = false; - bool busy = false; // no physical block available right now - int64_t physical_row = -1; - int position = -1; // logical position the fed token is written at - int32_t new_block = -1; - int new_block_index = -1; - }; - - // Allocate the next decode token's cache row, report any new block-table - // entry, and log it to sample_history. cur_pos waits for commit_step(). - StepAppend append_token(int slot, int32_t fed_token); - - // The batched step's compute succeeded: cur_pos++. - void commit_step(int slot); - - // Release the slot's blocks and clear its state. Safe on inactive slots - // and after a failed admission/prefill. - void retire(int slot); - - int slot_count() const { return (int)slots_.size(); } - int max_context() const { return max_ctx_; } - int decoding_count() const; - bool is_active(int slot) const; - bool is_prefilling(int slot) const; - bool has_prefill_prompt_at_least(int tokens) const; - Qwen35Slot & slot(int i) { return slots_[(size_t)i]; } - const Qwen35Slot & slot(int i) const { return slots_[(size_t)i]; } - -private: - // Logical extent whose block count includes the sequence's current pages - // plus one future page, capped at max_ctx. - uint32_t decode_headroom_capacity(int logical_tokens) const; - bool capacity_fits_pool(uint32_t token_capacity) const; - - // Atomically preflight and top up every decoding slot as one cohort before - // a younger sequence may reserve capacity. - PagedKvStatus protect_decode_headroom(); - - PagedKvPool & pool_; - int max_ctx_ = 0; - std::vector slots_; -}; +// Compatibility aliases keep the Qwen engine source stable while both Qwen +// and DeepSeek share the current model-neutral slot lifecycle implementation. +using Qwen35SlotPhase = SeqSlotPhase; +using Qwen35Slot = SeqSlot; +using Qwen35SlotManager = SeqSlotManager; } // namespace dflash::common diff --git a/server/test/test_feature_gate.cpp b/server/test/test_feature_gate.cpp index 404d7c4ee..c52c24e51 100644 --- a/server/test/test_feature_gate.cpp +++ b/server/test/test_feature_gate.cpp @@ -377,24 +377,25 @@ void test_feature_gate_layer_split_requires_supported_arch() { CHECK(gate_result(single, "qwen3", PlacementBackend::Cuda).empty()); } -void test_feature_gate_paged_attention_requires_qwen35_monolithic() { +void test_feature_gate_paged_attention_requires_monolithic_backend() { BackendArgs args; args.model_path = "/nonexistent/model.gguf"; args.paged_attention = true; CHECK(gate_result(args, "qwen35", PlacementBackend::Cuda).empty()); CHECK(gate_result(args, "qwen35", PlacementBackend::Hip).empty()); - // Only qwen35 has a paged decode path. qwen35moe shares Qwen35Config, so - // its rejection is this gate's job — the factory's field-presence - // cross-check cannot tell the two apart. - for (const char * arch : {"qwen35moe", "laguna", "qwen3", - "gemma4", "deepseek4"}) { + BackendArgs ds4 = gate_args_hip_deepseek4(); + ds4.paged_attention = true; + CHECK(gate_result(ds4, "deepseek4", PlacementBackend::Hip).empty()); + CHECK(!gate_result(ds4, "deepseek4", PlacementBackend::Cuda).empty()); + + // qwen35moe shares Qwen35Config, so its rejection is this gate's job — + // the factory's field-presence cross-check cannot tell the two apart. + for (const char * arch : {"qwen35moe", "laguna", "qwen3", "gemma4"}) { CHECK(!gate_result(args, arch, PlacementBackend::Cuda).empty()); } - // Only the monolithic qwen35 backend owns a paged K/V pool. Both - // placements are supported qwen35 launches without the flag, so the - // rejection has to come from the paged rule. + // Only monolithic Qwen and DeepSeek backends own paged K/V pools. BackendArgs split = args; CHECK(parse_placement_device_list("cuda:0,cuda:1", split.device)); CHECK(!gate_result(split, "qwen35", PlacementBackend::Cuda).empty()); @@ -503,6 +504,22 @@ void test_feature_gate_parallel_and_kv_pool_rules() { parallel.max_concurrency = 65; CHECK(!gate_result(parallel, "qwen35", PlacementBackend::Cuda).empty()); + BackendArgs ds4 = gate_args_hip_deepseek4(); + ds4.paged_attention = true; + ds4.max_concurrency = 16; + CHECK(gate_result(ds4, "deepseek4", PlacementBackend::Hip).empty()); + ds4.max_concurrency = 17; + CHECK(!gate_result(ds4, "deepseek4", PlacementBackend::Hip).empty()); + ds4.max_concurrency = 2; + ds4.ds4_prefill_mode = PrefillAttentionMode::Sparse; + CHECK(!gate_result(ds4, "deepseek4", PlacementBackend::Hip).empty()); + ds4.ds4_prefill_mode = PrefillAttentionMode::Exact; + ds4.ds4_fused_decode = true; + CHECK(!gate_result(ds4, "deepseek4", PlacementBackend::Hip).empty()); + ds4.ds4_fused_decode = false; + ds4.ds4_fused_verify_f16_kv = true; + CHECK(!gate_result(ds4, "deepseek4", PlacementBackend::Hip).empty()); + // --kv-pool-tokens sizes the shared pool, so it needs slots to share. BackendArgs pool = paged; pool.kv_pool_tokens = 4096; @@ -673,9 +690,11 @@ void test_model_capability_tables() { CHECK(!arch_supports_draft_swa("qwen36", false)); CHECK(!arch_supports_paged_attention("qwen36", false)); - // Paged decode lives in the monolithic qwen35 backend alone. + // Paged decode lives in the monolithic Qwen and DeepSeek backends. CHECK(arch_supports_paged_attention("qwen35", false)); CHECK(!arch_supports_paged_attention("qwen35", true)); + CHECK(arch_supports_paged_attention("deepseek4", false)); + CHECK(!arch_supports_paged_attention("deepseek4", true)); CHECK(!arch_supports_paged_attention("qwen35moe", false)); CHECK(arch_supports_draft_block_size("qwen35", false)); @@ -702,7 +721,7 @@ TEST_CASE(FeatureGateFixture, feature_gate_suite) { test_feature_gate_ds4_decode_options_require_monolithic_hip(); test_feature_gate_remote_draft_requires_supported_arch(); test_feature_gate_layer_split_requires_supported_arch(); - test_feature_gate_paged_attention_requires_qwen35_monolithic(); + test_feature_gate_paged_attention_requires_monolithic_backend(); test_feature_gate_paged_attention_requires_plain_ar_decode(); test_feature_gate_parallel_and_kv_pool_rules(); test_feature_warnings_silent_when_supported(); From aed2fcc214251f7229545e25e0115cb2a18bca83 Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 25 Aug 2026 10:33:30 +0000 Subject: [PATCH 4/5] fix(ds4): harden paged pool planning --- server/src/deepseek4/deepseek4_backend.cpp | 13 ++++---- .../src/deepseek4/deepseek4_paged_cache.cpp | 33 +++++++++++++++---- server/src/deepseek4/deepseek4_paged_cache.h | 8 +++++ server/test/test_deepseek4_paged_cache.cpp | 19 +++++++++++ 4 files changed, 61 insertions(+), 12 deletions(-) diff --git a/server/src/deepseek4/deepseek4_backend.cpp b/server/src/deepseek4/deepseek4_backend.cpp index 49e2ddb64..b4ef16392 100644 --- a/server/src/deepseek4/deepseek4_backend.cpp +++ b/server/src/deepseek4/deepseek4_backend.cpp @@ -1106,16 +1106,17 @@ bool DeepSeek4Backend::init() { const int max_ctx = cfg_.max_ctx > 0 ? cfg_.max_ctx : 8192; if (cfg_.paged_attention) { - uint64_t requested = cfg_.kv_pool_tokens > 0 + const uint64_t requested = cfg_.kv_pool_tokens > 0 ? (uint64_t)cfg_.kv_pool_tokens : (uint64_t)max_ctx * (uint64_t)cfg_.max_concurrency; - requested = std::max(requested, (uint64_t)max_ctx); - const uint64_t blocks64 = - (requested + DS4_PAGE_TOKENS - 1) / DS4_PAGE_TOKENS; - if (blocks64 == 0 || blocks64 > UINT32_MAX || + uint32_t physical_blocks = 0; + if (!plan_deepseek4_paged_pool_blocks( + (uint32_t)max_ctx, (uint32_t)cfg_.max_concurrency, + cfg_.kv_pool_tokens > 0 ? (uint64_t)cfg_.kv_pool_tokens : 0, + physical_blocks) || !create_deepseek4_paged_cache( backend_, w_, (uint32_t)cfg_.max_concurrency, - (uint32_t)max_ctx, (uint32_t)blocks64, paged_cache_)) { + (uint32_t)max_ctx, physical_blocks, paged_cache_)) { std::fprintf(stderr, "[deepseek4] paged cache allocation failed (ctx=%d slots=%d " "requested_pool_tokens=%llu); reduce --max-ctx/--max-concurrency " diff --git a/server/src/deepseek4/deepseek4_paged_cache.cpp b/server/src/deepseek4/deepseek4_paged_cache.cpp index caca04324..11cb18103 100644 --- a/server/src/deepseek4/deepseek4_paged_cache.cpp +++ b/server/src/deepseek4/deepseek4_paged_cache.cpp @@ -21,6 +21,21 @@ bool add_mul(uint64_t & dst, uint64_t a, uint64_t b) { } } +bool plan_deepseek4_paged_pool_blocks(uint32_t max_ctx, uint32_t slots, + uint64_t requested_tokens, + uint32_t & physical_blocks) { + physical_blocks = 0; + if (!max_ctx || !slots) return false; + const uint64_t pool_tokens = requested_tokens + ? requested_tokens + : uint64_t(max_ctx) * slots; + if (!pool_tokens) return false; + const uint64_t blocks = 1 + (pool_tokens - 1) / DS4_PAGE_TOKENS; + if (blocks > UINT32_MAX / DS4_PAGE_TOKENS) return false; + physical_blocks = static_cast(blocks); + return true; +} + bool prepare_deepseek4_gathered_lane_rows( const int32_t * slots, const int64_t * positions, uint32_t lanes, const int32_t * block_tables, uint32_t block_table_stride, @@ -54,6 +69,17 @@ bool prepare_deepseek4_gathered_lane_rows( ds4_raw_ring_row(pos); if (!ratio) continue; + // Validate the current logical page before reserving history. A + // malformed, very large position must fail without attempting an + // allocation proportional to that untrusted value. + const uint64_t current_logical_block = pos / DS4_PAGE_TOKENS; + if (current_logical_block >= block_table_stride) return false; + const int32_t current_physical = + block_tables[size_t(lane) * block_table_stride + current_logical_block]; + if (current_physical < 0 || uint32_t(current_physical) >= physical_blocks) { + return false; + } + // Every completed group before the current token contributes one // chronological row. Looking up each logical page (rather than // assuming contiguous physical pages) is the reference behaviour. @@ -72,13 +98,8 @@ bool prepare_deepseek4_gathered_lane_rows( row > uint64_t(INT64_MAX)) return false; rows.compressed_history.push_back(static_cast(row)); } - const uint64_t logical_block = pos / DS4_PAGE_TOKENS; - if (logical_block >= block_table_stride) return false; - const int32_t physical = - block_tables[size_t(lane) * block_table_stride + logical_block]; - if (physical < 0 || uint32_t(physical) >= physical_blocks) return false; uint64_t scatter = 0; - if (!ds4_compressed_page_row(pos, uint32_t(physical), ratio, scatter, + if (!ds4_compressed_page_row(pos, uint32_t(current_physical), ratio, scatter, rows.compressed_emitted) || scatter > uint64_t(INT64_MAX)) return false; if (rows.compressed_emitted) rows.compressed_scatter = int64_t(scatter); diff --git a/server/src/deepseek4/deepseek4_paged_cache.h b/server/src/deepseek4/deepseek4_paged_cache.h index d8bb3cc83..5dc047a89 100644 --- a/server/src/deepseek4/deepseek4_paged_cache.h +++ b/server/src/deepseek4/deepseek4_paged_cache.h @@ -36,6 +36,14 @@ struct DeepSeek4GatheredLaneRows { bool compressed_emitted = false; }; +// Convert the configured pool capacity into 128-token physical pages. A zero +// request selects max_ctx * slots; an explicit request is honored even when it +// is smaller than max_ctx. +bool plan_deepseek4_paged_pool_blocks(uint32_t max_ctx, + uint32_t slots, + uint64_t requested_tokens, + uint32_t & physical_blocks); + // block_tables is lane-major with block_table_stride entries per lane. // Physical block IDs may be fragmented and are validated against // physical_blocks. History excludes the current token; compressed history is diff --git a/server/test/test_deepseek4_paged_cache.cpp b/server/test/test_deepseek4_paged_cache.cpp index 11df40ef3..2889fc19c 100644 --- a/server/test/test_deepseek4_paged_cache.cpp +++ b/server/test/test_deepseek4_paged_cache.cpp @@ -1,10 +1,25 @@ #include "deepseek4/deepseek4_paged_cache.h" +#include "deepseek4/deepseek4_page_layout.h" #include "host_check.h" #include #include using namespace dflash::common; static int g_checks = 0; int main() { + uint32_t blocks = 0; + CHECK(plan_deepseek4_paged_pool_blocks(4096, 16, 0, blocks)); + CHECK(blocks == 512); + CHECK(plan_deepseek4_paged_pool_blocks(4096, 16, 128, blocks)); + CHECK(blocks == 1); + CHECK(plan_deepseek4_paged_pool_blocks(4096, 16, 129, blocks)); + CHECK(blocks == 2); + CHECK(!plan_deepseek4_paged_pool_blocks(0, 16, 128, blocks)); + CHECK(!plan_deepseek4_paged_pool_blocks(4096, 0, 128, blocks)); + const uint64_t overflowing_tokens = + uint64_t(UINT32_MAX / DS4_PAGE_TOKENS) * DS4_PAGE_TOKENS + 1; + CHECK(!plan_deepseek4_paged_pool_blocks( + 4096, 16, overflowing_tokens, blocks)); + DeepSeek4PagedCachePlan p, twice; CHECK(plan_deepseek4_paged_cache(512, 128, 3, 4096, 40, {0, 4, 128}, p)); CHECK(p.max_blocks_per_sequence == 32 && p.physical_rows[0] == 0); @@ -82,6 +97,10 @@ int main() { CHECK(rows[0].compressed_history.size() == 1); CHECK(rows[0].compressed_history[0] == 4); CHECK(rows[0].compressed_emitted && rows[0].compressed_scatter == 3); + + const int64_t invalid_large_pos[] = {INT64_MAX}; + CHECK(!prepare_deepseek4_gathered_lane_rows( + boundary_slot, invalid_large_pos, 1, boundary_table, 2, 2, 4, rows)); std::printf("OK test_deepseek4_paged_cache (%d checks)\n", g_checks); return 0; } From 4fa7112d5bbc72d2a25860f8d36b6cb748bffbbe Mon Sep 17 00:00:00 2001 From: Graffioh Date: Wed, 26 Aug 2026 14:36:22 +0000 Subject: [PATCH 5/5] feat(ds4): support six-slot heterogeneous concurrency --- README.md | 4 +- server/docs/DS4.md | 56 ++++++++++-- server/src/common/feature_gate.cpp | 5 +- server/src/common/paged_attention_config.h | 1 + server/src/deepseek4/deepseek4_backend.cpp | 91 ++++++++++++++----- server/src/deepseek4/deepseek4_graph.cpp | 3 +- server/src/deepseek4/deepseek4_internal.h | 3 +- server/src/deepseek4/deepseek4_seq_engine.cpp | 11 ++- server/test/test_deepseek4_paged_cache.cpp | 42 ++++----- server/test/test_feature_gate.cpp | 4 +- 10 files changed, 150 insertions(+), 70 deletions(-) diff --git a/README.md b/README.md index 77b220627..be60c6a07 100644 --- a/README.md +++ b/README.md @@ -375,8 +375,8 @@ When compression is on, multi-turn continuations automatically use **FlowKV**: a | `DFLASH_PREFILL_CACHE_SLOTS=N` | `0` | Container-entrypoint equivalent of `--prefill-cache-slots`; the native binary itself uses the CLI flag. | | `--kv-cache-dir ` | — | Persist prefix cache to disk | | `--kv-cache-budget N` | — | On-disk cache size cap | -| `--paged-attention` | off | Exact block-table attention for monolithic Qwen3.6-27B (16-token blocks) and DeepSeek4 on Strix Halo (128-token pages); see [paged attention](optimizations/paged_attention/README.md) and [DeepSeek4 concurrent serving](server/docs/DS4.md#strix-halo-concurrent-serving) | -| `--max-concurrency N` | `1` | Maximum concurrent sequence slots. Values above 1 enable paged attention automatically; Qwen supports up to 64 and DeepSeek4 up to 16. | +| `--paged-attention` | off | Exact block-table attention for monolithic Qwen3.6-27B (16-token blocks) and DeepSeek4 on Strix Halo or R9700 + Strix Halo (128-token pages); see [paged attention](optimizations/paged_attention/README.md) and [DeepSeek4 concurrent serving](server/docs/DS4.md#strix-halo-concurrent-serving) | +| `--max-concurrency N` | `1` | Maximum concurrent sequence slots. Values above 1 enable paged attention automatically; Qwen supports up to 64, while DeepSeek4 supports up to 6 on either monolithic Strix Halo or heterogeneous R9700 + Strix Halo. | | `--kv-pool-tokens N` | `0` (auto) | Shared physical K/V capacity for concurrent paged serving. Requires `--max-concurrency` greater than 1. Zero derives capacity from available device memory; explicit values are rounded to the backend's page size. | | `--admission-coalesce-ms N` | `20` | Idle-to-busy batching window for concurrent serving, from 0 to 1000 ms; `0` disables it. | diff --git a/server/docs/DS4.md b/server/docs/DS4.md index f8ab267a6..de7704abb 100644 --- a/server/docs/DS4.md +++ b/server/docs/DS4.md @@ -293,16 +293,23 @@ a throughput profile. ### Strix Halo concurrent serving -DeepSeek4 paged concurrency is deliberately a single-device path: one local -HIP target on Strix Halo (`gfx1151`), with the complete model and every expert -resident on that device. It does not use layer splitting, CUDA/HIP expert -ownership, host-streamed experts, or DSpark. +DeepSeek4 paged concurrency supports two resident HIP deployments: + +- one monolithic Strix Halo (`gfx1151`) target with every expert on that + device, through 6 lanes; and +- the in-process R9700 (`gfx1201`) target + Strix Halo expert-parallel + deployment, through 6 lanes. The target keeps dense work and its selected + experts while the secondary owns the remaining materialized experts. + +The heterogeneous mode is route-level expert parallelism. It is not an +explicit `--target-device hip:0,hip:1` layer split and does not use a remote +target shard or host-streamed experts. The backend keeps raw MLA rows, compressed rows, indexer state, sequence lengths, and block tables in a persistent 128-token paged cache. The shared HTTP scheduler performs admission, cancellation, slow-client isolation, and fair continuous batching. DeepSeek4 lowers each scheduler plan into one exact -gathered graph with up to 16 independent lanes. Decode rows share the weight +gathered graph with up to 6 independent lanes. Decode rows share the weight pass; each selected prompt advances by one exact token because the graph must not contain two rows from the same sequence. @@ -321,16 +328,45 @@ cmake --build server/build-hip -j /path/to/models/DeepSeek-V4-Flash-0731-ROCMFPX-MIX-STRIX.gguf \ --target-device hip:0 \ --paged-attention \ - --max-concurrency 16 \ - --kv-pool-tokens 8192 \ + --max-concurrency 6 \ + --kv-pool-tokens 24576 \ + --max-ctx 4096 \ + --ds4-prefill exact \ + --prefix-cache-slots 0 +``` + +For the R9700 + Strix Halo path, build one HIP binary for both architectures as +described above, expose the R9700 first, and select the static in-process expert +split: + +```bash +export HIP_VISIBLE_DEVICES=, +export DFLASH_DS4_MOE_TP=1 +export DFLASH_DS4_MOE_TP_INPROC=1 +export DFLASH_DS4_MOE_TP_GPU=1 +export DFLASH_EXPERT_BUDGET_MB=11700 + +./server/build-hip-dual/dflash_server \ + /path/to/models/DeepSeek-V4-Flash.gguf \ + --target-device hip:0 \ + --peer-access \ + --paged-attention \ + --max-concurrency 6 \ + --kv-pool-tokens 24576 \ --max-ctx 4096 \ --ds4-prefill exact \ --prefix-cache-slots 0 ``` -This mode fails closed for non-gfx1151 devices, CUDA, layer or remote target -splits, `DFLASH_DS4_MOE_TP`, drafts/DSpark, DDTree, PFlash/KVFlash, fused -decode, approximate prefill, windowed attention, and prefix-cache parking. +The heterogeneous paged cache and its full-context prefill staging allocation +are charged against the R9700 before selecting resident experts. Increase +`--kv-pool-tokens` only when the primary has enough memory for the larger +shared history pool. + +Paged concurrency fails closed for other primary/secondary architecture pairs, +CUDA or out-of-process expert ownership, explicit layer or remote target +splits, drafts/DSpark, DDTree, PFlash/KVFlash, fused decode, approximate +prefill, windowed attention, mutable expert caching, and prefix-cache parking. There is no automatic fallback to a slower or asymmetric execution mode. ### Local single-shard diff --git a/server/src/common/feature_gate.cpp b/server/src/common/feature_gate.cpp index 28ae4c69f..e9c4d30af 100644 --- a/server/src/common/feature_gate.cpp +++ b/server/src/common/feature_gate.cpp @@ -243,8 +243,9 @@ std::string check_feature_compatibility( return "--max-concurrency requires --paged-attention"; } // Qwen's graph is qualified through 64 lanes. DeepSeek's gathered - // whole-model graph is intentionally bounded to 16 independent lanes. - const int max_slots = arch == "deepseek4" ? 16 : 64; + // whole-model graph has a smaller, separately qualified ceiling. + const int max_slots = arch == "deepseek4" + ? DEEPSEEK4_MAX_PAGED_SEQUENCES : 64; if (args.max_concurrency > max_slots) { return "--max-concurrency must be at most " + std::to_string(max_slots) + " for " + arch; diff --git a/server/src/common/paged_attention_config.h b/server/src/common/paged_attention_config.h index c7768a5c2..b45b2b322 100644 --- a/server/src/common/paged_attention_config.h +++ b/server/src/common/paged_attention_config.h @@ -14,6 +14,7 @@ namespace dflash::common { constexpr int PAGED_BLOCK_SIZE = 16; +inline constexpr int DEEPSEEK4_MAX_PAGED_SEQUENCES = 6; constexpr int paged_block_count(int max_ctx) { return (max_ctx + PAGED_BLOCK_SIZE - 1) / PAGED_BLOCK_SIZE; diff --git a/server/src/deepseek4/deepseek4_backend.cpp b/server/src/deepseek4/deepseek4_backend.cpp index b4ef16392..2f05542f4 100644 --- a/server/src/deepseek4/deepseek4_backend.cpp +++ b/server/src/deepseek4/deepseek4_backend.cpp @@ -150,13 +150,14 @@ static bool env_int_in_range(const char * name, int fallback, return true; } -static bool is_gfx1151_device(int gpu) { +static bool is_gfx_device(int gpu, const char * arch) { #if defined(DFLASH27B_BACKEND_HIP) || defined(GGML_USE_HIP) cudaDeviceProp prop{}; return cudaGetDeviceProperties(&prop, gpu) == cudaSuccess && - std::strncmp(prop.gcnArchName, "gfx1151", 7) == 0; + std::strncmp(prop.gcnArchName, arch, std::strlen(arch)) == 0; #else (void) gpu; + (void) arch; return false; #endif } @@ -664,8 +665,9 @@ static bool fill_profiled_hot_placement(const DeepSeek4Weights & w, // uses authoritative router statistics and evaluates every selected expert. static bool compute_ds4_hybrid_budget_info(const DeepSeek4Weights & w, ggml_backend_t backend, - int max_ctx, + uint64_t kv_bytes, bool all_cold, + bool paged, Ds4HybridBudgetInfo & out, std::string * err) { out = {}; @@ -686,11 +688,13 @@ static bool compute_ds4_hybrid_budget_info(const DeepSeek4Weights & w, out.core_bytes = moe_hybrid_core_bytes_from_memory( "deepseek4", out.gpu_free, out.gpu_total); - out.kv_bytes = estimate_ds4_cache_bytes(w, max_ctx); + out.kv_bytes = kv_bytes; // In all-cold mode the KV cache is owned by the secondary (Strix) - // backend, so it must not consume the primary GPU's expert budget. - const uint64_t main_charge = all_cold ? 0 : out.kv_bytes; + // backend in the legacy contiguous path, so it does not consume the + // primary GPU's expert budget there. Paged serving always owns both its + // staging cache and persistent page tensors on the primary target. + const uint64_t main_charge = all_cold && !paged ? 0 : out.kv_bytes; if (out.gpu_total > out.core_bytes + main_charge + out.warm_bytes + out.safety_bytes) { out.expert_budget = out.gpu_total - out.core_bytes - main_charge - out.warm_bytes - out.safety_bytes; } @@ -793,11 +797,12 @@ bool DeepSeek4Backend::load_model() { ? compiled_placement_backend() : cfg_.device.backend; - // Paged concurrency, fused decode, and layer-major prefill require - // monolithic expert residency. Heterogeneous TP is the exception for - // non-paged modes: its fused graph owns the - // routed experts across two local GPU backends, so forcing a full load would - // disable the requested split before the TP runtime can initialize. + // Paged concurrency, fused decode, and layer-major prefill normally require + // monolithic expert residency. In-process heterogeneous TP is the explicit + // exception: its fused graph owns a fixed expert split across two local GPU + // backends, so forcing a full load would disable the requested placement + // before the TP runtime can initialize. init() has already rejected paged + // deployments outside the qualified R9700 + Strix Halo topology. const bool force_full = env_flag_enabled("DFLASH_DS4_FORCE_FULL_LOAD"); const bool heterogeneous_tp = env_flag_enabled("DFLASH_DS4_MOE_TP"); const bool need_monolithic = @@ -1046,17 +1051,31 @@ bool DeepSeek4Backend::init() { const PlacementBackend target_backend = cfg_.device.backend == PlacementBackend::Auto ? compiled_placement_backend() : cfg_.device.backend; - if (target_backend != PlacementBackend::Hip || - !is_gfx1151_device(cfg_.device.gpu)) { + const Ds4MoeTpConfig tp = ds4_moe_tp_config(cfg_.device.gpu); + if (target_backend != PlacementBackend::Hip) { std::fprintf(stderr, - "[deepseek4] paged concurrency currently requires one " - "local Strix Halo (gfx1151) HIP target\n"); + "[deepseek4] paged concurrency requires a local HIP target\n"); return false; } - if (env_flag_enabled("DFLASH_DS4_MOE_TP")) { + if (!tp.requested) { + if (!is_gfx_device(cfg_.device.gpu, "gfx1151")) { + std::fprintf(stderr, + "[deepseek4] monolithic paged concurrency requires one " + "local Strix Halo (gfx1151) target\n"); + return false; + } + } else if (!tp.in_process || !tp.backend_valid || + tp.secondary_backend != PlacementBackend::Hip) { std::fprintf(stderr, - "[deepseek4] paged concurrency keeps all experts resident " - "on Strix Halo and cannot use DFLASH_DS4_MOE_TP\n"); + "[deepseek4] heterogeneous paged concurrency requires " + "in-process HIP expert parallelism\n"); + return false; + } else if (tp.secondary_gpu == cfg_.device.gpu || + !is_gfx_device(cfg_.device.gpu, "gfx1201") || + !is_gfx_device(tp.secondary_gpu, "gfx1151")) { + std::fprintf(stderr, + "[deepseek4] heterogeneous paged concurrency requires an " + "R9700 (gfx1201) target and Strix Halo (gfx1151) secondary\n"); return false; } } @@ -1070,15 +1089,17 @@ bool DeepSeek4Backend::init() { configure_gfx1201_hybrid_sub_batch_default(cfg_.device.gpu); if (cfg_.paged_attention && - (cfg_.max_concurrency < 1 || cfg_.max_concurrency > 16 || + (cfg_.max_concurrency < 1 || + cfg_.max_concurrency > DEEPSEEK4_MAX_PAGED_SEQUENCES || cfg_.device.is_layer_split() || cfg_.prefill_mode != PrefillAttentionMode::Exact || cfg_.fused_decode || cfg_.fused_verify_f16_kv || env_flag_enabled("DFLASH_DS4_FUSED_DECODE") || env_flag_enabled("DFLASH_DS4_SPEC"))) { std::fprintf(stderr, - "[deepseek4] paged serving requires 1..16 local slots, exact " - "prefill, and autoregressive non-fused decode\n"); + "[deepseek4] paged serving requires 1..%d local slots, exact " + "prefill, and autoregressive non-fused decode\n", + DEEPSEEK4_MAX_PAGED_SEQUENCES); return false; } @@ -1177,9 +1198,10 @@ bool DeepSeek4Backend::init() { *this, *paged_cache_.pool, max_ctx, paged_cache_.plan.max_blocks_per_sequence); std::fprintf(stderr, - "[deepseek4-parallel] enabled %d slots, %u x %d-token physical " + "[deepseek4-parallel] enabled %d slots mode=%s, %u x %d-token physical " "blocks; prefill is exact reference mode at one prompt token per slot per scheduler iteration\n", - cfg_.max_concurrency, paged_cache_.plan.physical_blocks, + cfg_.max_concurrency, moe_hybrid_ ? "r9700+strix" : "strix", + paged_cache_.plan.physical_blocks, DS4_PAGE_TOKENS); } const int active_experts = @@ -1276,10 +1298,29 @@ bool DeepSeek4Backend::compute_uniform_hybrid_placement(const DeepSeek4Weights & MoeHybridPlacement * decode_out, std::string * err) const { if (decode_out) *decode_out = {}; + uint64_t kv_bytes = estimate_ds4_cache_bytes(w, max_ctx); + if (cfg_.paged_attention) { + uint32_t physical_blocks = 0; + DeepSeek4PagedCachePlan paged_plan; + if (!plan_deepseek4_paged_pool_blocks( + (uint32_t)max_ctx, (uint32_t)cfg_.max_concurrency, + cfg_.kv_pool_tokens > 0 ? (uint64_t)cfg_.kv_pool_tokens : 0, + physical_blocks) || + !plan_deepseek4_paged_cache( + (uint32_t)w.head_dim, (uint32_t)w.n_indexer_head_dim, + (uint32_t)cfg_.max_concurrency, (uint32_t)max_ctx, + physical_blocks, w.compress_ratios, paged_plan) || + kv_bytes > UINT64_MAX - paged_plan.total_persistent_bytes) { + if (err) *err = "failed to plan paged KV memory for hybrid placement"; + return false; + } + kv_bytes += paged_plan.total_persistent_bytes; + } Ds4HybridBudgetInfo budget; const Ds4MoeTpConfig tp = ds4_moe_tp_config(cfg_.device.gpu); - if (!compute_ds4_hybrid_budget_info(w, backend_, max_ctx, - tp.all_on_secondary, budget, err)) { + if (!compute_ds4_hybrid_budget_info( + w, backend_, kv_bytes, tp.all_on_secondary, + cfg_.paged_attention, budget, err)) { return false; } diff --git a/server/src/deepseek4/deepseek4_graph.cpp b/server/src/deepseek4/deepseek4_graph.cpp index a1a2b5a86..52e231b5f 100644 --- a/server/src/deepseek4/deepseek4_graph.cpp +++ b/server/src/deepseek4/deepseek4_graph.cpp @@ -7178,7 +7178,8 @@ bool deepseek4_paged_gathered_step( std::vector & out_argmax, MoeHybridStorage * hybrid, MoeHybridRoutingStats * routing_stats) { if (!backend || !embeddings || !positions || !slots || !block_tables || - lanes < 1 || lanes > 16 || cache.layers.size() != (size_t) w.n_layer || + lanes < 1 || lanes > DEEPSEEK4_MAX_PAGED_SEQUENCES || + cache.layers.size() != (size_t) w.n_layer || block_table_stride < cache.plan.max_blocks_per_sequence) return false; for (uint32_t lane = 0; lane < lanes; ++lane) { if (slots[lane] < 0) continue; diff --git a/server/src/deepseek4/deepseek4_internal.h b/server/src/deepseek4/deepseek4_internal.h index 9848775e9..fd94f14a1 100644 --- a/server/src/deepseek4/deepseek4_internal.h +++ b/server/src/deepseek4/deepseek4_internal.h @@ -25,6 +25,7 @@ #include "internal.h" #include "common/layer_split_utils.h" +#include "common/paged_attention_config.h" #include "common/prefill_attention_mode.h" #include "common/concurrency/paged_kv_pool.h" #include "deepseek4_paged_cache.h" @@ -373,7 +374,7 @@ bool create_deepseek4_paged_cache(ggml_backend_t backend, DeepSeek4PagedCache & out); void reset_deepseek4_paged_slot(DeepSeek4PagedCache & c, uint32_t slot); void free_deepseek4_paged_cache(DeepSeek4PagedCache & c); -// Exact gathered-reference decode for 1..16 independent lanes. Inputs are +// Exact gathered-reference decode for up to six independent lanes. Inputs are // lane-major; negative slots are inactive padding lanes. `out_logits` is // [n_vocab, lanes] and `out_argmax` is [lanes]. bool deepseek4_paged_gathered_step( diff --git a/server/src/deepseek4/deepseek4_seq_engine.cpp b/server/src/deepseek4/deepseek4_seq_engine.cpp index 9615a1440..e8c0437c8 100644 --- a/server/src/deepseek4/deepseek4_seq_engine.cpp +++ b/server/src/deepseek4/deepseek4_seq_engine.cpp @@ -20,10 +20,11 @@ bool DeepSeek4SeqEngine::token_is_eos(int32_t token) const { StepPlanLimits DeepSeek4SeqEngine::step_plan_limits( int decode_rows) const { - // The gathered graph accepts at most sixteen independent lanes and does + // The gathered graph accepts at most six independent lanes and does // not permit two rows from the same sequence. A prompt therefore advances // by one token while every live decoder still shares the same weight pass. - const int available = std::max(0, 16 - decode_rows); + const int available = std::max( + 0, DEEPSEEK4_MAX_PAGED_SEQUENCES - decode_rows); return {available, 1, available, 1}; } @@ -100,7 +101,7 @@ SeqEngine::StepResult DeepSeek4SeqEngine::step(const StepPlan & plan) { const StepPlanLimits limits = step_plan_limits((int)inputs.size()); if ((int)plan.prefills.size() > limits.max_prefill_sequences) { - return fail_step("DeepSeek4 step exceeds the sixteen-lane graph"); + return fail_step("DeepSeek4 step exceeds the six-lane graph"); } std::vector prefill_seen((size_t)n_slots, 0); for (const PrefillSlice & slice : plan.prefills) { @@ -189,8 +190,8 @@ SeqEngine::StepResult DeepSeek4SeqEngine::step(const StepPlan & plan) { } if (lane_tokens.empty()) return result; - if (lane_tokens.size() > 16) { - return fail_step("DeepSeek4 gathered step exceeds sixteen lanes"); + if (lane_tokens.size() > DEEPSEEK4_MAX_PAGED_SEQUENCES) { + return fail_step("DeepSeek4 gathered step exceeds six lanes"); } std::vector embeddings( diff --git a/server/test/test_deepseek4_paged_cache.cpp b/server/test/test_deepseek4_paged_cache.cpp index 2889fc19c..2acbc82de 100644 --- a/server/test/test_deepseek4_paged_cache.cpp +++ b/server/test/test_deepseek4_paged_cache.cpp @@ -7,18 +7,20 @@ using namespace dflash::common; static int g_checks = 0; int main() { uint32_t blocks = 0; - CHECK(plan_deepseek4_paged_pool_blocks(4096, 16, 0, blocks)); - CHECK(blocks == 512); - CHECK(plan_deepseek4_paged_pool_blocks(4096, 16, 128, blocks)); + CHECK(plan_deepseek4_paged_pool_blocks(4096, 6, 0, blocks)); + CHECK(blocks == 192); + CHECK(plan_deepseek4_paged_pool_blocks(131072, 6, 0, blocks)); + CHECK(blocks == 6144); + CHECK(plan_deepseek4_paged_pool_blocks(4096, 6, 128, blocks)); CHECK(blocks == 1); - CHECK(plan_deepseek4_paged_pool_blocks(4096, 16, 129, blocks)); + CHECK(plan_deepseek4_paged_pool_blocks(4096, 6, 129, blocks)); CHECK(blocks == 2); - CHECK(!plan_deepseek4_paged_pool_blocks(0, 16, 128, blocks)); + CHECK(!plan_deepseek4_paged_pool_blocks(0, 6, 128, blocks)); CHECK(!plan_deepseek4_paged_pool_blocks(4096, 0, 128, blocks)); const uint64_t overflowing_tokens = uint64_t(UINT32_MAX / DS4_PAGE_TOKENS) * DS4_PAGE_TOKENS + 1; CHECK(!plan_deepseek4_paged_pool_blocks( - 4096, 16, overflowing_tokens, blocks)); + 4096, 6, overflowing_tokens, blocks)); DeepSeek4PagedCachePlan p, twice; CHECK(plan_deepseek4_paged_cache(512, 128, 3, 4096, 40, {0, 4, 128}, p)); @@ -26,16 +28,12 @@ int main() { CHECK(p.physical_rows[1] == 1280 && p.physical_rows[2] == 40); CHECK(p.raw_bytes == uint64_t(3) * 512 * 128 * 3 * 2); CHECK(p.metadata_bytes == uint64_t(32 * 3 + 3 + 3) * 4); + CHECK(p.total_persistent_bytes == + p.metadata_bytes + p.raw_bytes + p.compressed_bytes + p.state_bytes); CHECK(plan_deepseek4_paged_cache(512, 128, 6, 4096, 40, {0, 4, 128}, twice)); // Paged rows are shared; only raw rings, metadata, and compressor state scale by slots. CHECK(twice.compressed_bytes == p.compressed_bytes); CHECK(twice.raw_bytes == p.raw_bytes * 2 && twice.state_bytes == p.state_bytes * 2); - DeepSeek4PagedCachePlan sixteen; - CHECK(plan_deepseek4_paged_cache(512, 128, 16, 4096, 40, - {0, 4, 128}, sixteen)); - CHECK(sixteen.slots == 16 && sixteen.max_blocks_per_sequence == 32); - CHECK(sixteen.compressed_bytes == p.compressed_bytes); - CHECK(sixteen.raw_bytes == p.raw_bytes / 3 * 16); CHECK(!plan_deepseek4_paged_cache(512, 128, 1, 4096, 40, {4, 16}, twice)); CHECK(!plan_deepseek4_paged_cache(512, 128, 1, 4096, std::numeric_limits::max(), {4}, twice)); @@ -64,18 +62,18 @@ int main() { CHECK(rows[2].raw_scatter == -1 && rows[2].compressed_scatter == -1); CHECK(rows[2].position == 0); - std::vector sixteen_slots(16); - std::vector sixteen_positions(16, 0); - std::vector sixteen_tables(16); - for (int i = 0; i < 16; ++i) { - sixteen_slots[(size_t) i] = i; - sixteen_tables[(size_t) i] = i; + std::vector six_slots(6); + std::vector six_positions(6, 0); + std::vector six_tables(6); + for (int i = 0; i < 6; ++i) { + six_slots[(size_t) i] = i; + six_tables[(size_t) i] = i; } CHECK(prepare_deepseek4_gathered_lane_rows( - sixteen_slots.data(), sixteen_positions.data(), 16, - sixteen_tables.data(), 1, 16, 4, rows)); - CHECK(rows.size() == 16); - for (int i = 0; i < 16; ++i) { + six_slots.data(), six_positions.data(), 6, + six_tables.data(), 1, 6, 4, rows)); + CHECK(rows.size() == 6); + for (int i = 0; i < 6; ++i) { CHECK(rows[(size_t) i].slot == i); CHECK(rows[(size_t) i].raw_history.empty()); CHECK(rows[(size_t) i].raw_scatter == int64_t(i * 128)); diff --git a/server/test/test_feature_gate.cpp b/server/test/test_feature_gate.cpp index c52c24e51..d0344ec99 100644 --- a/server/test/test_feature_gate.cpp +++ b/server/test/test_feature_gate.cpp @@ -506,9 +506,9 @@ void test_feature_gate_parallel_and_kv_pool_rules() { BackendArgs ds4 = gate_args_hip_deepseek4(); ds4.paged_attention = true; - ds4.max_concurrency = 16; + ds4.max_concurrency = DEEPSEEK4_MAX_PAGED_SEQUENCES; CHECK(gate_result(ds4, "deepseek4", PlacementBackend::Hip).empty()); - ds4.max_concurrency = 17; + ds4.max_concurrency = DEEPSEEK4_MAX_PAGED_SEQUENCES + 1; CHECK(!gate_result(ds4, "deepseek4", PlacementBackend::Hip).empty()); ds4.max_concurrency = 2; ds4.ds4_prefill_mode = PrefillAttentionMode::Sparse;