diff --git a/.github/script/ci/changes.py b/.github/script/ci/changes.py index 27749825d..ff6a076f3 100644 --- a/.github/script/ci/changes.py +++ b/.github/script/ci/changes.py @@ -82,10 +82,11 @@ ("qwen3_8mtp", "qwen3_8_mtp"): ("qwen3.8-mtp",), ("gemma", "gemma3"): ("gemma3",), ("gemma_text", "gemma3_text"): ("gemma3-text",), - # Gemma4e_Flash and Gemma4e_DFlash derive from Gemma4e and their engines include gemma4e_npu. - ("gemma4e",): ("gemma4e", "gemma4e-flash", "gemma4e-dflash"), + # Gemma4e_Flash, Gemma4e_DFlash and Gemma4e_DFlash_Flash derive from Gemma4e and their engines include gemma4e_npu. + ("gemma4e",): ("gemma4e", "gemma4e-flash", "gemma4e-dflash", "gemma4e-dflash-flash"), ("gemma4e_flash",): ("gemma4e-flash",), ("gemma4e_dflash",): ("gemma4e-dflash",), + ("gemma4e_dflash_flash",): ("gemma4e-dflash-flash",), ("gemma4_12b",): ("gemma4-12b",), ("hunyuan",): ("hunyuan",), ("gpt_oss",): ("gpt-oss",), diff --git a/.github/script/npu_test/plan.py b/.github/script/npu_test/plan.py index 367e194c9..7cc1c5b78 100644 --- a/.github/script/npu_test/plan.py +++ b/.github/script/npu_test/plan.py @@ -33,6 +33,7 @@ "gemma4e": "gemma4-it:e2b", "gemma4-12b": "gemma4-it:12b", "gemma4e-flash": "gemma4e-flash:e2b", + "gemma4e-dflash-flash": None, # same 9 GB nemotron3.5-bolt:e4b package; runs when the change touches it "gemma4e-dflash": None, # needs the 9 GB nemotron3.5-bolt:e4b package; runs when the change touches it "qwen3.5": "qwen3.5:0.8b", "minicpm-v-4.7": "minicpm-v-4.7:1b", diff --git a/docs/docs/benchmarks/flash_results.md b/docs/docs/benchmarks/flash_results.md index 1aab2fd8b..2a0ae7bbb 100644 --- a/docs/docs/benchmarks/flash_results.md +++ b/docs/docs/benchmarks/flash_results.md @@ -31,6 +31,7 @@ AMD Ryzen™ AI 9 370 (Strix Point) with 32 GB DRAM; performance is comparable t |-------------------------|-----------|-------:|--------:|--------:|--------:|---------:| | **gemma4e-flash:e2b** | XDNA2 | 0.20 | 0.22 | 0.36 | 0.73 | 1.53 | | **gemma4e-flash:e4b** | XDNA2 | 0.33 | 0.34 | 0.62 | 1.20 | 2.57 | +| **nemotron3.5-bolt-dflash-flash:e4b** | XDNA2 | 0.32 | 0.33 | 0.60 | 1.18 | 2.53 | | **qwen3vl-flash:4b** | XDNA2 | 0.24 | 0.24 | 0.45 | 0.85 | 1.83 | | **hy-mt2-flash:1.8b** | XDNA2 | 0.13 | 0.14 | 0.23 | 0.43 | 0.84 | @@ -44,6 +45,7 @@ Prompt tokens divided by TTFT (TTFT includes generating the first token). |-------------------------|-----------|-------:|--------:|--------:|--------:|---------:| | **gemma4e-flash:e2b** | XDNA2 | 322 | 593 | 721 | 705 | 668 | | **gemma4e-flash:e4b** | XDNA2 | 193 | 373 | 412 | 427 | 398 | +| **nemotron3.5-bolt-dflash-flash:e4b** | XDNA2 | 201 | 392 | 427 | 434 | 405 | | **qwen3vl-flash:4b** | XDNA2 | 271 | 526 | 570 | 601 | 558 | | **hy-mt2-flash:1.8b** | XDNA2 | 490 | 923 | 1,093 | 1,197 | 1,214 | @@ -55,7 +57,10 @@ Prompt tokens divided by TTFT (TTFT includes generating the first token). |-------------------------|-----------|-------:|------------| | **gemma4e-flash:e2b** | XDNA2 | 22.6 | [Gemma 4](../gemma4_results/) (`gemma4-it:e2b`) | | **gemma4e-flash:e4b** | XDNA2 | 12.6 | [Gemma 4](../gemma4_results/) (`gemma4-it:e4b`) | +| **nemotron3.5-bolt-dflash-flash:e4b** | XDNA2 | 18.2 | [Nemotron Bolt DFlash](../nemotron3.5-bolt-dflash_results/) (`nemotron3.5-bolt-dflash:e4b`) | | **qwen3vl-flash:4b** | XDNA2 | 19.6 | [Qwen3](../qwen3_results/) (Qwen 3 4B) | | **hy-mt2-flash:1.8b** | XDNA2 | 52.2 | [Hy-MT2](../hy-mt2_results/) | > Context is fixed at 1k tokens for every flash model, so 1k is the only applicable decode starting point. +> +> `nemotron3.5-bolt-dflash-flash:e4b` decodes with speculation, so its speed depends on the text: the value above is `flm bench` (about 30 decoded tokens, mean of 3 runs, 15 to 24 tokens/s between runs). Code and structured answers reach 33 to 40 tokens/s, open prose about 20 (see its [model card](../../models/nemotron3.5-bolt-dflash/#-model-card-nvidia-nemotron-35-bolt-e4b--dflash-flash)). The three rows of this model were measured on an AMD Ryzen™ AI MAX+ PRO 395 (Strix Halo) with the same method; on that machine `gemma4e-flash:e4b` gives 0.32 / 0.32 / 0.58 / 1.17 / 2.52 s. diff --git a/docs/docs/instructions/server/openapi.md b/docs/docs/instructions/server/openapi.md index 9a2745fb8..88a0422db 100644 --- a/docs/docs/instructions/server/openapi.md +++ b/docs/docs/instructions/server/openapi.md @@ -507,7 +507,7 @@ gc.collect() ## ⚡ Example: Flash Models (Single-Turn, Pinned System Prompt) -**Flash** models — `gemma4e-flash:e2b`, `gemma4e-flash:e4b`, `qwen3vl-flash:4b`, and `hy-mt2-flash:1.8b` — trade context length for a much faster time-to-first-token. They are built for high-throughput, short, independent requests: classification, captioning, extraction, routing, translation. +**Flash** models — `gemma4e-flash:e2b`, `gemma4e-flash:e4b`, `nemotron3.5-bolt-dflash-flash:e4b`, `qwen3vl-flash:4b`, and `hy-mt2-flash:1.8b` — trade context length for a much faster time-to-first-token. They are built for high-throughput, short, independent requests: classification, captioning, extraction, routing, translation. They are **single-turn**: every request starts from a clean KV state, so earlier turns are not carried over. The system prompt is the one exception — it is prefilled once and cached, so keep it fixed and vary only the user message (and, for vision models, the image). @@ -571,6 +571,6 @@ gc.collect() > 📏 Context is **fixed at 1k tokens** for every flash model; context-length overrides are ignored. > 🔧 **Tool calling is not supported** — any `tools` you pass are dropped. > 🌐 `hy-mt2-flash` is text-only and has no default system prompt — pin your own translation instruction as the system prompt, or put it in the user message. -> 🖼️ `gemma4e-flash` defaults to the smallest visual token budget (70) and keeps only the **first 30 seconds** of an audio clip. `qwen3vl-flash` always downscales images to a **256 px longer side**. +> 🖼️ `gemma4e-flash` and `nemotron3.5-bolt-dflash-flash` default to the smallest visual token budget (70) and keeps only the **first 30 seconds** of an audio clip. `qwen3vl-flash` always downscales images to a **256 px longer side**. -For the full per-model details, see the [`gemma4e-flash:e2b`](https://fastflowlm.com/docs/models/gemma/#-model-card-gemma-4-e2b-it--flash), [`gemma4e-flash:e4b`](https://fastflowlm.com/docs/models/gemma/#-model-card-gemma-4-e4b-it--flash), [`qwen3vl-flash:4b`](https://fastflowlm.com/docs/models/qwen/#-model-card-qwen3-vl-4b-instruct--flash), and [`hy-mt2-flash:1.8b`](https://fastflowlm.com/docs/models/hy-mt2/) model cards. Performance numbers are on the [Flash benchmarks](https://fastflowlm.com/docs/benchmarks/flash_results/) page. \ No newline at end of file +For the full per-model details, see the [`gemma4e-flash:e2b`](https://fastflowlm.com/docs/models/gemma/#-model-card-gemma-4-e2b-it--flash), [`gemma4e-flash:e4b`](https://fastflowlm.com/docs/models/gemma/#-model-card-gemma-4-e4b-it--flash), [`nemotron3.5-bolt-dflash-flash:e4b`](https://fastflowlm.com/docs/models/nemotron3.5-bolt-dflash/#-model-card-nvidia-nemotron-35-bolt-e4b--dflash-flash), [`qwen3vl-flash:4b`](https://fastflowlm.com/docs/models/qwen/#-model-card-qwen3-vl-4b-instruct--flash), and [`hy-mt2-flash:1.8b`](https://fastflowlm.com/docs/models/hy-mt2/) model cards. Performance numbers are on the [Flash benchmarks](https://fastflowlm.com/docs/benchmarks/flash_results/) page. \ No newline at end of file diff --git a/docs/docs/instructions/server/prefix_cache.md b/docs/docs/instructions/server/prefix_cache.md index 01e402f63..b916275d6 100644 --- a/docs/docs/instructions/server/prefix_cache.md +++ b/docs/docs/instructions/server/prefix_cache.md @@ -41,7 +41,7 @@ The other supported models gain the same way. Measured on shorter prompts (about | Endpoint | `POST /v1/chat/completions` of `flm serve` (what Hermes, Open WebUI and most agents use), streaming or not | | Not covered | `flm run` (CLI mode) and the Ollama endpoints `/api/chat` and `/api/generate`, which keep the older single-checkpoint reuse. A request on an Ollama endpoint also empties the prefix cache, so a client that mixes both APIs loses its cached prompts. | -Not covered: the single-turn models (`hy-mt2-flash`, `qwen3vl-flash`, `gemma4e-flash`: they take one request and pin their system prompt, so there is nothing to reuse), the developer-preview `*-core` backends, `lfm3`, and the speech, embedding and robotics models. They are unchanged. +Not covered: the single-turn models (`hy-mt2-flash`, `qwen3vl-flash`, `gemma4e-flash`, `nemotron3.5-bolt-dflash-flash`: they take one request and pin their system prompt, so there is nothing to reuse), the developer-preview `*-core` backends, `lfm3`, and the speech, embedding and robotics models. They are unchanged. It is on by default. Nothing to configure for a single chat or agent. diff --git a/docs/docs/instructions/server/request_parameters.md b/docs/docs/instructions/server/request_parameters.md index 82c3c698b..fe32b7349 100644 --- a/docs/docs/instructions/server/request_parameters.md +++ b/docs/docs/instructions/server/request_parameters.md @@ -70,7 +70,7 @@ A request that sends no sampling fields uses the model's own defaults, which dif | Models | `presence_penalty` at load | |---|---| | `qwen3.5:*`, `qwen3.5-core:*`, `qwen3.6-moe:35b-a3b`, `qwen3.6-moe-core:35b-a3b`, `qwen3.8-mtp:27b`, `minicpm-v-4.7:1b` | 1.5 | -| `gemma4-it:*`, `gemma4-it-core:*`, `gemma4e-flash:*`, `nemotron3.5-bolt:e4b` | 1.0 | +| `gemma4-it:*`, `gemma4-it-core:*`, `gemma4e-flash:*`, `nemotron3.5-bolt:e4b`, `nemotron3.5-bolt-dflash:e4b`, `nemotron3.5-bolt-dflash-flash:e4b` | 1.0 | | every other model | 0 | Sending `presence_penalty=0` turns the default off, and it stays off until the model is reloaded (see *Settings stick* above). @@ -260,7 +260,7 @@ Before generating, FLM reads the prompt into the model's KV cache. This is the * | Models | `max_prefill_len` | |---|---| -| Flash models (`gemma4e-flash:*`, `qwen3vl-flash:4b`, `hy-mt2-flash:1.8b`) | 1024 | +| Flash models (`gemma4e-flash:*`, `nemotron3.5-bolt-dflash-flash:e4b`, `qwen3vl-flash:4b`, `hy-mt2-flash:1.8b`) | 1024 | | Every other chat model | 4096 | **`max_prefill_len` is not a limit on the prompt.** A longer prompt just takes more chunks. A 32,000-token prompt on a model with 4096-token chunks is read in eight passes, seven of 4096 tokens and one of 3,328, and the server log shows each one: diff --git a/docs/docs/models/flash.md b/docs/docs/models/flash.md index 214e0fab4..c579e15c3 100644 --- a/docs/docs/models/flash.md +++ b/docs/docs/models/flash.md @@ -13,6 +13,7 @@ parent: Models |---------------|----------|----------------| | `gemma4e-flash:e2b` | Any-to-Text (image, audio) | [Gemma 4 E2B — Flash](https://fastflowlm.com/docs/models/gemma/#-model-card-gemma-4-e2b-it--flash) | | `gemma4e-flash:e4b` | Any-to-Text (image, audio) | [Gemma 4 E4B — Flash](https://fastflowlm.com/docs/models/gemma/#-model-card-gemma-4-e4b-it--flash) | +| `nemotron3.5-bolt-dflash-flash:e4b` | Any-to-Text (image, audio), speculative decoding | [Nemotron Bolt E4B — DFlash Flash](https://fastflowlm.com/docs/models/nemotron3.5-bolt-dflash/#-model-card-nvidia-nemotron-35-bolt-e4b--dflash-flash) | | `qwen3vl-flash:4b` | Image-Text-to-Text | [Qwen3-VL-4B — Flash](https://fastflowlm.com/docs/models/qwen/#-model-card-qwen3-vl-4b-instruct--flash) | | `hy-mt2-flash:1.8b` | Text-to-Text (Translation) | [Hy-MT2](https://fastflowlm.com/docs/models/hy-mt2/) | diff --git a/docs/docs/models/nemotron3.5-bolt-dflash.md b/docs/docs/models/nemotron3.5-bolt-dflash.md index 320c5065b..ad6f61806 100644 --- a/docs/docs/models/nemotron3.5-bolt-dflash.md +++ b/docs/docs/models/nemotron3.5-bolt-dflash.md @@ -58,3 +58,61 @@ Expected: `1, 2, 3, ..., 40` followed by `done`, at a visibly higher speed than - Speculation pauses while a tool-call or JSON grammar constrains the output, and whenever it would not be faster (for example at very long contexts); it resumes by itself. - `flm remove nemotron3.5-bolt-dflash:e4b` removes only the draft package; `nemotron3.5-bolt:e4b` stays installed. + +--- + +## 🧩 Model Card: [NVIDIA-Nemotron-3.5-Bolt-E4B — DFlash Flash](https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Bolt-E4B-4GB) + +- **Type:** Any-to-Text (image, audio), single-turn, with speculative decoding +- **What it is:** [`nemotron3.5-bolt-dflash:e4b`](#-model-card-nvidia-nemotron-35-bolt-e4b-with-the-dflash-draft) built for the + [Flash](https://fastflowlm.com/docs/models/flash/) serving style: the fast short-prompt prefill of `gemma4e-flash:e4b` together with + DFlash speculative decoding. Every request starts from a clean state; the system prompt is pinned and reused + ([how flash models work in server mode](https://fastflowlm.com/docs/instructions/server/openapi/#-example-flash-models-single-turn-pinned-system-prompt)) +- **Time to first token:** the same as `gemma4e-flash:e4b`, about 0.3 s for a short prompt, against 1.2 s for `nemotron3.5-bolt-dflash:e4b` +- **Decode speed:** 33 to 40 tokens/s on code and structured answers, 19 to 21 tokens/s on open prose, against 13 for `gemma4e-flash:e4b` +- **Think / Conversational mode / Vision / Audio:** as `nemotron3.5-bolt:e4b`. Tool calling is not supported (the `tools` field is dropped). + Images default to the smallest visual token budget (70) and only the first 30 seconds of an audio clip are used +- **Packages:** the same two downloads as `nemotron3.5-bolt-dflash:e4b` (`Nemotron-3.5-Bolt-E4B-NPU2`, about 9 GB, and the 0.27 GB draft + `Nemotron-3.5-Bolt-E4B-DFlash-NPU2`). Nothing new to download if you already have that tag +- **Max / Default Context Length:** 1k tokens (fixed, like every flash model) +- **License:** [OpenMDW-1.1](https://openmdw.ai/license/1-1/) +- **FastFlowLM:** 1.0.13 or later + +▶️ Run with FastFlowLM in PowerShell: + +```shell +flm run nemotron3.5-bolt-dflash-flash:e4b +``` + +Server mode: + +```shell +flm serve nemotron3.5-bolt-dflash-flash:e4b +``` + +📊 **Measured** (greedy, 18-token pinned system prompt, NPU, quiet machine): + +| Prompt length | `gemma4e-flash:e4b` | `nemotron3.5-bolt-dflash:e4b` | `nemotron3.5-bolt-dflash-flash:e4b` | +|---|---|---|---| +| ~50 tokens, time to first token | 0.31 s | 1.24 s | 0.32 s | +| ~140 tokens | 0.57 s | 1.24 s | 0.60 s | +| ~540 tokens | 1.48 s | 1.84 s | 1.49 s | +| ~960 tokens | 2.51 s | 2.15 s | 2.52 s | +| decode, code answer | 13 tokens/s | 38 to 40 tokens/s | 33 to 40 tokens/s | +| decode, prose answer | 13 tokens/s | 17 to 22 tokens/s | 19 to 21 tokens/s | + +📝 **Note:** + +- Flash prefill is tuned for short prompts: it is the faster choice up to a prompt of roughly 700 tokens. For longer prompts use + `nemotron3.5-bolt-dflash:e4b`, which also has a longer context. +- A long, fixed system prompt (instructions, examples) is where this tag gains most: it is prefilled once and the draft model sees it on + every request, which keeps the speculative steps long. Changing the system prompt between requests discards the pin. +- Sampling, `FLM_DFLASH=0` and the other behaviours of speculation are as on the [`nemotron3.5-bolt-dflash:e4b`](#-model-card-nvidia-nemotron-35-bolt-e4b-with-the-dflash-draft) card. + Greedy output can differ from `nemotron3.5-bolt:e4b` at near-ties. +- Accuracy (50-question samples, XRT and HRX give the same results): GSM8K 92%, the same as `nemotron3.5-bolt-dflash:e4b` and + `gemma4e-flash:e4b`. Document reading (DocVQA, ANLS) is 45, against 76 for `nemotron3.5-bolt-dflash:e4b`: flash models look at an image + with the smallest visual token budget (70 instead of 280), which is enough for captions and simple scenes but not for small text. For + documents and screenshots use `nemotron3.5-bolt-dflash:e4b`. +- Needs 9 of the NPU's 16 hardware contexts (vision and audio included), so other models can still be loaded next to it. +- `flm remove nemotron3.5-bolt-dflash-flash:e4b` removes only the draft package, shared with `nemotron3.5-bolt-dflash:e4b`: remove + both tags' use of it first if you want the space back. diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 789221bbe..c9f788895 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -514,6 +514,7 @@ list(APPEND FLM_ENGINE_LINK_LIBS list(APPEND FLM_ENGINE_LINK_LIBS gemma4e_flash) list(APPEND FLM_ENGINE_LINK_LIBS gemma4e_dflash) +list(APPEND FLM_ENGINE_LINK_LIBS gemma4e_dflash_flash) list(APPEND FLM_ENGINE_LINK_LIBS gemma4_12b_npu diff --git a/src/common/AutoModel/builtin_backends.cpp b/src/common/AutoModel/builtin_backends.cpp index 3cf755365..868c7edb6 100644 --- a/src/common/AutoModel/builtin_backends.cpp +++ b/src/common/AutoModel/builtin_backends.cpp @@ -54,6 +54,7 @@ void register_builtin_backends(BackendRegistry& registry) { RegisterFlm(registry, "gemma4e"); RegisterFlm(registry, "gemma4e-flash"); RegisterFlm(registry, "gemma4e-dflash"); + RegisterFlm(registry, "gemma4e-dflash-flash"); RegisterFlm(registry, "gemma4-12b"); RegisterFlm(registry, "hunyuan"); RegisterFlm(registry, "gpt-oss"); diff --git a/src/common/AutoModel/modeling_gemma4e.cpp b/src/common/AutoModel/modeling_gemma4e.cpp index 07e8a5297..c7234db29 100644 --- a/src/common/AutoModel/modeling_gemma4e.cpp +++ b/src/common/AutoModel/modeling_gemma4e.cpp @@ -704,7 +704,9 @@ bool Gemma4e::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std ? this->_shared_insert(meta_info, tokens, is_cancelled, &multi_modal_payload, last_image_token_index) : this->_shared_insert(meta_info, tokens, is_cancelled, nullptr); - if (this->enable_think) { + // Single-turn (flash): the only checkpoint is the pinned system prefix, which Gemma4e_Flash owns. Taking this one would + // overwrite it with the whole first prompt, and every later request would restore into that stale context. + if (this->enable_think && !this->single_turn) { checkpoint_his = token_history; int checkpoint_idx = this->lm_engine->checkpoint(); } diff --git a/src/common/AutoModel/modeling_gemma4e_dflash.cpp b/src/common/AutoModel/modeling_gemma4e_dflash.cpp index 4f4bed44d..e30bb2b79 100644 --- a/src/common/AutoModel/modeling_gemma4e_dflash.cpp +++ b/src/common/AutoModel/modeling_gemma4e_dflash.cpp @@ -50,47 +50,13 @@ void Gemma4e_DFlash::load_model(std::string model_path, json model_info, int def } } -std::string Gemma4e_DFlash::generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled) { - return this->_speculative_generate(meta_info, length_limit, os, is_cancelled); -} - -std::string Gemma4e_DFlash::generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os) { - if (!this->insert(meta_info, input)) { - return ""; - } - if (this->thinking_enabled()) { - os << "\n" << std::flush; - } - int checkpoint_idx = this->lm_engine->checkpoint(); - int restore_idx = this->lm_engine->restore(); - header_print_r("FLM", "Checkpoint before generation: " << checkpoint_idx << ", restore point: " << restore_idx << ", user context length: " << this->token_history.size()); - return this->_speculative_generate(meta_info, length_limit, os, [] { return false; }); -} - -void Gemma4e_DFlash::clear_context() { - Gemma4e::clear_context(); - if (auto* engine = dynamic_cast(this->lm_engine)) { - engine->reset_speculation_stats(); - } -} - -std::string Gemma4e_DFlash::show_profile() { - std::string profile = Gemma4e::show_profile(); - if (auto* engine = dynamic_cast(this->lm_engine)) { - if (engine->speculation_cycles() > 0) { - profile += engine->speculation_timing(); - } - } - return profile; -} - -std::string Gemma4e_DFlash::_speculative_generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled) { +template +std::string Gemma4e::_speculative_loop(Engine* engine, chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled, bool single_turn) { std::string result; assert(this->last_token != -1); stop_reason_t reason = EOT_DETECTED; - auto* engine = dynamic_cast(this->lm_engine); // Off under a decoding constraint (grammar / tool JSON, D14): a drafted token never passes the constraint's mask // through sample-and-compare unless every row is masked, which the plain loop already does cheaply. // draft_ready() is asked per cycle below: the engine's adaptive guard (it turns speculation off while it cannot win and re-probes @@ -136,7 +102,12 @@ std::string Gemma4e_DFlash::_speculative_generate(chat_meta_info_t& meta_info, i if (this->is_eos(token)) { meta_info.generated_tokens++; // The engine never ran the stop token: supply its KV row (it is the last token of a cycle, see below). - this->lm_engine->forward(token); + // Single-turn: the KV is discarded by the next request's restore to the pin, so skip it. + if (single_turn) { + this->token_history.pop_back(); // as Gemma4e_Flash::generate: the engine never ran the stop token + } else { + this->lm_engine->forward(token); + } return false; } meta_info.generated_tokens++; @@ -231,10 +202,105 @@ std::string Gemma4e_DFlash::_speculative_generate(chat_meta_info_t& meta_info, i // Cut short after its last token was sampled: no engine has run that one, so the next prompt has to carry it. this->token_history.pop_back(); } - if (!this->thinking_enabled()) { + // Single-turn: the only checkpoint is the pinned system prefix, which _pin_system_prefix() owns. + if (!single_turn && !this->thinking_enabled()) { // the history the checkpoint holds: an eos ending the turn was run, a cut-off token was not this->lm_engine->checkpoint(); checkpoint_his = token_history; } return result; } + +std::string Gemma4e_DFlash::generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled) { + return this->_speculative_loop(dynamic_cast(this->lm_engine), meta_info, length_limit, os, is_cancelled, false); +} + +std::string Gemma4e_DFlash::generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os) { + if (!this->insert(meta_info, input)) { + return ""; + } + if (this->thinking_enabled()) { + os << "\n" << std::flush; + } + int checkpoint_idx = this->lm_engine->checkpoint(); + int restore_idx = this->lm_engine->restore(); + header_print_r("FLM", "Checkpoint before generation: " << checkpoint_idx << ", restore point: " << restore_idx << ", user context length: " << this->token_history.size()); + return this->_speculative_loop(dynamic_cast(this->lm_engine), meta_info, length_limit, os, [] { return false; }, false); +} + +void Gemma4e_DFlash::clear_context() { + Gemma4e::clear_context(); + if (auto* engine = dynamic_cast(this->lm_engine)) { + engine->reset_speculation_stats(); + } +} + +std::string Gemma4e_DFlash::show_profile() { + std::string profile = Gemma4e::show_profile(); + if (auto* engine = dynamic_cast(this->lm_engine)) { + if (engine->speculation_cycles() > 0) { + profile += engine->speculation_timing(); + } + } + return profile; +} + +/************ Gemma4e_DFlash_Flash **************/ +gemma4e_engine_config_t Gemma4e_DFlash_Flash::engine_config() const { + return read_gemma4e_engine_config(this->lm_engine); +} + +const flm_state_api* Gemma4e_DFlash_Flash::engine_state_api() const { + return nullptr; +} + +void Gemma4e_DFlash_Flash::load_model(std::string model_path, json model_info, int default_context_length, bool enable_preemption, const std::string& backend) { + Gemma4e::load_model(model_path, model_info, default_context_length, enable_preemption, backend); + auto* engine = dynamic_cast(this->lm_engine); + if (engine == nullptr) { + header_print("WARNING", "Gemma4e_DFlash_Flash: the engine is not gemma4e_dflash_flash, speculation is off"); + return; + } + engine->set_stop_tokens(this->eos_token_ids); + if (this->draft_model_path.empty()) { + header_print("WARNING", "Gemma4e_DFlash_Flash: no draft package, decoding without speculation"); + return; + } + if (!dflash_env_enabled()) { + header_print("FLM", "DFlash speculation disabled by FLM_DFLASH=0"); + return; + } + if (engine->load_draft(this->draft_model_path)) { + header_print("FLM", "DFlash draft loaded from " << this->draft_model_path); + } else { + header_print("WARNING", "Gemma4e_DFlash_Flash: the draft package could not be loaded, decoding without speculation"); + } +} + +std::string Gemma4e_DFlash_Flash::generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled) { + return this->_speculative_loop(dynamic_cast(this->lm_engine), meta_info, length_limit, os, is_cancelled, true); +} + +std::string Gemma4e_DFlash_Flash::generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os) { + if (!this->insert(meta_info, input)) { + return ""; + } + return this->generate(meta_info, length_limit, os); +} + +void Gemma4e_DFlash_Flash::clear_context() { + Gemma4e::clear_context(); + if (auto* engine = dynamic_cast(this->lm_engine)) { + engine->reset_speculation_stats(); + } +} + +std::string Gemma4e_DFlash_Flash::show_profile() { + std::string profile = Gemma4e::show_profile(); + if (auto* engine = dynamic_cast(this->lm_engine)) { + if (engine->speculation_cycles() > 0) { + profile += engine->speculation_timing(); + } + } + return profile; +} diff --git a/src/include/AutoModel/all_models.hpp b/src/include/AutoModel/all_models.hpp index ccdce734d..6dfafbc6c 100644 --- a/src/include/AutoModel/all_models.hpp +++ b/src/include/AutoModel/all_models.hpp @@ -53,6 +53,7 @@ typedef enum { gemma4e, gemma4e_flash, gemma4e_dflash, + gemma4e_dflash_flash, gemma4_12b, gpt_oss, lfm2, @@ -89,6 +90,7 @@ inline std::pair> get_auto_model(const s {"gemma4e", SupportedModelFamily::gemma4e}, {"gemma4e-flash", SupportedModelFamily::gemma4e_flash}, {"gemma4e-dflash", SupportedModelFamily::gemma4e_dflash}, + {"gemma4e-dflash-flash", SupportedModelFamily::gemma4e_dflash_flash}, {"gemma4-12b", SupportedModelFamily::gemma4_12b}, {"gpt-oss", SupportedModelFamily::gpt_oss}, {"lfm2", SupportedModelFamily::lfm2}, @@ -154,6 +156,9 @@ inline std::pair> get_auto_model(const s case SupportedModelFamily::gemma4e_dflash: auto_chat_engine = std::make_unique(npu_device_inst); break; + case SupportedModelFamily::gemma4e_dflash_flash: + auto_chat_engine = std::make_unique(npu_device_inst); + break; case SupportedModelFamily::gemma4_12b: auto_chat_engine = std::make_unique(npu_device_inst); break; diff --git a/src/include/AutoModel/automodel.hpp b/src/include/AutoModel/automodel.hpp index b814bf1bb..f679567fb 100644 --- a/src/include/AutoModel/automodel.hpp +++ b/src/include/AutoModel/automodel.hpp @@ -42,6 +42,7 @@ #include "models/gemma4e/flm/aie2p/gemma4e_npu.hpp" #include "models/gemma4e_flash/flm/aie2p/gemma4e_flash.hpp" #include "models/gemma4e_dflash/flm/aie2p/gemma4e_dflash.hpp" +#include "models/gemma4e_dflash_flash/flm/aie2p/gemma4e_dflash_flash.hpp" #include "models/gemma4_12b/flm/aie2p/gemma4_12b_npu.hpp" #include "models/lfm2/flm/aie2p/lfm2_npu.hpp" #include "models/lfm3/flm/aie2p/lfm3_npu.hpp" diff --git a/src/include/AutoModel/modeling_gemma4e.hpp b/src/include/AutoModel/modeling_gemma4e.hpp index 3e5a305fe..d3e529f58 100644 --- a/src/include/AutoModel/modeling_gemma4e.hpp +++ b/src/include/AutoModel/modeling_gemma4e.hpp @@ -142,6 +142,15 @@ class Gemma4e : public AutoModel { /// \brief the saved-state table of the engine behind this wrapper (nullptr: the context cache stays off) virtual const flm_state_api* engine_state_api() const; + /// \brief the DFlash speculative decode loop (sample-and-compare, D11), shared by Gemma4e_DFlash and Gemma4e_DFlash_Flash. + /// \tparam Engine gemma4e_dflash or gemma4e_dflash_flash (same speculation API, unrelated types) + /// \param single_turn the flash contract: no checkpoint at the end (the pinned system prefix owns the only one) and no + /// KV row for the stop token (the next request restarts from the pin); the plain DFlash tag keeps both. + /// \note defined in modeling_gemma4e_dflash.cpp, the only place it is instantiated + template + std::string _speculative_loop(Engine* engine, chat_meta_info_t& meta_info, int length_limit, std::ostream& os, + std::function is_cancelled, bool single_turn); + /// \note Gemma4e and Gemma4e_Flash share this whole wrapper -- weights, /// tokenizer, sampler, chat template -- and differ only in the engine /// behind it. Which one to build is no longer decided here: the two @@ -306,9 +315,6 @@ class Gemma4e_DFlash : public Gemma4e { gemma4e_engine_config_t engine_config() const override; const flm_state_api* engine_state_api() const override; - /// \brief the speculative decode loop (sample-and-compare, D11); falls back to the plain loop step by step - std::string _speculative_generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled); - public: Gemma4e_DFlash(flm_rt::device* npu_device_inst) : Gemma4e(npu_device_inst) {} @@ -318,3 +324,24 @@ class Gemma4e_DFlash : public Gemma4e { void clear_context() override; std::string show_profile() override; }; + + +/************ Gemma4e_DFlash_Flash **************/ +/// Single-turn flash serving (Gemma4e_Flash: pinned system prefix, no tools, no continue_final_message) with DFlash +/// speculative decoding, backed by the gemma4e_dflash_flash engine: the gemma4e_flash prefill plus the DFlash draft/verify. +/// Weights are the nemotron3.5-bolt:e4b package, the draft comes from draft_model_path (see model_list base_model). +class Gemma4e_DFlash_Flash : public Gemma4e_Flash { +protected: + gemma4e_engine_config_t engine_config() const override; + /// \brief nullptr: the pinned system prefix (checkpoint/restore) owns the KV, the multi-slot prefix cache stays off + const flm_state_api* engine_state_api() const override; + +public: + Gemma4e_DFlash_Flash(flm_rt::device* npu_device_inst) : Gemma4e_Flash(npu_device_inst) {} + + void load_model(std::string model_path, json model_info, int default_context_length = -1, bool enable_preemption = false, const std::string& backend = "") override; + std::string generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }) override; + std::string generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os = std::cout) override; + void clear_context() override; + std::string show_profile() override; +}; diff --git a/src/include/models/gemma4e_dflash_flash/flm/aie2p/gemma4e_dflash_flash.hpp b/src/include/models/gemma4e_dflash_flash/flm/aie2p/gemma4e_dflash_flash.hpp new file mode 100644 index 000000000..72d74bb72 --- /dev/null +++ b/src/include/models/gemma4e_dflash_flash/flm/aie2p/gemma4e_dflash_flash.hpp @@ -0,0 +1,206 @@ +/// \file gemma4e_dflash_flash.hpp +/// \brief gemma4e_dflash_flash class +/// \author FastFlowLM Team +/// \date 2026-10-28 +/// \note Engine for Nemotron-3.5-Bolt-E4B with DFlash speculative decoding. +/// +/// gemma4e_dflash_flash starts as a byte-for-byte fork of gemma4e_npu (same Gemma4-E4B checkpoint). +/// The layer-type enum, the image/audio payload types and the GEMMA4E_* preprocessing constants are +/// shared with gemma4e_npu, so this header pulls them in rather than redefining them. +#pragma once +#include "models/gemma4e/flm/aie2p/gemma4e_npu.hpp" + +class gemma4e_dflash_flash : public causal_lm{ +public: + /// \brief initialize the qwen3vl_npu + /// \param config the configuration + /// \param npu_instance the npu instance + gemma4e_dflash_flash(LM_Config config, npu_xclbin_manager *npu_instance, int MAX_L = 4096); + ~gemma4e_dflash_flash(); + + /// \brief forward the qwen3vl_npu + /// \param ids the ids + /// \return the output tensor + buffer forward(int ids) override; + buffer prefill(std::vector& ids, void* payload = nullptr) override; + + /// \brief set the context length + /// \param L the context length + void set_context_length(int L) override; + + /// \brief load the weights + /// \param q4nx the q4nx + void load_weights(Q4NX& q4nx) override; + + /// \brief update the max length + void clear_context() override; + + /// \brief get the k cache + /// \param layer_idx the layer index + /// \param idx the index + /// \return the k cache + buffer get_k_cache(int layer_idx, int idx) override; + + /// \brief get the v cache + /// \param layer_idx the layer index + /// \param idx the index + /// \return the v cache + buffer get_v_cache(int layer_idx, int idx) override; + + /// \brief update the max length + /// \param MAX_L the max length + void update_max_length(uint32_t MAX_L) override; + + /// \brief get the current context length + /// \return the current context length + int get_current_context_length() override; + int checkpoint() override; + int restore() override; + + // ---- saved states (see flm_state_api.h) -------------------------------- + // A state holds the sliding-window ring of every sliding layer (valid at exactly the length it was read at: a + // ring cannot be rewound) and the K/V rows of the global layers. The layers that share another layer's cache + // hold no state. read_state() copies them out of the live buffers, write_state() puts a state back (and sets + // the context length), write_state_kv() puts back only the global rows (an ancestor's head, before a state + // taken further on is written). + std::unique_ptr new_state(); + int read_state(hybrid_engine_state& state, int kv_from); + int write_state(const hybrid_engine_state& state); + int write_state_kv(const hybrid_engine_state& state); + + // parameters for vision preprocessing in Gemma4e + unsigned int GEMMA4E_VISION_MAX_POSITION_EMBEDDINGS; + unsigned int GEMMA4E_VISION_NUM_HIDDEN_LAYERS; + unsigned int GEMMA4E_VISION_NUM_ATTENTION_HEADS; + unsigned int GEMMA4E_VISION_HIDDEN_SIZE; + unsigned int GEMMA4E_VISION_INTERMEDIATE_SIZE; + unsigned int GEMMA4E_VISION_HEAD_DIM; + unsigned int GEMMA4E_VISION_PATCH_SIZE; + float GEMMA4E_ROPE_THETA; + unsigned int GEMMA4E_POOLING_KERNEL_SIZE; + unsigned int GEMMA4E_POSITION_EMBEDDING_SIZE; + unsigned int GEMMA4E_VISION_IMAGE_OUTPUT_SIZE; + float GEMMA4E_VISION_RESCALE_FACTOR; + float GEMMA4E_VISION_IMAGE_MEAN; + float GEMMA4E_VISION_IMAGE_STD; + + // parameters for audio preprocessing in Gemma4e + unsigned int Audio_MM_TILE_M; + unsigned int Audio_MM_TILE_K; + unsigned int Audio_MM_TILE_N; + int Gemma4E_Audio_resample_rate; + float Gemma4E_Audio_gradient_clipping; + unsigned int Gemma4E_Audio_Multimodal_Output_SIZE; + unsigned int Gemma4E_Audio_language_projection_output_size; + unsigned int Gemma4E_Audio_HIDDEN_SIZE; + unsigned int Gemma4E_Audio_INTERMEDIATE_SIZE; + unsigned int Gemma4E_Audio_attention_chunk_size; + unsigned int Gemma4E_Audio_attention_context_left; + unsigned int Gemma4E_Audio_attention_context_right; + unsigned int Gemma4E_Audio_num_attention_heads; + unsigned int Gemma4E_Audio_num_attention_layers; + unsigned int Gemma4E_Audio_conv1d_kernel_size; + unsigned int Gemma4E_Audio_conv1d_stride; + unsigned int Gemma4E_Audio_conv2d_kernel_size; + unsigned int Gemma4E_Audio_conv2d_Stride; + unsigned int Gemma4e_Audio_conv2d_Padding; + unsigned int Gemma4E_Audio_subsampling_conv_channels_0; + unsigned int Gemma4E_Audio_subsampling_conv_channels_1; + float Gemma4E_Audio_attention_softcap; + + + inline void load_vision_preprocess_parameters(LM_Config& config){ + // Note: this should be called by Impl:: constructor + const nlohmann::json& vc = config.sub("vision_config"); + GEMMA4E_VISION_MAX_POSITION_EMBEDDINGS = vc.value("GEMMA4E_VISION_MAX_POSITION_EMBEDDINGS", -1); + GEMMA4E_VISION_NUM_HIDDEN_LAYERS = vc.value("GEMMA4E_VISION_NUM_HIDDEN_LAYERS", -1); + GEMMA4E_VISION_NUM_ATTENTION_HEADS = vc.value("GEMMA4E_VISION_NUM_ATTENTION_HEADS", -1); + GEMMA4E_VISION_HIDDEN_SIZE = vc.value("GEMMA4E_VISION_HIDDEN_SIZE", -1); + GEMMA4E_VISION_INTERMEDIATE_SIZE = vc.value("GEMMA4E_VISION_INTERMEDIATE_SIZE", -1); + GEMMA4E_VISION_HEAD_DIM = vc.value("GEMMA4E_VISION_HEAD_DIM", -1); + GEMMA4E_VISION_PATCH_SIZE = vc.value("GEMMA4E_VISION_PATCH_SIZE", -1); + GEMMA4E_ROPE_THETA = vc.value("GEMMA4E_ROPE_THETA", -1.0f); + GEMMA4E_POOLING_KERNEL_SIZE = vc.value("GEMMA4E_POOLING_KERNEL_SIZE", -1); + GEMMA4E_POSITION_EMBEDDING_SIZE = vc.value("GEMMA4E_POSITION_EMBEDDING_SIZE", -1); + GEMMA4E_VISION_IMAGE_OUTPUT_SIZE = vc.value("GEMMA4E_VISION_IMAGE_OUTPUT_SIZE", -1); + GEMMA4E_VISION_RESCALE_FACTOR = vc.value("GEMMA4E_VISION_RESCALE_FACTOR", -1.0f); + GEMMA4E_VISION_IMAGE_MEAN = vc.value("GEMMA4E_VISION_IMAGE_MEAN", -1.0f); + GEMMA4E_VISION_IMAGE_STD = vc.value("GEMMA4E_VISION_IMAGE_STD", -1.0f); + } + + inline void load_audio_preprocess_parameters(LM_Config& config){ + const nlohmann::json& ac = config.sub("audio_config"); + Audio_MM_TILE_M = ac.value("Audio_MM_TILE_M", 128); + Audio_MM_TILE_K = ac.value("Audio_MM_TILE_K", 512); + Audio_MM_TILE_N = ac.value("Audio_MM_TILE_N", 64); + Gemma4E_Audio_resample_rate = ac.value("Gemma4E_Audio_audio_resample_rate", -1); + Gemma4E_Audio_gradient_clipping = ac.value("Gemma4E_Audio_gradient_clipping", -1.0f); + Gemma4E_Audio_Multimodal_Output_SIZE = ac.value("Gemma4E_Audio_Multimodal_Output_SIZE", -1); + Gemma4E_Audio_language_projection_output_size = ac.value("Gemma4E_Audio_language_projection_output_size", -1); + Gemma4E_Audio_HIDDEN_SIZE = ac.value("Gemma4E_Audio_HIDDEN_SIZE", -1); + Gemma4E_Audio_INTERMEDIATE_SIZE = ac.value("Gemma4E_Audio_INTERMEDIATE_SIZE", -1); + Gemma4E_Audio_attention_chunk_size = ac.value("Gemma4E_Audio_attention_chunk_size", -1); + Gemma4E_Audio_attention_context_left = ac.value("Gemma4E_Audio_attention_context_left", -1); + Gemma4E_Audio_attention_context_right = ac.value("Gemma4E_Audio_attention_context_right", -1); + Gemma4E_Audio_num_attention_heads = ac.value("Gemma4E_Audio_num_attention_heads", -1); + Gemma4E_Audio_num_attention_layers = ac.value("Gemma4E_Audio_num_attention_layers", -1); + Gemma4E_Audio_conv1d_kernel_size = ac.value("Gemma4E_Audio_conv1d_kernel_size", -1); + Gemma4E_Audio_conv1d_stride = ac.value("Gemma4E_Audio_conv1d_stride", -1); + Gemma4E_Audio_conv2d_kernel_size = ac.value("Gemma4E_conv2d_kernel_size", -1); + Gemma4E_Audio_conv2d_Stride = ac.value("Gemma4E_conv2d_Stride", -1); + Gemma4e_Audio_conv2d_Padding = ac.value("Gemma4e_conv2d_Padding", -1); + Gemma4E_Audio_subsampling_conv_channels_0 = ac.value("Gemma4E_Audio_subsampling_conv_channels_0", -1); + Gemma4E_Audio_subsampling_conv_channels_1 = ac.value("Gemma4E_Audio_subsampling_conv_channels_1", -1); + Gemma4E_Audio_attention_softcap = ac.value("Gemma4E_Audio_attention_softcap", -1.0f); + } + + // existing causal_lm speculation hooks (greedy convenience path: draft + verify + argmax accept + commit) + bool supports_speculation() const override; + std::vector speculate(int last_token, int max_draft) override; + uint64_t last_speculation_prime_us() const override; + + // ---- DFlash speculative decoding ----------------------------------------------------------------- + // Everything below is NON-virtual on purpose (the vtable must stay identical to gemma4e_npu's, see causal_lm.hpp). + // The wrapper reaches these through dynamic_cast. + // + // One speculative cycle is split in three calls so the wrapper can run its own sampler on the verify rows + // (sample-and-compare, D11): dflash_draft -> dflash_verify -> (wrapper accepts k drafts) -> dflash_commit(k+1) + + /// \brief load the draft package (draft_dir/config.json + its draft_weight file); false = speculation stays off + bool load_draft(const std::string& draft_dir); + /// \brief true when the draft is loaded and FLM_DFLASH != 0 and no decline condition holds right now + bool draft_ready() const; + /// \brief token ids that end a generation (a verify batch is cut one past the first one inside the accepted prefix) + void set_stop_tokens(const std::vector& ids); + + /// \brief draft up to 7 tokens after `last_token` (the sampled token that has no KV row yet, at position = current context length) + /// \param drafts out: capacity >= 7 + /// \return number of drafts, 0 = decline (state untouched) + int dflash_draft(int last_token, int* drafts); + /// \brief run [last_token, drafts[0..n)] through the target at positions ctx..ctx+n; K/V rows are written, taps captured + /// \return pointer to (n+1) rows of `vocab_size()` bf16 logits (softcapped); row i = logits after row i's token. Valid until the next call + const bf16* dflash_verify(int last_token, const int* drafts, int n_drafts); + /// \brief keep the first n_keep verify rows (n_keep = accepted drafts + 1, >= 1): context length += n_keep, rejected rows are + /// invalidated (SWA ring slots restored), the draft context is advanced with the taps of the kept rows + void dflash_commit(int n_keep); + /// \brief vocabulary size of the logits rows returned by dflash_verify + int vocab_size() const; + /// \brief fp32-free access to the last verify rows for logprobs (same pointer rules as dflash_verify) + const bf16* last_speculation_logits(int& rows, int& vocab) const; + + // statistics (per session; reset_speculation_stats() is called by the wrapper's clear_context) + double mean_accepted_length() const; ///< tokens committed per cycle + double draft_hit_rate() const; ///< accepted drafts / drafted + int speculation_cycles() const; + std::string speculation_stats() const; ///< one line + std::string speculation_timing() const; ///< phase block (draft, verify, lm_head, accept, prime) + void reset_speculation_stats(); + +private: + struct Impl; + Impl* _impl; +}; + +/// \brief the saved-state table of this engine (flm_state_api.h); nullptr if `engine` is not a gemma4e_dflash_flash +extern "C" const flm_state_api* gemma4e_dflash_flash_state_api(void* engine); + diff --git a/src/inno/flm.iss b/src/inno/flm.iss index 6ca2d71bb..d8223029b 100644 --- a/src/inno/flm.iss +++ b/src/inno/flm.iss @@ -74,6 +74,7 @@ Source: "nanbeige_npu.dll"; DestDir: "{app}"; Flags: ignoreversion Source: "gemma4e_npu.dll"; DestDir: "{app}"; Flags: ignoreversion Source: "gemma4e_flash.dll"; DestDir: "{app}"; Flags: ignoreversion Source: "gemma4e_dflash.dll"; DestDir: "{app}"; Flags: ignoreversion +Source: "gemma4e_dflash_flash.dll"; DestDir: "{app}"; Flags: ignoreversion Source: "gemma4e_12b_npu.dll"; DestDir: "{app}"; Flags: ignoreversion Source: "hunyuan_npu.dll"; DestDir: "{app}"; Flags: ignoreversion Source: "lm_head.dll"; DestDir: "{app}"; Flags: ignoreversion diff --git a/src/lib/hrx/gemma4e_dflash_flash.dll b/src/lib/hrx/gemma4e_dflash_flash.dll new file mode 100644 index 000000000..6e5f41ca7 Binary files /dev/null and b/src/lib/hrx/gemma4e_dflash_flash.dll differ diff --git a/src/lib/hrx/gemma4e_dflash_flash.lib b/src/lib/hrx/gemma4e_dflash_flash.lib new file mode 100644 index 000000000..0325c5310 Binary files /dev/null and b/src/lib/hrx/gemma4e_dflash_flash.lib differ diff --git a/src/lib/hrx/libgemma4e_dflash_flash.so b/src/lib/hrx/libgemma4e_dflash_flash.so new file mode 100755 index 000000000..f9b119fcc Binary files /dev/null and b/src/lib/hrx/libgemma4e_dflash_flash.so differ diff --git a/src/lib/xrt/gemma4e_dflash_flash.dll b/src/lib/xrt/gemma4e_dflash_flash.dll new file mode 100644 index 000000000..f235abf4c Binary files /dev/null and b/src/lib/xrt/gemma4e_dflash_flash.dll differ diff --git a/src/lib/xrt/gemma4e_dflash_flash.lib b/src/lib/xrt/gemma4e_dflash_flash.lib new file mode 100644 index 000000000..20a3ad298 Binary files /dev/null and b/src/lib/xrt/gemma4e_dflash_flash.lib differ diff --git a/src/lib/xrt/libgemma4e_dflash_flash.so b/src/lib/xrt/libgemma4e_dflash_flash.so new file mode 100755 index 000000000..c4a25cbe5 Binary files /dev/null and b/src/lib/xrt/libgemma4e_dflash_flash.so differ diff --git a/src/model_info.json b/src/model_info.json index a698aa7aa..677520b39 100644 --- a/src/model_info.json +++ b/src/model_info.json @@ -4087,6 +4087,56 @@ "path": "dflash_weight.q4nx" } ], + "nemotron3.5-bolt-dflash-flash:e4b": [ + { + "type": "file", + "oid": "d117c32cf5bcf5c5f22d62d5e5c8249486afd5d9", + "size": 1574, + "path": ".gitattributes" + }, + { + "type": "file", + "oid": "ec297ac5456384786644013ec196da33b916be97", + "size": 2619, + "path": "LICENSE.OpenMDW-1.1" + }, + { + "type": "file", + "oid": "0c962bbc53cd5ecc1f7cf36cbb50af3efe1f0363", + "size": 325, + "path": "MANIFEST.sha256" + }, + { + "type": "file", + "oid": "2ea5e5f6b3b44533daebb1be130dfb4f3c2bff73", + "size": 3729, + "path": "README.md" + }, + { + "type": "file", + "oid": "4021fcef0c1e178beccd273b96a154ace237870c", + "size": 663, + "path": "config.json" + }, + { + "type": "file", + "oid": "19f27431df94a86e3d39d293878ab58970bc8876", + "size": 50, + "path": "configuration.json" + }, + { + "type": "file", + "oid": "c9861cfc8a360dd79c223aaf7c5177ce7ce1171a", + "size": 270406816, + "lfs": { + "oid": "e9548041bef37788cd3d2b38de5a6879d2d790c0b5ca821c4543cae86c04c165", + "size": 270406816, + "pointerSize": 134 + }, + "xetHash": "543a8e40205cb293da499715d7f07c6b6d6020578515a68cdc5cec6c7e386801", + "path": "dflash_weight.q4nx" + } + ], "nemotron3.5-bolt:e4b": [ { "type": "file", diff --git a/src/model_list.json b/src/model_list.json index f645763f8..4137e0d69 100644 --- a/src/model_list.json +++ b/src/model_list.json @@ -320,6 +320,43 @@ "footprint": 9.5 } }, + "nemotron3.5-bolt-dflash-flash": { + "e4b": { + "supported_platforms": ["aie2p"], + "name": "Nemotron-3.5-Bolt-E4B-DFlash-NPU2", + "base_model": "nemotron3.5-bolt:e4b", + "url": "https://huggingface.co/FastFlowLM/Nemotron-3.5-Bolt-E4B-DFlash-NPU2", + "file_url": "https://huggingface.co/api/models/FastFlowLM/Nemotron-3.5-Bolt-E4B-DFlash-NPU2/tree/main", + "ms_url": "https://modelscope.cn/models/amd/Nemotron-3.5-Bolt-E4B-DFlash-NPU2", + "size": 270410098, + "flm_min_version": "1.0.12", + "files": [ + "config.json", + "dflash_weight.q4nx", + "LICENSE.OpenMDW-1.1" + ], + "vlm": true, + "asr": true, + "default_context_length": 1024, + "max_prefill_len": 1024, + "details": { + "format": "NPU2", + "family": "gemma4e-dflash-flash", + "think": true, + "parameter_size": "8B+0.36B", + "quantization_level": "Q4_1" + }, + "label": [ + "audio", + "vision", + "reasoning", + "chat-transcription", + "single-turn", + "speculative" + ], + "footprint": 9.5 + } + }, "gemma4e-flash": { "e2b": { "supported_platforms": ["aie2p"], diff --git a/src/test/gemma4e_dflash_flash/CMakeLists.txt b/src/test/gemma4e_dflash_flash/CMakeLists.txt new file mode 100644 index 000000000..1f7e1be65 --- /dev/null +++ b/src/test/gemma4e_dflash_flash/CMakeLists.txt @@ -0,0 +1,70 @@ +cmake_minimum_required(VERSION 3.22) +project(gemma4e_dflash_flash VERSION 1.0.0 LANGUAGES CXX) + +include(${CMAKE_CURRENT_LIST_DIR}/../CMakeLists.txt) +npu_test_setup() + +# Find and enable OpenMP for multi-threading +find_package(OpenMP) +if(OpenMP_CXX_FOUND) + message(STATUS "OpenMP found - enabling multi-threading support for test") +else() + message(WARNING "OpenMP not found - some optimizations will run single-threaded") +endif() + +add_npu_test( + test_gemma4e_dflash_flash + test/gemma4e_dflash_flash + USE_AUTOMODEL + USE_TOKENIZER + USE_SAMPLER + USE_AVX512 + SOURCES + "${CMAKE_SOURCE_DIR}/../../common/AutoModel/modeling_gemma4e.cpp" + "${CMAKE_SOURCE_DIR}/../../common/AutoModel/modeling_gemma4e_dflash.cpp" + "${CMAKE_SOURCE_DIR}/../../common/AutoModel/modeling_gemma4e_image.cpp" + "${CMAKE_SOURCE_DIR}/../../common/AutoModel/modeling_gemma4e_audio.cpp" + "${CMAKE_SOURCE_DIR}/../../common/image_process_utils/imageproc.cpp" + "${CMAKE_SOURCE_DIR}/../../common/image_process_utils/imageprocAVX512.cpp" + "${CMAKE_SOURCE_DIR}/../../common/image/image_reader.cpp" + "${CMAKE_SOURCE_DIR}/../../common/audio/audio_reader.cpp" + "${CMAKE_SOURCE_DIR}/../../common/audio_process_utils/audioproc.cpp" + "${CMAKE_SOURCE_DIR}/../../common/audio_process_utils/audioprocAVX512.cpp" +) + +target_compile_definitions(test_gemma4e_dflash_flash PUBLIC + USEAVX512=1 + __FLM_VERSION__="0.9.39" +) + +# Add OpenMP compiler flags if available +if(OpenMP_CXX_FOUND) + target_compile_options(test_gemma4e_dflash_flash PUBLIC + $<$:/openmp> + $<$>:-fopenmp> + ) +endif() +target_link_directories(test_gemma4e_dflash_flash PUBLIC + ${CMAKE_SOURCE_DIR}/../../external_libs +) + +target_link_libraries(test_gemma4e_dflash_flash PUBLIC + gemma4e_dflash_flash + # modeling_gemma4e.cpp serves every Gemma4e wrapper, so all three engines must be linked. + gemma4e_npu + gemma4e_flash + xrt_coreutil + zlib +) + +# Link OpenMP if available +if(OpenMP_CXX_FOUND) + target_link_libraries(test_gemma4e_dflash_flash PUBLIC OpenMP::OpenMP_CXX) +endif() + +# Add test target +add_custom_target(test_gemma4e_dflash_flash_target + DEPENDS test_gemma4e_dflash_flash + COMMENT "Building test_gemma4e_dflash_flash executable" +) + diff --git a/src/test/gemma4e_dflash_flash/Makefile b/src/test/gemma4e_dflash_flash/Makefile new file mode 100644 index 000000000..cdafb653b --- /dev/null +++ b/src/test/gemma4e_dflash_flash/Makefile @@ -0,0 +1,113 @@ +# ============================================================================= +# Qwen NPU Library Makefile +# ============================================================================= +# +# Author: Zhenyu Xu (zxu3@clemson.edu) +# Created: 2024 +# +# This Makefile is used to build the Qwen NPU library components. +# +# Usage: +# make - Build all targets +# make clean - Remove all built files +# make help - Show this help message +# +# ============================================================================= +-include ../common.mk + + +SOURCES += test.cpp +SOURCES += ../../common/AutoModel/automodel.cpp +SOURCES += $(CHAT_SOURCES) +SOURCES += ../../common/AutoModel/modeling_gemma4e.cpp +SOURCES += ../../common/AutoModel/modeling_gemma4e_dflash.cpp +SOURCES += ../../common/AutoModel/modeling_gemma4e_image.cpp +SOURCES += ../../common/AutoModel/modeling_gemma4e_audio.cpp +SOURCES += ../../common/image_process_utils/imageproc.cpp +SOURCES += ../../common/image_process_utils/imageprocAVX512.cpp +SOURCES += ../../common/audio_process_utils/audioproc.cpp +SOURCES += ../../common/audio_process_utils/audioprocAVX512.cpp +SOURCES += ../../common/image/image_reader.cpp +SOURCES += ../../include/lodepng/lodepng.cpp +SOURCES += ../../common/audio/audio_reader.cpp +SOURCES += ../../common/audio/audio_decoder.cpp +SOURCES += ../../common/audio/audio_resampler.cpp +SOURCES += ../../common/tokenizer/tokenizer.cpp +SOURCES += ../../common/modules/sampler.cpp + +HEADERS += ../../include/models/gemma4e_dflash_flash/flm/aie2p/gemma4e_dflash_flash.hpp +HEADERS += ../../include/models/gemma4e_dflash/flm/aie2p/gemma4e_dflash.hpp +HEADERS += ../../include/models/gemma4e_flash/flm/aie2p/gemma4e_flash.hpp +HEADERS += ../../include/models/gemma4e/flm/aie2p/gemma4e_npu.hpp + +# Test task selection; text mode is default +# 0: text +# 1: images +# 2: audios +# 3: omni (image + audio) +# 4: long prompt from prompt.txt (prefill only) +TYPE ?= 0 + + + +ifeq ($(WSL), 0) + +LDFLAGS += -lgemma4e_dflash_flash +LDFLAGS += -lgemma4e_dflash +LDFLAGS += -lgemma4e_flash +# modeling_gemma4e.cpp serves both wrappers, so both engines must be linked. +LDFLAGS += -lgemma4e_npu +LDFLAGS += -ljpeg -lvorbisfile -lvorbis -logg -lopus + +CPP_SOURCES := $(filter %.cpp,$(SOURCES)) +OBJECTS := $(addprefix $(BUILD_DIR)/,$(notdir $(CPP_SOURCES:.cpp=.o))) +DEPS := $(OBJECTS:.o=.d) +VPATH := $(sort $(dir $(CPP_SOURCES))) + +all: $(BUILD_DIR)/test_gemma4e_dflash_flash + +$(BUILD_DIR): + mkdir -p $(BUILD_DIR) + +$(BUILD_DIR)/%.o: %.cpp | $(BUILD_DIR) + $(CXX) $(CXX_FLAGS) -c $< -o $@ + +$(BUILD_DIR)/test_gemma4e_dflash_flash: $(OBJECTS) + $(CXX) $(CXX_FLAGS) -o $@ $^ $(LDFLAGS) $(DEPENDENCY_LDFLAGS) + +clean: + rm -rf $(BUILD_DIR) + +test: $(BUILD_DIR)/test_gemma4e_dflash_flash + cp ../../model_list.json $(BUILD_DIR)/model_list.json + cd $(BUILD_DIR) && export LD_LIBRARY_PATH=../../../lib && ./test_gemma4e_dflash_flash --model nemotron3.5-bolt-dflash-flash:e4b --type $(TYPE) + +-include $(DEPS) +.PHONY: all clean test + +else + +# WSL build environment +# Use CMake to invoke the Visual Studio +PWSH := powershell.exe + +all: directories test + +directories: + mkdir -p $(BUILD_DIR) + +$(BUILD_DIR)/test_gemma4e_dflash_flash.exe: $(SOURCES) + cd $(BUILD_DIR) && $(PWSH) -Command "cmake ../../../test/gemma4e_dflash_flash" + cd $(BUILD_DIR) && $(PWSH) -Command "cmake --build . --config Release --target test_gemma4e_dflash_flash" + +clean: + rm -rf $(BUILD_DIR) + + +test: directories $(BUILD_DIR)/test_gemma4e_dflash_flash.exe + cp ../../model_list.json $(BUILD_DIR)/model_list.json + cd $(BUILD_DIR) && ${PWSH} -Command "\$$env:PATH = '..\..\..\lib;' + \$$env:PATH; .\test_gemma4e_dflash_flash.exe -m nemotron3.5-bolt-dflash-flash:e4b -t $(TYPE) -p 0" + +.PHONY: all clean test directories + +endif diff --git a/src/test/gemma4e_dflash_flash/test.cpp b/src/test/gemma4e_dflash_flash/test.cpp new file mode 100644 index 000000000..63d99eb02 --- /dev/null +++ b/src/test/gemma4e_dflash_flash/test.cpp @@ -0,0 +1,131 @@ +#include +#include +#include +#include "utils/utils.hpp" +#include "utils/vm_args.hpp" +#include "AutoModel/modeling_gemma4e.hpp" +#include "model_list.hpp" + +flm_rt::device npu_device_global; + +int main(int argc, char* argv[]) { + #ifdef __WINDOWS__ + SetConsoleOutputCP(CP_UTF8); + SetConsoleCP(CP_UTF8); + // Set thread priority to low + SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_LOWEST); + #endif + + arg_utils::po::options_description desc("Allowed options"); + arg_utils::po::variables_map vm; + desc.add_options()("model,m", arg_utils::po::value()->required(), "Model file"); + desc.add_options()("type,t", arg_utils::po::value()->default_value(0), "\t0: text mode\n\t1: image only\n\t2: audio only\n\t3: omni mode\n\t4: long prompt from prompt.txt\n\t"); + desc.add_options()("Preemption,p", arg_utils::po::value()->default_value(false), "Preemption"); + desc.add_options()("Greedy,g", arg_utils::po::value()->default_value(true), "Greedy decoding"); + desc.add_options()("Length,l", arg_utils::po::value()->default_value(512), "Max generation length"); + desc.add_options()("Speculate,s", arg_utils::po::value()->default_value(1), "1: DFlash speculative decoding, 0: plain decoding (sets FLM_DFLASH=0)"); + desc.add_options()("Think", arg_utils::po::value()->default_value(false), "Enable thinking"); + arg_utils::po::store(arg_utils::po::parse_command_line(argc, argv, desc), vm); + + std::string tag = vm["model"].as(); + int type = vm["type"].as(); + bool preemption = vm["Preemption"].as(); + bool greedy = vm["Greedy"].as(); + int length_limit = vm["Length"].as(); + const int speculate = vm["Speculate"].as(); + const bool think = vm["Think"].as(); + if (speculate == 0) { + // Read by the wrapper and the engine at load time: the tag then decodes exactly like nemotron3.5-bolt:e4b. +#ifdef __WINDOWS__ + _putenv_s("FLM_DFLASH", "0"); +#else + setenv("FLM_DFLASH", "0", 1); +#endif + } + std::cout << "Model: " << tag << std::endl; + + std::string exe_dir = utils::get_executable_directory(); + std::string model_dir = utils::get_models_directory(); + std::string model_list_path = exe_dir + "/model_list.json"; + model_list model_list(model_list_path, model_dir); + + header_print("info", "Initializing chat model..."); + // a DFlash tag loads from its base package (main weights, tokenizer, towers); model_info carries the draft package path + std::string model_path = model_list.get_load_path(tag); + std::pair model_info_pair = model_list.get_model_info(tag); + nlohmann::json model_info = model_info_pair.second; + std::cout << "Model path: " << model_path << std::endl; + + std::unique_ptr chat = std::make_unique(&npu_device_global); + std::cout << "Chat model initialized" << std::endl; + npu_device_global = flm_rt::device(0); + std::cout << "NPU Device initialized: " << npu_device_global.get_info() << std::endl; + chat->load_model(model_path, model_info, -1, preemption); + header_print("info", "Model loaded"); + + chat_meta_info_t meta_info; + lm_uniform_input_t uniformed_input; + if (greedy) { + chat->set_topk(1); // deterministic (greedy) for a smoke test + } + chat->configure_parameter("enable_think", think); + + switch (type) { + case 0: + // one clear answer, and long enough to run many speculative cycles + uniformed_input.prompt = "List the integers from 1 to 40 separated by commas, then write the word done."; + break; + case 1: + uniformed_input.prompt = "Describe the image briefly in 16 tokens"; + uniformed_input.images.push_back("../../../tb_files/amd_256s.png"); + length_limit = 16; + break; + case 2: + uniformed_input.prompt = "Transcribe the following speech segment in its original language. Follow these specific instructions for formatting the answer:\n* Only output the transcription, with no newlines.\n* When transcribing numbers, write the digits, i.e. write 1.7 and not one point seven, and write 3 instead of three."; + uniformed_input.audios.push_back("../../../tb_files/nvidia.mp3"); + break; + case 3: + uniformed_input.prompt = "Answer the question and further describe what is in the image."; + uniformed_input.images.push_back("../../../tb_files/Zootopia.jpg"); + uniformed_input.audios.push_back("../../../tb_files/Recording.wav"); + break; + case 4: { + // Long prompt: read the text from prompt.txt; prefill only. + std::ifstream file("../../../../prompt.txt", std::ios::binary); + if (!file.is_open()) { + std::cout << "Failed to open prompt file" << std::endl; + return 1; + } + file.seekg(0, std::ios::end); + uniformed_input.prompt.resize(file.tellg()); + file.seekg(0, std::ios::beg); + file.read(uniformed_input.prompt.data(), uniformed_input.prompt.size()); + file.close(); + break; + } + default: + header_print("info", "Unknown test type, exit 0;"); + exit(0); + } + + std::cout << "Prompt: " << uniformed_input.prompt << std::endl; + std::cout << "Response: " << std::endl; + + chat->start_total_timer(); + if (type == 4) { + // Prefill only -- this path exercises long-context prefill. + chat->insert(meta_info, uniformed_input); + } else { + std::string response = chat->generate_with_prompt(meta_info, uniformed_input, length_limit, std::cout); + } + chat->stop_total_timer(); + std::cout << std::endl; + std::cout << std::endl; + std::cout << chat->show_profile() << std::endl; + + std::pair> history = chat->get_history(); + std::cout << "History length: " << history.second.size() << std::endl; + std::cout << std::endl; + + return 0; +} diff --git a/src/test/phi4_core/test_model_downloader.cpp b/src/test/phi4_core/test_model_downloader.cpp index 4b4a7f6cb..9ffddf10e 100644 --- a/src/test/phi4_core/test_model_downloader.cpp +++ b/src/test/phi4_core/test_model_downloader.cpp @@ -2,6 +2,7 @@ #include "model_downloader.hpp" #include "test_support.hpp" +#include #include #include #include @@ -785,9 +786,23 @@ void TestShippedCatalogDeclaresTheDflashDependant() { const auto base = models.get_model_info("nemotron3.5-bolt:e4b").second; TEST_REQUIRE(base.at("name") == "Nemotron-3.5-Bolt-E4B-NPU2"); TEST_REQUIRE(!base.contains("base_model") && !base.contains("draft_model_path")); + // the flash variant shares the base and the draft package with the plain DFlash tag + const char* dflash_flash = "nemotron3.5-bolt-dflash-flash:e4b"; + TEST_REQUIRE(models.is_model_supported(dflash_flash)); + TEST_REQUIRE(models.get_base_model(dflash_flash) == std::string("nemotron3.5-bolt:e4b")); + const auto flash = models.get_model_info(dflash_flash).second; + TEST_REQUIRE(flash.at("name") == "Nemotron-3.5-Bolt-E4B-DFlash-NPU2"); + TEST_REQUIRE(flash.at("details").at("family") == "gemma4e-dflash-flash"); + TEST_REQUIRE(flash.at("xclbin_folder") == "Nemotron-3.5-Bolt-E4B-DFlash-NPU2"); + TEST_REQUIRE(flash.at("default_context_length") == 1024); + TEST_REQUIRE(flash.at("max_prefill_len") == 1024); + const auto labels = flash.at("label").get>(); + TEST_REQUIRE(std::find(labels.begin(), labels.end(), "single-turn") != labels.end()); + TEST_REQUIRE(std::find(labels.begin(), labels.end(), "tool-calling") == labels.end()); // a platform without the base prunes the dependant with it model_list dev04(path, exe_dir, "dev04", {"flm"}); TEST_REQUIRE(!dev04.is_model_supported(dflash)); + TEST_REQUIRE(!dev04.is_model_supported(dflash_flash)); } download_utils::DownloadRequest Request(const fs::path& source, const fs::path& destination, diff --git a/src/wix/flm.wxs b/src/wix/flm.wxs index b3382e11b..f7af3b2c7 100644 --- a/src/wix/flm.wxs +++ b/src/wix/flm.wxs @@ -197,6 +197,9 @@ + + + diff --git a/src/xclbins/Nemotron-3.5-Bolt-E4B-DFlash-NPU2/fused_prefill.xclbin b/src/xclbins/Nemotron-3.5-Bolt-E4B-DFlash-NPU2/fused_prefill.xclbin new file mode 100644 index 000000000..c886e37cd Binary files /dev/null and b/src/xclbins/Nemotron-3.5-Bolt-E4B-DFlash-NPU2/fused_prefill.xclbin differ