feat: 支持 Gemma 4 与原生多模态推理

README.md

@@ -71,12 +71,16 @@ Read the method overview and integration rules in
 | Qwen3.5 / Qwen3.6 | ✅ |
 | Qwen3.5 / Qwen3.6 MoE | ✅ |
 | GLM-4.7-Flash | ✅ |
+| Gemma 4 Dense / MoE | ✅ |
 | Llama 3 / 3.1 | ✅ |
 | MiniMax M2.7 | ✅ |
 
 See [Supported Models](docs/en/features/supported-models.md) for the precision,
 parallelism, and sparse-method compatibility matrices.
 
+Native image, video, and audio inputs are enabled per checkpoint with
+`enable_multimodal=True`; see the supported-model matrix for media coverage.
+
 ## Documentation
 
 | Topic | Link |

docs/en/features/supported-models.md

@@ -18,6 +18,7 @@ parallel size must use that value.
 | Qwen3.5 / Qwen3.6 | `qwen3_5` | BF16 / block FP8 | ✅ | 1 only | 1 only |
 | Qwen3.6 MoE | `qwen3_5_moe` | BF16 / block FP8 | ✅ | 1 only | ✅ |
 | GLM-4.7-Flash | `glm4_moe_lite` | BF16 | 1 / 2 / 4 (H100 only)⁵ | 1 only | 1 / 2 / 4⁵ |
+| Gemma 4 Dense / MoE | `gemma4` | BF16 / FP16 | ✅ | 1 only | ✅ (MoE only) |
 | Llama 3 / 3.1 | `llama` | BF16 / FP16 | ✅ | 1 only | 1 only |
 | MiniMax M2.7 | `minimax_m2` | block FP8 with BF16 non-quantized weights | ✅ | 1 only | ✅ |
 
@@ -59,6 +60,7 @@ layer.
 | Qwen3.5 / Qwen3.6 | ✅ | ✅ | ✅ | Experimental⁴ | ✅ | ✅ | ✅ | ✅ | — | Matched checkpoint³ |
 | Qwen3.6 MoE | ✅ | ✅ | ✅ | Experimental⁴ | ✅ | ✅ | ✅ | ✅ | — | — |
 | GLM-4.7-Flash | ✅⁵ | ✅⁵ | ✅⁵ | Experimental⁴⁵ | — | ✅⁵ | — | ✅⁵ | — | — |
+| Gemma 4 Dense / MoE | ✅ | ✅⁶ | — | — | — | ✅ | — | — | — | — |
 | Llama 3 / 3.1 | ✅ | ✅ | ✅ | Experimental⁴ | ✅ | ✅ | ✅ | ✅ | Selected checkpoint¹ | Compressor required² |
 | MiniMax M2.7 | ✅ | ✅ | ✅ | Experimental⁴ | ✅ | ✅ | ✅ | ✅ | — | — |
 
@@ -81,4 +83,19 @@ global-head selection. Model-specific TP, EP, and DP restrictions still apply.
 cross-rank sparse-index aggregation, so their selection semantics are not
 guaranteed to match `TP=1`.
 
+⁶ Gemma 4 checkpoints with shared KV layers reject per-layer StreamingLLM
+eviction. Vanilla and OmniKV remain supported.
+
+## Native Multimodal Support
+
+Set `enable_multimodal=True` to use a checkpoint's native media towers.
+Sparse-vLLM accepts OpenAI-compatible Chat and Responses content parts and
+uses the checkpoint processor and chat template. Unsupported media fail
+explicitly during admission.
+
+| Model family | Image | Video | Audio |
+| --- | :---: | :---: | :---: |
+| Qwen3.5 / Qwen3.6 Dense and MoE | ✅ | ✅ | — |
+| Gemma 4 Dense and MoE | ✅ | ✅ | — |
+
 `—` means that the combination is not currently supported.

docs/zh/features/supported-models.md

@@ -14,6 +14,7 @@
 | Qwen3.5 / Qwen3.6 | `qwen3_5` | BF16 / 块级 FP8 | ✅ | 仅支持 1 | 仅支持 1 |
 | Qwen3.6 MoE | `qwen3_5_moe` | BF16 / 块级 FP8 | ✅ | 仅支持 1 | ✅ |
 | GLM-4.7-Flash | `glm4_moe_lite` | BF16 | 1 / 2 / 4（仅 H100）⁵ | 仅支持 1 | 1 / 2 / 4⁵ |
+| Gemma 4 Dense / MoE | `gemma4` | BF16 / FP16 | ✅ | 仅支持 1 | ✅（仅 MoE） |
 | Llama 3 / 3.1 | `llama` | BF16 / FP16 | ✅ | 仅支持 1 | 仅支持 1 |
 | MiniMax M2.7 | `minimax_m2` | 块级 FP8，非量化权重使用 BF16 | ✅ | 仅支持 1 | ✅ |
 
@@ -50,6 +51,7 @@ vanilla 和 OmniKV 使用 radix 模式，对 StreamingLLM、SnapKV、H2O 和 R-K
 | Qwen3.5 / Qwen3.6 | ✅ | ✅ | ✅ | 实验性⁴ | ✅ | ✅ | ✅ | ✅ | — | 匹配的 checkpoint³ |
 | Qwen3.6 MoE | ✅ | ✅ | ✅ | 实验性⁴ | ✅ | ✅ | ✅ | ✅ | — | — |
 | GLM-4.7-Flash | ✅⁵ | ✅⁵ | ✅⁵ | 实验性⁴⁵ | — | ✅⁵ | — | ✅⁵ | — | — |
+| Gemma 4 Dense / MoE | ✅ | ✅⁶ | — | — | — | ✅ | — | — | — | — |
 | Llama 3 / 3.1 | ✅ | ✅ | ✅ | 实验性⁴ | ✅ | ✅ | ✅ | ✅ | 指定 checkpoint¹ | 需要 compressor² |
 | MiniMax M2.7 | ✅ | ✅ | ✅ | 实验性⁴ | ✅ | ✅ | ✅ | ✅ | — | — |
 
@@ -70,4 +72,19 @@ TP、EP、DP 限制仍然适用。
 `TP>1` 时，基于 head 评分的稀疏方法使用 TP-local selection，不跨 rank
 聚合 sparse index，因此其选择语义不保证与 `TP=1` 相同。
 
+⁶ 带共享 KV 层的 Gemma 4 checkpoint 不支持逐层 StreamingLLM eviction；
+Vanilla 和 OmniKV 仍受支持。
+
+## 原生多模态支持
+
+设置 `enable_multimodal=True` 后，可使用 checkpoint 自带的媒体塔。
+Sparse-vLLM 接受 OpenAI 兼容的 Chat 与 Responses content part，并使用
+checkpoint 自身的 processor 和 chat template；不受支持的媒体会在接纳阶段
+明确报错。
+
+| 模型家族 | 图片 | 视频 | 音频 |
+| --- | :---: | :---: | :---: |
+| Qwen3.5 / Qwen3.6 Dense 与 MoE | ✅ | ✅ | — |
+| Gemma 4 Dense 与 MoE | ✅ | ✅ | — |
+
 `—` 表示当前不支持该组合。

pyproject.toml

@@ -21,6 +21,7 @@ dependencies = [
     "apache-tvm-ffi==0.1.10",
     "nvidia-cutlass-dsl>=4.6,<5",
     "pillow",
+    "torchvision",
     "einops",
     "sglang-kernel>=0.4.5,<0.4.6",
     "tqdm",

src/sparsevllm/__init__.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-__all__ = ["LLM", "SamplingParams"]
+__all__ = ["LLM", "MultiModalPrompt", "SamplingParams"]
 
 
 def __getattr__(name: str):
@@ -12,4 +12,8 @@ def __getattr__(name: str):
         from sparsevllm.sampling_params import SamplingParams
 
         return SamplingParams
+    if name == "MultiModalPrompt":
+        from sparsevllm.multimodal import MultiModalPrompt
+
+        return MultiModalPrompt
     raise AttributeError(f"module 'sparsevllm' has no attribute {name!r}")

src/sparsevllm/configs/model.py

@@ -97,6 +97,7 @@ def load_and_validate_model(config) -> None:
         config.tensor_parallel_size,
         config.expert_parallel_size,
         config.data_parallel_size,
+        config.hf_config,
     )
     if config.tiny_random:
         from sparsevllm.debug.tiny_random import apply_tiny_random_overrides

src/sparsevllm/configs/runtime.py

@@ -72,6 +72,7 @@ class Config(
     # least one synchronous loading path when the budget is smaller.
     weight_loading_workers: int = 1
     enforce_eager: bool = True
+    enable_multimodal: bool = True
     hf_config: AutoConfig | None = None
     outer_hf_config: Any | None = None
     runtime_layout: RuntimeLayout | None = None

src/sparsevllm/configs/sparse.py

@@ -12,6 +12,7 @@
 )
 from sparsevllm.utils.log import logger, log_once
 
+
 def normalize_sparse_method_name(config) -> bool:
     raw_sparse_method = config.vllm_sparse_method
     raw_sparse_method_normalized = "" if raw_sparse_method is None else str(raw_sparse_method).strip().lower()
@@ -175,6 +176,15 @@ def _normalize_skipkv(config) -> None:
         )
 
 def normalize_sparse_methods(config) -> None:
+    if (
+        getattr(config.hf_config, "model_type", "") == "gemma4_text"
+        and int(getattr(config.hf_config, "num_kv_shared_layers", 0) or 0)
+        and config.vllm_sparse_method == "streamingllm"
+    ):
+        raise NotImplementedError(
+            "Gemma 4 StreamingLLM requires independent per-layer KV caches; "
+            "KV-sharing variants support vanilla and OmniKV."
+        )
     _normalize_quest(config)
     _normalize_h2o(config)
     _normalize_rkv(config)

src/sparsevllm/engine/cache_manager/base.py

@@ -244,8 +244,18 @@ def __init__(self, config: Config, parallel_context: ParallelContext):
             raise ValueError("CacheManager requires config.runtime_layout.")
         self.num_kv_layers = int(self.runtime_layout.num_kv_layers)
 
-        self.num_kv_heads = self.hf_config.num_key_value_heads // self.tp_size
-        self.head_dim = resolve_attention_qk_head_dim(self.hf_config)
+        layout_heads = tuple(getattr(self.runtime_layout, "kv_num_heads", ()))
+        layout_dims = tuple(getattr(self.runtime_layout, "kv_head_dims", ()))
+        self.num_kv_heads = (
+            int(layout_heads[0]) // self.tp_size
+            if layout_heads
+            else int(self.hf_config.num_key_value_heads) // self.tp_size
+        )
+        self.head_dim = (
+            int(layout_dims[0])
+            if layout_dims
+            else resolve_attention_qk_head_dim(self.hf_config)
+        )
 
         self.max_model_len = config.max_model_len
         resident_buffer_rows = int(config.max_num_seqs_in_gpu)

src/sparsevllm/engine/cache_manager/prefix_cache_mixin.py

@@ -296,6 +296,8 @@ def _record_prefix_materialization(
         token_ids: list[int],
         slots: torch.Tensor,
     ) -> None:
+        if getattr(seq, "multimodal_digest", None) is not None:
+            return
         if not self.enable_prefix_caching or self.prefix_cache is None:
             return
         if len(token_ids) != int(slots.numel()):

src/sparsevllm/engine/cache_manager/standard.py

@@ -38,6 +38,7 @@
 )
 from .storage import (
     ExplicitKVStorage,
+    HeterogeneousExplicitKVStorage,
     create_attention_cache_storage,
 )
 
@@ -139,7 +140,7 @@ def _init_prefix_offload(self) -> None:
         host_size_gb = getattr(self.config, "prefix_cache_host_size_gb", None)
         if host_size_gb is None:
             raise RuntimeError("Prefix cache offload requires prefix_cache_host_size_gb.")
-        storage = self._require_explicit_storage("Prefix cache offload")
+        storage = self._require_uniform_explicit_storage("Prefix cache offload")
         kv_cache = storage.cache
         bytes_per_block = int(
             self.prefix_cache_block_size
@@ -186,7 +187,12 @@ def allocate_kv_cache(self):
         available_memory, slot_bytes_per_layer = self._get_available_slots_info()
         num_layers = self.num_kv_layers
 
-        slot_bytes = num_layers * slot_bytes_per_layer
+        storage = self.attention_cache_storage
+        slot_bytes = (
+            storage.bytes_per_slot()
+            if isinstance(storage, HeterogeneousExplicitKVStorage)
+            else num_layers * slot_bytes_per_layer
+        )
         self.config.num_kvcache_slots = available_memory // slot_bytes
         assert self.config.num_kvcache_slots > 0, "可用显存不足以分配 KV Cache"
 
@@ -198,27 +204,39 @@ def allocate_kv_cache(self):
             num_slots=self.config.num_kvcache_slots,
             device=self.device,
         )
-        self.kv_cache = (
-            self.attention_cache_storage.kv_cache
-            if isinstance(self.attention_cache_storage, ExplicitKVStorage)
-            else None
-        )
+        self.kv_cache = getattr(self.attention_cache_storage, "kv_cache", None)
 
     def attention_cache_bytes_per_slot_per_layer(self) -> int:
         storage = getattr(self, "attention_cache_storage", None)
         if storage is None:
             return super().attention_cache_bytes_per_slot_per_layer()
         return int(storage.bytes_per_slot_per_layer())
 
-    def _require_explicit_storage(self, operation: str) -> ExplicitKVStorage:
+    def _logical_live_kv_bytes(self) -> int:
+        storage = getattr(self, "attention_cache_storage", None)
+        if not isinstance(storage, HeterogeneousExplicitKVStorage):
+            return super()._logical_live_kv_bytes()
+        return int(self.row_seq_lens.sum()) * storage.bytes_per_slot()
+
+    def _require_explicit_storage(
+        self, operation: str
+    ) -> ExplicitKVStorage | HeterogeneousExplicitKVStorage:
         storage = self.attention_cache_storage
-        if not isinstance(storage, ExplicitKVStorage):
+        if not isinstance(storage, (ExplicitKVStorage, HeterogeneousExplicitKVStorage)):
             raise TypeError(
                 f"{operation} requires ExplicitKVStorage, got "
                 f"{type(storage).__name__}."
             )
         return storage
 
+    def _require_uniform_explicit_storage(self, operation: str) -> ExplicitKVStorage:
+        storage = self.attention_cache_storage
+        if not isinstance(storage, ExplicitKVStorage):
+            raise NotImplementedError(
+                f"{operation} does not support heterogeneous per-layer KV shapes."
+            )
+        return storage
+
     def get_layer_batch_states(self, layer_idx: int) -> LayerBatchStates:
         return self.layer_batch_state
 
@@ -771,6 +789,9 @@ def prefix_kv_payload_nbytes(self, payload: object) -> int:
             raise RuntimeError("Standard mixed prefix KV payload is missing token slots.")
         if not isinstance(payload.token_slots, torch.Tensor):
             raise RuntimeError("Standard mixed prefix KV payload has no device slots.")
+        storage = self.attention_cache_storage
+        if isinstance(storage, HeterogeneousExplicitKVStorage):
+            return int(payload.token_slots.numel()) * storage.bytes_per_slot()
         dtype_size = self._cache_slot_dtype_size()
         return int(
             payload.token_slots.numel()

src/sparsevllm/engine/cache_manager/storage/__init__.py

@@ -4,6 +4,7 @@
 
 from .base import AttentionCacheStorage, CacheLayout
 from .explicit_kv import ExplicitKVStorage
+from .heterogeneous_explicit_kv import HeterogeneousExplicitKVStorage
 
 if TYPE_CHECKING:
     from .mla_latent import MlaLatentStorage
@@ -23,6 +24,18 @@ def create_attention_cache_storage(
     )
     dtype = config.hf_config.torch_dtype
     if layout is CacheLayout.EXPLICIT_KV:
+        runtime_layout = getattr(config, "runtime_layout", None)
+        parallel_topology = getattr(config, "parallel_topology", None)
+        layer_shapes = (
+            runtime_layout.local_kv_shapes(parallel_topology.attention_tp_size)
+            if runtime_layout is not None and parallel_topology is not None
+            else ()
+        )
+        if len(set(layer_shapes)) > 1:
+            return HeterogeneousExplicitKVStorage(
+                layer_shapes=layer_shapes,
+                dtype=dtype,
+            )
         return ExplicitKVStorage(
             num_kv_heads=num_kv_heads,
             head_dim=head_dim,
@@ -51,6 +64,7 @@ def __getattr__(name: str):
     "AttentionCacheStorage",
     "CacheLayout",
     "ExplicitKVStorage",
+    "HeterogeneousExplicitKVStorage",
     "MlaLatentStorage",
     "create_attention_cache_storage",
 ]