From 926af3723cc492ae844e38b74585816d9d2fac11 Mon Sep 17 00:00:00 2001 From: Zohaib Moti Date: Thu, 30 Jul 2026 12:45:37 -0600 Subject: [PATCH] Fix get_head_size() when config materializes head_dim as 0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cherry-pick of the get_head_size() portion of upstream commit ceae5bcbda ("[ROCm][CI] Fix nixl tests", vllm-project/vllm#45219). DeepseekVLV2TextConfig materializes a missing head_dim as 0 rather than leaving it unset. The existing guard only rejected None, so 0 was passed through as the architecture head size while the true head size is hidden_size / num_attention_heads = 1280 / 10 = 128. The resulting zero-sized KV cache made the first inference request abort the engine with hipErrorLaunchFailure ("CUDA error: unspecified launch failure"). This is a latent bug rather than a recent stack regression: the guard has only ever tested for None, and the DeepSeek-VL2-tiny configs happened to survive earlier because different code consumed the head size. Verified on gfx1151 with the nightly's own benchmark arguments, before vs after, identical args: DeepSeek-VL2-tiny_4096 before: exit 1, 0 successful requests, 5x "unspecified launch failure" after: exit 0, 2 successful requests, prefill 22084 tok/s, decode 138 tok/s DeepSeek-VL2-tiny_VLM before: engine dead, 0 tok/s after: prefill 5167 tok/s, decode 154 tok/s Changes: - The upstream commit also carries nixl/CI test changes that are unrelated to this crash; only the get_head_size() fix and its regression test are taken here to keep the later upstream merge (AIESW-37860) conflict-free. - DeepSeek-VL2-tiny_VLM still fails its multimodal sanity check after this fix, but that check was already failing on the Jun 26 nightly while the benchmark itself passed, so it is a separate pre-existing issue. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- tests/config/test_model_arch_config.py | 17 +++++++++++++++++ .../model_arch_config_convertor.py | 9 ++++++--- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/tests/config/test_model_arch_config.py b/tests/config/test_model_arch_config.py index e172983b54f4..46790be6e4e1 100644 --- a/tests/config/test_model_arch_config.py +++ b/tests/config/test_model_arch_config.py @@ -6,6 +6,7 @@ from pathlib import Path import pytest +from transformers import PretrainedConfig from vllm.config import ModelConfig, ParallelConfig, SpeculativeConfig from vllm.transformers_utils.model_arch_config_convertor import ( @@ -114,6 +115,22 @@ def _assert_model_config_methods( assert model_config.get_head_size() == expected["head_size"] +def test_head_size_falls_back_when_head_dim_is_zero(): + """Regression test for configs that materialize missing head_dim as 0.""" + hf_config = PretrainedConfig( + model_type="deepseek_vl_v2", + hidden_size=1280, + num_attention_heads=10, + num_key_value_heads=10, + head_dim=0, + kv_lora_rank=None, + ) + + convertor = ModelArchConfigConvertorBase(hf_config, hf_config) + + assert convertor.get_head_size() == 128 + + @pytest.mark.parametrize("model", BASE_MODELS_TO_TEST) def test_base_model_arch_config(model: str): """Test model architecture config for base models.""" diff --git a/vllm/transformers_utils/model_arch_config_convertor.py b/vllm/transformers_utils/model_arch_config_convertor.py index 250aee503786..fa0fe7d5d66b 100644 --- a/vllm/transformers_utils/model_arch_config_convertor.py +++ b/vllm/transformers_utils/model_arch_config_convertor.py @@ -58,9 +58,12 @@ def get_head_size(self) -> int: if qk_rope_head_dim and qk_nope_head_dim: return qk_rope_head_dim + qk_nope_head_dim - # NOTE: Some configs may set head_dim=None in the config - if getattr(self.hf_text_config, "head_dim", None) is not None: - return self.hf_text_config.head_dim + # NOTE: Some config classes may set head_dim=None or materialize a missing + # head_dim as 0 (for example, DeepseekVLV2TextConfig). + if ( + head_dim := getattr(self.hf_text_config, "head_dim", None) + ) is not None and head_dim > 0: + return head_dim # NOTE: Some models (such as PLaMo2.1) use `hidden_size_per_head` if getattr(self.hf_text_config, "hidden_size_per_head", None) is not None: