From 02bde4771030f2b99a72592cad38a3d90e68ea39 Mon Sep 17 00:00:00 2001 From: Xiake Sun Date: Thu, 8 Jan 2026 15:45:07 +0800 Subject: [PATCH 1/4] Fix fix pa metadata args type mismatch --- python/sglang/srt/layers/attention/aiter_backend.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/sglang/srt/layers/attention/aiter_backend.py b/python/sglang/srt/layers/attention/aiter_backend.py index 5f2f584f69fa..832334122ddd 100755 --- a/python/sglang/srt/layers/attention/aiter_backend.py +++ b/python/sglang/srt/layers/attention/aiter_backend.py @@ -595,7 +595,7 @@ def _build_pa_metadata_for_decode( get_pa_metadata_v1( seqlens_qo_indptr=qo_indptr, pages_kv_indptr=pages_kv_indptr, - context_lens=context_lens, + context_lens=context_lens.int(), num_heads_per_head_k=tp_q_head_num // self.num_kv_head, num_heads_k=self.num_kv_head, is_causal=True, From b8384c3bdb6853a4923fe8a3c35d2ae44479c45e Mon Sep 17 00:00:00 2001 From: Xiake Sun Date: Thu, 8 Jan 2026 15:46:00 +0800 Subject: [PATCH 2/4] Increase --cuda-graph-max-bs 16 to match concurrency used in mmmu acc test --- scripts/ci/sglang_benchmark_workflow.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/ci/sglang_benchmark_workflow.sh b/scripts/ci/sglang_benchmark_workflow.sh index 36b70865045c..0767be7932cf 100755 --- a/scripts/ci/sglang_benchmark_workflow.sh +++ b/scripts/ci/sglang_benchmark_workflow.sh @@ -105,7 +105,7 @@ if [[ "${TYPE}" == "launch" ]]; then --mem-fraction-static 0.85 \ --disable-radix-cache \ --max-prefill-tokens 32768 \ - --cuda-graph-max-bs 8 \ + --cuda-graph-max-bs 16 \ --page-size 1024 \ --mm-enable-dp-encoder \ --kv-cache-dtype fp8_e4m3 \ @@ -236,4 +236,4 @@ else echo echo "========== SGLANG BENCHMARK ${TYPE} FAILED WITH EXIT CODE $exit_code ==========" exit $exit_code -fi \ No newline at end of file +fi From a31d1592e9638feec61a8cf0f06ad8de863e5b13 Mon Sep 17 00:00:00 2001 From: Xiake Sun Date: Thu, 8 Jan 2026 16:56:05 +0800 Subject: [PATCH 3/4] Pass q/k/v descale=None to flash_attn_varlen_fp8_pertensor_func to improve acc --- python/sglang/srt/layers/attention/aiter_backend.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/sglang/srt/layers/attention/aiter_backend.py b/python/sglang/srt/layers/attention/aiter_backend.py index 832334122ddd..aaece8d2a3ad 100755 --- a/python/sglang/srt/layers/attention/aiter_backend.py +++ b/python/sglang/srt/layers/attention/aiter_backend.py @@ -1149,9 +1149,9 @@ def forward_extend( max_seqlen_k=self.forward_metadata.max_kv_len, softmax_scale=layer.scaling, causal=True, - q_descale=layer.k_scale, - k_descale=layer.k_scale, - v_descale=layer.v_scale, + q_descale=None, + k_descale=None, + v_descale=None, ) return o.view(-1, layer.tp_q_head_num * layer.head_dim) From fc6abff203be3754be6cd33f6edd6dc4619dd63a Mon Sep 17 00:00:00 2001 From: Xiake Sun Date: Fri, 9 Jan 2026 13:19:29 +0000 Subject: [PATCH 4/4] Add temp wa for hip runtime bug during cuda graph capture in doc, update CI config --- evaluation/Qwen3_Qmni_README.md | 10 +++++++--- scripts/ci/sglang_benchmark_workflow.sh | 3 +++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/evaluation/Qwen3_Qmni_README.md b/evaluation/Qwen3_Qmni_README.md index 659bb53160e5..b56223cb9f7e 100644 --- a/evaluation/Qwen3_Qmni_README.md +++ b/evaluation/Qwen3_Qmni_README.md @@ -57,6 +57,7 @@ echo "launching ${model}" echo "TP=${TP}" # AITER_MOE_PADDING_SIZE=128 \ only for fp8 tp8 + TORCH_NCCL_BLOCKING_WAIT=1 \ AITER_MOE_SMALL_BATCH=1 \ SGLANG_ROCM_USE_AITER_LINEAR_FP8HIPB=0 \ ROCM_QUICK_REDUCE_QUANTIZATION=INT4 \ @@ -77,15 +78,18 @@ --mm-attention-backend "aiter_attn" \ --max-prefill-tokens 32768 \ --disable-radix-cache \ - --page-size 32 \ + --page-size 1024 \ + --kv-cache-dtype fp8_e4m3 \ --mm-enable-dp-encoder \ --enable-aiter-allreduce-fusion \ --max-running-requests 128 \ - --cuda-graph-max-bs 8 + --cuda-graph-max-bs 16 2>&1 | tee log.server.log & ``` - Please add **AITER_MOE_PADDING_SIZE=128** when launch server for fp8 model with tp8. + Please note + - add **AITER_MOE_PADDING_SIZE=128** when launch server for fp8 model with tp8. + - add **TORCH_NCCL_BLOCKING_WAIT=1** as temporal workaround due to a known HIP runtime bug during cuda graph caputure, which that is tracked here: [ROCm/hip#3876](https://github.com/ROCm/hip/issues/3876). # Curl request diff --git a/scripts/ci/sglang_benchmark_workflow.sh b/scripts/ci/sglang_benchmark_workflow.sh index 0767be7932cf..877c88a3977b 100755 --- a/scripts/ci/sglang_benchmark_workflow.sh +++ b/scripts/ci/sglang_benchmark_workflow.sh @@ -28,6 +28,7 @@ if [[ "${TYPE}" == "launch" ]]; then echo "========== LAUNCHING SERVER ========" if [[ "${model_name}" == "Qwen3-VL-235B" ]]; then export SGLANG_USE_AITER=1 + export TORCH_NCCL_BLOCKING_WAIT=1 echo "********** AOT Prebuild aiter kernel start ... **********" cd /aiter python3 op_tests/test_rope.py @@ -86,6 +87,7 @@ if [[ "${TYPE}" == "launch" ]]; then export SGLANG_ROCM_USE_AITER_LINEAR_SHUFFLE=1 export SGLANG_ROCM_USE_AITER_LINEAR_FP8HIPB=0 export ROCM_QUICK_REDUCE_QUANTIZATION=INT4 + export TORCH_NCCL_BLOCKING_WAIT=1 echo "********** AOT Prebuild aiter kernel start ... **********" cd /aiter python3 op_tests/test_rope.py @@ -119,6 +121,7 @@ if [[ "${TYPE}" == "launch" ]]; then export SGLANG_ROCM_USE_AITER_PA_ASM_PRESHUFFLE_LAYOUT=0 export SGLANG_ROCM_USE_AITER_LINEAR_SHUFFLE=1 export SGLANG_ROCM_USE_AITER_LINEAR_FP8HIPB=1 + export TORCH_NCCL_BLOCKING_WAIT=1 python3 -m sglang.launch_server \ --model-path "${model_path}" \ --host localhost \