From 18882271d189a305fdcf983c45a951eff9ef6933 Mon Sep 17 00:00:00 2001 From: Xiake Sun Date: Thu, 2 Jul 2026 03:20:55 +0000 Subject: [PATCH 01/11] Update Qwen3.5 CI test base docker image --- .github/workflows/sglang_benchmark_workflow.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sglang_benchmark_workflow.yaml b/.github/workflows/sglang_benchmark_workflow.yaml index 786fccd738a4..d0573bbebd3f 100644 --- a/.github/workflows/sglang_benchmark_workflow.yaml +++ b/.github/workflows/sglang_benchmark_workflow.yaml @@ -141,13 +141,13 @@ jobs: case "${model_name}" in offical_qwen3p5_397B_ptpc) model_path="/models/offical_qwen3p5_397B_ptpc" - model_image="rocm/ali-private:ubuntu22.04_rocm7.2.0.43_cp310_torch2.9.1_sglang_8e85368_aiter_303a583_qwen3_5_20260417_v2" + model_image="rocm/ali-private:ubuntu22.04_rocm7.2.0.43_cp310_torch2.9.1_sglang_9b3cca1_aiter_dd9d1b_ck_c61dc42_qwen3.5_20260625_2" model_tp="8" model_vlm_cache_size_mb="0" ;; Qwen3.5-27B-PTPC-compressor) model_path="/models/Qwen3.5-27B-PTPC-compressor" - model_image="rocm/ali-private:ubuntu22.04_rocm7.2.0.43_cp310_torch2.9.1_sglang_8e85368_aiter_303a583_qwen3_5_20260417_v2" + model_image="rocm/ali-private:ubuntu22.04_rocm7.2.0.43_cp310_torch2.9.1_sglang_9b3cca1_aiter_dd9d1b_ck_c61dc42_qwen3.5_20260625_2" model_tp="1" model_vlm_cache_size_mb="8192" ;; From 6257607ca426c273e58cde4fa82a7de2eea1436c Mon Sep 17 00:00:00 2001 From: Xiake Sun Date: Tue, 7 Jul 2026 14:42:14 +0000 Subject: [PATCH 02/11] Add docker login step Signed-off-by: Xiake Sun --- .../workflows/sglang_benchmark_workflow.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/sglang_benchmark_workflow.yaml b/.github/workflows/sglang_benchmark_workflow.yaml index d0573bbebd3f..6557a63dab98 100644 --- a/.github/workflows/sglang_benchmark_workflow.yaml +++ b/.github/workflows/sglang_benchmark_workflow.yaml @@ -172,6 +172,23 @@ jobs: echo "Resolved model_tp: ${model_tp}" echo "Resolved model_vlm_cache_size_mb: ${model_vlm_cache_size_mb}" + - name: Docker login + env: + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + run: | + for attempt in 1 2 3; do + if echo "$DOCKER_PASSWORD" | docker login -u rocmshared --password-stdin; then + echo "Docker login succeeded on attempt ${attempt}" + exit 0 + fi + echo "Docker login attempt ${attempt} failed" + if [ "${attempt}" != 3 ]; then + sleep 10 + fi + done + echo "Docker login failed after 3 attempts, continuing anyway" + exit 0 + - name: Prepare SGLang image run: | set -euo pipefail From 5c574245e1cb2594e57f1250cbe13398ed8a0645 Mon Sep 17 00:00:00 2001 From: Xiake Sun Date: Tue, 7 Jul 2026 14:48:51 +0000 Subject: [PATCH 03/11] Update condition Signed-off-by: Xiake Sun --- .github/workflows/sglang_benchmark_workflow.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/sglang_benchmark_workflow.yaml b/.github/workflows/sglang_benchmark_workflow.yaml index 6557a63dab98..76ea449ff0a3 100644 --- a/.github/workflows/sglang_benchmark_workflow.yaml +++ b/.github/workflows/sglang_benchmark_workflow.yaml @@ -173,6 +173,7 @@ jobs: echo "Resolved model_vlm_cache_size_mb: ${model_vlm_cache_size_mb}" - name: Docker login + if: ${{ !github.event.pull_request || !github.event.pull_request.head.repo.fork) }} env: DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} run: | From a4ebcd2aea14460cf133e7cb12bdc9dd64a9a319 Mon Sep 17 00:00:00 2001 From: Xiake Sun Date: Tue, 7 Jul 2026 14:49:26 +0000 Subject: [PATCH 04/11] Fix format Signed-off-by: Xiake Sun --- .github/workflows/sglang_benchmark_workflow.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sglang_benchmark_workflow.yaml b/.github/workflows/sglang_benchmark_workflow.yaml index 76ea449ff0a3..8b3f5346b910 100644 --- a/.github/workflows/sglang_benchmark_workflow.yaml +++ b/.github/workflows/sglang_benchmark_workflow.yaml @@ -173,7 +173,7 @@ jobs: echo "Resolved model_vlm_cache_size_mb: ${model_vlm_cache_size_mb}" - name: Docker login - if: ${{ !github.event.pull_request || !github.event.pull_request.head.repo.fork) }} + if: ${{ !github.event.pull_request || !github.event.pull_request.head.repo.fork }} env: DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} run: | From 218314bcaa6bb87d284f5bbbfac8dfe581f9d4ae Mon Sep 17 00:00:00 2001 From: Xiake Sun Date: Wed, 8 Jul 2026 00:24:00 +0000 Subject: [PATCH 05/11] change user name Signed-off-by: Xiake Sun --- .github/workflows/sglang_benchmark_workflow.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sglang_benchmark_workflow.yaml b/.github/workflows/sglang_benchmark_workflow.yaml index 8b3f5346b910..7dd6c4c61737 100644 --- a/.github/workflows/sglang_benchmark_workflow.yaml +++ b/.github/workflows/sglang_benchmark_workflow.yaml @@ -173,12 +173,12 @@ jobs: echo "Resolved model_vlm_cache_size_mb: ${model_vlm_cache_size_mb}" - name: Docker login - if: ${{ !github.event.pull_request || !github.event.pull_request.head.repo.fork }} env: DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} run: | for attempt in 1 2 3; do - if echo "$DOCKER_PASSWORD" | docker login -u rocmshared --password-stdin; then + echo "$DOCKER_PASSWORD" + if echo "$DOCKER_PASSWORD" | docker login -u alishared --password-stdin; then echo "Docker login succeeded on attempt ${attempt}" exit 0 fi From 1ec37d7a1eb2a786855f5321752820d2b21fb6ce Mon Sep 17 00:00:00 2001 From: Xiake Sun Date: Wed, 8 Jul 2026 00:56:24 +0000 Subject: [PATCH 06/11] Add condition back for docker login action Signed-off-by: Xiake Sun --- .github/workflows/sglang_benchmark_workflow.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sglang_benchmark_workflow.yaml b/.github/workflows/sglang_benchmark_workflow.yaml index 7dd6c4c61737..faa27e51f08b 100644 --- a/.github/workflows/sglang_benchmark_workflow.yaml +++ b/.github/workflows/sglang_benchmark_workflow.yaml @@ -173,11 +173,11 @@ jobs: echo "Resolved model_vlm_cache_size_mb: ${model_vlm_cache_size_mb}" - name: Docker login + if: ${{ !github.event.pull_request || !github.event.pull_request.head.repo.fork }} env: DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} run: | for attempt in 1 2 3; do - echo "$DOCKER_PASSWORD" if echo "$DOCKER_PASSWORD" | docker login -u alishared --password-stdin; then echo "Docker login succeeded on attempt ${attempt}" exit 0 From ed48505f8141eb084bcd74e1359d8813f5f9b082 Mon Sep 17 00:00:00 2001 From: Xiake Sun Date: Wed, 8 Jul 2026 00:59:21 +0000 Subject: [PATCH 07/11] Refine condition Signed-off-by: Xiake Sun --- .github/workflows/sglang_benchmark_workflow.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sglang_benchmark_workflow.yaml b/.github/workflows/sglang_benchmark_workflow.yaml index faa27e51f08b..579fc730a325 100644 --- a/.github/workflows/sglang_benchmark_workflow.yaml +++ b/.github/workflows/sglang_benchmark_workflow.yaml @@ -173,7 +173,7 @@ jobs: echo "Resolved model_vlm_cache_size_mb: ${model_vlm_cache_size_mb}" - name: Docker login - if: ${{ !github.event.pull_request || !github.event.pull_request.head.repo.fork }} + if: ${{ !github.event.pull_request } env: DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} run: | From 263228aba06397c772eee36666490b1f9e937b4a Mon Sep 17 00:00:00 2001 From: Xiake Sun Date: Wed, 8 Jul 2026 01:01:27 +0000 Subject: [PATCH 08/11] Refine condition Signed-off-by: Xiake Sun --- .github/workflows/sglang_benchmark_workflow.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sglang_benchmark_workflow.yaml b/.github/workflows/sglang_benchmark_workflow.yaml index 579fc730a325..9b232a6a7978 100644 --- a/.github/workflows/sglang_benchmark_workflow.yaml +++ b/.github/workflows/sglang_benchmark_workflow.yaml @@ -173,7 +173,7 @@ jobs: echo "Resolved model_vlm_cache_size_mb: ${model_vlm_cache_size_mb}" - name: Docker login - if: ${{ !github.event.pull_request } + if: ${{ !github.event.pull_request }} env: DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} run: | From f8e8ed4dd26789dfcab9796a0da6516fd414300c Mon Sep 17 00:00:00 2001 From: Xiake Sun Date: Wed, 8 Jul 2026 01:04:06 +0000 Subject: [PATCH 09/11] Refine condition Signed-off-by: Xiake Sun --- .github/workflows/sglang_benchmark_workflow.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sglang_benchmark_workflow.yaml b/.github/workflows/sglang_benchmark_workflow.yaml index 9b232a6a7978..cb6e903f32aa 100644 --- a/.github/workflows/sglang_benchmark_workflow.yaml +++ b/.github/workflows/sglang_benchmark_workflow.yaml @@ -173,7 +173,7 @@ jobs: echo "Resolved model_vlm_cache_size_mb: ${model_vlm_cache_size_mb}" - name: Docker login - if: ${{ !github.event.pull_request }} + if: ${{ github.event.pull_request }} env: DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} run: | From a9dbca23e6d0142274e4587a664e3a1d92a7be77 Mon Sep 17 00:00:00 2001 From: Xiake Sun Date: Fri, 10 Jul 2026 08:21:25 +0000 Subject: [PATCH 10/11] Update Qwen3.5 397B and 27B launch server cmd Signed-off-by: Xiake Sun --- scripts/ci/sglang_benchmark_workflow.sh | 45 ++++++++++++++++++------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/scripts/ci/sglang_benchmark_workflow.sh b/scripts/ci/sglang_benchmark_workflow.sh index 0a5f398ecdc8..8cce4a4a8bfe 100644 --- a/scripts/ci/sglang_benchmark_workflow.sh +++ b/scripts/ci/sglang_benchmark_workflow.sh @@ -19,16 +19,25 @@ BENCHMARK_EXAMPLE_ROOT=${SGLANG_BENCHMARK_EXAMPLE_ROOT:-/models/benchamark_examp export SGLANG_DISABLE_CUDNN_CHECK=1 export SGLANG_USE_CUDA_IPC_TRANSPORT=1 -# Override to 0 for Ali-specific runs when required. export SGLANG_VLM_CACHE_SIZE_MB="${SGLANG_VLM_CACHE_SIZE_MB:-8192}" export SGLANG_USE_AITER=1 export SGLANG_ROCM_USE_AITER_LINEAR_SHUFFLE=1 export SGLANG_ROCM_USE_AITER_LINEAR_FP8HIPB=1 export SGLANG_USE_AITER_NEW_CA=false - -if [[ "${MODEL_NAME}" == "offical_qwen3p5_397B_ptpc" ]]; then - export AITER_MOE_PADDING_SIZE="${AITER_MOE_PADDING_SIZE:-256}" +export USE_HIP_LINEAR_ATTN="${USE_HIP_LINEAR_ATTN:-1}" +export SGLANG_USE_IPC_POOL_HANDLE_CACHE="${SGLANG_USE_IPC_POOL_HANDLE_CACHE:-1}" +export HIP_GDN_SORT_IDX_BS="${HIP_GDN_SORT_IDX_BS:-32768}" + +# Model-specific aiter tuning (from the validated launch references). +if [[ "${MODEL_NAME}" == "Qwen3.5-27B-PTPC-compressor" ]]; then + export AITER_QUICK_REDUCE_QUANTIZATION="${AITER_QUICK_REDUCE_QUANTIZATION:-INT8}" + export SGLANG_DISABLE_AITER_FUSED_AR_RMSNORM="${SGLANG_DISABLE_AITER_FUSED_AR_RMSNORM:-1}" +elif [[ "${MODEL_NAME}" == "offical_qwen3p5_397B_ptpc" ]]; then + export AITER_QUICK_REDUCE_QUANTIZATION="${AITER_QUICK_REDUCE_QUANTIZATION:-INT6}" + export USE_AITER_COMM="${USE_AITER_COMM:-1}" fi +# Fallback quick-reduce level for any other model. +export AITER_QUICK_REDUCE_QUANTIZATION="${AITER_QUICK_REDUCE_QUANTIZATION:-INT6}" echo "Detect TYPE: ${TYPE}" echo "Detect model_name: ${MODEL_NAME}" @@ -82,8 +91,15 @@ print_launch_recipe() { echo "export SGLANG_ROCM_USE_AITER_LINEAR_SHUFFLE=${SGLANG_ROCM_USE_AITER_LINEAR_SHUFFLE}" echo "export SGLANG_ROCM_USE_AITER_LINEAR_FP8HIPB=${SGLANG_ROCM_USE_AITER_LINEAR_FP8HIPB}" echo "export SGLANG_USE_AITER_NEW_CA=${SGLANG_USE_AITER_NEW_CA}" - if [[ -n "${AITER_MOE_PADDING_SIZE:-}" ]]; then - echo "export AITER_MOE_PADDING_SIZE=${AITER_MOE_PADDING_SIZE}" + echo "export USE_HIP_LINEAR_ATTN=${USE_HIP_LINEAR_ATTN}" + echo "export SGLANG_USE_IPC_POOL_HANDLE_CACHE=${SGLANG_USE_IPC_POOL_HANDLE_CACHE}" + echo "export HIP_GDN_SORT_IDX_BS=${HIP_GDN_SORT_IDX_BS}" + echo "export AITER_QUICK_REDUCE_QUANTIZATION=${AITER_QUICK_REDUCE_QUANTIZATION}" + if [[ -n "${USE_AITER_COMM:-}" ]]; then + echo "export USE_AITER_COMM=${USE_AITER_COMM}" + fi + if [[ -n "${SGLANG_DISABLE_AITER_FUSED_AR_RMSNORM:-}" ]]; then + echo "export SGLANG_DISABLE_AITER_FUSED_AR_RMSNORM=${SGLANG_DISABLE_AITER_FUSED_AR_RMSNORM}" fi echo @@ -102,13 +118,15 @@ nohup python3 -m sglang.launch_server \\ --mem-fraction-static 0.9 \\ --max-prefill-tokens 32768 \\ --max-running-requests 128 \\ + --cuda-graph-max-bs 128 \\ + --kv-cache-dtype fp8_e4m3 \\ + --disable-radix-cache \\ + --mm-attention-backend aiter_attn \\ EOF - if [[ "${MODEL_NAME}" == "offical_qwen3p5_397B_ptpc" ]]; then + if [[ "${MODEL_NAME}" == "Qwen3.5-27B-PTPC-compressor" ]]; then echo " --disable-custom-all-reduce \\" fi cat < ${SERVER_LOG} 2>&1 & EOF } @@ -390,9 +408,6 @@ if [[ "${TYPE}" == "launch" ]]; then rm -f "${SERVER_LOG}" model="${MODEL_PATH}" attention_backend="aiter" - if [[ "${MODEL_NAME}" == "Qwen3.5-27B-PTPC-compressor" ]]; then - attention_backend="triton" - fi launch_args=( --port "${PORT}" @@ -407,11 +422,15 @@ if [[ "${TYPE}" == "launch" ]]; then --mem-fraction-static 0.9 --max-prefill-tokens 32768 --max-running-requests 128 + --cuda-graph-max-bs 128 + --kv-cache-dtype fp8_e4m3 --disable-radix-cache --mm-attention-backend aiter_attn ) - if [[ "${MODEL_NAME}" == "offical_qwen3p5_397B_ptpc" ]]; then + # 27B disables the aiter custom all-reduce (paired with + # SGLANG_DISABLE_AITER_FUSED_AR_RMSNORM=1 above); 397B keeps it. + if [[ "${MODEL_NAME}" == "Qwen3.5-27B-PTPC-compressor" ]]; then launch_args+=(--disable-custom-all-reduce) fi From d745c7187e3bbbf4099036bae33eeff86409d33b Mon Sep 17 00:00:00 2001 From: Xiake Sun Date: Mon, 13 Jul 2026 05:34:46 +0000 Subject: [PATCH 11/11] Update CI base image 0710 Signed-off-by: Xiake Sun --- .github/workflows/sglang_benchmark_workflow.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sglang_benchmark_workflow.yaml b/.github/workflows/sglang_benchmark_workflow.yaml index cb6e903f32aa..01e40f7b332b 100644 --- a/.github/workflows/sglang_benchmark_workflow.yaml +++ b/.github/workflows/sglang_benchmark_workflow.yaml @@ -141,13 +141,13 @@ jobs: case "${model_name}" in offical_qwen3p5_397B_ptpc) model_path="/models/offical_qwen3p5_397B_ptpc" - model_image="rocm/ali-private:ubuntu22.04_rocm7.2.0.43_cp310_torch2.9.1_sglang_9b3cca1_aiter_dd9d1b_ck_c61dc42_qwen3.5_20260625_2" + model_image="rocm/ali-private:ubuntu22.04_rocm7.2.0.43_cp310_torch2.9.1_sglang_3038fa5_aiter_ea051b0_qwen3_5_20260710_v2" model_tp="8" model_vlm_cache_size_mb="0" ;; Qwen3.5-27B-PTPC-compressor) model_path="/models/Qwen3.5-27B-PTPC-compressor" - model_image="rocm/ali-private:ubuntu22.04_rocm7.2.0.43_cp310_torch2.9.1_sglang_9b3cca1_aiter_dd9d1b_ck_c61dc42_qwen3.5_20260625_2" + model_image="rocm/ali-private:ubuntu22.04_rocm7.2.0.43_cp310_torch2.9.1_sglang_3038fa5_aiter_ea051b0_qwen3_5_20260710_v2" model_tp="1" model_vlm_cache_size_mb="8192" ;;