[NPU] Sync Ascend with main and consolidate NPU adaptations

.buildkite/README.md

@@ -8,15 +8,18 @@ build (PR and push to `main`):
 | Step | Purpose | Queue (machine) |
 |---|---|---|
 | `pre-commit` | pre-commit gate | `small_cpu_queue_premerge` (r6in.large) |
-| `plugin-contracts` | plugin contract tests (19 files) | `medium_cpu_queue_premerge` (r6in.4xlarge) |
-| `agent-adapter` | agent adapter tests (3 files) | `small_cpu_queue_premerge` |
+| `plugin-contracts` | plugin contracts and CPU tests (27 files) | `medium_cpu_queue_premerge` (r6in.4xlarge) |
+| `agent-adapter` | agent adapter tests (4 files) | `small_cpu_queue_premerge` |
+| `upstream-sync-cpu` | mechanically synchronized upstream CPU tests | `medium_cpu_queue_premerge` |
 | `utils` | utils tests (`pytest tests/utils`) | `medium_cpu_queue_premerge` |
 
-The three test steps depend on the pre-commit gate. Each suite runs its files
+The four test steps depend on the pre-commit gate. Each suite runs its files
 sequentially inside one step because these queues boot a fresh EC2 instance
-per job — a per-file matrix would be mostly boot + pip-install time. All
-always-on CPU steps use the standard `python:3.11` image and install their
-lightweight dependencies at runtime.
+per job — a per-file matrix would be mostly boot + pip-install time.
+Most always-on CPU steps use the standard `python:3.11` image and install their
+lightweight dependencies at runtime. `upstream-sync-cpu` uses `VIME_CI_IMAGE`
+(defaulting to `vllm/vime:latest`) because the synchronized GLM and checkpoint
+tests import the image-pinned Megatron stack even though they do not allocate a GPU.
 
 ## Creating the pipeline (one-time, Buildkite UI)
 
@@ -49,7 +52,7 @@ No secrets are required for these steps (WANDB etc. is GPU-suite only).
 
 The GPU suites are behind a **block step** (`:rocket: Run GPU test suites?`):
 click it in the Buildkite UI, multi-select the suites (`short`,
-`vllm-config`, `megatron`, `precision`, `ckpt`), and the follow-up step
+`vllm-config`, `megatron`, `vime-customized`, `precision`, `ckpt`), and the follow-up step
 generates one job per test via [`gpu_suites.py`](./gpu_suites.py) — the same
 `gpu_lock_exec.py` + `docker run` invocations used by the GPU jobs, including
 the per-test `VIME_TEST_USE_DEEPEP` / `VIME_TEST_USE_FP8_ROLLOUT` /
@@ -61,12 +64,15 @@ reports a passing commit status even if nobody unblocks the GPU gate.
 GPU jobs run on the shared **`mithril-h100-pool`** queue, following the same
 pattern vllm-omni uses for it: each job is a Kubernetes pod (agent-stack-k8s
 `kubernetes` plugin) on an H100 SXM node, with GPUs allocated via
-`nvidia.com/gpu` limits (4 or 8), a memory-backed `/dev/shm`, and the node's
+`nvidia.com/gpu` limits (1 to 8), a memory-backed `/dev/shm`, and the node's
 `/mnt/hf-cache` mounted as `HF_HOME`. vime tests `hf download` their models at
 startup, so a warm HF cache is all they need. `WANDB_API_KEY` is not wired up
 yet; runs report without wandb until it's added (e.g. as a k8s secret in the
 pod spec).
 
+Set `VIME_CI_IMAGE` to an immutable candidate digest for image-backed jobs;
+otherwise they use `vllm/vime:latest`. Do not update `latest` before merge.
+
 ## Keeping it in sync
 
 The test lists live in `pipeline.yml` and `gpu_suites.py`; update both together

.buildkite/gpu_suites.py

@@ -23,31 +23,16 @@
 import subprocess
 
 GPU_QUEUE = "mithril-h100-pool"
-CI_IMAGE = "inferactinc/public:vime-latest"
+CI_IMAGE = os.environ.get("VIME_CI_IMAGE", "vllm/vime:latest")
 HF_CACHE_HOST_PATH = "/mnt/hf-cache"
 HF_HOME = "/root/.cache/huggingface"
 NODE_INSTANCE_TYPE = "gpu-h100-sxm"
 
-# Known hardware-fit failures on the pool's 80 GB H100s — test-level issues,
-# not pipeline ones (PR #239, builds #6/#7):
-#   * gsm8k_async_short: FIXED — max-tokens-per-gpu reduced 9216→2048 (peak
-#     39.6 GB on H200, well within H100 80 GB). Root cause was Qwen3.5 248k
-#     vocab × 5 logits copies in calculate_log_probs_and_entropy.
-#   * parallel_check: cross-layout grad-norm invariance (TP4+per-token-loss)
-#     diverges ~12% on ~11% of rollout data (bimodal: most <1.5%, outliers
-#     10-20%). Confirmed same behavior in slime — Megatron FP reduction-order
-#     non-invariance, not a vime bug.
-# soft_fail keeps them running and visible (orange) without failing the build.
-SOFT_FAIL_ON_H100 = {
-    "test_qwen3_0.6B_parallel_check.py",
-}
-
 # (test_file, num_gpus, extra_args, env overrides)
 SUITES = {
     "short": [
         ("test_qwen3.5_0.8B_gsm8k_async_short.py", 4, "", {}),
         ("test_qwen3.5_0.8B_gsm8k_short.py", 4, "", {}),
-        ("test_qwen2.5_0.5B_ppo_critic_only_short.py", 4, "", {}),
         ("test_qwen2.5_0.5B_fully_async_short.py", 4, "", {}),
     ],
     "vllm-config": [
@@ -57,26 +42,48 @@
         ("test_vllm_config_mixed_offload_ft.py", 8, "", {}),
     ],
     "megatron": [
+        ("test_full_disk_weight_update.py", 4, "", {}),
         ("test_quick_start_glm4_9B.py", 8, "", {}),
         ("test_glm4.7_30B_A3B_pd_mooncake.py", 8, "", {}),
-        ("test_qwen3_30B_A3B.py", 8, "", {"USE_DEEPEP": "1", "USE_FP8_ROLLOUT": "1"}),
+        (
+            "test_qwen3_30B_A3B.py",
+            8,
+            "",
+            {"USE_DEEPEP": "1", "USE_FP8_ROLLOUT": "1"},
+        ),
         ("test_qwen3.6_35B_A3B_pd_mooncake.py", 8, "", {"USE_DEEPEP": "1"}),
         ("test_qwen3_30B_A3B_r3.py", 8, "", {"USE_DEEPEP": "1", "USE_FP8_ROLLOUT": "1", "ENABLE_EVAL": "0"}),
         ("test_qwen3_30B_A3B_r3.py", 8, "", {"ENABLE_EVAL": "0"}),
         ("test_qwen3_4B_ppo.py", 8, "", {}),
         ("test_qwen3_4B_ppo_disaggregate.py", 8, "", {}),
         ("test_qwen3_4B_ppo_train_critic_only.py", 8, "", {}),
+        ("test_ppo_logprob_entropy_gpu.py", 2, "", {}),
+        ("test_release_train.py", 4, "", {}),
         ("test_qwen3_4B_streaming_partial_rollout.py", 8, "", {}),
         ("test_moonlight_16B_A3B.py", 8, "", {}),
         ("test_moonlight_16B_A3B_r3.py", 8, "", {"ENABLE_EVAL": "0"}),
+        ("test_mimo_7B_mtp_only_grad.py", 8, "", {}),
         ("test_qwen2.5_0.5B_debug_rollout_then_train.py", 8, "", {}),
         ("test_qwen2.5_0.5B_opd_vllm.py", 8, "", {}),
+        ("test_qwen2.5_0.5B_fanout_short.py", 4, "", {}),
+        ("test_qwen2.5_0.5B_debug_train_dump_e2e.py", 8, "", {}),
+        ("test_qwen3_4B_external_pd.py", 6, "", {"VIME_TEST_UPDATE_MODE": "delta"}),
+    ],
+    "vime-customized": [
+        ("test_qwen2_5_0_5B_non_colocate_pp.py", 4, "", {}),
+        ("test_geo3k_vlm_multi_turn_e2e.py", 1, "", {}),
+        ("test_qwen2.5_vl_3B_ep_disaggregation.py", 3, "", {}),
+        ("test_moonlight_16B_A3B_non_colocate_nccl.py", 8, "", {}),
+        ("test_qwen3_5_0_8B_top_p_cp2.py", 4, "", {}),
     ],
     "precision": [
         ("test_qwen3_0.6B_parallel_check.py", 8, "", {}),
     ],
     "ckpt": [
-        ("test_qwen3_4B_ckpt.py", 8, "", {}),
+        ("test_qwen3_4B_ckpt.py", 8, "--save-optimizer gpu --load-optimizer gpu", {}),
+        ("test_qwen3_4B_ckpt.py", 8, "--save-optimizer gpu --load-optimizer cpu", {}),
+        ("test_qwen3_4B_ckpt.py", 8, "--save-optimizer cpu --load-optimizer cpu", {}),
+        ("test_qwen3_4B_ckpt.py", 8, "--save-optimizer cpu --load-optimizer gpu", {}),
         ("test_qwen3_4B_ckpt.py", 8, "--async-save", {}),
     ],
 }
@@ -107,9 +114,10 @@ def gpu_step(suite: str, test_file: str, num_gpus: int, extra_args: str, env: di
         {"name": "VIME_TEST_USE_DEEPEP", "value": vime_flags.get("USE_DEEPEP", "0")},
         {"name": "VIME_TEST_USE_FP8_ROLLOUT", "value": vime_flags.get("USE_FP8_ROLLOUT", "0")},
         {"name": "VIME_TEST_ENABLE_EVAL", "value": vime_flags.get("ENABLE_EVAL", "1")},
+        {"name": "NCCL_NVLS_ENABLE", "value": env.get("NCCL_NVLS_ENABLE", "0")},
     ]
     # anything else in env is passed to the pod verbatim (e.g. allocator knobs)
-    pod_env += [{"name": k, "value": v} for k, v in env.items() if k not in vime_flags]
+    pod_env += [{"name": k, "value": v} for k, v in env.items() if k not in vime_flags and k != "NCCL_NVLS_ENABLE"]
     # Set a stable commit identifier for downstream tooling.
     command = "\n".join(
         [
@@ -130,7 +138,12 @@ def gpu_step(suite: str, test_file: str, num_gpus: int, extra_args: str, env: di
         "command": command,
         "agents": {"queue": GPU_QUEUE},
         "timeout_in_minutes": 360,
-        "retry": {"automatic": [{"exit_status": -1, "limit": 2}]},
+        "retry": {
+            "automatic": [
+                {"exit_status": -1, "limit": 2},
+                {"exit_status": 1, "limit": 2},
+            ]
+        },
         "plugins": [
             {
                 "kubernetes": {
@@ -159,9 +172,6 @@ def gpu_step(suite: str, test_file: str, num_gpus: int, extra_args: str, env: di
             }
         ],
     }
-    if test_file in SOFT_FAIL_ON_H100:
-        step["soft_fail"] = True
-        step["label"] = ":warning: " + step["label"]
     return step
 
 

.buildkite/npu_suites.py

@@ -31,7 +31,6 @@
         ("test_qwen3_4B_npu.py", "npu-8", "", {}),
         ("test_qwen3_30B_A3B_npu.py", "npu-16", "", {}),
         ("test_qwen3_vl_8B_npu.py", "npu-8", "", {}),
-        ("test_qwen3.5_35B_A3B_npu.py", "npu-16", "", {}),
         ("test_glm4.7_30B_A3B_npu.py", "npu-16", "", {}),
     ],
     "nightly": [],

.buildkite/pipeline-npu-image.yaml

@@ -78,7 +78,6 @@ steps:
         --local context=. \
         --local dockerfile=./docker \
         --opt filename=Dockerfile.npu \
-        --opt build-arg:BASE_IMAGE=swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/vllm-ascend/vllm-ascend \
         --opt build-arg:APTMIRROR=http://cache-service.nginx-pypi-cache.svc.cluster.local:8081 \
         --opt build-arg:PIP_INDEX_URL=http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
         --secret id=dockerconfig,src=/home/user/.docker/config.json \

.buildkite/pipeline.yml

@@ -26,6 +26,10 @@ steps:
       automatic:
         - exit_status: -1  # agent lost (fresh instance failed to boot)
           limit: 2
+        - exit_status: 1  # transient package-index failure
+          limit: 2
+        - exit_status: 3  # pre-commit environment install failure
+          limit: 2
     command: |
       docker run --rm \
         -e GIT_CONFIG_PARAMETERS="'safe.directory=/workspace'" \
@@ -46,6 +50,8 @@ steps:
       automatic:
         - exit_status: -1
           limit: 2
+        - exit_status: 1
+          limit: 2
     command: |
       # GLOO/TP_SOCKET_IFNAME=lo: the torch.distributed tests (e.g.
       # test_metric_report_dist) rendezvous over localhost; inside a
@@ -58,11 +64,13 @@ steps:
         python:3.11 bash -c '
           set -euo pipefail
           pip install -q torch --index-url https://download.pytorch.org/whl/cpu
-          pip install -q pytest numpy packaging pyyaml omegaconf tqdm httpx requests ray pybase64 pylatexenc sympy aiohttp pillow safetensors transformers cloudpickle
+          pip install -q pytest numpy packaging pyyaml omegaconf tqdm httpx requests ray pybase64 pylatexenc sympy aiohttp pillow safetensors transformers cloudpickle blake3 xxhash zstandard psutil wandb
           pip install -q -e . --no-deps
           python tests/test_megatron_argument_validation.py
           python tests/test_value_temperature.py
-          python tests/test_rollout_validation.py
+          python tests/test_docs_consistency.py
+          python tests/test_placement_group.py
+          python tests/test_external_vllm_engines.py
           python tests/plugin_contracts/test_plugin_rollout_contracts.py
           python tests/plugin_contracts/test_plugin_runtime_hook_contracts.py
           python tests/plugin_contracts/test_plugin_path_loading_contracts.py
@@ -78,6 +86,12 @@ steps:
           python tests/test_metric_report_dist.py
           python tests/test_loss_cp_invariance.py
           python tests/test_sample.py
+          python tests/test_cispo_loss.py
+          python tests/test_logprob_response_spans.py
+          python tests/test_empty_colocated_weight_bucket.py
+          python tests/test_expert_routing.py
+          python tests/test_reloadable_process_group_memory_check.py
+          python tests/test_ppo_logprob_entropy.py
           python tests/utils/test_hf_checkpoint_saver.py
         '
 
@@ -91,19 +105,78 @@ steps:
       automatic:
         - exit_status: -1
           limit: 2
+        - exit_status: 1
+          limit: 2
     command: |
       docker run --rm --shm-size=2g \
         -e GIT_CONFIG_PARAMETERS="'safe.directory=/workspace'" \
         -v "$$PWD:/workspace" -w /workspace \
         python:3.11 bash -c '
           set -euo pipefail
           pip install -q torch --index-url https://download.pytorch.org/whl/cpu
-          pip install -q pytest numpy packaging pyyaml omegaconf tqdm httpx requests ray pybase64 pylatexenc sympy aiohttp pillow safetensors transformers cloudpickle
+          pip install -q pytest numpy packaging pyyaml omegaconf tqdm httpx requests ray pybase64 pylatexenc sympy aiohttp pillow safetensors transformers cloudpickle blake3 xxhash zstandard wandb
           pip install -q openai openai-agents anthropic
           pip install -q -e . --no-deps
-          python tests/test_agent_trajectory.py
-          python tests/test_agent_adapters.py
-          python tests/test_agent_sdk_adapters.py
+          python tests/test_agent/test_adapters.py
+          python tests/test_agent/test_harness.py
+          python tests/test_agent/test_trajectory_manager_branching.py
+          python tests/test_agent/test_agent_rollout_cpu.py
+        '
+
+  - label: ":pytest: synchronized upstream CPU tests"
+    key: upstream-sync-cpu
+    depends_on: pre-commit
+    agents:
+      queue: medium_cpu_queue_premerge
+    timeout_in_minutes: 45
+    retry:
+      automatic:
+        - exit_status: -1
+          limit: 2
+        - exit_status: 1
+          limit: 2
+    command: |
+      docker run --rm --network host --ipc=host \
+        -e GIT_CONFIG_PARAMETERS="'safe.directory=/workspace'" \
+        -e GLOO_SOCKET_IFNAME=lo -e TP_SOCKET_IFNAME=lo \
+        -v "$$PWD:/workspace" -w /workspace \
+        "$${VIME_CI_IMAGE:-vllm/vime:latest}" bash -lc '
+          set -euo pipefail
+          pip install -q -e . --no-deps --break-system-packages
+          for test_file in \
+            tests/test_advantage_whiten_cp.py \
+            tests/test_accelerator.py \
+            tests/test_block_fp8_zero_block.py \
+            tests/test_deep_ep_tms_patch.py \
+            tests/test_discounted_returns.py \
+            tests/test_eval_config.py \
+            tests/test_filter_long_prompt.py \
+            tests/test_fully_async_rollout.py \
+            tests/test_glm52_layerwise_comparison.py \
+            tests/test_glm5_indexer_q_norm.py \
+            tests/test_glm5_indexer_short_context.py \
+            tests/test_hf_to_megatron.py \
+            tests/test_layerwise_alignment.py \
+            tests/test_model_provider_freeze.py \
+            tests/test_policy_loss.py \
+            tests/test_ppo_kl_metric.py \
+            tests/test_process_rollout_data.py \
+            tests/test_qwen3_5_vl_native.py \
+            tests/test_qwen3_linear_attention_cu_seqlens.py \
+            tests/test_read_file_slicing.py \
+            tests/test_reloadable_process_group_world.py \
+            tests/test_rollout_data_utils.py \
+            tests/test_rollout_metrics.py \
+            tests/test_rollout_sample_hooks.py \
+            tests/test_vllm_rollout.py \
+            tests/test_agent/test_sandbox_exec_and_wait.py \
+            tests/test_stateless_adam.py \
+            tests/test_tau_bench_token_delta.py \
+            tests/test_train_data_utils.py \
+            tests/test_update_weight_factory.py \
+            tests/observability/test_trace_utils.py; do
+            python -m pytest "$$test_file"
+          done
         '
 
   - label: ":pytest: utils tests"
@@ -116,14 +189,16 @@ steps:
       automatic:
         - exit_status: -1
           limit: 2
+        - exit_status: 1
+          limit: 2
     command: |
       docker run --rm --network host --ipc=host \
         -e GIT_CONFIG_PARAMETERS="'safe.directory=/workspace'" \
         -v "$$PWD:/workspace" -w /workspace \
         python:3.11 bash -lc '
           set -euo pipefail
           pip install -q torch --index-url https://download.pytorch.org/whl/cpu
-          pip install -q pytest numpy packaging pyyaml omegaconf tqdm httpx requests ray pybase64 pylatexenc sympy aiohttp pillow safetensors transformers cloudpickle
+          pip install -q pytest numpy packaging pyyaml omegaconf tqdm httpx requests ray pybase64 pylatexenc sympy aiohttp pillow safetensors transformers cloudpickle blake3 xxhash zstandard
           pip install -q -e . --no-deps
           python -m pytest tests/utils
         '
@@ -143,15 +218,17 @@ steps:
         multiple: true
         required: true
         options:
-          - label: "run-ci-short — 4 GPU, 4 tests"
+          - label: "run-ci-short — 4 GPU, 3 tests"
             value: short
           - label: "run-ci-vllm-config — 8 GPU, 4 tests"
             value: vllm-config
-          - label: "run-ci-megatron — 8 GPU, 14 runs"
+          - label: "run-ci-megatron — up to 8 GPU, 21 runs"
             value: megatron
+          - label: "run-ci-vime-customized — 1–8 GPU, 6 tests"
+            value: vime-customized
           - label: "run-ci-precision — 8 GPU, 1 test"
             value: precision
-          - label: "run-ci-ckpt — 8 GPU, 2 runs"
+          - label: "run-ci-ckpt — 8 GPU, 5 runs"
             value: ckpt
 
   - label: ":pipeline: upload selected GPU suites"