[Sync] Support request-scoped streaming aborts

.buildkite/README.md

@@ -17,9 +17,9 @@ The four test steps depend on the pre-commit gate. Each suite runs its files
 sequentially inside one step because these queues boot a fresh EC2 instance
 per job — a per-file matrix would be mostly boot + pip-install time.
 Most always-on CPU steps use the standard `python:3.11` image and install their
-lightweight dependencies at runtime. `upstream-sync-cpu` uses `VIME_CI_IMAGE`
-(defaulting to `vllm/vime:latest`) because the synchronized GLM and checkpoint
-tests import the image-pinned Megatron stack even though they do not allocate a GPU.
+lightweight dependencies at runtime. `upstream-sync-cpu` uses
+`vllm/vime:latest` because the synchronized GLM and checkpoint tests import the
+image-pinned Megatron stack even though they do not allocate a GPU.
 
 ## Creating the pipeline (one-time, Buildkite UI)
 
@@ -70,8 +70,8 @@ startup, so a warm HF cache is all they need. `WANDB_API_KEY` is not wired up
 yet; runs report without wandb until it's added (e.g. as a k8s secret in the
 pod spec).
 
-Set `VIME_CI_IMAGE` to an immutable candidate digest for image-backed jobs;
-otherwise they use `vllm/vime:latest`. Do not update `latest` before merge.
+GPU jobs use `vllm/vime:latest`. Rebuild and publish that image before validating
+Dockerfile or vLLM patch changes.
 
 ## Keeping it in sync
 

.buildkite/gpu_suites.py

@@ -23,7 +23,7 @@
 import subprocess
 
 GPU_QUEUE = "mithril-h100-pool"
-CI_IMAGE = os.environ.get("VIME_CI_IMAGE", "vllm/vime:latest")
+CI_IMAGE = "vllm/vime:latest"
 HF_CACHE_HOST_PATH = "/mnt/hf-cache"
 HF_HOME = "/root/.cache/huggingface"
 NODE_INSTANCE_TYPE = "gpu-h100-sxm"
@@ -43,24 +43,24 @@
     ],
     "megatron": [
         ("test_full_disk_weight_update.py", 4, "", {}),
-        ("test_quick_start_glm4_9B.py", 8, "", {}),
+        ("test_quick_start_glm4_9B.py", 8, "", {"ENABLE_EVAL": "0"}),
         ("test_glm4.7_30B_A3B_pd_mooncake.py", 8, "", {}),
         (
             "test_qwen3_30B_A3B.py",
             8,
             "",
-            {"USE_DEEPEP": "1", "USE_FP8_ROLLOUT": "1"},
+            {"USE_DEEPEP": "1", "USE_FP8_ROLLOUT": "1", "ENABLE_EVAL": "0"},
         ),
         ("test_qwen3.6_35B_A3B_pd_mooncake.py", 8, "", {"USE_DEEPEP": "1"}),
         ("test_qwen3_30B_A3B_r3.py", 8, "", {"USE_DEEPEP": "1", "USE_FP8_ROLLOUT": "1", "ENABLE_EVAL": "0"}),
         ("test_qwen3_30B_A3B_r3.py", 8, "", {"ENABLE_EVAL": "0"}),
-        ("test_qwen3_4B_ppo.py", 8, "", {}),
-        ("test_qwen3_4B_ppo_disaggregate.py", 8, "", {}),
-        ("test_qwen3_4B_ppo_train_critic_only.py", 8, "", {}),
+        ("test_qwen3_4B_ppo.py", 8, "", {"ENABLE_EVAL": "0"}),
+        ("test_qwen3_4B_ppo_disaggregate.py", 8, "", {"ENABLE_EVAL": "0"}),
+        ("test_qwen3_4B_ppo_train_critic_only.py", 8, "", {"ENABLE_EVAL": "0"}),
         ("test_ppo_logprob_entropy_gpu.py", 2, "", {}),
         ("test_release_train.py", 4, "", {}),
         ("test_qwen3_4B_streaming_partial_rollout.py", 8, "", {}),
-        ("test_moonlight_16B_A3B.py", 8, "", {}),
+        ("test_moonlight_16B_A3B.py", 8, "", {"ENABLE_EVAL": "0"}),
         ("test_moonlight_16B_A3B_r3.py", 8, "", {"ENABLE_EVAL": "0"}),
         ("test_mimo_7B_mtp_only_grad.py", 8, "", {}),
         ("test_qwen2.5_0.5B_debug_rollout_then_train.py", 8, "", {}),

.buildkite/pipeline.yml

@@ -140,7 +140,7 @@ steps:
         -e GIT_CONFIG_PARAMETERS="'safe.directory=/workspace'" \
         -e GLOO_SOCKET_IFNAME=lo -e TP_SOCKET_IFNAME=lo \
         -v "$$PWD:/workspace" -w /workspace \
-        "$${VIME_CI_IMAGE:-vllm/vime:latest}" bash -lc '
+        vllm/vime:latest bash -lc '
           set -euo pipefail
           pip install -q -e . --no-deps --break-system-packages
           for test_file in \
@@ -224,7 +224,7 @@ steps:
             value: vllm-config
           - label: "run-ci-megatron — up to 8 GPU, 21 runs"
             value: megatron
-          - label: "run-ci-vime-customized — 1–8 GPU, 6 tests"
+          - label: "run-ci-vime-customized — 1–8 GPU, 5 tests"
             value: vime-customized
           - label: "run-ci-precision — 8 GPU, 1 test"
             value: precision

.claude/skills/add-tests-and-ci/SKILL.md

@@ -37,23 +37,16 @@ if __name__ == "__main__":
     raise SystemExit(pytest.main([__file__]))
 ```
 
-- `run-ci-changed` extracts a top-level `NUM_GPUS = <N>` constant from added/modified `tests/test_*.py` and `tests/plugin_contracts/test_*.py`; if missing, it defaults to 8 GPUs. Set `NUM_GPUS = 0` for CPU-only tests.
+- Set `NUM_GPUS = 0` for CPU-only tests, following the existing test metadata convention.
 - For GPU/e2e tests, follow the nearby file pattern (`prepare()`, `execute()`, `NUM_GPUS`, and any model/dataset constants).
 
-### Step 3: Register Tests in GitHub CI
+### Step 3: Register Tests in Buildkite CI
 
-Whenever adding, moving, or renaming a test file, update the GitHub workflow template before finishing:
+Whenever adding, moving, or renaming a test file, update its Buildkite registration before finishing:
 
-1. Add the test to the appropriate matrix in `.github/workflows/pr-test.yml.j2`.
-   - CPU-only pytest/unit tests usually belong in `cpu-unittest` with `num_gpus: 0`.
-   - GPU/e2e tests should be placed beside the nearest similar model/path test with the matching `num_gpus` and environment fields.
-2. Regenerate workflows:
-
-```bash
-python .github/workflows/generate_github_workflows.py
-```
-
-3. Include both `.github/workflows/pr-test.yml.j2` and the generated `.github/workflows/pr-test.yml` in the change set.
+1. Register CPU test files in the appropriate command list in `.buildkite/pipeline.yml`, beside similar tests. Agent CPU tests belong in `agent-adapter`.
+2. Register GPU/e2e tests in `.buildkite/gpu_suites.py`, with the matching GPU count and environment settings. Update `.buildkite/pipeline.yml` when changing suite selection or wiring.
+3. Include the registration changes with the tests. These files are the source of truth; there is no GitHub workflow regeneration step.
 
 Only skip fixed matrix registration when the test is intentionally helper-only or manually invoked; state that reason in the final response.
 
@@ -64,34 +57,29 @@ Only skip fixed matrix registration when the test is intentionally helper-only o
 - Run repository-wide checks only when they are already part of the task or workflow.
 - Avoid documenting placeholder test commands that may not exist in the current tree.
 
-### Step 5: Keep Workflow Template as Source of Truth
+### Step 5: Keep Buildkite Sources in Sync
 
 For CI workflow changes unrelated to a new, moved, or renamed test:
 
-1. Edit `.github/workflows/pr-test.yml.j2`
-2. Regenerate workflows:
-
-```bash
-python .github/workflows/generate_github_workflows.py
-```
-
-3. Include both the template and generated workflow file in the change set (`.j2` and `.yml`). If the user asked for a commit, commit both.
+1. Edit `.buildkite/pipeline.yml` for always-on CPU commands and pipeline wiring.
+2. Edit `.buildkite/gpu_suites.py` for generated GPU jobs rather than editing its generated output.
+3. Keep suite definitions, selection, and `.buildkite/README.md` consistent when changing suites.
 
 ### Step 6: Provide Verifiable PR Notes
 
 Include:
 
 - Which tests were added/changed
-- Where each new/renamed test was registered in `.github/workflows/pr-test.yml.j2`
+- Where each new/renamed test was registered in `.buildkite/pipeline.yml` or `.buildkite/gpu_suites.py`
 - Exact commands executed
 - GPU assumptions for each test path
 - Why this coverage protects against regression
 
 ## Common Mistakes
 
-- Editing generated workflow file only
-- Relying on `run-ci-changed` discovery for a new test that should run in the regular PR matrix
-- Forgetting `NUM_GPUS = 0` on a CPU-only changed test, causing `run-ci-changed` to default to 8 GPUs
+- Editing generated GPU jobs instead of their source
+- Relying on pytest discovery for a new test in a suite with an explicit file list
+- Treating a green CPU build as GPU validation; GPU suites require the manual Buildkite gate
 - Adding a CPU pytest file that passes under `pytest tests/foo.py` but fails under CI's `python tests/foo.py`
 - Adding tests without following existing constants/conventions
 - Making tests too large or non-deterministic
@@ -101,5 +89,6 @@ Include:
 
 - Pytest config: `pyproject.toml`
 - Tests: `tests/`
-- CI template: `.github/workflows/pr-test.yml.j2`
+- CI sources: `.buildkite/pipeline.yml`, `.buildkite/gpu_suites.py`
+- Buildkite guide: `.buildkite/README.md`
 - CI guide: `docs/en/developer_guide/ci.md`

.claude/skills/release/SKILL.md

@@ -0,0 +1,77 @@
+---
+name: release
+description: Prepare and verify a vime release, including version bumps, stable Docker patch snapshots, Docker/conda dependency alignment, and release-specific validation. Use when cutting or auditing a vime release.
+---
+
+# Release vime
+
+Prepare a release without publishing, tagging, pushing images, or changing the
+dependency baseline unless the user explicitly requests those external or
+scope-expanding actions.
+
+## Establish the release baseline
+
+- Inspect the worktree and preserve unrelated user changes.
+- Compare the previous release tag and release commit to identify the current
+  repository conventions.
+- Confirm the requested vime version and the current stable vLLM version
+  from `docker/Dockerfile` and `docker/README.md`.
+- Do not pull in an unmerged vLLM/Docker upgrade merely because a newer
+  branch exists. Treat that as a separate decision.
+
+## Update release versions
+
+- Set the package version in `setup.py`.
+- Set the documentation version in `docs/conf.py`.
+- Bump `docker/version.txt` to a unique image version following its existing
+  dated naming convention.
+- Search the repository for the old vime version and review every remaining
+  occurrence instead of replacing unrelated dependency versions.
+
+## Freeze the stable Docker patches
+
+- Treat `docker/patch/latest/` as the patch stack for the current Docker base.
+- Snapshot it exactly into `docker/patch/<stable-vllm-version>/`. At release
+  time, the two directories must contain the same patch filenames and bytes.
+- Preserve older vLLM patch directories. Remove an obsolete file from the
+  current stable snapshot only after confirming it is absent from `latest`.
+- Verify the stable patch stack applies in Dockerfile order to clean checkouts
+  of the pinned vLLM and Megatron commits. Do not validate against a dirty
+  developer checkout.
+
+## Audit Docker and conda together
+
+Compare `build_conda.sh` with `docker/Dockerfile`, `docker/justfile`, and the
+stable patch snapshot. Check at least:
+
+- vLLM version, commit, CUDA variant, `sglang-kernel`, and `sgl-deep-gemm`;
+- Megatron, torch-memory-saver, FlashQLA, and other shared source pins;
+- `PATCH_VERSION`, patch filenames, application order, optional patches, and
+  failure-on-conflict behavior;
+- PyTorch, torchvision, torchaudio, CUDA Python, Transformer Engine, router,
+  NumPy, and SciPy pins;
+- whether dependency resolution can undo a compatibility pin later in the
+  script; reassert and validate such pins after the resolving install;
+- intentional differences such as conda being CUDA-12-only, omitting FA3, or
+  not rebuilding feature-specific DeepGEMM/DeepEP forks. Keep a difference
+  only when the release CI scope makes it intentional.
+
+Prefer direct loops over duplicated patch-application blocks while preserving
+required-versus-optional semantics and useful failure messages.
+
+## Validate before handoff
+
+Run the checks that are available locally:
+
+- `python .claude/skills/release/scripts/check_release.py --repo .
+  --expected-version <version>`;
+- `python setup.py --version`;
+- `bash -n build_conda.sh`;
+- byte-for-byte comparison of `docker/patch/latest/` and the stable snapshot;
+- patch parsing plus clean-checkout application against the pinned upstream
+  commits;
+- `git diff --check` and a final review of the complete release diff.
+
+Run the release conda CI and relevant Docker builds when the environment and
+requested scope permit. The conda workflow is selected by a PR title containing
+`[release]`. Explicitly report any full build or GPU validation that was not run.

.claude/skills/release/scripts/check_release.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+"""Check local vime release metadata and Docker/conda patch alignment."""
+
+import argparse
+import ast
+import re
+import sys
+from pathlib import Path
+
+
+def _setup_version(path: Path) -> str:
+    tree = ast.parse(path.read_text())
+    for node in ast.walk(tree):
+        if not isinstance(node, ast.Call) or getattr(node.func, "id", None) != "setup":
+            continue
+        for keyword in node.keywords:
+            if keyword.arg == "version":
+                return ast.literal_eval(keyword.value)
+    raise ValueError(f"setup version not found in {path}")
+
+
+def _assigned_string(path: Path, name: str) -> str:
+    tree = ast.parse(path.read_text())
+    for node in tree.body:
+        if not isinstance(node, ast.Assign):
+            continue
+        if any(isinstance(target, ast.Name) and target.id == name for target in node.targets):
+            return ast.literal_eval(node.value)
+    raise ValueError(f"{name} not found in {path}")
+
+
+def _shell_exports(text: str) -> dict[str, str]:
+    return dict(re.findall(r'^export ([A-Z][A-Z0-9_]*)="([^"]+)"$', text, re.MULTILINE))
+
+
+def _docker_args(text: str) -> dict[str, str]:
+    return dict(re.findall(r"^ARG ([A-Z][A-Z0-9_]*)=(\S+)$", text, re.MULTILINE))
+
+
+def _loop_items(text: str, variable: str) -> list[list[str]]:
+    return [items.split() for items in re.findall(rf"for {variable} in ([^;]+); do", text)]
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--repo", type=Path, default=Path.cwd())
+    parser.add_argument("--expected-version")
+    args = parser.parse_args()
+
+    repo = args.repo.resolve()
+    errors: list[str] = []
+
+    setup_version = _setup_version(repo / "setup.py")
+    docs_version = _assigned_string(repo / "docs/conf.py", "__version__")
+    if setup_version != docs_version:
+        errors.append(f"setup.py={setup_version} but docs/conf.py={docs_version}")
+    if args.expected_version and setup_version != args.expected_version:
+        errors.append(f"release version is {setup_version}, expected {args.expected_version}")
+
+    docker_text = (repo / "docker/Dockerfile").read_text()
+    conda_text = (repo / "build_conda.sh").read_text()
+    readme_text = (repo / "docker/README.md").read_text()
+    justfile_text = (repo / "docker/justfile").read_text()
+    docker_args = _docker_args(docker_text)
+    conda_exports = _shell_exports(conda_text)
+
+    image_tag = docker_args.get("VLLM_IMAGE_TAG", "")
+    docker_vllm_version = re.sub(r"-cu\d+$", "", image_tag)
+    conda_vllm_version = conda_exports.get("VLLM_VERSION", "")
+    if docker_vllm_version != conda_vllm_version:
+        errors.append(
+            f"Docker vLLM={docker_vllm_version or '<missing>'}, " f"conda vLLM={conda_vllm_version or '<missing>'}"
+        )
+
+    stable_match = re.search(r"current stable version is:\s*\n- vllm (v\S+)", readme_text)
+    readme_vllm_version = stable_match.group(1) if stable_match else ""
+    if readme_vllm_version != conda_vllm_version:
+        errors.append(
+            f"README stable vLLM={readme_vllm_version or '<missing>'}, "
+            f"conda vLLM={conda_vllm_version or '<missing>'}"
+        )
+
+    for tag in re.findall(r"VLLM_IMAGE_TAG=(v[^'\"\s]+)", justfile_text):
+        if re.sub(r"-cu\d+$", "", tag) != conda_vllm_version:
+            errors.append(f"docker/justfile uses inconsistent vLLM tag {tag}")
+
+    for pin in ("MEGATRON_COMMIT", "TMS_COMMIT", "FLASH_QLA_COMMIT"):
+        if docker_args.get(pin) != conda_exports.get(pin):
+            errors.append(
+                f"{pin}: Docker={docker_args.get(pin, '<missing>')}, conda={conda_exports.get(pin, '<missing>')}"
+            )
+
+    patch_version = conda_exports.get("PATCH_VERSION", "")
+    if patch_version != conda_vllm_version:
+        errors.append(f"PATCH_VERSION={patch_version or '<missing>'}, expected {conda_vllm_version}")
+
+    latest_dir = repo / "docker/patch/latest"
+    stable_dir = repo / f"docker/patch/{patch_version}"
+    latest = {path.name: path.read_bytes() for path in latest_dir.glob("*.patch")}
+    stable = {path.name: path.read_bytes() for path in stable_dir.glob("*.patch")}
+    if latest.keys() != stable.keys():
+        errors.append(
+            "stable patch filenames differ from latest: "
+            f"only_latest={sorted(latest.keys() - stable.keys())}, "
+            f"only_stable={sorted(stable.keys() - latest.keys())}"
+        )
+    for name in latest.keys() & stable.keys():
+        if latest[name] != stable[name]:
+            errors.append(f"stable patch differs from latest: {name}")
+
+    docker_vllm_loops = _loop_items(docker_text, "patch")
+    conda_loops = _loop_items(conda_text, "patch_name")
+    docker_vllm_order = docker_vllm_loops[0] if docker_vllm_loops else []
+    conda_vllm_order = conda_loops[0] if conda_loops else []
+    expected_vllm = {name for name in latest if name.startswith("vllm")}
+    if docker_vllm_order != conda_vllm_order:
+        errors.append("Docker and conda vLLM patch order differs")
+    if set(docker_vllm_order) != expected_vllm:
+        errors.append("Docker/conda vLLM patch loop does not cover the latest patch set")
+
+    docker_megatron_order = re.findall(r"git apply (megatron[^ ]*\.patch)", docker_text)
+    conda_megatron_order = conda_loops[1] if len(conda_loops) > 1 else []
+    expected_megatron = {name for name in latest if name.startswith("megatron")}
+    if docker_megatron_order != conda_megatron_order:
+        errors.append("Docker and conda Megatron patch order differs")
+    if set(docker_megatron_order) != expected_megatron:
+        errors.append("Docker/conda Megatron patch logic does not cover the latest patch set")
+
+    docker_version = (repo / "docker/version.txt").read_text().strip()
+    if not re.fullmatch(r"nightly-dev-\d{8}[a-z]", docker_version):
+        errors.append(f"unexpected docker/version.txt format: {docker_version}")
+
+    if errors:
+        for error in errors:
+            print(f"ERROR: {error}", file=sys.stderr)
+        return 1
+
+    print(f"release={setup_version}, vllm={conda_vllm_version}, docker={docker_version}, patches={len(latest)}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())