diff --git a/docker/cmp170hx/Dockerfile b/docker/cmp170hx/Dockerfile new file mode 100644 index 000000000000..0282a3271601 --- /dev/null +++ b/docker/cmp170hx/Dockerfile @@ -0,0 +1,108 @@ +# vLLM (Morrowmake/vllm-cmp170hx, branch ampere-glm53) for GLM-5.3-Flash W4A16 +# on 4x NVIDIA CMP 170HX, with optional sm_80 Marlin. +# +# The engine is installed the way the recipe's native install does it: Python +# 3.12, torch 2.13.0 (CUDA 13.0 build), the fork checked out at the pinned +# commit and installed editable with upstream's precompiled extensions +# (VLLM_USE_PRECOMPILED=1), then the runtime extras the engine pins in +# requirements/cuda.txt (FlashInfer 0.7.0, humming-kernels 0.1.16, TileLang +# 0.1.12). The pin sits on upstream e55d076f89, which has no nightly wheel of +# its own; the extensions come from the wheel of b6761e8ded, whose C++, CUDA +# and Rust sources are identical to e55d076f89 (the two commits between them +# are Python and tests). Model weights are not in the image; mount them (see +# README.md in this folder for the run command). +# +# The CUDA 13.3.1 devel base matches the toolkit the release was validated +# with (Triton, TileLang, FlashInfer and the host-memory all-reduce extension +# compile against it at runtime, so nvcc and g++ must be present). +# NVIDIA_REQUIRE_CUDA is relaxed to cuda>=13.0 so the container starts on any +# driver 580 or newer, the same requirement as the recipe. +# +# Entry point: the vllm CLI. The recipe runs the image with its own serve.sh +# mounted and used as the entry point, so the layout (LAYOUT=tp4 or pp4) and +# every setting are chosen at run time (container.env in this folder). +# +# docker build -t vllm-cmp170hx:1.6.0-3a2bf16dae docker/ + +ARG BASE=nvidia/cuda:13.3.1-devel-ubuntu24.04@sha256:4ff859525f99de5782aa73607ce24219b07dddd48d12b97c1c301d7e1cfb0a87 + +ARG VLLM_COMMIT=3a2bf16dae8b97f5ff2c7e9bc5809d24545e6340 + +FROM ${BASE} AS build +ARG VLLM_REPO=https://github.com/Morrowmake/vllm-cmp170hx.git +ARG VLLM_BRANCH=ampere-glm53 +ARG VLLM_COMMIT +ARG VLLM_WHEEL_COMMIT=b6761e8ded57ef85b708f34af8cab1649eae1069 +ARG PYTHON_VERSION=3.12 +ARG UV_VERSION=0.12.11 +ENV UV_LINK_MODE=copy \ + UV_PYTHON_INSTALL_DIR=/opt/python-dl \ + PYTHONDONTWRITEBYTECODE=1 \ + CUDA_HOME=/usr/local/cuda +RUN apt-get update \ + && apt-get install -y --no-install-recommends git ca-certificates curl \ + && rm -rf /var/lib/apt/lists/* +RUN curl -LsSf "https://astral.sh/uv/${UV_VERSION}/install.sh" | env UV_INSTALL_DIR=/usr/local/bin sh +RUN uv python install "${PYTHON_VERSION}" \ + && mv /opt/python-dl/cpython-${PYTHON_VERSION}.*-linux-x86_64-gnu /opt/python \ + && rm -rf /opt/python-dl \ + && grep -rlI --null /opt/python-dl/ /opt/python \ + | xargs -0 -r sed -i -E "s#/opt/python-dl/cpython-[^/\"' ]+#/opt/python#g" \ + && uv venv --relocatable --python /opt/python/bin/python${PYTHON_VERSION} /opt/venv +# Full transport clone: version construction and wheel lookup need history. +RUN git clone --no-local --no-tags --single-branch --branch "${VLLM_BRANCH}" "${VLLM_REPO}" /opt/vllm-src \ + && git -C /opt/vllm-src checkout -B "${VLLM_BRANCH}" "${VLLM_COMMIT}" \ + && git -C /opt/vllm-src update-ref "refs/remotes/origin/${VLLM_BRANCH}" "${VLLM_COMMIT}" \ + && git -C /opt/vllm-src fetch --no-tags https://github.com/vllm-project/vllm.git \ + refs/tags/v0.30.1rc0:refs/tags/v0.30.1rc0 \ + && git -C /opt/vllm-src merge-base --is-ancestor v0.30.1rc0 HEAD \ + && git -C /opt/vllm-src submodule update --init --recursive --depth 1 \ + && git -C /opt/vllm-src reflog expire --expire=now --all \ + && git -C /opt/vllm-src repack -ad \ + && git -C /opt/vllm-src prune --expire=now +COPY runtime_tree.py /tmp/runtime_tree.py +ENV VIRTUAL_ENV=/opt/venv +RUN uv pip install --extra-index-url https://flashinfer.ai/whl/ \ + "torch==2.13.0" "torchvision==0.28.0" "torchaudio==2.11.0" +RUN VLLM_BUILD_AMPERE_MARLIN=0 VLLM_USE_PRECOMPILED=1 VLLM_PRECOMPILED_WHEEL_COMMIT="${VLLM_WHEEL_COMMIT}" \ + VLLM_PRECOMPILED_WHEEL_VARIANT=cu130 \ + uv pip install --extra-index-url https://flashinfer.ai/whl/ -e /opt/vllm-src +RUN uv pip install --extra-index-url https://flashinfer.ai/whl/ \ + -r /opt/vllm-src/requirements/cuda.txt \ + "torch==2.13.0" "torchvision==0.28.0" "torchaudio==2.11.0" \ + "flashinfer-python==0.7.0" "flashinfer-cubin==0.7.0" \ + "tilelang==0.1.12" "apache-tvm-ffi==0.1.11" "ninja" "huggingface_hub[hf_xet]>=1.0" \ + && uv pip freeze > /opt/venv/requirements.lock.txt +RUN test -f /opt/vllm-src/csrc/libtorch_stable/moe/ampere_marlin/build_standalone.py \ + || (echo "Set VLLM_REPO, VLLM_BRANCH and VLLM_COMMIT to a source with optional Marlin support" >&2; exit 1) +RUN cd /opt/vllm-src && CUDA_VISIBLE_DEVICES= VLLM_BUILD_AMPERE_MARLIN=1 \ + PATH=/opt/venv/bin:$PATH /opt/venv/bin/python \ + csrc/libtorch_stable/moe/ampere_marlin/build_standalone.py --out /opt/vllm-src/vllm \ + --build-dir /tmp/ampere-marlin-build \ + && rm -rf /tmp/ampere-marlin-build +RUN CUDA_VISIBLE_DEVICES= /opt/venv/bin/python -c "from vllm.ampere_marlin import require_extension; require_extension()" +RUN mkdir -p /tmp/cudastub && ln -sf /usr/local/cuda/lib64/stubs/libcuda.so /tmp/cudastub/libcuda.so.1 && LD_LIBRARY_PATH=/tmp/cudastub CUDA_VISIBLE_DEVICES= /opt/venv/bin/python -c "import torch, vllm, vllm._C_stable_libtorch, vllm._moe_C_stable_libtorch, vllm._custom_ops, triton, tilelang, flashinfer, humming; from vllm.v1.attention.backends.mla import triton_mla_sparse; from vllm.v1.attention.ops import triton_mqa_logits, triton_e4m3; from vllm.v1.worker.gpu import prologue_fuse; from vllm.distributed.device_communicators import host_shm_all_reduce; print(vllm.__version__, torch.__version__, torch.version.cuda)" \ + && find /opt/python /opt/venv /opt/vllm-src -name __pycache__ -type d -prune -exec rm -rf {} + \ + && /opt/venv/bin/python /tmp/runtime_tree.py finalize /opt "${VLLM_COMMIT}" \ + && /opt/venv/bin/python /tmp/runtime_tree.py check /opt "${VLLM_COMMIT}" + +FROM ${BASE} +ARG VLLM_COMMIT +COPY --from=build /opt/python /opt/python +COPY --from=build /opt/venv /opt/venv +COPY --from=build /opt/vllm-src /opt/vllm-src +ENV PATH=/opt/venv/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ + VIRTUAL_ENV=/opt/venv \ + CUDA_HOME=/usr/local/cuda \ + NVIDIA_REQUIRE_CUDA=cuda>=13.0 +LABEL org.opencontainers.image.title="vllm-cmp170hx" \ + org.opencontainers.image.description="vLLM fork for GLM-5.3-Flash W4A16 on 4x NVIDIA CMP 170HX (sm_80) with optional sm_80 Marlin; weights not included" \ + org.opencontainers.image.source="https://github.com/Morrowmake/vllm-cmp170hx" \ + org.opencontainers.image.revision="${VLLM_COMMIT}" \ + org.opencontainers.image.url="https://github.com/Morrowmake/glm53-flash-cmp170hx-recipe" \ + org.opencontainers.image.version="1.6.0" \ + org.opencontainers.image.licenses="Apache-2.0" \ + org.opencontainers.image.base.name="docker.io/nvidia/cuda:13.3.1-devel-ubuntu24.04" \ + org.opencontainers.image.base.digest="sha256:4ff859525f99de5782aa73607ce24219b07dddd48d12b97c1c301d7e1cfb0a87" +EXPOSE 8000 +ENTRYPOINT ["/opt/venv/bin/vllm"] diff --git a/docker/cmp170hx/Dockerfile.overlay b/docker/cmp170hx/Dockerfile.overlay new file mode 100644 index 000000000000..5cb2f2c06f94 --- /dev/null +++ b/docker/cmp170hx/Dockerfile.overlay @@ -0,0 +1,10 @@ +# Overlay for the copy-drafts build measured in PixelML/club-170hx (2026-10-02). +# Build context: a checkout of PixelML/sm80vllm at c93c274c86543d513c67243859480aeec46bc951. +# BASE_IMAGE: the full build of 127c6f0761b2c81d2c0ba7e2d8a5cf71e3247903 (see README.md). +# c93c274c8 changes only these four Python files relative to 127c6f076. +ARG BASE_IMAGE=sm80vllm-cmp170hx:127c6f076 +FROM ${BASE_IMAGE} +COPY tests/v1/spec_decode/test_glm5_copy_drafts.py /opt/vllm-src/tests/v1/spec_decode/test_glm5_copy_drafts.py +COPY vllm/envs.py /opt/vllm-src/vllm/envs.py +COPY vllm/v1/worker/gpu/model_runner.py /opt/vllm-src/vllm/v1/worker/gpu/model_runner.py +COPY vllm/v1/worker/gpu/spec_decode/copy_drafts.py /opt/vllm-src/vllm/v1/worker/gpu/spec_decode/copy_drafts.py diff --git a/docker/cmp170hx/README.md b/docker/cmp170hx/README.md new file mode 100644 index 000000000000..f2ff516d2519 --- /dev/null +++ b/docker/cmp170hx/README.md @@ -0,0 +1,50 @@ +# sm80vllm container for 4 × CMP 170HX (GLM-5.3-Flash W4A16) + +Pinned build recipe for the images used in the +[PixelML/club-170hx](https://github.com/PixelML/club-170hx) GLM-5.3-Flash notebooks. + +## Pull (exact bits that were measured) + +| image | what it is | digest | +|---|---|---| +| `ghcr.io/pixelml/sm80vllm:tf-learnings-c93c274c8` | engine + copy drafts (`VLLM_GLM5_COPY_DRAFTS`) | `sha256:f0dbb483b000f85ac66adb6f190c11d1af395914df67a0d9b7e54dfe2258fa86` | +| `ghcr.io/pixelml/sm80vllm:tf-learnings-127c6f076` | engine only (Morrowmake `mm/ampere` @ `3a2bf16da` merged into the DSV4 branch) | `sha256:704cbb8841be7c99bd1d5d6a8136fc118ca22cfc85457ef4a669a07ef957e8ff` | + +Always pull by digest: `docker pull ghcr.io/pixelml/sm80vllm@sha256:...`. + +## Build it yourself + +Every input is pinned: the CUDA base image by digest, the engine source by full commit, +the precompiled-extension wheel by full commit (see the header of `Dockerfile`). + +```bash +# 1. Engine image at 127c6f0761b2c81d2c0ba7e2d8a5cf71e3247903 +git clone https://github.com/PixelML/sm80vllm.git && cd sm80vllm +docker build -f docker/cmp170hx/Dockerfile \ + --build-arg VLLM_REPO=https://github.com/PixelML/sm80vllm.git \ + --build-arg VLLM_BRANCH=glm53-sm80-tf-learnings \ + --build-arg VLLM_COMMIT=127c6f0761b2c81d2c0ba7e2d8a5cf71e3247903 \ + -t sm80vllm-cmp170hx:127c6f076 docker/cmp170hx + +# 2. Copy-drafts overlay at c93c274c86543d513c67243859480aeec46bc951 (four Python files) +git checkout c93c274c86543d513c67243859480aeec46bc951 +docker build -f docker/cmp170hx/Dockerfile.overlay \ + --build-arg BASE_IMAGE=sm80vllm-cmp170hx:127c6f076 \ + -t sm80vllm-cmp170hx:c93c274c8 . +``` + +A self-built image is functionally the same but not byte-identical (build timestamps). +Use the published digests when you need the exact measured bits. + +## Run + +The image is a drop-in for the `IMAGE=` line of +[Morrowmake/glm53-flash-cmp170hx-recipe](https://github.com/Morrowmake/glm53-flash-cmp170hx-recipe) +v1.6.0 (`a242b4f`): set `IMAGE=ghcr.io/pixelml/sm80vllm@sha256:...` in the recipe's `.env`. +Copy drafts are off unless `VLLM_GLM5_COPY_DRAFTS=1`. + +## Credits and licences + +`Dockerfile` and `runtime_tree.py` are from the Morrowmake recipe (MIT) and build +[Morrowmake/vllm-cmp170hx](https://github.com/Morrowmake/vllm-cmp170hx) (Apache-2.0); +this repository's changes are Apache-2.0. Model weights are not in the image. diff --git a/docker/cmp170hx/runtime_tree.py b/docker/cmp170hx/runtime_tree.py new file mode 100644 index 000000000000..a003b7e7212c --- /dev/null +++ b/docker/cmp170hx/runtime_tree.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python3 +"""Construct and check the Git-free runtime payload before layer creation.""" +import argparse +import bz2 +import gzip +import lzma +import tarfile +import tempfile +import zipfile +import ast +import json +import os +from pathlib import Path +import re +import shutil +import subprocess + + +def git_integrity(root, pin): + def git(*args): + return subprocess.check_output(["git", "-C", str(root), *args], text=True).strip() + if git("rev-parse", "HEAD") != pin: + raise ValueError("Source HEAD differs from the image pin") + stores = [root / ".git"] + modules = root / ".git/modules" + if modules.exists(): + stores.extend(p.parent for p in modules.rglob("HEAD") if (p.parent / "objects").is_dir()) + for store in stores: + if not store.is_dir() or (store / "objects/info/alternates").exists(): + raise ValueError("Git object store must be self-contained") + result = subprocess.run(["git", "--git-dir", str(store), "fsck", "--full", + "--unreachable", "--no-reflogs"], capture_output=True, text=True) + if result.returncode or result.stdout.strip() or result.stderr.strip(): + raise ValueError("Git object integrity/reachability check failed") + + +def version(root): + # The installed version was generated while Git was available. Do not + # recompute it after removing history, or import the editable installation. + tree = ast.parse((root / "vllm/_version.py").read_text()) + for node in tree.body: + if isinstance(node, ast.Assign) and any(isinstance(t, ast.Name) and t.id == "__version__" for t in node.targets): + return ast.literal_eval(node.value) + raise ValueError("Installed engine version is missing") + + + +def scan_stream(stream, needles, label, depth=0): + """Scan bytes and standard nested archives, without extracting to the tree.""" + if depth > 12: + raise ValueError("Archive nesting limit exceeded") + overlap = max(map(len, needles), default=1) - 1 + previous = b"" + for chunk in iter(lambda: stream.read(4 * 1024 * 1024), b""): + data = previous + chunk.lower() + if any(n in data for n in needles): + raise ValueError("Forbidden identifier in runtime file: " + label) + previous = data[-overlap:] if overlap else b"" + stream.seek(0) + header = stream.read(512) + stream.seek(0) + compressed = None + if header.startswith(b"\x1f\x8b"): + compressed = gzip.GzipFile(fileobj=stream) + elif header.startswith(b"BZh"): + compressed = bz2.BZ2File(stream) + elif header.startswith(b"\xfd7zXZ\x00"): + compressed = lzma.LZMAFile(stream) + if compressed is not None: + with compressed, tempfile.TemporaryFile() as expanded: + shutil.copyfileobj(compressed, expanded, 4 * 1024 * 1024) + expanded.seek(0) + scan_stream(expanded, needles, label + "!compressed", depth + 1) + elif header.startswith(b"PK\x03\x04"): + with zipfile.ZipFile(stream) as archive: + check_stores(m.filename for m in archive.infolist()) + for member in archive.infolist(): + check_member(member.filename, needles) + if not member.is_dir(): + with archive.open(member) as source, tempfile.TemporaryFile() as expanded: + shutil.copyfileobj(source, expanded, 4 * 1024 * 1024) + expanded.seek(0) + scan_stream(expanded, needles, label + "!" + member.filename, depth + 1) + elif header[257:262] == b"ustar": + with tarfile.open(fileobj=stream, mode="r:") as archive: + members = set() + for member in archive: + check_member(member.name, needles) + check_member(member.linkname, needles) + members.add(member.name.rstrip("/")) + if member.isfile(): + with archive.extractfile(member) as source, tempfile.TemporaryFile() as expanded: + shutil.copyfileobj(source, expanded, 4 * 1024 * 1024) + expanded.seek(0) + scan_stream(expanded, needles, label + "!" + member.name, depth + 1) + check_stores(members) + + +def check_stores(names): + paths = {Path(name) for name in names} + heads = {p.parent for p in paths if p.name == "HEAD"} + roots = {parent.parent for p in paths for parent in (p, *p.parents) + if parent.name == "objects"} + if heads & roots: + raise ValueError("Retained archived Git object store") + + +def check_member(name, needles): + parts = Path(name).parts + if ".git" in parts or any(p.startswith("pack-") and p.endswith((".pack", ".idx")) for p in parts): + raise ValueError("Retained archived Git metadata") + if any(n in name.lower().encode() for n in needles): + raise ValueError("Forbidden identifier in archive path") + +def check(opt, pin, forbidden): + root = opt / "vllm-src" + record = json.loads((root / "provenance.json").read_text()) + expected = dict(schema=1, engine_commit=pin, + engine_repository="https://github.com/Morrowmake/vllm-cmp170hx.git", + engine_version=version(root)) + if record != expected or (root / "provenance.json").stat().st_mode & 0o222: + raise ValueError("Runtime provenance mismatch or writable provenance") + needles = [s.encode().lower() for s in forbidden if s] + for directory, dirs, files in os.walk(opt, followlinks=False): + parent = Path(directory) + if {"HEAD", "objects"} <= set(dirs + files) or any(name.endswith((".pack", ".idx")) and name.startswith("pack-") for name in files): + raise ValueError("Retained Git object store") + for name in dirs + files: + path = parent / name + if name == ".git": + raise ValueError("Retained Git metadata") + metadata = str(path.relative_to(opt)) + if path.is_symlink(): + metadata += "\n" + os.readlink(path) + if any(n in metadata.lower().encode() for n in needles): + raise ValueError("Forbidden identifier in runtime path") + if path.is_file() and not path.is_symlink(): + with path.open("rb") as stream: + scan_stream(stream, needles, str(path.relative_to(opt))) + return record + + +def finalize(opt, pin, forbidden): + root = opt / "vllm-src" + git_integrity(root, pin) + record = dict(schema=1, engine_commit=pin, + engine_repository="https://github.com/Morrowmake/vllm-cmp170hx.git", + engine_version=version(root)) + for directory, dirs, files in os.walk(opt, topdown=True, followlinks=False): + if ".git" in dirs: + path = Path(directory) / ".git" + if path.is_symlink(): + path.unlink() + else: + shutil.rmtree(path) + dirs.remove(".git") + if ".git" in files: + (Path(directory) / ".git").unlink() + target = root / "provenance.json" + target.write_text(json.dumps(record, sort_keys=True) + "\n") + target.chmod(0o444) + return check(opt, pin, forbidden) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("mode", choices=("finalize", "check")) + parser.add_argument("opt", type=Path) + parser.add_argument("pin") + parser.add_argument("--forbid", action="append", default=[]) + args = parser.parse_args() + if not re.fullmatch(r"[0-9a-f]{40}", args.pin): + parser.error("pin must be a full lowercase commit hash") + forbidden = args.forbid + json.loads(os.environ.get("IMAGE_FORBIDDEN_STRINGS", "[]")) + action = finalize if args.mode == "finalize" else check + print(json.dumps(action(args.opt, args.pin, forbidden), sort_keys=True)) + + +if __name__ == "__main__": + main()