diff --git a/skills.yaml b/skills.yaml index e4838dc..0464bfd 100644 --- a/skills.yaml +++ b/skills.yaml @@ -313,7 +313,7 @@ - name: vllm-xpu-run maintainer: "xaleryb" external-repo: "https://github.com/intel/gpu-ai-skills" - external-commit: "0b4fafd09c5eb4cc5daf532d915ef5984a919775" + external-commit: "6e7bce819e2e44a4aac692b0a49253f39c0f6957" external-path: "plugins/intel-gpu-ai-skills/skills/vllm-xpu-run" external-license: "Apache-2.0" intel-products: "Intel GPU" diff --git a/skills/vllm-xpu-run/.source.json b/skills/vllm-xpu-run/.source.json index 301b1e5..56bafd1 100644 --- a/skills/vllm-xpu-run/.source.json +++ b/skills/vllm-xpu-run/.source.json @@ -1,6 +1,6 @@ { "repo": "https://github.com/intel/gpu-ai-skills", "path": "plugins/intel-gpu-ai-skills/skills/vllm-xpu-run", - "commit": "0b4fafd09c5eb4cc5daf532d915ef5984a919775", + "commit": "6e7bce819e2e44a4aac692b0a49253f39c0f6957", "license": "Apache-2.0" } diff --git a/skills/vllm-xpu-run/SKILL.md b/skills/vllm-xpu-run/SKILL.md index 63b3380..9d10068 100644 --- a/skills/vllm-xpu-run/SKILL.md +++ b/skills/vllm-xpu-run/SKILL.md @@ -51,7 +51,7 @@ general vLLM, CUDA, or Transformers support. | `vllm/vllm-openai:latest` | `vllm/vllm-openai-xpu:latest` | | `--gpus all` | `--device /dev/dri` + `-v /dev/dri/by-path:/dev/dri/by-path:ro` + `--ipc=host` | | `--dtype auto` | `--dtype bfloat16` (explicit) | -| CUDA graphs default | `--enforce-eager` | +| CUDA graphs default | graphs also capture by default from vLLM v0.31.0 — emit nothing. Add `--enforce-eager` only for pre-v0.31.0 images. | | `--tensor-parallel-size N` | same; pin N XPUs in `ZE_AFFINITY_MASK` | ## Quickstart (single GPU) @@ -83,7 +83,6 @@ docker run -d --name vllm-xpu \ vllm/vllm-openai-xpu:latest \ Qwen/Qwen2.5-1.5B-Instruct \ --dtype bfloat16 \ - --enforce-eager \ --block-size=64 \ --max-model-len 4096 \ --gpu-memory-utilization 0.85 @@ -114,7 +113,7 @@ remote-box workflow), see `references/remote-deploy.md`. | Flag | Why | |---|---| | `--dtype bfloat16` | Battlemage runs bf16 better than fp16; `auto` may pick fp16 from the checkpoint config. Set for unquantized serving only — combining with `--quantization` produces conflicts. | -| `--enforce-eager` | Conservative default. Graph capture / `torch.compile` on XPU is experimental. After a stable eager run, drop it and re-bench; keep only if TPOT/TTFT improve and content stays correct. | +| `--enforce-eager` | **Do not emit this on vLLM v0.31.0 or newer.** There, XPU graphs capture by default and `VLLM_XPU_ENABLE_XPU_GRAPH` no longer exists, so the flag disables exactly what the version enables — measured cost roughly 2x decode (TPOT 64 ms vs 36 ms, 4x Arc Pro B70 at concurrency 8). It does not change TTFT, so eager is never the explanation for a slow first token. Emit it only for pre-v0.31.0 images, where graph capture was experimental, or to isolate a suspected graph-capture bug. Check the image first: `docker run --rm --entrypoint python3 -c "import vllm; print(vllm.__version__)"`. | | `--block-size=64` | Validated default for the XPU paged-attention path on Battlemage. Bench higher (128, 256) once 64 is correct. | | `--gpu-memory-utilization 0.85` | Default 0.92 fails on workstations with active GUI sessions. Drop to 0.70 with browsers open; raise to 0.92 on headless servers. | | `-v /dev/dri/by-path:/dev/dri/by-path:ro` | Some oneCCL/device-discovery configurations scan the host's `/dev/dri/by-path` symlinks even for a single-GPU launch. Include this read-only mount to support those configurations; if it is omitted, affected images can abort at engine initialization with `opendir failed: could not open device directory`. | diff --git a/skills/vllm-xpu-run/references/multi-gpu-and-tuning.md b/skills/vllm-xpu-run/references/multi-gpu-and-tuning.md index 646e775..bf0ccd1 100644 --- a/skills/vllm-xpu-run/references/multi-gpu-and-tuning.md +++ b/skills/vllm-xpu-run/references/multi-gpu-and-tuning.md @@ -39,7 +39,6 @@ docker run -d --name vllm-xpu \ vllm/vllm-openai-xpu:latest \ \ --dtype bfloat16 \ - --enforce-eager \ --block-size=64 \ --disable-custom-all-reduce \ --tensor-parallel-size "$TP" diff --git a/skills/vllm-xpu-run/references/remote-deploy.md b/skills/vllm-xpu-run/references/remote-deploy.md index af935db..93cdb43 100644 --- a/skills/vllm-xpu-run/references/remote-deploy.md +++ b/skills/vllm-xpu-run/references/remote-deploy.md @@ -38,7 +38,6 @@ ssh "$GPU_HOST" 'docker run -d --name vllm-xpu \ vllm/vllm-openai-xpu:latest \ \ --dtype bfloat16 \ - --enforce-eager \ --block-size=64 \ --max-model-len 4096 \ --gpu-memory-utilization 0.85' diff --git a/skills/vllm-xpu-run/scripts/emit_launch.sh b/skills/vllm-xpu-run/scripts/emit_launch.sh index c558f71..8596296 100644 --- a/skills/vllm-xpu-run/scripts/emit_launch.sh +++ b/skills/vllm-xpu-run/scripts/emit_launch.sh @@ -20,6 +20,7 @@ image="${VLLM_XPU_IMAGE:-vllm/vllm-openai-xpu:latest}" port=8000 container_name="vllm-xpu" trust_remote_code=0 +enforce_eager=0 usage() { cat <<'EOF' @@ -38,6 +39,9 @@ Options: --port N First host port to publish (default: 8000) --container-name N Base container name (default: vllm-xpu) --trust-remote-code Add --trust-remote-code + --enforce-eager Add --enforce-eager, disabling XPU graph capture. + Only for images before vLLM v0.31.0, where graph + capture was experimental. Omitted by default. -h, --help Show this help EOF } @@ -72,6 +76,7 @@ while [ "$#" -gt 0 ]; do --port) port="${2:-}"; shift 2 ;; --container-name) container_name="${2:-}"; shift 2 ;; --trust-remote-code) trust_remote_code=1; shift ;; + --enforce-eager) enforce_eager=1; shift ;; -h|--help) usage; exit 0 ;; *) die "unknown argument: $1" ;; esac @@ -165,11 +170,21 @@ if [ "$required_xpus" -gt "$num_xpus" ]; then die "topology requires $required_xpus XPU(s), but --num-xpus is $num_xpus" fi -serve_flags="--enforce-eager \\ - --block-size=64 \\ +serve_flags="--block-size=64 \\ --max-model-len $ctx \\ --gpu-memory-utilization 0.85" +# XPU graph capture is left on by default. From vLLM v0.31.0 graphs capture +# automatically and VLLM_XPU_ENABLE_XPU_GRAPH no longer exists; --enforce-eager +# turns off exactly what that version enables, costing roughly 2x decode +# (measured TPOT 64 ms vs 36 ms, 4x Arc Pro B70 at concurrency 8). It does not +# change TTFT, so eager is never the explanation for a slow first token. Pass +# --enforce-eager only for pre-v0.31.0 images, where capture was experimental. +if [ "$enforce_eager" -eq 1 ]; then + serve_flags="--enforce-eager \\ + $serve_flags" +fi + case "$quant" in bf16|fp16) serve_flags="--dtype bfloat16 \\