From 614ee6cf587af3cd3d2674d478985410b1904f82 Mon Sep 17 00:00:00 2001 From: KrystalRay Date: Mon, 2 Nov 2026 19:15:09 +0800 Subject: [PATCH 01/20] feat: bootstrap synchronous Ouro execution on Ascend NPU --- tests/test_device.py | 55 +++++++++++++++++++++++++++++++ vllm_rlt/core/kv_cache_manager.py | 3 +- vllm_rlt/core/memory.py | 2 ++ vllm_rlt/device.py | 13 ++++++++ vllm_rlt/engine/stage_engine.py | 24 ++++++++++++++ vllm_rlt/engine/worker.py | 7 +++- vllm_rlt/models/ouro.py | 2 ++ vllm_rlt/worker/model_runner.py | 10 ++++++ 8 files changed, 114 insertions(+), 2 deletions(-) create mode 100644 tests/test_device.py create mode 100644 vllm_rlt/device.py diff --git a/tests/test_device.py b/tests/test_device.py new file mode 100644 index 00000000..fa0cd828 --- /dev/null +++ b/tests/test_device.py @@ -0,0 +1,55 @@ +"""Device bootstrap and fail-fast NPU scope without a device dependency.""" + +import importlib +from types import SimpleNamespace + +import pytest +import torch + +from tests.helpers import tiny_ouro_config +from vllm_rlt import CacheConfig, ExecutionConfig, ExitConfig, SchedulerConfig +from vllm_rlt.core.memory import plan_cache +from vllm_rlt.device import resolve_device +from vllm_rlt.engine.stage_engine import StageEngine + + +def test_cpu_device_resolution_does_not_import_optional_backend(monkeypatch): + monkeypatch.setattr( + importlib, "import_module", lambda name: pytest.fail(f"unexpected import: {name}") + ) + assert resolve_device("cpu") == torch.device("cpu") + assert resolve_device("cuda:2") == torch.device("cuda:2") + + +def npu_model_stub(**overrides): + parameter = SimpleNamespace( + device=SimpleNamespace(type="npu"), dtype=torch.bfloat16, element_size=lambda: 2 + ) + return SimpleNamespace( + parameters=lambda: iter([parameter]), config=tiny_ouro_config(**overrides) + ) + + +@pytest.mark.parametrize( + "options", + [ + dict(execution_config=ExecutionConfig(async_scheduling=True)), + dict(execution_config=ExecutionConfig(cuda_graphs=True)), + dict(cache_config=CacheConfig(num_blocks=32, enable_prefix_caching=True)), + dict(scheduler_config=SchedulerConfig(enable_preemption=True)), + dict(exit_config=ExitConfig(mode="ouro_delayed")), + ], +) +def test_npu_unsupported_execution_rejected_before_allocation(options): + with pytest.raises(ValueError, match="NPU Ouro requires synchronous"): + StageEngine(npu_model_stub(), **options) + + +def test_npu_does_not_use_cpu_default_memory_budget(): + model = npu_model_stub() + with pytest.raises(ValueError, match="explicit num_blocks"): + plan_cache(model, CacheConfig(), SchedulerConfig(), ExecutionConfig(), "ascend") + blocks, info = plan_cache( + model, CacheConfig(num_blocks=17), SchedulerConfig(), ExecutionConfig(), "ascend" + ) + assert blocks == 17 and info["source"] == "blocks" diff --git a/vllm_rlt/core/kv_cache_manager.py b/vllm_rlt/core/kv_cache_manager.py index dfcdf0df..b0b28be3 100644 --- a/vllm_rlt/core/kv_cache_manager.py +++ b/vllm_rlt/core/kv_cache_manager.py @@ -23,6 +23,7 @@ create_backend, validate_backend_name, ) +from vllm_rlt.device import resolve_device from vllm_rlt.request import ExecutionBatch from vllm_rlt.worker.buffers import stage_metadata from vllm_rlt.worker.metadata import DeviceKVMetadata, KVMetadata, MetadataBuilder @@ -133,7 +134,7 @@ def __init__( ): raise ValueError("recurrent_layers must be valid cache layer indices") self.storage_depths = max_loops if layout == "last_exited" else 1 - self.device = torch.device(device) + self.device = resolve_device(device) self.dtype = dtype self.attention = attention or create_backend( backend, self.device, dtype, head_dim, block_size diff --git a/vllm_rlt/core/memory.py b/vllm_rlt/core/memory.py index e9d2d4cf..21beea57 100644 --- a/vllm_rlt/core/memory.py +++ b/vllm_rlt/core/memory.py @@ -69,6 +69,8 @@ def plan_cache( if cache.kv_cache_memory_bytes is not None: blocks = budget_blocks(cache.kv_cache_memory_bytes, per_block) return blocks, {"source": "bytes", "bytes_per_block": per_block} + if device.type == "npu": + raise ValueError("NPU requires explicit num_blocks or kv_cache_memory_bytes") if device.type != "cuda": return 256, {"source": "cpu_default", "bytes_per_block": per_block} torch.cuda.synchronize(device) diff --git a/vllm_rlt/device.py b/vllm_rlt/device.py new file mode 100644 index 00000000..41244158 --- /dev/null +++ b/vllm_rlt/device.py @@ -0,0 +1,13 @@ +"""Register optional device types only when explicitly requested.""" + +import importlib + +import torch + + +def resolve_device(device) -> torch.device: + if isinstance(device, str) and device.split(":", 1)[0] == "npu": + # torch.device cannot parse private-use names until torch-npu registers + # them. Do this in each spawned worker, without importing it on CPU/CUDA. + importlib.import_module("torch_npu") + return torch.device(device) diff --git a/vllm_rlt/engine/stage_engine.py b/vllm_rlt/engine/stage_engine.py index a83d5101..3001eee9 100644 --- a/vllm_rlt/engine/stage_engine.py +++ b/vllm_rlt/engine/stage_engine.py @@ -70,6 +70,26 @@ def __init__( self.execution_config = resolve_cuda_graph_rows( execution_config or ExecutionConfig(), scheduler_config, speculative_config ) + if parameter.device.type == "npu": + if config.to_dict()["model_type"] != "ouro" or config.total_ut_steps != 4: + raise ValueError("NPU currently supports Ouro with four loops") + if str(parameter.dtype) != "torch.bfloat16": + raise ValueError("NPU Ouro requires BF16 weights") + if ( + self.execution_config.async_scheduling + or self.execution_config.cuda_graphs + or self.execution_config.prefill_uva + or speculative_config is not None + or self.stage_config.role != "prefill_decode" + or scheduler_config.enable_preemption + or cache_config.enable_prefix_caching + or cache_config.layout != "last_exited" + or self.exit_config.mode != "ouro" + ): + raise ValueError( + "NPU Ouro requires synchronous local execution, ouro exit mode and " + "last_exited KV without speculation, graphs, preemption or prefix caching" + ) staged_prefill = requires_staged_prefill( scheduler_config, produces_tokens=self.stage_config.produces_tokens ) @@ -323,6 +343,10 @@ def _add_request(self, request_id, prompt_token_ids, sampling_params, *, trace_i if any(type(t) is not int or not 0 <= t < config.vocab_size for t in prompt_token_ids): raise ValueError("prompt token IDs must be integers within the model vocabulary") max_loops = params.max_loops or config.total_ut_steps + if next(self.model.parameters()).device.type == "npu" and ( + max_loops != 4 or params.exit_threshold != 1.0 or params.temperature != 0 + ): + raise ValueError("NPU Ouro requests require four loops, exit_threshold=1 and greedy") if max_loops > config.total_ut_steps or params.min_loops > max_loops: raise ValueError("requested loop bounds exceed the model's supported depth") if self.speculative_config is not None and ( diff --git a/vllm_rlt/engine/worker.py b/vllm_rlt/engine/worker.py index 9a2b2805..80e98985 100644 --- a/vllm_rlt/engine/worker.py +++ b/vllm_rlt/engine/worker.py @@ -13,6 +13,7 @@ import torch +from vllm_rlt.device import resolve_device from vllm_rlt.engine.connector import TransferConnector from vllm_rlt.engine.stage_config import StageConfig from vllm_rlt.engine.stage_engine import StageEngine @@ -33,12 +34,16 @@ def __init__( ): if type(max_control_messages) is not int or max_control_messages < 1: raise ValueError("max_control_messages must be a positive integer") - device = torch.device(f"cuda:{device}" if isinstance(device, int) else device) + device = resolve_device(f"cuda:{device}" if isinstance(device, int) else device) torch.set_num_threads(1) if device.type == "cuda": if device.index is None: device = torch.device("cuda", torch.cuda.current_device()) torch.cuda.set_device(device) + elif device.type == "npu": + if device.index is None: + device = torch.device("npu", torch.npu.current_device()) + torch.npu.set_device(device) torch.manual_seed(options["seed"]) self.channel = channel self.max_control_messages = max_control_messages diff --git a/vllm_rlt/models/ouro.py b/vllm_rlt/models/ouro.py index 31b37f9c..5041fdf3 100644 --- a/vllm_rlt/models/ouro.py +++ b/vllm_rlt/models/ouro.py @@ -19,6 +19,7 @@ import torch from torch import nn +from vllm_rlt.device import resolve_device from vllm_rlt.kernels import silu_and_mul from vllm_rlt.layers import ( RMSNorm, @@ -309,6 +310,7 @@ def from_pretrained( """ from safetensors import safe_open + device = resolve_device(device) if dtype not in (torch.float32, torch.float16, torch.bfloat16): raise ValueError("dtype must be float32, float16, or bfloat16") folder = Path(path_or_repo).expanduser() diff --git a/vllm_rlt/worker/model_runner.py b/vllm_rlt/worker/model_runner.py index ada23d20..93d59dac 100644 --- a/vllm_rlt/worker/model_runner.py +++ b/vllm_rlt/worker/model_runner.py @@ -1152,6 +1152,9 @@ def _readback_slot(self, result): raise RuntimeError("readback buffers exhausted; retire completed submissions") def _synchronize_submission(self, stream, prepared): + if self.device.type == "npu": + torch.npu.synchronize(self.device) + return if self.device.type != "cuda": return (stream or torch.cuda.current_stream(self.device)).synchronize() @@ -1254,6 +1257,10 @@ def submit(self, scheduled_batch: SchedulerOutput | None = None, *, batch: Execu for span in batch.spans: self.events[span.request.request_id] = compute_done self.submissions.complete(compute_done) + if self.device.type == "npu": + # No CUDA completion events protect NPU metadata snapshots. + # Drain before pools/host inputs can be recycled or KV freed. + torch.npu.synchronize(self.device) if split_prefill: self.split_prefill_submissions += 1 return Submission( @@ -1390,6 +1397,9 @@ def release(self, request_id): self.rng.drop(request_id) def synchronize(self): + if self.device.type == "npu": + torch.npu.synchronize(self.device) + return for stream in ( self.core_stream, self.boundary_stream, From dc798cb061ed6c2f8edee8fe53cef8dcfa009c9c Mon Sep 17 00:00:00 2001 From: KrystalRay Date: Mon, 2 Nov 2026 23:51:35 +0800 Subject: [PATCH 02/20] feat: expose synchronous BF16 Ouro on Ascend through Torch attention --- README.md | 2 +- docs/recipes/ouro_ascend.md | 113 +++++++++++++++++++++ pyproject.toml | 5 +- tests/conftest.py | 6 ++ tests/test_ascend.py | 171 ++++++++++++++++++++++++++++++++ tests/test_ascend_checkpoint.py | 113 +++++++++++++++++++++ tests/test_ascend_serving.py | 135 +++++++++++++++++++++++++ tests/test_device.py | 16 ++- vllm_rlt/engine/stage_engine.py | 6 +- vllm_rlt/entrypoints/cli.py | 2 + vllm_rlt/entrypoints/serve.py | 4 +- vllm_rlt/profiling.py | 2 + 12 files changed, 569 insertions(+), 6 deletions(-) create mode 100644 docs/recipes/ouro_ascend.md create mode 100644 tests/test_ascend.py create mode 100644 tests/test_ascend_checkpoint.py create mode 100644 tests/test_ascend_serving.py diff --git a/README.md b/README.md index 00ed4e17..4240b3fd 100644 --- a/README.md +++ b/README.md @@ -134,7 +134,7 @@ No PyPI publication is implied. | Checkpoint | Implementation | Documented validation scope | Guide | | --- | --- | --- | --- | -| Ouro-1.4B | Native Ouro | Existing runtime and benchmark coverage under their recorded configurations | [Runtime](docs/runtime.md), [accuracy](docs/accuracy.md) | +| Ouro-1.4B | Native Ouro | Existing runtime and benchmark coverage under their recorded configurations; initial single-device Ascend 910B3 BF16/fixed-four-loop synchronous path with explicit Torch attention | [Runtime](docs/runtime.md), [accuracy](docs/accuracy.md), [Ascend recipe](docs/recipes/ouro_ascend.md) | | Ouro-2.6B | Shared Ouro | Shares the Ouro implementation; not separately qualified, and 1.4B quality/performance results are not inherited | [Scope notes](docs/recipes/ouro_thinking.md#6-evidence-and-limitations) | | Ouro-1.4B-Thinking | Reuses Ouro | Bounded BF16, fixed-four-loop synchronous basic integration | [Shared recipe](docs/recipes/ouro_thinking.md) | | Ouro-2.6B-Thinking | Reuses Ouro | Bounded BF16, fixed-four-loop synchronous basic integration | [Shared recipe](docs/recipes/ouro_thinking.md) | diff --git a/docs/recipes/ouro_ascend.md b/docs/recipes/ouro_ascend.md new file mode 100644 index 00000000..1c27dd9d --- /dev/null +++ b/docs/recipes/ouro_ascend.md @@ -0,0 +1,113 @@ +# Ouro-1.4B on Ascend 910B3: initial synchronous support + +This path reuses the native Ouro model and shared engine on one Ascend NPU. +Select **--attention-backend torch explicitly**. Weights, ordinary activations +and KV storage remain BF16; the existing Torch attention computes QK, softmax +and PV intermediates in FP32 through Torch-NPU. This is not a fused CANN +attention integration and makes no acceleration claim. + +## Supported scope + +- Ouro, fixed four-loop greedy inference, synchronous local execution, + LAST_EXITED depth-aware paged KV, prefill and incremental decode. +- Explicit num_blocks or kv_cache_memory_bytes; automatic NPU memory sizing + is not implemented. +- Cancellation, request ID reuse, EOS/length handling, HTTP completions and SSE + reuse the existing engine and serving lifecycle. +- Async scheduling, CUDA graphs, speculation, preemption, prefix caching, PD, + adaptive exit and stochastic sampling are rejected for this initial path. +- Qualification targets Ouro-1.4B on 910B3. Other checkpoints and Ascend devices + require separate validation. + +The shared scheduler, model and KV owner are retained. NPU submissions drain +before metadata buffers or request pages can be reused. CPU/CUDA execution +does not import the optional Torch-NPU package. Built-in profiling supports +CPU/CUDA; use torch_npu.profiler for NPU device traces. + +## Environment and installation + +The development environment is Linux aarch64, Python 3.11.13, Torch 2.9.0+cpu, +torch-npu 2.9.0, CANN 9.0.0, driver 26.1.0.b050 and Transformers 4.57.6. +Keep Torch and Torch-NPU matched; activate vendor CANN before the project Python. + +~~~bash +source /path/to/cann/set_env.sh +source /path/to/isolated-npu-environment/bin/activate +export ASCEND_RT_VISIBLE_DEVICES=4 # replace with your assigned physical NPU +python -c 'import torch, torch_npu; print(torch.__version__, torch_npu.__version__); print(torch.npu.get_device_name(0))' +python -m pip install 'numpy>=1.24' 'safetensors>=0.4.3' 'huggingface-hub>=0.25' \ + 'transformers>=4.54' 'aiohttp>=3.11,<4' setuptools wheel +python -m pip install --no-deps --no-build-isolation -e . +~~~ + +The no-deps install preserves the vendor Torch pair and avoids the project's +Linux Triton dependency. Neither Triton nor a native CUDA extension is required +for this explicit Torch backend. No InferRT or MindIE-SD integration is involved. + +Use the complete ByteDance/Ouro-1.4B checkpoint at revision +574fa66cb8bf5abdc979642d01cf2b79b16bfab1. The model.safetensors SHA256 is +58872a72616c736595b8b7662079c5b12c5a162ec16eae94f21c348dfa9885af. + +## Start the service + +~~~bash +python -m vllm_rlt.entrypoints.serve \ + --model /path/to/Ouro-1.4B --served-model-name ouro \ + --device npu --dtype bfloat16 --attention-backend torch \ + --num-blocks 64 --block-size 16 --max-num-seqs 2 \ + --max-num-batched-tokens 32 --port 8123 +~~~ + +~~~bash +curl http://127.0.0.1:8123/health +curl http://127.0.0.1:8123/v1/completions \ + -H 'Content-Type: application/json' \ + -d '{"model":"ouro","prompt":"The capital of France is","max_tokens":4,"temperature":0,"min_loops":4,"max_loops":4,"exit_threshold":1}' +~~~ + +Add stream=true for SSE. Python callers use device="npu" and +attention_backend="torch"; the spawned worker registers Torch-NPU itself. +The assigned physical device becomes logical npu:0. Increase KV capacity for +longer prompts, output limits or concurrency. This small service command is a +smoke configuration, not a performance benchmark. + +## Reproduce validation + +~~~bash +python -m pip install pytest ruff pre-commit +pre-commit run --all-files +export VLLM_RLT_OURO_CHECKPOINT=/path/to/Ouro-1.4B +export VLLM_RLT_ASCEND_RESULTS=/path/to/validation-results +python -m pytest tests/test_ascend.py tests/test_ascend_checkpoint.py \ + tests/test_ascend_serving.py --run-npu -q +~~~ + +The NPU flag requires assigned ASCEND_RT_VISIBLE_DEVICES; CPU collection skips +device tests. Checks cover a CPU attention oracle, GQA and causal page +boundaries, incremental/full KV allocation, four-depth state ownership, +chunking, cancellation/reuse, EOS/length behavior, pinned checkpoint generation +and a real spawned HTTP/SSE worker. + +Metadata-only allocation changes require exact hidden/logit/KV equality. +The BF16 diagnostic bounds remain relative L2 at most 2% and elementwise +atol=0.02, rtol=0.02. These are this validation's bounds, not a universal quality +guarantee. Task quality needs separately pinned paired evaluation against the +official code; short generation does not establish broader quality. +Performance characterization should retain input lengths, concurrency, +TTFT/TPOT/throughput, memory and repetitions; the existing +[serving benchmark guide](../serving.md#reuse-the-upstream-benchmark) describes +the compatible upstream client. + +## Fused attention remains separate + +An experimental BF16 paged npu_fused_infer_attention_score adapter is outside +this change. It failed frozen whole-model numerical checks and lost a correct +answer in a ten-question smoke. Promoting attention intermediates in +npu_fusion_attention improved local mathematical agreement but still failed +the original whole-model joint gate. Those failures have not been relaxed or +hidden; this recipe does not qualify that fused backend. + +This contribution covers a bounded part of +[#69 Ascend support](https://github.com/InferMatrixLab/vllm-rlt/issues/69). +It does not complete CANN fused adaptation or other Ouro runtime features on +Ascend. Measured qualification results are recorded separately from this scope. diff --git a/pyproject.toml b/pyproject.toml index 40652f8c..b9e8980a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,7 +43,10 @@ license-files = ["LICENSE", "NOTICE"] [tool.pytest.ini_options] testpaths = ["tests"] pythonpath = ["."] -markers = ["gpu: requires an explicitly selected CUDA device and --run-gpu"] +markers = [ + "gpu: requires an explicitly selected CUDA device and --run-gpu", + "npu: requires an assigned Ascend device and torch-npu/CANN", +] [tool.ruff] line-length = 100 diff --git a/tests/conftest.py b/tests/conftest.py index f65e8a42..6981e499 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,15 +8,21 @@ def pytest_addoption(parser): parser.addoption( "--run-gpu", action="store_true", help="Run CUDA tests on explicitly selected devices" ) + parser.addoption("--run-npu", action="store_true", help="Run Ascend tests on an assigned NPU") def pytest_collection_modifyitems(config, items): enabled = config.getoption("--run-gpu") if enabled and not os.environ.get("CUDA_VISIBLE_DEVICES"): raise pytest.UsageError("--run-gpu requires explicit CUDA_VISIBLE_DEVICES") + npu_enabled = config.getoption("--run-npu") + if npu_enabled and not os.environ.get("ASCEND_RT_VISIBLE_DEVICES"): + raise pytest.UsageError("--run-npu requires assigned ASCEND_RT_VISIBLE_DEVICES") for item in items: if "gpu" in item.keywords and not enabled: item.add_marker(pytest.mark.skip(reason="set CUDA_VISIBLE_DEVICES and use --run-gpu")) + if "npu" in item.keywords and not npu_enabled: + item.add_marker(pytest.mark.skip(reason="use --run-npu on an assigned NPU")) @pytest.fixture(autouse=True, scope="session") diff --git a/tests/test_ascend.py b/tests/test_ascend.py new file mode 100644 index 00000000..4632c7f1 --- /dev/null +++ b/tests/test_ascend.py @@ -0,0 +1,171 @@ +"""Opt-in synchronous BF16 Ouro checks using explicit Torch attention on 910B3.""" + +import pytest +import torch + +from tests.helpers import tiny_ouro_config +from vllm_rlt import CacheConfig, SamplingParams, SchedulerConfig +from vllm_rlt.core.kv_cache_manager import KVCacheManager +from vllm_rlt.device import resolve_device +from vllm_rlt.engine.stage_engine import StageEngine +from vllm_rlt.kernels.paged_attention import torch_paged_attention +from vllm_rlt.models import OuroForCausalLM + +pytestmark = pytest.mark.npu + + +@pytest.fixture +def npu_device(): + resolve_device("npu") + torch.npu.set_device(0) + assert torch.npu.get_device_name(0) == "Ascend910B3" + yield torch.device("npu:0") + torch.npu.synchronize() + torch.npu.empty_cache() + + +def assert_bf16_close(actual, expected): + a, b = actual.detach().float().cpu(), expected.detach().float().cpu() + assert torch.isfinite(a).all() and torch.isfinite(b).all() + assert (a - b).norm() / b.norm().clamp_min(1e-8) <= 0.02 + torch.testing.assert_close(a, b, atol=0.02, rtol=0.02) + + +@pytest.mark.parametrize("kv_heads", [1, 2, 4]) +def test_torch_attention_on_npu_matches_cpu_prefix_oracle(npu_device, kv_heads): + torch.manual_seed(83) + keys = torch.randn(8, 2, 16, kv_heads, 64, dtype=torch.bfloat16) + values = torch.randn_like(keys) + q = torch.randn(64, 4, 64, dtype=torch.bfloat16) + lengths = torch.tensor([0, 1, 15, 16, 17, 31, 32, 3] * 8, dtype=torch.int32) + tables = torch.tensor([[5, 2], [1, 7]] * 32, dtype=torch.int32) + expected = torch_paged_attention(q, keys[:, 1], values[:, 1], tables, lengths) + actual = torch_paged_attention( + q.to(npu_device), + keys.to(npu_device)[:, 1], + values.to(npu_device)[:, 1], + tables.to(npu_device), + lengths.to(npu_device), + ) + assert_bf16_close(actual, expected) + assert torch.count_nonzero(actual[0]).item() == 0 + + +def make_npu_model(device): + torch.manual_seed(41) + config = tiny_ouro_config(hidden_size=256, intermediate_size=512, head_dim=64) + return OuroForCausalLM(config).to(device=device, dtype=torch.bfloat16).eval() + + +@torch.inference_mode() +def recurrent_states(model, chunks, *, tokens=None, incremental=False): + c = model.config + cache = KVCacheManager( + c.num_hidden_layers, + c.num_key_value_heads, + c.head_dim, + 32, + 16, + 4, + device=next(model.parameters()).device, + dtype=torch.bfloat16, + backend="torch", + incremental_allocation=incremental, + ) + tokens = tokens if tokens is not None else ([3, 7, 2, 11, 5, 17, 8] * 3)[:21] + assert cache.allocate("a", len(tokens), initial_tokens=16 if incremental else None) + states, kv, start = {}, {}, 0 + try: + for size in chunks: + assert cache.ensure_capacity("a", start + size) + positions = list(range(start, start + size)) + hidden = model.prelude(torch.tensor(tokens[start : start + size], device=cache.device)) + for depth in range(4): + hidden, _ = model.recurrent(hidden, ["a"] * size, [depth] * size, positions, cache) + states[(start, depth)] = (hidden.clone(), model.coda(hidden).clone()) + start += size + assert start == len(tokens) + for depth in range(4): + for layer in range(c.num_hidden_layers): + assert all(cache.token_written("a", p, depth) for p in range(len(tokens))) + kv[(layer, depth)] = tuple( + t.clone() for t in cache.read(layer, "a", depth, len(tokens)) + ) + assert len({tuple(cache.get_block_table("a", d)) for d in range(4)}) == 4 + return states, kv + finally: + cache.free("a") + assert cache.num_used_blocks == 0 + + +def test_npu_chunked_prefill_and_incremental_kv(npu_device): + model = make_npu_model(npu_device) + expected, expected_kv = recurrent_states(model, [19, 1, 1]) + incremental, incremental_kv = recurrent_states(model, [19, 1, 1], incremental=True) + for key in expected: + for a, b in zip(incremental[key], expected[key], strict=True): + torch.testing.assert_close(a, b, atol=0, rtol=0) + for key in expected_kv: + for a, b in zip(incremental_kv[key], expected_kv[key], strict=True): + torch.testing.assert_close(a, b, atol=0, rtol=0) + chunked, chunked_kv = recurrent_states(model, [5, 14, 1, 1], incremental=True) + for key in ((19, d) for d in range(4)): + for a, b in zip(chunked[key], expected[key], strict=True): + assert_bf16_close(a, b) + for key in expected_kv: + for a, b in zip(chunked_kv[key], expected_kv[key], strict=True): + assert_bf16_close(a, b) + + +def test_npu_cancel_reuse_and_multiple_requests(npu_device): + engine = StageEngine( + make_npu_model(npu_device), + attention_backend="torch", + cache_config=CacheConfig(num_blocks=32, block_size=16, incremental_allocation=True), + scheduler_config=SchedulerConfig(max_num_seqs=2, max_num_batched_tokens=32), + ) + try: + params = SamplingParams(max_tokens=3, temperature=0, ignore_eos=True) + engine.add_request("cancel", [3, 7, 2] * 6, params) + engine.add_request("keep", [5, 9, 11], params) + engine.step() + engine.abort_request("cancel") + engine.add_request("cancel", [13, 14, 15], params) + finished = {} + for _ in range(200): + for output in engine.step(): + if output.finished: + finished[output.request_id] = output + if not engine.has_unfinished_requests(): + break + assert set(finished) == {"keep", "cancel"} + assert all( + o.exit_depths == [4] * 3 and o.finish_reason == "length" for o in finished.values() + ) + assert engine.cache_manager.num_used_blocks == 0 + finally: + engine.close() + + +@pytest.mark.parametrize("ignore_eos", [False, True]) +def test_npu_eos_and_length_handling(npu_device, ignore_eos): + model = make_npu_model(npu_device) + model.lm_head.weight.zero_() # Argmax 0 is the tiny model's actual EOS token. + engine = StageEngine(model, attention_backend="torch", cache_config=CacheConfig(num_blocks=16)) + try: + engine.add_request( + "eos", [3, 7, 2], SamplingParams(max_tokens=3, temperature=0, ignore_eos=ignore_eos) + ) + output = None + for _ in range(100): + for item in engine.step(): + if item.finished: + output = item + if not engine.has_unfinished_requests(): + break + assert output is not None + assert output.token_ids == [0] * (3 if ignore_eos else 1) + assert output.finish_reason == ("length" if ignore_eos else "stop") + assert engine.cache_manager.num_used_blocks == 0 + finally: + engine.close() diff --git a/tests/test_ascend_checkpoint.py b/tests/test_ascend_checkpoint.py new file mode 100644 index 00000000..6b08c1d3 --- /dev/null +++ b/tests/test_ascend_checkpoint.py @@ -0,0 +1,113 @@ +"""Pinned real-weight NPU checks for explicit Torch attention, not fused CANN.""" + +import hashlib +import json +import os +from pathlib import Path + +import pytest +import torch + +from tests.test_ascend import assert_bf16_close, recurrent_states +from vllm_rlt import CacheConfig, SamplingParams, SchedulerConfig +from vllm_rlt.device import resolve_device +from vllm_rlt.engine.stage_engine import StageEngine +from vllm_rlt.models import OuroForCausalLM, load_tokenizer + +pytestmark = pytest.mark.npu + + +@pytest.fixture(scope="module") +def checkpoint(): + source = os.environ.get("VLLM_RLT_OURO_CHECKPOINT") + if not source: + pytest.skip("set VLLM_RLT_OURO_CHECKPOINT to the pinned local Ouro-1.4B checkpoint") + resolve_device("npu") + torch.npu.set_device(0) + assert torch.npu.get_device_name(0) == "Ascend910B3" + with (Path(source) / "model.safetensors").open("rb") as stream: + digest = hashlib.file_digest(stream, "sha256").hexdigest() + assert digest == "58872a72616c736595b8b7662079c5b12c5a162ec16eae94f21c348dfa9885af" + model = OuroForCausalLM.from_pretrained(source, device="npu", dtype=torch.bfloat16) + assert all(p.dtype == torch.bfloat16 for p in model.parameters()) + yield model, load_tokenizer(source) + torch.npu.synchronize() + + +def save_record(name, record): + folder = os.environ.get("VLLM_RLT_ASCEND_RESULTS") + if folder: + path = Path(folder) + path.mkdir(parents=True, exist_ok=True) + (path / name).write_text(json.dumps(record, indent=2)) + + +@torch.inference_mode() +def test_checkpoint_incremental_pages_preserve_hidden_logits_and_kv(checkpoint): + model, tokenizer = checkpoint + tokens = tokenizer.encode( + "The capital of France is Paris. The capital of Japan is Tokyo. " + "A language model predicts the next token in a sequence.", + add_special_tokens=False, + )[:21] + assert len(tokens) == 21 + expected, expected_kv = recurrent_states(model, [19, 1, 1], tokens=tokens) + actual, actual_kv = recurrent_states(model, [19, 1, 1], tokens=tokens, incremental=True) + pairs = {} + for key in expected: + for name, a, b in zip(("hidden", "logits"), actual[key], expected[key], strict=True): + pairs[f"{key}.{name}"] = (a, b) + for key in expected_kv: + for name, a, b in zip(("key", "value"), actual_kv[key], expected_kv[key], strict=True): + pairs[f"{key}.{name}"] = (a, b) + save_record( + "checkpoint-numerics.json", + dict( + tokens=tokens, + states=len(pairs), + exact=all(torch.equal(a, b) for a, b in pairs.values()), + finite=all( + bool(torch.isfinite(a).all() and torch.isfinite(b).all()) for a, b in pairs.values() + ), + ), + ) + for a, b in pairs.values(): + assert_bf16_close(a, b) + torch.testing.assert_close(a, b, atol=0, rtol=0) + + +@torch.inference_mode() +def test_checkpoint_greedy_generation_and_cleanup(checkpoint): + model, tokenizer = checkpoint + engine = StageEngine( + model, + attention_backend="torch", + cache_config=CacheConfig(num_blocks=64), + scheduler_config=SchedulerConfig(max_num_seqs=2, max_num_batched_tokens=32), + ) + prompts = ["The capital of France is", "One plus one equals"] + finished = {} + try: + for index, prompt in enumerate(prompts): + engine.add_request( + str(index), + tokenizer.encode(prompt, add_special_tokens=False), + SamplingParams(max_tokens=4, temperature=0, ignore_eos=True), + ) + for _ in range(200): + for output in engine.step(): + if output.finished: + assert output.exit_depths == [4] * 4 + assert output.finish_reason == "length" + finished[output.request_id] = dict( + tokens=output.token_ids, text=tokenizer.decode(output.token_ids) + ) + if not engine.has_unfinished_requests(): + break + assert set(finished) == {"0", "1"} + assert finished["0"]["tokens"] == [7042, 30, 198, 198] + assert finished["1"]["tokens"] == [827, 30, 198, 198] + assert engine.cache_manager.num_used_blocks == 0 + save_record("checkpoint-generation.json", finished) + finally: + engine.close() diff --git a/tests/test_ascend_serving.py b/tests/test_ascend_serving.py new file mode 100644 index 00000000..90f596c0 --- /dev/null +++ b/tests/test_ascend_serving.py @@ -0,0 +1,135 @@ +"""Real socket + spawned NPU worker smoke, opt-in with pinned local weights.""" + +import asyncio +import json +import os +import signal +import socket +import subprocess +import sys +import time +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.npu + + +def test_checkpoint_http_completions_stream_and_disconnect(tmp_path): + import aiohttp + + source = os.environ.get("VLLM_RLT_OURO_CHECKPOINT") + if not source: + pytest.skip("set VLLM_RLT_OURO_CHECKPOINT to the pinned local Ouro-1.4B checkpoint") + folder = Path(os.environ.get("VLLM_RLT_ASCEND_RESULTS", tmp_path)) + folder.mkdir(parents=True, exist_ok=True) + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + port = listener.getsockname()[1] + command = [ + sys.executable, + "-m", + "vllm_rlt.entrypoints.serve", + "--model", + source, + "--served-model-name", + "ouro-ascend", + "--device", + "npu", + "--dtype", + "bfloat16", + "--attention-backend", + "torch", + "--num-blocks", + "64", + "--max-num-seqs", + "2", + "--max-num-batched-tokens", + "32", + "--port", + str(port), + ] + with (folder / "service.log").open("w") as log: + process = subprocess.Popen( + command, stdout=log, stderr=subprocess.STDOUT, start_new_session=True + ) + + async def exercise(): + url = f"http://127.0.0.1:{port}" + async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=60)) as client: + deadline = time.monotonic() + 120 + while True: + assert process.poll() is None, (folder / "service.log").read_text() + try: + async with client.get(url + "/health") as response: + if response.status == 200: + break + except aiohttp.ClientConnectionError: + pass + assert time.monotonic() < deadline, "NPU service startup timed out" + await asyncio.sleep(0.25) + + def body(prompt, **extra): + return dict( + model="ouro-ascend", + prompt=prompt, + max_tokens=4, + temperature=0, + min_loops=4, + max_loops=4, + exit_threshold=1, + ignore_eos=True, + **extra, + ) + + async def complete(prompt): + async with client.post(url + "/v1/completions", json=body(prompt)) as response: + result = await response.json() + assert response.status == 200, result + assert result["usage"]["completion_tokens"] == 4 + return result + + prompts = ["The capital of France is", "One plus one equals"] + normal = await asyncio.gather(*(complete(prompt) for prompt in prompts)) + events = [] + async with client.post( + url + "/v1/completions", + json=body(prompts[0], stream=True, stream_options={"include_usage": True}), + ) as response: + assert response.status == 200 + async for line in response.content: + if line.startswith(b"data: "): + data = line[6:].strip().decode() + events.append(data if data == "[DONE]" else json.loads(data)) + assert events[-1] == "[DONE]" + text = "".join( + choice["text"] for event in events[:-1] for choice in event.get("choices", []) + ) + assert text == normal[0]["choices"][0]["text"] + assert any( + isinstance(event, dict) + and (event.get("usage") or {}).get("completion_tokens") == 4 + for event in events + ) + # Close a real streaming connection early, then verify the worker + # remains usable. Direct engine tests check page reclamation. + async with client.post( + url + "/v1/completions", json=body(prompts[1], stream=True) + ) as response: + assert response.status == 200 + await response.content.readline() + after_disconnect = await complete(prompts[1]) + assert after_disconnect["choices"][0]["text"] == normal[1]["choices"][0]["text"] + return dict(normal=normal, stream=events, after_disconnect=after_disconnect) + + try: + result = asyncio.run(exercise()) + (folder / "service-results.json").write_text(json.dumps(result, indent=2)) + finally: + process.terminate() + try: + process.wait(timeout=30) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait(timeout=10) + assert process.returncode == 0, (folder / "service.log").read_text() diff --git a/tests/test_device.py b/tests/test_device.py index fa0cd828..02672ce0 100644 --- a/tests/test_device.py +++ b/tests/test_device.py @@ -11,6 +11,7 @@ from vllm_rlt.core.memory import plan_cache from vllm_rlt.device import resolve_device from vllm_rlt.engine.stage_engine import StageEngine +from vllm_rlt.profiling import ProfileConfig def test_cpu_device_resolution_does_not_import_optional_backend(monkeypatch): @@ -48,8 +49,19 @@ def test_npu_unsupported_execution_rejected_before_allocation(options): def test_npu_does_not_use_cpu_default_memory_budget(): model = npu_model_stub() with pytest.raises(ValueError, match="explicit num_blocks"): - plan_cache(model, CacheConfig(), SchedulerConfig(), ExecutionConfig(), "ascend") + plan_cache(model, CacheConfig(), SchedulerConfig(), ExecutionConfig(), "torch") blocks, info = plan_cache( - model, CacheConfig(num_blocks=17), SchedulerConfig(), ExecutionConfig(), "ascend" + model, CacheConfig(num_blocks=17), SchedulerConfig(), ExecutionConfig(), "torch" ) assert blocks == 17 and info["source"] == "blocks" + + +@pytest.mark.parametrize("backend", ["auto", "triton", "flash_attn_4"]) +def test_npu_requires_explicit_torch_before_allocation(backend): + with pytest.raises(ValueError, match="explicit attention_backend='torch'"): + StageEngine(npu_model_stub(), attention_backend=backend) + + +def test_builtin_profiler_does_not_mislabel_npu_as_cpu_only(tmp_path): + with pytest.raises(ValueError, match="torch_npu.profiler"): + ProfileConfig(str(tmp_path)).resolve_activities("npu:0") diff --git a/vllm_rlt/engine/stage_engine.py b/vllm_rlt/engine/stage_engine.py index 3001eee9..dd13576a 100644 --- a/vllm_rlt/engine/stage_engine.py +++ b/vllm_rlt/engine/stage_engine.py @@ -2,6 +2,8 @@ from dataclasses import dataclass, replace from typing import TYPE_CHECKING, cast +import torch + from vllm_rlt.attention import BackendCapabilities, create_backend from vllm_rlt.config import ( CacheConfig, @@ -71,9 +73,11 @@ def __init__( execution_config or ExecutionConfig(), scheduler_config, speculative_config ) if parameter.device.type == "npu": + if attention_backend != "torch": + raise ValueError("NPU execution requires explicit attention_backend='torch'") if config.to_dict()["model_type"] != "ouro" or config.total_ut_steps != 4: raise ValueError("NPU currently supports Ouro with four loops") - if str(parameter.dtype) != "torch.bfloat16": + if parameter.dtype != torch.bfloat16: raise ValueError("NPU Ouro requires BF16 weights") if ( self.execution_config.async_scheduling diff --git a/vllm_rlt/entrypoints/cli.py b/vllm_rlt/entrypoints/cli.py index a9b8025c..5099ef26 100644 --- a/vllm_rlt/entrypoints/cli.py +++ b/vllm_rlt/entrypoints/cli.py @@ -100,6 +100,8 @@ def main(): parser.error("--toy uses built-in token ID prompts; omit --prompt") if not args.toy and not args.prompt: parser.error("provide --prompt or use --toy for a CPU smoke test") + if args.device.split(":", 1)[0] == "npu" and args.attention_backend != "torch": + parser.error("NPU execution requires explicit --attention-backend torch") profiling = profile_config_from_args(args) if profiling is not None: profiling.resolve_activities(args.device) diff --git a/vllm_rlt/entrypoints/serve.py b/vllm_rlt/entrypoints/serve.py index cd452d92..713bab30 100644 --- a/vllm_rlt/entrypoints/serve.py +++ b/vllm_rlt/entrypoints/serve.py @@ -82,7 +82,7 @@ def main(): parser.add_argument("--tokenizer") parser.add_argument("--tokenizer-revision") parser.add_argument("--served-model-name") - parser.add_argument("--device", choices=["cpu", "cuda"], default="cuda") + parser.add_argument("--device", choices=["cpu", "cuda", "npu"], default="cuda") parser.add_argument("--dtype", choices=["bfloat16", "float32"], default="bfloat16") parser.add_argument( "--attention-backend", @@ -115,6 +115,8 @@ def main(): limits = ServingLimits(**{name: getattr(args, name) for name in defaults}) if args.device == "cpu" and args.attention_backend not in ("auto", "torch"): parser.error("CPU execution requires --attention-backend torch or auto") + if args.device == "npu" and args.attention_backend != "torch": + parser.error("NPU execution requires explicit --attention-backend torch") torch.set_num_threads(args.cpu_threads) logging.basicConfig(level=logging.INFO) app = create_app( diff --git a/vllm_rlt/profiling.py b/vllm_rlt/profiling.py index 41245be8..757bbafd 100644 --- a/vllm_rlt/profiling.py +++ b/vllm_rlt/profiling.py @@ -51,6 +51,8 @@ def __post_init__(self): object.__setattr__(self, "activities", values) def resolve_activities(self, device): + if str(device).startswith("npu"): + raise ValueError("NPU device profiling requires torch_npu.profiler") names = self.activities or (("cpu", "cuda") if str(device).startswith("cuda") else ("cpu",)) values = [getattr(torch.profiler.ProfilerActivity, name.upper()) for name in names] if not set(values) <= torch.profiler.supported_activities(): From 629e8450e1f9a2d38c24edcaf10e399c3035eb1e Mon Sep 17 00:00:00 2001 From: KrystalRay Date: Tue, 3 Nov 2026 01:27:27 +0800 Subject: [PATCH 03/20] docs: record Ascend validation results and quality blockers --- README.md | 2 +- docs/recipes/ouro_ascend.md | 77 +++++++++++++++++++++++++++++++++++-- 2 files changed, 75 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 4240b3fd..183cc29e 100644 --- a/README.md +++ b/README.md @@ -134,7 +134,7 @@ No PyPI publication is implied. | Checkpoint | Implementation | Documented validation scope | Guide | | --- | --- | --- | --- | -| Ouro-1.4B | Native Ouro | Existing runtime and benchmark coverage under their recorded configurations; initial single-device Ascend 910B3 BF16/fixed-four-loop synchronous path with explicit Torch attention | [Runtime](docs/runtime.md), [accuracy](docs/accuracy.md), [Ascend recipe](docs/recipes/ouro_ascend.md) | +| Ouro-1.4B | Native Ouro | Existing runtime and benchmark coverage under their recorded configurations; experimental single-device Ascend 910B3 BF16/fixed-four-loop synchronous Torch path, with paired quality regressions pending correction | [Runtime](docs/runtime.md), [accuracy](docs/accuracy.md), [Ascend recipe](docs/recipes/ouro_ascend.md) | | Ouro-2.6B | Shared Ouro | Shares the Ouro implementation; not separately qualified, and 1.4B quality/performance results are not inherited | [Scope notes](docs/recipes/ouro_thinking.md#6-evidence-and-limitations) | | Ouro-1.4B-Thinking | Reuses Ouro | Bounded BF16, fixed-four-loop synchronous basic integration | [Shared recipe](docs/recipes/ouro_thinking.md) | | Ouro-2.6B-Thinking | Reuses Ouro | Bounded BF16, fixed-four-loop synchronous basic integration | [Shared recipe](docs/recipes/ouro_thinking.md) | diff --git a/docs/recipes/ouro_ascend.md b/docs/recipes/ouro_ascend.md index 1c27dd9d..43d58995 100644 --- a/docs/recipes/ouro_ascend.md +++ b/docs/recipes/ouro_ascend.md @@ -6,6 +6,11 @@ and KV storage remain BF16; the existing Torch attention computes QK, softmax and PV intermediates in FP32 through Torch-NPU. This is not a fused CANN attention integration and makes no acceleration claim. +**Quality status:** the selected same-device paired screen is HF 63/87 versus +native 61/87 (losses: 203 and 881; gains: none). The frozen no-net-loss gate +failed. This implementation remains experimental and is not ready for +submission as a quality-qualified backend. + ## Supported scope - Ouro, fixed four-loop greedy inference, synchronous local execution, @@ -35,8 +40,8 @@ source /path/to/cann/set_env.sh source /path/to/isolated-npu-environment/bin/activate export ASCEND_RT_VISIBLE_DEVICES=4 # replace with your assigned physical NPU python -c 'import torch, torch_npu; print(torch.__version__, torch_npu.__version__); print(torch.npu.get_device_name(0))' -python -m pip install 'numpy>=1.24' 'safetensors>=0.4.3' 'huggingface-hub>=0.25' \ - 'transformers>=4.54' 'aiohttp>=3.11,<4' setuptools wheel +python -m pip install 'numpy==1.26.4' 'safetensors==0.8.0' 'huggingface-hub==0.36.2' \ + 'transformers==4.57.6' 'tokenizers==0.22.2' 'aiohttp==3.14.3' setuptools wheel python -m pip install --no-deps --no-build-isolation -e . ~~~ @@ -110,4 +115,70 @@ hidden; this recipe does not qualify that fused backend. This contribution covers a bounded part of [#69 Ascend support](https://github.com/InferMatrixLab/vllm-rlt/issues/69). It does not complete CANN fused adaptation or other Ouro runtime features on -Ascend. Measured qualification results are recorded separately from this scope. +Ascend. Measured validation outcomes and remaining blockers are recorded below. + +## Measured validation (2026-11-03) + +Runtime source: 4734e1963bdf7fd4e0fd66b2612511d6d4173550, against base +989921ac1d3897c5c43e27ec8cddbc91fa953194. Later recipe updates do not change +the measured runtime sources. + +- Ruff lint/format and the CPU regression hook passed. +- Opt-in real 910B3 tests: 10 passed in 44.41s, including actual HTTP/SSE. +- Incremental/preallocated real-weight KV differential: 216 finite states, all + exactly equal; attention/chunking diagnostics retain their original joint bounds. +- Same-device paired selected GSM8K-87: official 63/87, native 61/87; + losses [203, 881], gains []; 63/87 exact token sequences. + Parse failures: {'hf_eager': 19, 'rlt_target': 19}; length limits: {'hf_eager': 0, 'rlt_target': 0}; + execution limits: {'hf_eager': 0, 'rlt_target': 0}. +- Each pair runs on the same physical 910B3; all 87 cases pool three cards. + The original ten are regression smoke; 77 cases were held out. This is + selected-case evidence, not full GSM8K or adaptive/concurrent quality. + +The quality run uses official unmodified HF eager, Transformers 4.57.6, +Tokenizers 0.22.2, lm-eval 0.4.9.2 strict scoring, three-shot raw CoT, +greedy four-loop BF16, max 1024 output tokens and source-ranked seed0 IDs. +Dataset revision is 740312add88f781978c0658806c59bc2815b9866. + +The table below is a separate **engine-only short-burst** characterization: +8 output tokens/request, ignore EOS, one shape-specific warmup and three +unprofiled repetitions. Values are medians of per-burst request means; output +throughput includes prefill and drain. TTFT starts before engine admission. +TPOT is first-to-last observed output time divided by seven. + +| Input | Concurrency | Output tok/s (range) | TTFT ms | TPOT ms | Peak allocated / reserved GiB | +|---|---:|---:|---:|---:|---:| +| 16 | 1 | 2.726 (2.700–2.901) | 1238.01 | 241.94 | 4.970 / 5.309 | +| 16 | 2 | 3.793 (3.657–3.810) | 2122.58 | 298.19 | 4.970 / 5.311 | +| 16 | 8 | 5.031 (5.016–5.086) | 7980.90 | 675.01 | 4.973 / 5.314 | +| 64 | 1 | 1.367 (1.342–1.372) | 4176.89 | 242.77 | 4.972 / 5.314 | +| 64 | 2 | 1.590 (1.569–1.599) | 7964.07 | 300.45 | 4.974 / 5.314 | +| 64 | 8 | 1.789 (1.786–1.793) | 23417.86 | 1756.95 | 4.984 / 5.314 | +| 256 | 1 | 0.370 (0.364–0.373) | 19472.72 | 307.45 | 4.978 / 5.318 | +| 256 | 2 | 0.372 (0.371–0.377) | 39959.74 | 431.61 | 4.988 / 5.318 | +| 256 | 8 | 0.379 (0.378–0.380) | 125114.88 | 6228.64 | 4.988 / 5.318 | + +Configuration: 768 blocks, block size 16, max 8 sequences / 256 batched tokens, +incremental allocation, no async/graphs/UVA/prefix caching/speculation. +Measured requests: 99, output tokens: 792, failures: 0. Live logical KV blocks +after each completed burst: 0. +Torch allocator peaks include model/cache/workspace and exclude some driver +or runtime memory. Other cards ran independent quality shards; CPU/NUMA +affinity was not pinned. No steady-state throughput, HTTP/p99, accelerator +hardware-only comparison or acceleration claim is made. + +Separate ops-only/full profiles capture a complete warmed second-token decode +iteration, with actual NPU device kernels. Full mode enables shapes, stacks +and memory; exported shapes, readable Python stacks and memory CSV rows +were checked separately from performance timing. +Full prefill plus all eight output tokens is outside these diagnostic captures. + +The conservative Torch path has substantial prefill/dispatch overhead. Fused +attention optimization remains a later step and must retain the quality gates. + +The no-net-loss quality criterion failed on this selected set. Case 203 +changes a quantity from 7 to 5 and produces 27 instead of 31. Case 881 writes +'The answer is 10 + 6 = 16'; the unchanged strict scorer extracts 10. It remains +a failure under the frozen protocol. Neither the scorer nor the gate was +relaxed. A diagnostic HF-style BF16 attention alternative also failed the +original numerical bounds and was not incorporated. From b571af66e4b346c7b93de63a6042e28ebf1564bd Mon Sep 17 00:00:00 2001 From: KrystalRay Date: Tue, 3 Nov 2026 02:51:19 +0800 Subject: [PATCH 04/20] perf: use native Ascend scatter for paged KV writes --- tests/test_ascend_kv.py | 51 +++++++++++++++++++++++++++++++ vllm_rlt/core/kv_cache_manager.py | 18 +++++++++-- vllm_rlt/kernels/ascend_kv.py | 41 +++++++++++++++++++++++++ 3 files changed, 108 insertions(+), 2 deletions(-) create mode 100644 tests/test_ascend_kv.py create mode 100644 vllm_rlt/kernels/ascend_kv.py diff --git a/tests/test_ascend_kv.py b/tests/test_ascend_kv.py new file mode 100644 index 00000000..8445e0f0 --- /dev/null +++ b/tests/test_ascend_kv.py @@ -0,0 +1,51 @@ +"""Assigned-device regressions for native KV copies and descriptor lifetime.""" + +import pytest +import torch + +from vllm_rlt.core.kv_cache_manager import KVCacheManager +from vllm_rlt.device import resolve_device + +pytestmark = pytest.mark.npu + + +@pytest.mark.parametrize("rows", [1, 3]) +@torch.inference_mode() +def test_native_scatter_preserves_inputs_other_layers_and_reused_ids(rows): + device = resolve_device("npu:0") + torch.npu.set_device(device) + cache = KVCacheManager(2, 2, 64, 32, 16, 4, device=device, dtype=torch.bfloat16) + cache.key_cache.zero_() + cache.value_cache.zero_() + assert cache.allocate("a", 17) + assert cache.allocate("b", 3) + ids, depths, positions = ["a", "b", "a"][:rows], [0, 2, 3][:rows], [15, 0, 16][:rows] + batch = cache._prepare_batch(ids, depths, positions) + torch.manual_seed(99) + k = torch.randn(rows, 2, 64, device=device, dtype=torch.bfloat16) + v = torch.randn_like(k) + saved_k, saved_v = k.clone(), v.clone() + expected_k, expected_v = cache.key_cache.clone(), cache.value_cache.clone() + for layer in range(2): + expected_k[batch.write_blocks, layer, batch.write_offsets] = k + expected_v[batch.write_blocks, layer, batch.write_offsets] = v + cache._write_prepared(layer, batch, k, v) + torch.testing.assert_close(cache.key_cache, expected_k, atol=0, rtol=0) + torch.testing.assert_close(cache.value_cache, expected_v, atol=0, rtol=0) + torch.testing.assert_close(k, saved_k, atol=0, rtol=0) + torch.testing.assert_close(v, saved_v, atol=0, rtol=0) + cache.free("a") + assert cache.allocate("a", 17) + with pytest.raises((ValueError, RuntimeError)): + cache._write_prepared(0, batch, k, v) + new_batch = cache._prepare_batch(["a"], [1], [0]) + expected_k, expected_v = cache.key_cache.clone(), cache.value_cache.clone() + expected_k[new_batch.write_blocks, 1, new_batch.write_offsets] = k[:1] + expected_v[new_batch.write_blocks, 1, new_batch.write_offsets] = v[:1] + cache._write_prepared(1, new_batch, k[:1], v[:1]) + torch.testing.assert_close(cache.key_cache, expected_k, atol=0, rtol=0) + torch.testing.assert_close(cache.value_cache, expected_v, atol=0, rtol=0) + cache.free("a") + cache.free("b") + assert cache.num_used_blocks == 0 + torch.npu.synchronize() diff --git a/vllm_rlt/core/kv_cache_manager.py b/vllm_rlt/core/kv_cache_manager.py index b0b28be3..37c1352a 100644 --- a/vllm_rlt/core/kv_cache_manager.py +++ b/vllm_rlt/core/kv_cache_manager.py @@ -684,8 +684,22 @@ def _write_prepared( self._validate_tensor(v, len(batch.position_ids), "v") if not batch.rows: return - self.key_cache[batch.write_blocks, layer, batch.write_offsets] = k[: len(batch.rows)] - self.value_cache[batch.write_blocks, layer, batch.write_offsets] = v[: len(batch.rows)] + if self.device.type == "npu": + if not hasattr(self, "_npu_writer"): + from vllm_rlt.kernels.ascend_kv import AscendKVWriter + + self._npu_writer = AscendKVWriter() + self._npu_writer.write( + self.key_cache, + self.value_cache, + layer, + batch, + k[: len(batch.rows)], + v[: len(batch.rows)], + ) + else: + self.key_cache[batch.write_blocks, layer, batch.write_offsets] = k[: len(batch.rows)] + self.value_cache[batch.write_blocks, layer, batch.write_offsets] = v[: len(batch.rows)] for allocation, depth, position in batch.rows: allocation.written[self._plane(depth)][layer].add(position) diff --git a/vllm_rlt/kernels/ascend_kv.py b/vllm_rlt/kernels/ascend_kv.py new file mode 100644 index 00000000..fc41189c --- /dev/null +++ b/vllm_rlt/kernels/ascend_kv.py @@ -0,0 +1,41 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Exact KV copies through native Scatter ND on Ascend.""" + +from __future__ import annotations + +import weakref + +import torch + + +class AscendKVWriter: + """Reuse immutable traversal indices across layers without retaining the descriptor.""" + + def __init__(self): + self._metadata = None + self._indices = None + + def write(self, keys, values, layer, batch, k, v): + import torch_npu + + if self._metadata is None or self._metadata() is not batch: + owner = batch.owner + addresses = [ + ( + allocation.block_tables[owner._plane(depth)][position // owner.block_size], + position % owner.block_size, + ) + for allocation, depth, position in batch.rows + ] + self._indices = torch.tensor( + [ + [[block, index, offset] for block, offset in addresses] + for index in range(keys.shape[1]) + ], + device=keys.device, + dtype=torch.int32, + ) + self._metadata = weakref.ref(batch) + indices = self._indices[layer] + torch_npu.npu_scatter_nd_update_(keys, indices, k) + torch_npu.npu_scatter_nd_update_(values, indices, v) From 65bf157873afb029ef04920898176c3587c49c1d Mon Sep 17 00:00:00 2001 From: KrystalRay Date: Tue, 3 Nov 2026 02:51:19 +0800 Subject: [PATCH 05/20] feat: add explicit BF16 eager attention on Ascend --- tests/test_ascend_eager.py | 127 ++++++++++++++++++++++++++++++ tests/test_ascend_serving.py | 7 +- vllm_rlt/attention/backend.py | 14 +++- vllm_rlt/core/kv_cache_manager.py | 8 +- vllm_rlt/engine/stage_engine.py | 6 +- vllm_rlt/entrypoints/cli.py | 8 +- vllm_rlt/entrypoints/serve.py | 5 +- vllm_rlt/kernels/ascend_eager.py | 76 ++++++++++++++++++ 8 files changed, 235 insertions(+), 16 deletions(-) create mode 100644 tests/test_ascend_eager.py create mode 100644 vllm_rlt/kernels/ascend_eager.py diff --git a/tests/test_ascend_eager.py b/tests/test_ascend_eager.py new file mode 100644 index 00000000..3698f2d6 --- /dev/null +++ b/tests/test_ascend_eager.py @@ -0,0 +1,127 @@ +"""Causal/GQA semantics for the separately selected BF16 eager contract.""" + +import pytest +import torch + +from vllm_rlt.kernels.ascend_eager import AscendEagerAttention + + +@pytest.mark.parametrize("kv_heads", [1, 2, 4]) +def test_eager_groups_match_independent_causal_queries(kv_heads): + torch.manual_seed(730) + keys = torch.randn(8, 16, kv_heads, 64, dtype=torch.bfloat16) + values = torch.randn_like(keys) + q = torch.randn(7, 4, 64, dtype=torch.bfloat16) + lengths = torch.tensor([0, 1, 15, 16, 17, 3, 31], dtype=torch.int32) + tables = torch.tensor([[4, 1], [4, 1], [4, 1], [4, 1], [4, 1], [7, 2], [7, 2]]) + actual = AscendEagerAttention()(q, keys, values, tables, lengths) + expected = torch.zeros_like(q) + for row, length in enumerate(lengths.tolist()): + if not length: + continue + pos = torch.arange(length) + k = keys[tables[row, pos // 16], pos % 16].repeat_interleave(4 // kv_heads, dim=1) + v = values[tables[row, pos // 16], pos % 16].repeat_interleave(4 // kv_heads, dim=1) + scores = torch.einsum("hd,thd->ht", q[row], k) * (64**-0.5) + probs = scores.softmax(-1, dtype=torch.float32).to(q.dtype) + expected[row] = torch.einsum("ht,thd->hd", probs, v) + torch.testing.assert_close(actual, expected, atol=0.02, rtol=0.02) + assert torch.isfinite(actual).all() + assert torch.count_nonzero(actual[0]) == 0 + + +@pytest.mark.npu +@pytest.mark.parametrize("chunk_size", [4096, 128]) +@torch.inference_mode() +def test_engine_matches_the_same_manual_prefill_chunks(chunk_size): + from tests.helpers import tiny_ouro_config + from vllm_rlt.models import OuroForCausalLM + from vllm_rlt import CacheConfig, SamplingParams, SchedulerConfig + from vllm_rlt.core.kv_cache_manager import KVCacheManager + from vllm_rlt.device import resolve_device + from vllm_rlt.engine.stage_engine import StageEngine + + device = resolve_device("npu:0") + torch.npu.set_device(device) + torch.manual_seed(41) + model = OuroForCausalLM( + tiny_ouro_config( + hidden_size=256, intermediate_size=512, head_dim=64, max_position_embeddings=256 + ) + ).to(device=device, dtype=torch.bfloat16) + c = model.config + tokens = ([3, 7, 2, 11, 5, 17, 8] * 22)[:149] + cache = KVCacheManager( + c.num_hidden_layers, + c.num_key_value_heads, + c.head_dim, + 128, + 16, + 4, + device=device, + dtype=torch.bfloat16, + backend="ascend_eager", + ) + assert cache.allocate("manual", len(tokens) + 3) + expected, manual_states = [], [] + handle = model.model.norm.register_forward_hook( + lambda _m, _i, output: manual_states.append(output.detach().cpu().clone()) + ) + try: + for start in range(0, len(tokens), chunk_size): + end = min(start + chunk_size, len(tokens)) + hidden = model.prelude(torch.tensor(tokens[start:end], device=device)) + for depth in range(4): + hidden, _ = model.recurrent( + hidden, + ["manual"] * (end - start), + [depth] * (end - start), + list(range(start, end)), + cache, + compute_gate=False, + ) + for step in range(3): + expected.append(int(model.coda(hidden[-1:])[0].argmax())) + if step < 2: + pos = len(tokens) + step + hidden = model.prelude(torch.tensor([expected[-1]], device=device)) + for depth in range(4): + hidden, _ = model.recurrent( + hidden, ["manual"], [depth], [pos], cache, compute_gate=False + ) + finally: + handle.remove() + cache.free("manual") + actual_states = [] + handle = model.model.norm.register_forward_hook( + lambda _m, _i, output: actual_states.append(output.detach().cpu().clone()) + ) + engine = StageEngine( + model, + attention_backend="ascend_eager", + cache_config=CacheConfig(num_blocks=128), + scheduler_config=SchedulerConfig( + max_num_seqs=1, max_num_batched_tokens=4096, prefill_chunk_size=chunk_size + ), + ) + try: + engine.add_request( + "engine", + tokens, + SamplingParams(max_tokens=3, temperature=0, min_loops=4, max_loops=4, ignore_eos=True), + ) + result = None + while engine.has_unfinished_requests(): + for out in engine.step(): + if out.finished: + result = out + assert result is not None + assert result.token_ids == expected + assert result.exit_depths == [4] * 3 + assert len(actual_states) == len(manual_states) + for actual, reference in zip(actual_states, manual_states, strict=True): + torch.testing.assert_close(actual, reference, atol=0, rtol=0) + assert engine.cache_manager.num_used_blocks == 0 + finally: + handle.remove() + engine.close() diff --git a/tests/test_ascend_serving.py b/tests/test_ascend_serving.py index 90f596c0..384eef37 100644 --- a/tests/test_ascend_serving.py +++ b/tests/test_ascend_serving.py @@ -15,13 +15,14 @@ pytestmark = pytest.mark.npu -def test_checkpoint_http_completions_stream_and_disconnect(tmp_path): +@pytest.mark.parametrize("backend", ["torch", "ascend_eager"]) +def test_checkpoint_http_completions_stream_and_disconnect(tmp_path, backend): import aiohttp source = os.environ.get("VLLM_RLT_OURO_CHECKPOINT") if not source: pytest.skip("set VLLM_RLT_OURO_CHECKPOINT to the pinned local Ouro-1.4B checkpoint") - folder = Path(os.environ.get("VLLM_RLT_ASCEND_RESULTS", tmp_path)) + folder = Path(os.environ.get("VLLM_RLT_ASCEND_RESULTS", tmp_path)) / backend folder.mkdir(parents=True, exist_ok=True) with socket.socket() as listener: listener.bind(("127.0.0.1", 0)) @@ -39,7 +40,7 @@ def test_checkpoint_http_completions_stream_and_disconnect(tmp_path): "--dtype", "bfloat16", "--attention-backend", - "torch", + backend, "--num-blocks", "64", "--max-num-seqs", diff --git a/vllm_rlt/attention/backend.py b/vllm_rlt/attention/backend.py index 210e766e..48a06383 100644 --- a/vllm_rlt/attention/backend.py +++ b/vllm_rlt/attention/backend.py @@ -36,6 +36,12 @@ class AttentionBackend: def __call__(self, *args, **kwargs): return self.kernel(*args, **kwargs) + def attend_metadata(self, q, keys, values, batch): + method = getattr(self.kernel, "attend_metadata", None) + if method is not None: + return method(q, keys, values, batch) + return self(q, keys, values, batch.block_tables, batch.context_lengths) + def prefill(self, *args, **kwargs): return self.kernel.prefill(*args, **kwargs) @@ -68,11 +74,17 @@ def backend_capabilities(implementation) -> BackendCapabilities: def validate_backend_name(backend: str) -> None: - if backend not in {"auto", "torch", "triton", *FLASH_BACKENDS}: + if backend not in {"auto", "torch", "triton", "ascend_eager", *FLASH_BACKENDS}: raise ValueError("unknown attention backend") def _create_explicit(backend, device, dtype, head_dim, block_size): + if backend == "ascend_eager": + if device.type != "npu" or dtype != torch.bfloat16: + raise ValueError("ascend_eager requires an NPU and BF16") + from vllm_rlt.kernels.ascend_eager import AscendEagerAttention + + return AttentionBackend(AscendEagerAttention(), _TORCH_CAPABILITIES, {"backend": backend}) if backend == "triton": if device.type != "cuda": raise ValueError("the Triton attention backend requires a CUDA or ROCm device") diff --git a/vllm_rlt/core/kv_cache_manager.py b/vllm_rlt/core/kv_cache_manager.py index 37c1352a..3a88f39e 100644 --- a/vllm_rlt/core/kv_cache_manager.py +++ b/vllm_rlt/core/kv_cache_manager.py @@ -755,12 +755,8 @@ def _attend_prepared( batch.cu_seqlens_q, batch.max_seqlen_q, ) - return self.attention( - q, - self.key_cache[:, layer], - self.value_cache[:, layer], - batch.block_tables, - batch.context_lengths, + return self.attention.attend_metadata( + q, self.key_cache[:, layer], self.value_cache[:, layer], batch ) @torch.no_grad() diff --git a/vllm_rlt/engine/stage_engine.py b/vllm_rlt/engine/stage_engine.py index dd13576a..e1b25f5c 100644 --- a/vllm_rlt/engine/stage_engine.py +++ b/vllm_rlt/engine/stage_engine.py @@ -73,8 +73,10 @@ def __init__( execution_config or ExecutionConfig(), scheduler_config, speculative_config ) if parameter.device.type == "npu": - if attention_backend != "torch": - raise ValueError("NPU execution requires explicit attention_backend='torch'") + if attention_backend not in ("torch", "ascend_eager"): + raise ValueError( + "NPU execution requires explicit attention_backend='torch' or 'ascend_eager'" + ) if config.to_dict()["model_type"] != "ouro" or config.total_ut_steps != 4: raise ValueError("NPU currently supports Ouro with four loops") if parameter.dtype != torch.bfloat16: diff --git a/vllm_rlt/entrypoints/cli.py b/vllm_rlt/entrypoints/cli.py index 5099ef26..29a96295 100644 --- a/vllm_rlt/entrypoints/cli.py +++ b/vllm_rlt/entrypoints/cli.py @@ -67,6 +67,7 @@ def main(): "--attention-backend", type=str.lower, choices=[ + "ascend_eager", "auto", "torch", "triton", @@ -100,8 +101,11 @@ def main(): parser.error("--toy uses built-in token ID prompts; omit --prompt") if not args.toy and not args.prompt: parser.error("provide --prompt or use --toy for a CPU smoke test") - if args.device.split(":", 1)[0] == "npu" and args.attention_backend != "torch": - parser.error("NPU execution requires explicit --attention-backend torch") + if args.device.split(":", 1)[0] == "npu" and args.attention_backend not in ( + "torch", + "ascend_eager", + ): + parser.error("NPU execution requires explicit --attention-backend torch or ascend_eager") profiling = profile_config_from_args(args) if profiling is not None: profiling.resolve_activities(args.device) diff --git a/vllm_rlt/entrypoints/serve.py b/vllm_rlt/entrypoints/serve.py index 713bab30..70a708f7 100644 --- a/vllm_rlt/entrypoints/serve.py +++ b/vllm_rlt/entrypoints/serve.py @@ -88,6 +88,7 @@ def main(): "--attention-backend", type=str.lower, choices=[ + "ascend_eager", "auto", "torch", "triton", @@ -115,8 +116,8 @@ def main(): limits = ServingLimits(**{name: getattr(args, name) for name in defaults}) if args.device == "cpu" and args.attention_backend not in ("auto", "torch"): parser.error("CPU execution requires --attention-backend torch or auto") - if args.device == "npu" and args.attention_backend != "torch": - parser.error("NPU execution requires explicit --attention-backend torch") + if args.device == "npu" and args.attention_backend not in ("torch", "ascend_eager"): + parser.error("NPU execution requires explicit --attention-backend torch or ascend_eager") torch.set_num_threads(args.cpu_threads) logging.basicConfig(level=logging.INFO) app = create_app( diff --git a/vllm_rlt/kernels/ascend_eager.py b/vllm_rlt/kernels/ascend_eager.py new file mode 100644 index 00000000..2ab08478 --- /dev/null +++ b/vllm_rlt/kernels/ascend_eager.py @@ -0,0 +1,76 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Explicit BF16 eager attention on Ascend, matching released Ouro arithmetic.""" + +import math +import weakref + +import torch + + +class AscendEagerAttention: + """Batch each request/depth's queries; borrow only validated host metadata.""" + + def __init__(self): + self._metadata = None + self._groups = None + + def _plan(self, keys, lengths, device): + grouped = {} + for row, (key, length) in enumerate(zip(keys, lengths, strict=True)): + if length: + grouped.setdefault(key, []).append(row) + planned = [] + for rows in grouped.values(): + maximum = max(lengths[row] for row in rows) + indices = torch.tensor(rows, dtype=torch.long, device=device) + positions = torch.arange(maximum, device=device) + valid = positions.unsqueeze(0) < torch.tensor( + [lengths[row] for row in rows], device=device + ).unsqueeze(1) + mask = torch.zeros((len(rows), maximum), dtype=torch.bfloat16, device=device) + mask.masked_fill_(~valid, torch.finfo(torch.bfloat16).min) + initial = len(rows) == maximum and [lengths[row] for row in rows] == list( + range(1, maximum + 1) + ) + planned.append((rows[0], indices, positions, mask, initial)) + return planned + + def _forward(self, q, keys, values, tables, groups): + result = torch.zeros_like(q) + block_size = keys.shape[1] + head_groups = q.shape[1] // keys.shape[2] + for first, indices, positions, mask, initial in groups: + blocks = tables[first, positions // block_size] + offsets = positions % block_size + k, v = keys[blocks, offsets], values[blocks, offsets] + if head_groups != 1: + k = k.repeat_interleave(head_groups, dim=1) + v = v.repeat_interleave(head_groups, dim=1) + query = q.index_select(0, indices).transpose(0, 1).unsqueeze(0).contiguous() + key = k.transpose(0, 1).unsqueeze(0) + value = v.transpose(0, 1).unsqueeze(0) + # HF's initial V is a projection view; subsequent cache concatenation + # makes K/V contiguous. Preserve that operation layout on this path. + if not initial: + key, value = key.contiguous(), value.contiguous() + scores = torch.matmul(query, key.transpose(-2, -1)) * (1.0 / math.sqrt(q.shape[-1])) + # Keep the BF16 score/scaling and probability rounding boundaries. + scores = scores + mask[None, None, :, :] + probabilities = torch.softmax(scores, dim=-1, dtype=torch.float32).to(q.dtype) + output = torch.matmul(probabilities, value).squeeze(0).transpose(0, 1) + result.index_copy_(0, indices, output) + return result + + def __call__(self, q, keys, values, tables, lengths): + groups = self._plan([tuple(row) for row in tables.tolist()], lengths.tolist(), q.device) + return self._forward(q, keys, values, tables, groups) + + def attend_metadata(self, q, keys, values, batch): + if self._metadata is None or self._metadata() is not batch: + self._groups = self._plan( + [(id(allocation), depth) for allocation, depth, _ in batch.rows], + [position + 1 for _, _, position in batch.rows], + q.device, + ) + self._metadata = weakref.ref(batch) + return self._forward(q, keys, values, batch.block_tables, self._groups) From 87f9a21d9798603a61f961f17c6b4fae71e81e07 Mon Sep 17 00:00:00 2001 From: KrystalRay Date: Tue, 3 Nov 2026 03:21:33 +0800 Subject: [PATCH 06/20] test: guard Ascend BF16 attention rounding contract --- tests/test_ascend_eager.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/tests/test_ascend_eager.py b/tests/test_ascend_eager.py index 3698f2d6..37e5f6ce 100644 --- a/tests/test_ascend_eager.py +++ b/tests/test_ascend_eager.py @@ -35,11 +35,11 @@ def test_eager_groups_match_independent_causal_queries(kv_heads): @torch.inference_mode() def test_engine_matches_the_same_manual_prefill_chunks(chunk_size): from tests.helpers import tiny_ouro_config - from vllm_rlt.models import OuroForCausalLM from vllm_rlt import CacheConfig, SamplingParams, SchedulerConfig from vllm_rlt.core.kv_cache_manager import KVCacheManager from vllm_rlt.device import resolve_device from vllm_rlt.engine.stage_engine import StageEngine + from vllm_rlt.models import OuroForCausalLM device = resolve_device("npu:0") torch.npu.set_device(device) @@ -125,3 +125,30 @@ def test_engine_matches_the_same_manual_prefill_chunks(chunk_size): finally: handle.remove() engine.close() + + +@pytest.mark.parametrize("device_name", ["cpu", pytest.param("npu", marks=pytest.mark.npu)]) +@torch.inference_mode() +def test_bf16_probability_rounding_is_part_of_the_eager_contract(device_name): + from vllm_rlt.device import resolve_device + from vllm_rlt.kernels.paged_attention import torch_paged_attention + + device = resolve_device(device_name) + if device_name == "npu": + torch.npu.set_device(0) + keys = torch.zeros(1, 16, 1, 64, dtype=torch.bfloat16, device=device) + values = torch.zeros_like(keys) + keys[0, 0].fill_(1.03125) + keys[0, 1].fill_(1) + values[0, 0].fill_(128) + values[0, 1].fill_(-128) + query = torch.ones(1, 1, 64, dtype=torch.bfloat16, device=device) + table = torch.tensor([[0]], dtype=torch.int32, device=device) + lengths = torch.tensor([2], dtype=torch.int32, device=device) + # QK/sqrt(64) is [8.25,8]. BF16 probabilities are [.5625,.4375], + # so their weighted value is exactly16. FP32 probabilities retain + # [.5621765,.4378235] and round the final value to15.9375 instead. + actual = AscendEagerAttention()(query, keys, values, table, lengths) + torch.testing.assert_close(actual, torch.full_like(actual, 16), atol=0, rtol=0) + old_contract = torch_paged_attention(query, keys, values, table, lengths) + assert not torch.equal(actual, old_contract) From 9c6c1ccbd08acc4cd7f80576bcc6a0bd52acffc1 Mon Sep 17 00:00:00 2001 From: KrystalRay Date: Tue, 3 Nov 2026 03:24:48 +0800 Subject: [PATCH 07/20] fix: keep native KV writes opt-in with Ascend eager --- tests/test_ascend_kv.py | 18 ++++++++++++++++-- vllm_rlt/core/kv_cache_manager.py | 2 +- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/tests/test_ascend_kv.py b/tests/test_ascend_kv.py index 8445e0f0..298d0c1a 100644 --- a/tests/test_ascend_kv.py +++ b/tests/test_ascend_kv.py @@ -10,11 +10,24 @@ @pytest.mark.parametrize("rows", [1, 3]) +@pytest.mark.parametrize("backend", ["torch", "ascend_eager"]) @torch.inference_mode() -def test_native_scatter_preserves_inputs_other_layers_and_reused_ids(rows): +def test_native_scatter_preserves_inputs_other_layers_and_reused_ids(rows, backend, monkeypatch): device = resolve_device("npu:0") torch.npu.set_device(device) - cache = KVCacheManager(2, 2, 64, 32, 16, 4, device=device, dtype=torch.bfloat16) + import torch_npu + + native = torch_npu.npu_scatter_nd_update_ + calls = [] + + def observed(*args, **kwargs): + calls.append(True) + return native(*args, **kwargs) + + monkeypatch.setattr(torch_npu, "npu_scatter_nd_update_", observed) + cache = KVCacheManager( + 2, 2, 64, 32, 16, 4, device=device, dtype=torch.bfloat16, backend=backend + ) cache.key_cache.zero_() cache.value_cache.zero_() assert cache.allocate("a", 17) @@ -49,3 +62,4 @@ def test_native_scatter_preserves_inputs_other_layers_and_reused_ids(rows): cache.free("b") assert cache.num_used_blocks == 0 torch.npu.synchronize() + assert len(calls) == (6 if backend == "ascend_eager" else 0) diff --git a/vllm_rlt/core/kv_cache_manager.py b/vllm_rlt/core/kv_cache_manager.py index 3a88f39e..2ee380eb 100644 --- a/vllm_rlt/core/kv_cache_manager.py +++ b/vllm_rlt/core/kv_cache_manager.py @@ -684,7 +684,7 @@ def _write_prepared( self._validate_tensor(v, len(batch.position_ids), "v") if not batch.rows: return - if self.device.type == "npu": + if self.device.type == "npu" and self.attention.info.get("backend") == "ascend_eager": if not hasattr(self, "_npu_writer"): from vllm_rlt.kernels.ascend_kv import AscendKVWriter From 227e966bc2cc5f86eab5c5f4b2b31afb63c7c66f Mon Sep 17 00:00:00 2001 From: KrystalRay Date: Tue, 3 Nov 2026 03:45:15 +0800 Subject: [PATCH 08/20] docs: record Ascend BF16 contract and measured validation --- README.md | 2 +- docs/recipes/ouro_ascend.md | 275 +++++++++++++++++------------------- 2 files changed, 132 insertions(+), 145 deletions(-) diff --git a/README.md b/README.md index 183cc29e..9b4f90db 100644 --- a/README.md +++ b/README.md @@ -134,7 +134,7 @@ No PyPI publication is implied. | Checkpoint | Implementation | Documented validation scope | Guide | | --- | --- | --- | --- | -| Ouro-1.4B | Native Ouro | Existing runtime and benchmark coverage under their recorded configurations; experimental single-device Ascend 910B3 BF16/fixed-four-loop synchronous Torch path, with paired quality regressions pending correction | [Runtime](docs/runtime.md), [accuracy](docs/accuracy.md), [Ascend recipe](docs/recipes/ouro_ascend.md) | +| Ouro-1.4B | Native Ouro | Existing runtime and benchmark coverage under their recorded configurations; experimental single-device Ascend 910B3 BF16/fixed-four-loop synchronous path with explicit HF-style attention; selected full-prefill paired regression matches official HF (63/87 each, 87/87 exact sequences) | [Runtime](docs/runtime.md), [accuracy](docs/accuracy.md), [Ascend recipe](docs/recipes/ouro_ascend.md) | | Ouro-2.6B | Shared Ouro | Shares the Ouro implementation; not separately qualified, and 1.4B quality/performance results are not inherited | [Scope notes](docs/recipes/ouro_thinking.md#6-evidence-and-limitations) | | Ouro-1.4B-Thinking | Reuses Ouro | Bounded BF16, fixed-four-loop synchronous basic integration | [Shared recipe](docs/recipes/ouro_thinking.md) | | Ouro-2.6B-Thinking | Reuses Ouro | Bounded BF16, fixed-four-loop synchronous basic integration | [Shared recipe](docs/recipes/ouro_thinking.md) | diff --git a/docs/recipes/ouro_ascend.md b/docs/recipes/ouro_ascend.md index 43d58995..659b8e3e 100644 --- a/docs/recipes/ouro_ascend.md +++ b/docs/recipes/ouro_ascend.md @@ -1,66 +1,67 @@ -# Ouro-1.4B on Ascend 910B3: initial synchronous support +# Ouro-1.4B on Ascend 910B3 -This path reuses the native Ouro model and shared engine on one Ascend NPU. -Select **--attention-backend torch explicitly**. Weights, ordinary activations -and KV storage remain BF16; the existing Torch attention computes QK, softmax -and PV intermediates in FP32 through Torch-NPU. This is not a fused CANN -attention integration and makes no acceleration claim. +This experimental path runs native Ouro on one NPU with BF16, synchronous +greedy decoding and four recurrent loops. Select +`--attention-backend ascend_eager` explicitly. It batches causal queries by +request/depth through Torch-NPU/CANN MatMul and Softmax primitives and uses +native ScatterND for paged KV writes. It preserves released Ouro HF eager +rounding: BF16 QK/scale, FP32 softmax rounded to BF16, then BF16 PV. -**Quality status:** the selected same-device paired screen is HF 63/87 versus -native 61/87 (losses: 203 and 881; gains: none). The frozen no-net-loss gate -failed. This implementation remains experimental and is not ready for -submission as a quality-qualified backend. +The older explicit `torch` backend remains available with its original +FP32 attention intermediates and indexed KV writes. Its arithmetic is a +different reference contract. Automatic backend selection is unchanged. ## Supported scope -- Ouro, fixed four-loop greedy inference, synchronous local execution, - LAST_EXITED depth-aware paged KV, prefill and incremental decode. -- Explicit num_blocks or kv_cache_memory_bytes; automatic NPU memory sizing - is not implemented. -- Cancellation, request ID reuse, EOS/length handling, HTTP completions and SSE - reuse the existing engine and serving lifecycle. -- Async scheduling, CUDA graphs, speculation, preemption, prefix caching, PD, - adaptive exit and stochastic sampling are rejected for this initial path. -- Qualification targets Ouro-1.4B on 910B3. Other checkpoints and Ascend devices - require separate validation. - -The shared scheduler, model and KV owner are retained. NPU submissions drain -before metadata buffers or request pages can be reused. CPU/CUDA execution -does not import the optional Torch-NPU package. Built-in profiling supports -CPU/CUDA; use torch_npu.profiler for NPU device traces. +- Native Ouro-1.4B, BF16 parameters/ordinary activations/KV, fixed four loops, + synchronous single-device greedy generation. +- Per-request/depth/layer paged KV; incremental/full allocation, causal prefill, + decode, cancellation and request-ID reuse. +- Existing spawned HTTP/SSE serving lifecycle. Explicit KV capacity is required. +- Async scheduling, graphs, speculation, PD, preemption, prefix caching, + adaptive depth and stochastic sampling are rejected for this initial path. +- Other checkpoints, Ascend models/devices and concurrent task quality need + separate qualification. The quality regression below uses a full prompt + forward; chunked generation can round differently. + +The shared scheduler/model/KV owner are retained. NPU submissions drain before +metadata or pages can be reused. CPU/CUDA do not import Torch-NPU unconditionally. +Use `torch_npu.profiler` for NPU device profiling; built-in profiling targets +CPU/CUDA. ## Environment and installation -The development environment is Linux aarch64, Python 3.11.13, Torch 2.9.0+cpu, -torch-npu 2.9.0, CANN 9.0.0, driver 26.1.0.b050 and Transformers 4.57.6. -Keep Torch and Torch-NPU matched; activate vendor CANN before the project Python. +Measured environment: Linux aarch64, Python 3.11.13, Torch 2.9.0+cpu, +Torch-NPU 2.9.0, CANN 9.0.0, driver 26.1.0.b050, Transformers 4.57.6, +Tokenizers 0.22.2 and NumPy 1.26.4. Keep the vendor Torch pair matched. ~~~bash source /path/to/cann/set_env.sh source /path/to/isolated-npu-environment/bin/activate -export ASCEND_RT_VISIBLE_DEVICES=4 # replace with your assigned physical NPU +export ASCEND_RT_VISIBLE_DEVICES=4 # use your assigned physical NPU python -c 'import torch, torch_npu; print(torch.__version__, torch_npu.__version__); print(torch.npu.get_device_name(0))' python -m pip install 'numpy==1.26.4' 'safetensors==0.8.0' 'huggingface-hub==0.36.2' \ 'transformers==4.57.6' 'tokenizers==0.22.2' 'aiohttp==3.14.3' setuptools wheel python -m pip install --no-deps --no-build-isolation -e . ~~~ -The no-deps install preserves the vendor Torch pair and avoids the project's -Linux Triton dependency. Neither Triton nor a native CUDA extension is required -for this explicit Torch backend. No InferRT or MindIE-SD integration is involved. +The no-deps install preserves vendor Torch and avoids the Linux Triton +dependency. No native CUDA extension, custom compiled kernel or InferRT is used. +Current upstream ScatterPA examples use an API/layout unavailable in this +Torch-NPU/CANN pair; the native writer retains the existing ND cache. -Use the complete ByteDance/Ouro-1.4B checkpoint at revision -574fa66cb8bf5abdc979642d01cf2b79b16bfab1. The model.safetensors SHA256 is -58872a72616c736595b8b7662079c5b12c5a162ec16eae94f21c348dfa9885af. +Use ByteDance/Ouro-1.4B revision 574fa66cb8bf5abdc979642d01cf2b79b16bfab1. +Weights SHA256: +`58872a72616c736595b8b7662079c5b12c5a162ec16eae94f21c348dfa9885af`. -## Start the service +## Start and call the service ~~~bash python -m vllm_rlt.entrypoints.serve \ --model /path/to/Ouro-1.4B --served-model-name ouro \ - --device npu --dtype bfloat16 --attention-backend torch \ - --num-blocks 64 --block-size 16 --max-num-seqs 2 \ - --max-num-batched-tokens 32 --port 8123 + --device npu --dtype bfloat16 --attention-backend ascend_eager \ + --num-blocks 1024 --block-size 16 --max-num-seqs 1 \ + --max-num-batched-tokens 4096 --prefill-chunk-size 4096 --port 8123 ~~~ ~~~bash @@ -70,115 +71,101 @@ curl http://127.0.0.1:8123/v1/completions \ -d '{"model":"ouro","prompt":"The capital of France is","max_tokens":4,"temperature":0,"min_loops":4,"max_loops":4,"exit_threshold":1}' ~~~ -Add stream=true for SSE. Python callers use device="npu" and -attention_backend="torch"; the spawned worker registers Torch-NPU itself. -The assigned physical device becomes logical npu:0. Increase KV capacity for -longer prompts, output limits or concurrency. This small service command is a -smoke configuration, not a performance benchmark. +Add `stream=true` for SSE. Python callers select `device="npu"` and +`attention_backend="ascend_eager"`; the spawned worker registers Torch-NPU. +The assigned physical NPU becomes logical `npu:0`. Capacity and token budgets +must fit your prompts/outputs/concurrency; this command is a reproduction +configuration, not certification of every 4096-token input. -## Reproduce validation +Both `prefill_chunk_size` and `max_num_batched_tokens` must accommodate the +whole prompt to compare with official full-prefill. Raising only the latter +retains the scheduler's default 128-token chunks. + +## Validation (2026-11-03) + +Base 989921ac1d3897c5c43e27ec8cddbc91fa953194 rejects `--device npu`. +Runtime 45220081159627ff4fba36a6b0b3ac10787214c8 supplies the new backend. +Final runtime be037ed5cb9344b59830da2449b5590f3dd6c231 adds an arithmetic +regression and makes native writes exclusive to the new backend; selected +eager execution and original Torch execution were separately revalidated. +Later documentation changes preserve these runtime/test contents. ~~~bash -python -m pip install pytest ruff pre-commit -pre-commit run --all-files +pre-commit run --all-files --verbose export VLLM_RLT_OURO_CHECKPOINT=/path/to/Ouro-1.4B -export VLLM_RLT_ASCEND_RESULTS=/path/to/validation-results +export VLLM_RLT_ASCEND_RESULTS=/path/to/results python -m pytest tests/test_ascend.py tests/test_ascend_checkpoint.py \ - tests/test_ascend_serving.py --run-npu -q + tests/test_ascend_serving.py tests/test_ascend_kv.py \ + tests/test_ascend_eager.py --run-npu -q ~~~ -The NPU flag requires assigned ASCEND_RT_VISIBLE_DEVICES; CPU collection skips -device tests. Checks cover a CPU attention oracle, GQA and causal page -boundaries, incremental/full KV allocation, four-depth state ownership, -chunking, cancellation/reuse, EOS/length behavior, pinned checkpoint generation -and a real spawned HTTP/SSE worker. - -Metadata-only allocation changes require exact hidden/logit/KV equality. -The BF16 diagnostic bounds remain relative L2 at most 2% and elementwise -atol=0.02, rtol=0.02. These are this validation's bounds, not a universal quality -guarantee. Task quality needs separately pinned paired evaluation against the -official code; short generation does not establish broader quality. -Performance characterization should retain input lengths, concurrency, -TTFT/TPOT/throughput, memory and repetitions; the existing -[serving benchmark guide](../serving.md#reuse-the-upstream-benchmark) describes -the compatible upstream client. - -## Fused attention remains separate - -An experimental BF16 paged npu_fused_infer_attention_score adapter is outside -this change. It failed frozen whole-model numerical checks and lost a correct -answer in a ten-question smoke. Promoting attention intermediates in -npu_fusion_attention improved local mathematical agreement but still failed -the original whole-model joint gate. Those failures have not been relaxed or -hidden; this recipe does not qualify that fused backend. - -This contribution covers a bounded part of -[#69 Ascend support](https://github.com/InferMatrixLab/vllm-rlt/issues/69). -It does not complete CANN fused adaptation or other Ouro runtime features on -Ascend. Measured validation outcomes and remaining blockers are recorded below. - -## Measured validation (2026-11-03) - -Runtime source: 4734e1963bdf7fd4e0fd66b2612511d6d4173550, against base -989921ac1d3897c5c43e27ec8cddbc91fa953194. Later recipe updates do not change -the measured runtime sources. - -- Ruff lint/format and the CPU regression hook passed. -- Opt-in real 910B3 tests: 10 passed in 44.41s, including actual HTTP/SSE. -- Incremental/preallocated real-weight KV differential: 216 finite states, all - exactly equal; attention/chunking diagnostics retain their original joint bounds. -- Same-device paired selected GSM8K-87: official 63/87, native 61/87; - losses [203, 881], gains []; 63/87 exact token sequences. - Parse failures: {'hf_eager': 19, 'rlt_target': 19}; length limits: {'hf_eager': 0, 'rlt_target': 0}; - execution limits: {'hf_eager': 0, 'rlt_target': 0}. -- Each pair runs on the same physical 910B3; all 87 cases pool three cards. - The original ten are regression smoke; 77 cases were held out. This is - selected-case evidence, not full GSM8K or adaptive/concurrent quality. - -The quality run uses official unmodified HF eager, Transformers 4.57.6, -Tokenizers 0.22.2, lm-eval 0.4.9.2 strict scoring, three-shot raw CoT, -greedy four-loop BF16, max 1024 output tokens and source-ranked seed0 IDs. -Dataset revision is 740312add88f781978c0658806c59bc2815b9866. - -The table below is a separate **engine-only short-burst** characterization: -8 output tokens/request, ignore EOS, one shape-specific warmup and three -unprofiled repetitions. Values are medians of per-burst request means; output -throughput includes prefill and drain. TTFT starts before engine admission. -TPOT is first-to-last observed output time divided by seven. - -| Input | Concurrency | Output tok/s (range) | TTFT ms | TPOT ms | Peak allocated / reserved GiB | -|---|---:|---:|---:|---:|---:| -| 16 | 1 | 2.726 (2.700–2.901) | 1238.01 | 241.94 | 4.970 / 5.309 | -| 16 | 2 | 3.793 (3.657–3.810) | 2122.58 | 298.19 | 4.970 / 5.311 | -| 16 | 8 | 5.031 (5.016–5.086) | 7980.90 | 675.01 | 4.973 / 5.314 | -| 64 | 1 | 1.367 (1.342–1.372) | 4176.89 | 242.77 | 4.972 / 5.314 | -| 64 | 2 | 1.590 (1.569–1.599) | 7964.07 | 300.45 | 4.974 / 5.314 | -| 64 | 8 | 1.789 (1.786–1.793) | 23417.86 | 1756.95 | 4.984 / 5.314 | -| 256 | 1 | 0.370 (0.364–0.373) | 19472.72 | 307.45 | 4.978 / 5.318 | -| 256 | 2 | 0.372 (0.371–0.377) | 39959.74 | 431.61 | 4.988 / 5.318 | -| 256 | 8 | 0.379 (0.378–0.380) | 125114.88 | 6228.64 | 4.988 / 5.318 | - -Configuration: 768 blocks, block size 16, max 8 sequences / 256 batched tokens, -incremental allocation, no async/graphs/UVA/prefix caching/speculation. -Measured requests: 99, output tokens: 792, failures: 0. Live logical KV blocks -after each completed burst: 0. -Torch allocator peaks include model/cache/workspace and exclude some driver -or runtime memory. Other cards ran independent quality shards; CPU/NUMA -affinity was not pinned. No steady-state throughput, HTTP/p99, accelerator -hardware-only comparison or acceleration claim is made. - -Separate ops-only/full profiles capture a complete warmed second-token decode -iteration, with actual NPU device kernels. Full mode enables shapes, stacks -and memory; exported shapes, readable Python stacks and memory CSV rows -were checked separately from performance timing. -Full prefill plus all eight output tokens is outside these diagnostic captures. - -The conservative Torch path has substantial prefill/dispatch overhead. Fused -attention optimization remains a later step and must retain the quality gates. - -The no-net-loss quality criterion failed on this selected set. Case 203 -changes a quantity from 7 to 5 and produces 27 instead of 31. Case 881 writes -'The answer is 10 + 6 = 16'; the unchanged strict scorer extracts 10. It remains -a failure under the frozen protocol. Neither the scorer nor the gate was -relaxed. A diagnostic HF-style BF16 attention alternative also failed the -original numerical bounds and was not incorporated. +- Ruff/pre-commit passed; CPU 926 passed / 602 skipped. Device cases were run + separately:22 passed, comprising18 NPU and 4 CPU cases. +- Final eager vs unmodified official HF:216 selected states and2016 layer + events exactly equal; restored Torch vs original Torch is also exact. +- The arithmetic regression distinguishes an analytical output16 in BF16 + eager from15.9375 in the original FP32-intermediate path, on CPU andNPU. +- Real spawned health/completions/SSE, termination/usage, disconnect recovery + and shutdown pass for both explicit backends. +- At4522008,87 selected same-device HF/native pairs score63/87 each, + losses[]/gains[], with87/87 complete token sequences exactly equal. + Parse failures19 each; length/execution limits0 each. Every pair shares a + physical910B3, pooled across cards4/5/6. These are reused regression cases, + not a held-out/full-GSM8K estimate. No threshold or scorer was relaxed. +- Quality uses one full-prefill chunk4096 and4096 token budget. Chunked128, + concurrent/adaptive task quality and other models remain unqualified. + +Numerical bounds remain joint relative L2<=2% and atol=rtol=0.02. +The new explicit backend is checked against released HF BF16 eager. +It still fails 209/216 states against the old FP32-intermediate Torch reference; +that failure remains recorded and is not called passed. + +The earlier63/87 HF vs61/87 native observation combined different arithmetic +with native default 128-token chunking. Changing either alone did not fix +question 203. Matching both restores203/881/526 and the full87 regression. +Manual question 203 replay has15 exact logits and192 exact populated KV hashes. + +## Measured performance + +Same physical910B3, profiler disabled, P64/C1 orC8, fixed4 loops, ignore EOS, +exactly 8 outputs/request,768 blocks,max 8 sequences / 256 batched tokens,explicit +128-token chunks. One warmup/shape,3repetitions/run; A/C/C/A gives 6 observations +per arm/shape. TTFT/TPOT are medians of per-burst request means, not pooled +individual-request percentiles. Throughput includes prefill/drain; no HTTP. + +|Concurrency|Old Torch tok/s|New backend tok/s|Old/new TTFT ms|Old/new TPOT ms| +|---:|---:|---:|---:|---:| +|1|1.326|4.711|4266.35 /221.03|246.81 /210.99| +|8|1.720|12.778|24363.00 /702.72|1818.05 /583.61| + +Pair-specific throughput ratios are 3.47–3.81x at C1 and 7.11–8.05x at C8, +with non-overlapping observed ranges. This combined backend/arithmetic change +does not isolate ScatterND. A KV-only A/B regressed 1.23% and 6.28%; +the final runtime therefore keeps original writes on the Torch path. +New backend native-writing optimality is not established. + +Final runtime be037ed confirmation on the sameNPU gives medians 4.752tok/s +at C1 and13.705tok/s at C8 (three reps each, measured after ABBA). +Allocated peaks are about 4.97–4.99GiB; reserved peaks about 5.33GiB. +All bursts finish with 0 live logical KV blocks and no failed repetitions. + +A separate 9-shape candidate characterization covers 16/64/256 input tokens and +concurrency 1/2/8. These are short bursts, not saturation/HTTP/SLO or hardware-only +claims. CPU/NUMA placement is not pinned; independent CPU/device checks overlap +part of the shared-host timing. Raw ranges and controls must accompany reuse. + +Separate ops-only/full warmed decode traces confirm actual native ScatterND +dispatch and 96 softmax calls. Full mode has readable shapes, stacks and memory +exports. Complete warmed request traces additionally cover prefill and all 8 +outputs. Profile durations are diagnostic and excluded from speed claims. + +## Fused operators remain experimental + +BF16 FIA and promoted FP32 FlashAttentionScore did not pass earlier whole-model +gates. Native RMSNorm changes also failed selected states despite local speed +gains; SwiGLU/RotaryMul candidates were not adopted. +The current backend uses native primitives and batching, not fused attention. + +This is a bounded contribution toward +[#69](https://github.com/InferMatrixLab/vllm-rlt/issues/69), with task quality +qualified only for the stated regression configuration. From 3f0bb13b082f03febe71064e91754440d3896741 Mon Sep 17 00:00:00 2001 From: KrystalRay Date: Tue, 3 Nov 2026 09:59:54 +0800 Subject: [PATCH 09/20] fix: remove Ascend SKU assumptions and clarify backend contracts --- README.md | 2 +- docs/recipes/ouro_ascend.md | 8 +++- tests/test_ascend.py | 3 +- tests/test_ascend_checkpoint.py | 6 ++- tests/test_attention_module.py | 70 +++++++++++++++++++++++++++++++ vllm_rlt/attention/backend.py | 5 +++ vllm_rlt/core/kv_cache_manager.py | 2 +- 7 files changed, 89 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 9b4f90db..569373db 100644 --- a/README.md +++ b/README.md @@ -134,7 +134,7 @@ No PyPI publication is implied. | Checkpoint | Implementation | Documented validation scope | Guide | | --- | --- | --- | --- | -| Ouro-1.4B | Native Ouro | Existing runtime and benchmark coverage under their recorded configurations; experimental single-device Ascend 910B3 BF16/fixed-four-loop synchronous path with explicit HF-style attention; selected full-prefill paired regression matches official HF (63/87 each, 87/87 exact sequences) | [Runtime](docs/runtime.md), [accuracy](docs/accuracy.md), [Ascend recipe](docs/recipes/ouro_ascend.md) | +| Ouro-1.4B | Native Ouro | Ascend inference support | [Runtime](docs/runtime.md), [Ascend recipe](docs/recipes/ouro_ascend.md) | | Ouro-2.6B | Shared Ouro | Shares the Ouro implementation; not separately qualified, and 1.4B quality/performance results are not inherited | [Scope notes](docs/recipes/ouro_thinking.md#6-evidence-and-limitations) | | Ouro-1.4B-Thinking | Reuses Ouro | Bounded BF16, fixed-four-loop synchronous basic integration | [Shared recipe](docs/recipes/ouro_thinking.md) | | Ouro-2.6B-Thinking | Reuses Ouro | Bounded BF16, fixed-four-loop synchronous basic integration | [Shared recipe](docs/recipes/ouro_thinking.md) | diff --git a/docs/recipes/ouro_ascend.md b/docs/recipes/ouro_ascend.md index 659b8e3e..124855b7 100644 --- a/docs/recipes/ouro_ascend.md +++ b/docs/recipes/ouro_ascend.md @@ -1,4 +1,4 @@ -# Ouro-1.4B on Ascend 910B3 +# Ouro-1.4B on Ascend This experimental path runs native Ouro on one NPU with BF16, synchronous greedy decoding and four recurrent loops. Select @@ -29,6 +29,12 @@ metadata or pages can be reused. CPU/CUDA do not import Torch-NPU unconditionall Use `torch_npu.profiler` for NPU device profiling; built-in profiling targets CPU/CUDA. +Hardware selection does not inspect or whitelist device model names. +The backend requires BF16 and the Torch-NPU/CANN operations used below. +The measurements in this recipe were collected on 910B3; other Ascend devices +can run the same opt-in checks without a model-name assertion. Their numerical +and performance results require validation on the actual device/runtime pair. + ## Environment and installation Measured environment: Linux aarch64, Python 3.11.13, Torch 2.9.0+cpu, diff --git a/tests/test_ascend.py b/tests/test_ascend.py index 4632c7f1..fc4209d7 100644 --- a/tests/test_ascend.py +++ b/tests/test_ascend.py @@ -1,4 +1,4 @@ -"""Opt-in synchronous BF16 Ouro checks using explicit Torch attention on 910B3.""" +"""Opt-in synchronous BF16 Ouro checks using explicit Torch attention on Ascend.""" import pytest import torch @@ -18,7 +18,6 @@ def npu_device(): resolve_device("npu") torch.npu.set_device(0) - assert torch.npu.get_device_name(0) == "Ascend910B3" yield torch.device("npu:0") torch.npu.synchronize() torch.npu.empty_cache() diff --git a/tests/test_ascend_checkpoint.py b/tests/test_ascend_checkpoint.py index 6b08c1d3..6beb91b2 100644 --- a/tests/test_ascend_checkpoint.py +++ b/tests/test_ascend_checkpoint.py @@ -24,9 +24,11 @@ def checkpoint(): pytest.skip("set VLLM_RLT_OURO_CHECKPOINT to the pinned local Ouro-1.4B checkpoint") resolve_device("npu") torch.npu.set_device(0) - assert torch.npu.get_device_name(0) == "Ascend910B3" + checksum = hashlib.sha256() with (Path(source) / "model.safetensors").open("rb") as stream: - digest = hashlib.file_digest(stream, "sha256").hexdigest() + for block in iter(lambda: stream.read(8 * 1024 * 1024), b""): + checksum.update(block) + digest = checksum.hexdigest() assert digest == "58872a72616c736595b8b7662079c5b12c5a162ec16eae94f21c348dfa9885af" model = OuroForCausalLM.from_pretrained(source, device="npu", dtype=torch.bfloat16) assert all(p.dtype == torch.bfloat16 for p in model.parameters()) diff --git a/tests/test_attention_module.py b/tests/test_attention_module.py index 5c4de6dd..823cf322 100644 --- a/tests/test_attention_module.py +++ b/tests/test_attention_module.py @@ -155,3 +155,73 @@ def stage(values, dtype): assert len({batch.cu_seqlens_q.data_ptr() for batch in batches}) == 1 assert batches[0].cu_seqlens_q.tolist() == [0, 2, 3] assert batches[0].max_seqlen_q == 2 + + +def test_prepared_attention_keeps_the_legacy_kernel_signature(): + cache = KVCacheManager(1, 1, 4, 4, 2, 1) + assert cache.allocate("a", 3) + batch = cache._prepare_batch(["a"] * 3, [0] * 3, [0, 1, 2]) + keys = torch.zeros(3, 1, 4) + values = torch.arange(12, dtype=torch.float32).reshape(3, 1, 4) + query = torch.zeros(3, 2, 4) + cache._write_prepared(0, batch, keys, values) + kernel = cache.attention.kernel + expected = kernel( + query, + cache.key_cache[:, 0], + cache.value_cache[:, 0], + batch.block_tables, + batch.context_lengths, + ) + seen = [] + + def legacy(*args): + seen.append(args) + return kernel(*args) + + cache.attention.kernel = legacy + actual = cache._attend_prepared(0, batch, query) + torch.testing.assert_close(actual, expected, atol=0, rtol=0) + assert len(seen) == 1 and len(seen[0]) == 5 + assert seen[0][0] is query + assert seen[0][3] is batch.block_tables + assert seen[0][4] is batch.context_lengths + cache.free("a") + + +def test_metadata_attention_borrows_only_a_live_validated_descriptor(): + cache = KVCacheManager(1, 1, 4, 4, 2, 1) + assert cache.allocate("a", 3) + batch = cache._prepare_batch(["a"] * 3, [0] * 3, [0, 1, 2]) + keys = torch.zeros(3, 1, 4) + values = torch.arange(12, dtype=torch.float32).reshape(3, 1, 4) + query = torch.zeros(3, 2, 4) + cache._write_prepared(0, batch, keys, values) + legacy = cache.attention.kernel + expected = legacy( + query, + cache.key_cache[:, 0], + cache.value_cache[:, 0], + batch.block_tables, + batch.context_lengths, + ) + seen = [] + + class MetadataKernel: + def __call__(self, *args): + pytest.fail("metadata-aware kernels must not read device metadata back") + + def attend_metadata(self, q, k, v, descriptor): + seen.append(descriptor) + return legacy(q, k, v, descriptor.block_tables, descriptor.context_lengths) + + cache.attention.kernel = MetadataKernel() + actual = cache._attend_prepared(0, batch, query) + torch.testing.assert_close(actual, expected, atol=0, rtol=0) + assert len(seen) == 1 and seen[0] is batch + cache.free("a") + assert cache.allocate("a", 3) + with pytest.raises((ValueError, RuntimeError)): + cache._attend_prepared(0, batch, query) + assert len(seen) == 1 + cache.free("a") diff --git a/vllm_rlt/attention/backend.py b/vllm_rlt/attention/backend.py index 48a06383..e04d9bac 100644 --- a/vllm_rlt/attention/backend.py +++ b/vllm_rlt/attention/backend.py @@ -37,6 +37,11 @@ def __call__(self, *args, **kwargs): return self.kernel(*args, **kwargs) def attend_metadata(self, q, keys, values, batch): + """Borrow validated host metadata when supported; retain the legacy tensor API. + + Metadata-aware kernels can reuse host planning across layers without + reading device tables or lengths back to the host. + """ method = getattr(self.kernel, "attend_metadata", None) if method is not None: return method(q, keys, values, batch) diff --git a/vllm_rlt/core/kv_cache_manager.py b/vllm_rlt/core/kv_cache_manager.py index 2ee380eb..67b21e2e 100644 --- a/vllm_rlt/core/kv_cache_manager.py +++ b/vllm_rlt/core/kv_cache_manager.py @@ -684,7 +684,7 @@ def _write_prepared( self._validate_tensor(v, len(batch.position_ids), "v") if not batch.rows: return - if self.device.type == "npu" and self.attention.info.get("backend") == "ascend_eager": + if self.device.type == "npu" and self.backend == "ascend_eager": if not hasattr(self, "_npu_writer"): from vllm_rlt.kernels.ascend_kv import AscendKVWriter From f9cfe1b2e808d649e5e6963d87de667e8e520eeb Mon Sep 17 00:00:00 2001 From: KrystalRay Date: Tue, 3 Nov 2026 10:05:06 +0800 Subject: [PATCH 10/20] docs: distinguish portability checks from historical measurements --- docs/recipes/ouro_ascend.md | 82 ++++++++++++++++++++++++------------- 1 file changed, 53 insertions(+), 29 deletions(-) diff --git a/docs/recipes/ouro_ascend.md b/docs/recipes/ouro_ascend.md index 124855b7..bedfeeb7 100644 --- a/docs/recipes/ouro_ascend.md +++ b/docs/recipes/ouro_ascend.md @@ -89,12 +89,20 @@ retains the scheduler's default 128-token chunks. ## Validation (2026-11-03) -Base 989921ac1d3897c5c43e27ec8cddbc91fa953194 rejects `--device npu`. -Runtime 45220081159627ff4fba36a6b0b3ac10787214c8 supplies the new backend. -Final runtime be037ed5cb9344b59830da2449b5590f3dd6c231 adds an arithmetic -regression and makes native writes exclusive to the new backend; selected -eager execution and original Torch execution were separately revalidated. -Later documentation changes preserve these runtime/test contents. +Recorded sources: + +- Base `989921a` rejects `--device npu`. +- Backend revision `4522008` was used for the paired 87-question quality + campaign and primary performance comparison below. +- Routing revision `be037ed` adds an arithmetic regression and makes native + writes exclusive to the new backend. Selected eager and original Torch + execution were separately revalidated at that revision. +- Portability/interface revision `8bb68fb` removes device-name assertions, + supports Python 3.10 checksum calculation, and adds legacy/metadata-aware + attention dispatch regressions. + +Each result retains its measured source. Earlier quality, numerical and +performance campaigns are not relabeled as tests of a newer revision. ~~~bash pre-commit run --all-files --verbose @@ -105,34 +113,50 @@ python -m pytest tests/test_ascend.py tests/test_ascend_checkpoint.py \ tests/test_ascend_eager.py --run-npu -q ~~~ -- Ruff/pre-commit passed; CPU 926 passed / 602 skipped. Device cases were run - separately:22 passed, comprising18 NPU and 4 CPU cases. -- Final eager vs unmodified official HF:216 selected states and2016 layer - events exactly equal; restored Torch vs original Torch is also exact. -- The arithmetic regression distinguishes an analytical output16 in BF16 - eager from15.9375 in the original FP32-intermediate path, on CPU andNPU. -- Real spawned health/completions/SSE, termination/usage, disconnect recovery - and shutdown pass for both explicit backends. -- At4522008,87 selected same-device HF/native pairs score63/87 each, - losses[]/gains[], with87/87 complete token sequences exactly equal. - Parse failures19 each; length/execution limits0 each. Every pair shares a - physical910B3, pooled across cards4/5/6. These are reused regression cases, - not a held-out/full-GSM8K estimate. No threshold or scorer was relaxed. -- Quality uses one full-prefill chunk4096 and4096 token budget. Chunked128, +### Portability/interface checks at `8bb68fb` + +- Ruff/pre-commit passed; CPU **928 passed / 602 skipped**. +- Assigned-device suite: **22 passed**, comprising 18 NPU and 4 CPU cases. + Both explicit backends pass spawned HTTP/SSE, termination/usage, disconnect + recovery and shutdown checks. +- Legacy kernels retain their five tensor arguments; metadata-aware kernels + receive the validated descriptor, and stale descriptors remain rejected. +- The actual fixture checksum block was also executed successfully under + Python 3.10.12, where `hashlib.file_digest` is unavailable. +- Arithmetic kernels are unchanged by this revision. The full 87-question + campaign and whole-model HF layer-event comparison were not repeated here. + +### Earlier numerical and quality evidence + +- At `be037ed`, eager versus unmodified official HF has 216 selected states + and 2016 layer events exactly equal. Restored Torch versus original Torch + is also exact. +- The analytical rounding regression distinguishes output 16 in BF16 eager + from 15.9375 in the original FP32-intermediate path, on CPU and NPU. +- At `4522008`, 87 selected same-device HF/native pairs score 63/87 each, + with no losses or gains and 87/87 complete token sequences exactly equal. + Parse failures are 19 per arm; length and execution limits are zero. + Each pair shares a physical 910B3, pooled across three cards. These are + reused regression cases, not a held-out or full-GSM8K estimate. +- Quality uses one full-prefill chunk and token budget of 4096. Chunked, concurrent/adaptive task quality and other models remain unqualified. -Numerical bounds remain joint relative L2<=2% and atol=rtol=0.02. -The new explicit backend is checked against released HF BF16 eager. -It still fails 209/216 states against the old FP32-intermediate Torch reference; -that failure remains recorded and is not called passed. +Numerical bounds remain joint relative L2 <= 2% and atol=rtol=0.02. +The explicit BF16 backend targets released HF eager. At `be037ed`, it still +fails 209/216 states against the old FP32-intermediate Torch reference; +that historical failure remains recorded and is not relabeled as passed. -The earlier63/87 HF vs61/87 native observation combined different arithmetic -with native default 128-token chunking. Changing either alone did not fix -question 203. Matching both restores203/881/526 and the full87 regression. -Manual question 203 replay has15 exact logits and192 exact populated KV hashes. +The earlier HF 63/87 versus native 61/87 observation combined different +arithmetic with native default 128-token chunking. Changing either alone did +not fix question 203. Matching both restores questions 203/881/526 and the +87-question regression. The manual question 203 replay has 15 exact logits +and 192 exact populated KV hashes. ## Measured performance +The primary comparison uses original Torch at `4734e19` and the explicit +backend at `4522008`; the later confirmation uses `be037ed`. + Same physical910B3, profiler disabled, P64/C1 orC8, fixed4 loops, ignore EOS, exactly 8 outputs/request,768 blocks,max 8 sequences / 256 batched tokens,explicit 128-token chunks. One warmup/shape,3repetitions/run; A/C/C/A gives 6 observations @@ -150,7 +174,7 @@ does not isolate ScatterND. A KV-only A/B regressed 1.23% and 6.28%; the final runtime therefore keeps original writes on the Torch path. New backend native-writing optimality is not established. -Final runtime be037ed confirmation on the sameNPU gives medians 4.752tok/s +Routing-revision be037ed confirmation on the sameNPU gives medians 4.752tok/s at C1 and13.705tok/s at C8 (three reps each, measured after ABBA). Allocated peaks are about 4.97–4.99GiB; reserved peaks about 5.33GiB. All bursts finish with 0 live logical KV blocks and no failed repetitions. From 95f4999cd4dc548260d8625bd70adc97b229779e Mon Sep 17 00:00:00 2001 From: KrystalRay Date: Tue, 3 Nov 2026 10:21:22 +0800 Subject: [PATCH 11/20] test: reuse native checkpoint loading on Ascend --- tests/test_ascend_checkpoint.py | 91 ++++++++++++++------------------- 1 file changed, 38 insertions(+), 53 deletions(-) diff --git a/tests/test_ascend_checkpoint.py b/tests/test_ascend_checkpoint.py index 6beb91b2..c26e8fe8 100644 --- a/tests/test_ascend_checkpoint.py +++ b/tests/test_ascend_checkpoint.py @@ -1,7 +1,5 @@ -"""Pinned real-weight NPU checks for explicit Torch attention, not fused CANN.""" +"""Opt-in local-checkpoint KV and lifecycle regressions on Ascend.""" -import hashlib -import json import os from pathlib import Path @@ -21,29 +19,17 @@ def checkpoint(): source = os.environ.get("VLLM_RLT_OURO_CHECKPOINT") if not source: - pytest.skip("set VLLM_RLT_OURO_CHECKPOINT to the pinned local Ouro-1.4B checkpoint") + pytest.skip("set VLLM_RLT_OURO_CHECKPOINT to a prepared local Ouro checkpoint") + if not Path(source).is_dir(): + pytest.fail("VLLM_RLT_OURO_CHECKPOINT must be a prepared local directory") resolve_device("npu") torch.npu.set_device(0) - checksum = hashlib.sha256() - with (Path(source) / "model.safetensors").open("rb") as stream: - for block in iter(lambda: stream.read(8 * 1024 * 1024), b""): - checksum.update(block) - digest = checksum.hexdigest() - assert digest == "58872a72616c736595b8b7662079c5b12c5a162ec16eae94f21c348dfa9885af" model = OuroForCausalLM.from_pretrained(source, device="npu", dtype=torch.bfloat16) assert all(p.dtype == torch.bfloat16 for p in model.parameters()) yield model, load_tokenizer(source) torch.npu.synchronize() -def save_record(name, record): - folder = os.environ.get("VLLM_RLT_ASCEND_RESULTS") - if folder: - path = Path(folder) - path.mkdir(parents=True, exist_ok=True) - (path / name).write_text(json.dumps(record, indent=2)) - - @torch.inference_mode() def test_checkpoint_incremental_pages_preserve_hidden_logits_and_kv(checkpoint): model, tokenizer = checkpoint @@ -62,54 +48,53 @@ def test_checkpoint_incremental_pages_preserve_hidden_logits_and_kv(checkpoint): for key in expected_kv: for name, a, b in zip(("key", "value"), actual_kv[key], expected_kv[key], strict=True): pairs[f"{key}.{name}"] = (a, b) - save_record( - "checkpoint-numerics.json", - dict( - tokens=tokens, - states=len(pairs), - exact=all(torch.equal(a, b) for a, b in pairs.values()), - finite=all( - bool(torch.isfinite(a).all() and torch.isfinite(b).all()) for a, b in pairs.values() - ), - ), - ) for a, b in pairs.values(): assert_bf16_close(a, b) torch.testing.assert_close(a, b, atol=0, rtol=0) +@pytest.mark.parametrize("backend", ["torch", "ascend_eager"]) @torch.inference_mode() -def test_checkpoint_greedy_generation_and_cleanup(checkpoint): +def test_checkpoint_greedy_replay_and_request_cleanup(checkpoint, backend): model, tokenizer = checkpoint engine = StageEngine( model, - attention_backend="torch", + attention_backend=backend, cache_config=CacheConfig(num_blocks=64), scheduler_config=SchedulerConfig(max_num_seqs=2, max_num_batched_tokens=32), ) prompts = ["The capital of France is", "One plus one equals"] - finished = {} + loops = model.config.total_ut_steps + params = SamplingParams( + max_tokens=4, temperature=0, min_loops=loops, max_loops=loops, ignore_eos=True + ) + expected = None try: - for index, prompt in enumerate(prompts): - engine.add_request( - str(index), - tokenizer.encode(prompt, add_special_tokens=False), - SamplingParams(max_tokens=4, temperature=0, ignore_eos=True), - ) - for _ in range(200): - for output in engine.step(): - if output.finished: - assert output.exit_depths == [4] * 4 - assert output.finish_reason == "length" - finished[output.request_id] = dict( - tokens=output.token_ids, text=tokenizer.decode(output.token_ids) - ) - if not engine.has_unfinished_requests(): - break - assert set(finished) == {"0", "1"} - assert finished["0"]["tokens"] == [7042, 30, 198, 198] - assert finished["1"]["tokens"] == [827, 30, 198, 198] - assert engine.cache_manager.num_used_blocks == 0 - save_record("checkpoint-generation.json", finished) + # Repeat the same packed workload and reuse IDs after all pages are freed. + # Expected IDs come from this checkpoint, not a particular weight file. + for _ in range(2): + finished = {} + for index, prompt in enumerate(prompts): + engine.add_request( + str(index), tokenizer.encode(prompt, add_special_tokens=False), params + ) + for _ in range(200): + for output in engine.step(): + if output.finished: + assert len(output.token_ids) == params.max_tokens + assert all( + 0 <= token < model.config.vocab_size for token in output.token_ids + ) + assert output.exit_depths == [loops] * params.max_tokens + assert output.finish_reason == "length" + finished[output.request_id] = list(output.token_ids) + if not engine.has_unfinished_requests(): + break + assert set(finished) == {str(index) for index in range(len(prompts))} + assert engine.cache_manager.num_used_blocks == 0 + if expected is None: + expected = finished + else: + assert finished == expected finally: engine.close() From 7b34b4d5747a65ab7b3a7148de69257adb26f26b Mon Sep 17 00:00:00 2001 From: KrystalRay Date: Tue, 3 Nov 2026 10:21:22 +0800 Subject: [PATCH 12/20] docs: simplify the Ascend setup recipe --- docs/recipes/ouro_ascend.md | 224 ++++++++---------------------------- 1 file changed, 47 insertions(+), 177 deletions(-) diff --git a/docs/recipes/ouro_ascend.md b/docs/recipes/ouro_ascend.md index bedfeeb7..f5e280e7 100644 --- a/docs/recipes/ouro_ascend.md +++ b/docs/recipes/ouro_ascend.md @@ -1,201 +1,71 @@ -# Ouro-1.4B on Ascend - -This experimental path runs native Ouro on one NPU with BF16, synchronous -greedy decoding and four recurrent loops. Select -`--attention-backend ascend_eager` explicitly. It batches causal queries by -request/depth through Torch-NPU/CANN MatMul and Softmax primitives and uses -native ScatterND for paged KV writes. It preserves released Ouro HF eager -rounding: BF16 QK/scale, FP32 softmax rounded to BF16, then BF16 PV. - -The older explicit `torch` backend remains available with its original -FP32 attention intermediates and indexed KV writes. Its arithmetic is a -different reference contract. Automatic backend selection is unchanged. - -## Supported scope - -- Native Ouro-1.4B, BF16 parameters/ordinary activations/KV, fixed four loops, - synchronous single-device greedy generation. -- Per-request/depth/layer paged KV; incremental/full allocation, causal prefill, - decode, cancellation and request-ID reuse. -- Existing spawned HTTP/SSE serving lifecycle. Explicit KV capacity is required. -- Async scheduling, graphs, speculation, PD, preemption, prefix caching, - adaptive depth and stochastic sampling are rejected for this initial path. -- Other checkpoints, Ascend models/devices and concurrent task quality need - separate qualification. The quality regression below uses a full prompt - forward; chunked generation can round differently. - -The shared scheduler/model/KV owner are retained. NPU submissions drain before -metadata or pages can be reused. CPU/CUDA do not import Torch-NPU unconditionally. -Use `torch_npu.profiler` for NPU device profiling; built-in profiling targets -CPU/CUDA. - -Hardware selection does not inspect or whitelist device model names. -The backend requires BF16 and the Torch-NPU/CANN operations used below. -The measurements in this recipe were collected on 910B3; other Ascend devices -can run the same opt-in checks without a model-name assertion. Their numerical -and performance results require validation on the actual device/runtime pair. - -## Environment and installation - -Measured environment: Linux aarch64, Python 3.11.13, Torch 2.9.0+cpu, -Torch-NPU 2.9.0, CANN 9.0.0, driver 26.1.0.b050, Transformers 4.57.6, -Tokenizers 0.22.2 and NumPy 1.26.4. Keep the vendor Torch pair matched. +# Ouro on Ascend + +Run Ouro on one Ascend NPU with BF16, synchronous greedy decoding and four +recurrent loops. Use the explicit `ascend_eager` attention backend. + +## 1. Prepare the environment and model + +Use a compatible Torch, Torch-NPU and CANN environment with the project's +Python dependencies installed. Keep the vendor Torch pair; CUDA/Triton +packages are not needed for this backend. ~~~bash source /path/to/cann/set_env.sh -source /path/to/isolated-npu-environment/bin/activate -export ASCEND_RT_VISIBLE_DEVICES=4 # use your assigned physical NPU -python -c 'import torch, torch_npu; print(torch.__version__, torch_npu.__version__); print(torch.npu.get_device_name(0))' -python -m pip install 'numpy==1.26.4' 'safetensors==0.8.0' 'huggingface-hub==0.36.2' \ - 'transformers==4.57.6' 'tokenizers==0.22.2' 'aiohttp==3.14.3' setuptools wheel +source /path/to/npu-environment/bin/activate python -m pip install --no-deps --no-build-isolation -e . +export MODEL=/path/to/prepared/Ouro-1.4B ~~~ -The no-deps install preserves vendor Torch and avoids the Linux Triton -dependency. No native CUDA extension, custom compiled kernel or InferRT is used. -Current upstream ScatterPA examples use an API/layout unavailable in this -Torch-NPU/CANN pair; the native writer retains the existing ND cache. +Use the checkpoint's own config and tokenizer. Model loading uses the shared +native Ouro loader, including its supported sharded checkpoint layout. -Use ByteDance/Ouro-1.4B revision 574fa66cb8bf5abdc979642d01cf2b79b16bfab1. -Weights SHA256: -`58872a72616c736595b8b7662079c5b12c5a162ec16eae94f21c348dfa9885af`. +## 2. Start the server -## Start and call the service +Select an allocated physical NPU using `ASCEND_RT_VISIBLE_DEVICES`. ~~~bash +ASCEND_RT_VISIBLE_DEVICES= \ python -m vllm_rlt.entrypoints.serve \ - --model /path/to/Ouro-1.4B --served-model-name ouro \ + --model "$MODEL" --tokenizer "$MODEL" --served-model-name ouro \ --device npu --dtype bfloat16 --attention-backend ascend_eager \ - --num-blocks 1024 --block-size 16 --max-num-seqs 1 \ - --max-num-batched-tokens 4096 --prefill-chunk-size 4096 --port 8123 + --num-blocks 1024 --max-num-seqs 1 \ + --max-num-batched-tokens 4096 --prefill-chunk-size 4096 \ + --host 127.0.0.1 --port 8123 ~~~ +Explicit KV capacity is required. Adjust it for the model, available memory, +prompt/output lengths and concurrency. To compare with full-prefill inference, +both the chunk size and batched-token budget must fit the whole prompt. + +## 3. Send a request + ~~~bash -curl http://127.0.0.1:8123/health -curl http://127.0.0.1:8123/v1/completions \ +curl --fail http://127.0.0.1:8123/health +curl --fail http://127.0.0.1:8123/v1/completions \ -H 'Content-Type: application/json' \ - -d '{"model":"ouro","prompt":"The capital of France is","max_tokens":4,"temperature":0,"min_loops":4,"max_loops":4,"exit_threshold":1}' + -d '{"model":"ouro","prompt":"The capital of France is","max_tokens":8,"temperature":0,"min_loops":4,"max_loops":4,"exit_threshold":1}' ~~~ -Add `stream=true` for SSE. Python callers select `device="npu"` and -`attention_backend="ascend_eager"`; the spawned worker registers Torch-NPU. -The assigned physical NPU becomes logical `npu:0`. Capacity and token budgets -must fit your prompts/outputs/concurrency; this command is a reproduction -configuration, not certification of every 4096-token input. +Wait for HTTP 200 before sending requests. Add `"stream": true` for SSE. +The `ascend_eager` backend follows HF-style BF16 attention rounding; +`--attention-backend torch` selects the existing FP32-intermediate reference. -Both `prefill_chunk_size` and `max_num_batched_tokens` must accommodate the -whole prompt to compare with official full-prefill. Raising only the latter -retains the scheduler's default 128-token chunks. +## 4. Run checks -## Validation (2026-11-03) - -Recorded sources: - -- Base `989921a` rejects `--device npu`. -- Backend revision `4522008` was used for the paired 87-question quality - campaign and primary performance comparison below. -- Routing revision `be037ed` adds an arithmetic regression and makes native - writes exclusive to the new backend. Selected eager and original Torch - execution were separately revalidated at that revision. -- Portability/interface revision `8bb68fb` removes device-name assertions, - supports Python 3.10 checksum calculation, and adds legacy/metadata-aware - attention dispatch regressions. - -Each result retains its measured source. Earlier quality, numerical and -performance campaigns are not relabeled as tests of a newer revision. +With an assigned NPU visible to the test process: ~~~bash -pre-commit run --all-files --verbose -export VLLM_RLT_OURO_CHECKPOINT=/path/to/Ouro-1.4B -export VLLM_RLT_ASCEND_RESULTS=/path/to/results -python -m pytest tests/test_ascend.py tests/test_ascend_checkpoint.py \ - tests/test_ascend_serving.py tests/test_ascend_kv.py \ - tests/test_ascend_eager.py --run-npu -q +pre-commit run --all-files +VLLM_RLT_OURO_CHECKPOINT="$MODEL" python -m pytest \ + tests/test_ascend.py tests/test_ascend_checkpoint.py \ + tests/test_ascend_eager.py tests/test_ascend_kv.py \ + tests/test_ascend_serving.py --run-npu -q ~~~ -### Portability/interface checks at `8bb68fb` - -- Ruff/pre-commit passed; CPU **928 passed / 602 skipped**. -- Assigned-device suite: **22 passed**, comprising 18 NPU and 4 CPU cases. - Both explicit backends pass spawned HTTP/SSE, termination/usage, disconnect - recovery and shutdown checks. -- Legacy kernels retain their five tensor arguments; metadata-aware kernels - receive the validated descriptor, and stale descriptors remain rejected. -- The actual fixture checksum block was also executed successfully under - Python 3.10.12, where `hashlib.file_digest` is unavailable. -- Arithmetic kernels are unchanged by this revision. The full 87-question - campaign and whole-model HF layer-event comparison were not repeated here. - -### Earlier numerical and quality evidence - -- At `be037ed`, eager versus unmodified official HF has 216 selected states - and 2016 layer events exactly equal. Restored Torch versus original Torch - is also exact. -- The analytical rounding regression distinguishes output 16 in BF16 eager - from 15.9375 in the original FP32-intermediate path, on CPU and NPU. -- At `4522008`, 87 selected same-device HF/native pairs score 63/87 each, - with no losses or gains and 87/87 complete token sequences exactly equal. - Parse failures are 19 per arm; length and execution limits are zero. - Each pair shares a physical 910B3, pooled across three cards. These are - reused regression cases, not a held-out or full-GSM8K estimate. -- Quality uses one full-prefill chunk and token budget of 4096. Chunked, - concurrent/adaptive task quality and other models remain unqualified. - -Numerical bounds remain joint relative L2 <= 2% and atol=rtol=0.02. -The explicit BF16 backend targets released HF eager. At `be037ed`, it still -fails 209/216 states against the old FP32-intermediate Torch reference; -that historical failure remains recorded and is not relabeled as passed. - -The earlier HF 63/87 versus native 61/87 observation combined different -arithmetic with native default 128-token chunking. Changing either alone did -not fix question 203. Matching both restores questions 203/881/526 and the -87-question regression. The manual question 203 replay has 15 exact logits -and 192 exact populated KV hashes. - -## Measured performance - -The primary comparison uses original Torch at `4734e19` and the explicit -backend at `4522008`; the later confirmation uses `be037ed`. - -Same physical910B3, profiler disabled, P64/C1 orC8, fixed4 loops, ignore EOS, -exactly 8 outputs/request,768 blocks,max 8 sequences / 256 batched tokens,explicit -128-token chunks. One warmup/shape,3repetitions/run; A/C/C/A gives 6 observations -per arm/shape. TTFT/TPOT are medians of per-burst request means, not pooled -individual-request percentiles. Throughput includes prefill/drain; no HTTP. - -|Concurrency|Old Torch tok/s|New backend tok/s|Old/new TTFT ms|Old/new TPOT ms| -|---:|---:|---:|---:|---:| -|1|1.326|4.711|4266.35 /221.03|246.81 /210.99| -|8|1.720|12.778|24363.00 /702.72|1818.05 /583.61| - -Pair-specific throughput ratios are 3.47–3.81x at C1 and 7.11–8.05x at C8, -with non-overlapping observed ranges. This combined backend/arithmetic change -does not isolate ScatterND. A KV-only A/B regressed 1.23% and 6.28%; -the final runtime therefore keeps original writes on the Torch path. -New backend native-writing optimality is not established. - -Routing-revision be037ed confirmation on the sameNPU gives medians 4.752tok/s -at C1 and13.705tok/s at C8 (three reps each, measured after ABBA). -Allocated peaks are about 4.97–4.99GiB; reserved peaks about 5.33GiB. -All bursts finish with 0 live logical KV blocks and no failed repetitions. - -A separate 9-shape candidate characterization covers 16/64/256 input tokens and -concurrency 1/2/8. These are short bursts, not saturation/HTTP/SLO or hardware-only -claims. CPU/NUMA placement is not pinned; independent CPU/device checks overlap -part of the shared-host timing. Raw ranges and controls must accompany reuse. - -Separate ops-only/full warmed decode traces confirm actual native ScatterND -dispatch and 96 softmax calls. Full mode has readable shapes, stacks and memory -exports. Complete warmed request traces additionally cover prefill and all 8 -outputs. Profile durations are diagnostic and excluded from speed claims. - -## Fused operators remain experimental - -BF16 FIA and promoted FP32 FlashAttentionScore did not pass earlier whole-model -gates. Native RMSNorm changes also failed selected states despite local speed -gains; SwiGLU/RotaryMul candidates were not adopted. -The current backend uses native primitives and batching, not fused attention. - -This is a bounded contribution toward -[#69](https://github.com/InferMatrixLab/vllm-rlt/issues/69), with task quality -qualified only for the stated regression configuration. +These checks cover causal attention, cross-page KV, request reuse and HTTP/SSE. +They do not establish model quality or performance; use the +[accuracy guide](../accuracy.md) and [serving benchmark guide](../serving.md) +for those evaluations. + +Async scheduling, graphs, speculation, PD, preemption, prefix caching and +adaptive exit are currently unsupported on this path. From 3e115dfeeb1665e28ae031a6421a94c0894dc030 Mon Sep 17 00:00:00 2001 From: KrystalRay Date: Mon, 9 Nov 2026 11:16:55 +0800 Subject: [PATCH 13/20] perf: batch Ascend eager attention and build KV write indices on device Merge the per-group KV-read refactor and the follow-up batching work into one reviewable change. The arithmetic contract is unchanged: BF16 score scaling, additive BF16 causal mask, FP32 softmax cast back to BF16 probabilities, then a BF16 matmul. - Build the Scatter ND write index from staged device tensors with a fail-closed row/address check. - Batch equal-length whole-prompt prefill and independent single-row decode groups; cache write slots, ragged plans and the prefill mask per batch. - Keep a single decode row on the original per-group path. - Clamp pooled-KV gather slots before index_select: plan_attention_metadata pads short block tables with -1, which CANN gather rejects. The additive mask already zeroes those columns, so clamping cannot change the output. Covered by a BF16 mixed-length ragged test. - Add CPU/NPU parity tests and record matched engine-level and full W1-W7 serving results in the Ascend recipe and README support matrix. --- README.md | 2 +- docs/recipes/ouro_ascend.md | 138 ++++++++++++++++++++++ tests/test_ascend_eager.py | 140 +++++++++++++++++++++- vllm_rlt/kernels/ascend_eager.py | 197 +++++++++++++++++++++++++++++-- vllm_rlt/kernels/ascend_kv.py | 36 +++--- 5 files changed, 485 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 569373db..78ebbe6f 100644 --- a/README.md +++ b/README.md @@ -134,7 +134,7 @@ No PyPI publication is implied. | Checkpoint | Implementation | Documented validation scope | Guide | | --- | --- | --- | --- | -| Ouro-1.4B | Native Ouro | Ascend inference support | [Runtime](docs/runtime.md), [Ascend recipe](docs/recipes/ouro_ascend.md) | +| Ouro-1.4B | Native Ouro | Ascend BF16, fixed-four-loop synchronous full-prefill support; HF-eager-aligned attention with batched prefill/decode; chunked prefill unresolved | [Runtime](docs/runtime.md), [Ascend recipe](docs/recipes/ouro_ascend.md#5-attention-batching-and-device-side-kv-writes) | | Ouro-2.6B | Shared Ouro | Shares the Ouro implementation; not separately qualified, and 1.4B quality/performance results are not inherited | [Scope notes](docs/recipes/ouro_thinking.md#6-evidence-and-limitations) | | Ouro-1.4B-Thinking | Reuses Ouro | Bounded BF16, fixed-four-loop synchronous basic integration | [Shared recipe](docs/recipes/ouro_thinking.md) | | Ouro-2.6B-Thinking | Reuses Ouro | Bounded BF16, fixed-four-loop synchronous basic integration | [Shared recipe](docs/recipes/ouro_thinking.md) | diff --git a/docs/recipes/ouro_ascend.md b/docs/recipes/ouro_ascend.md index f5e280e7..bd67f977 100644 --- a/docs/recipes/ouro_ascend.md +++ b/docs/recipes/ouro_ascend.md @@ -69,3 +69,141 @@ for those evaluations. Async scheduling, graphs, speculation, PD, preemption, prefix caching and adaptive exit are currently unsupported on this path. + +## 5. Attention batching and device-side KV writes + +Two follow-up changes reduce host submissions on this path without changing the +BF16 arithmetic contract: + +- `ascend_kv.py` builds the Scatter ND write index from the batch's staged + device tensors (`write_blocks` / `write_offsets`) instead of a Python loop + over `batch.rows`; a fail-closed check rejects a row/address count mismatch. +- `ascend_eager.py` merges equal-length whole-prompt prefill into one + `[size, width]` attention call, and merges independent single-row decode + groups into one ragged gather. Write slots, ragged plans and the prefill mask + are cached per batch. A single decode row keeps the original per-group path + so it does not pay for plan construction. + +The arithmetic is unchanged: BF16 score scaling, additive BF16 causal mask, +FP32 softmax cast back to BF16 probabilities, then a BF16 matmul. The default +backend is still `AscendEagerAttention`; no fused CANN attention operator is +used, because those operators do not reproduce the released HF eager rounding. + +### Measurement conditions + +One Ascend 910B3, CANN 9.0.0, Torch 2.9.0+cpu with torch_npu 2.9.0, +Ouro-1.4B BF16, fixed four recurrent loops, synchronous greedy decoding, +full prefill, frozen prompt IDs, forced 32-token output, profiler off. +Three arms share one process on the same card (NPU 6, `taskset -c 64-71`): +official HF eager as reference, the pre-change per-group eager attention, +and the merged implementation. Each arm gets one warmup and three timed +repetitions; throughput, TTFT and TPOT are medians over request samples. +P2048/C8 uses a 16384 token budget to preserve full-prefill geometry. + +### Before/after results (same hardware, different card assignment) + +| Input / output / concurrency | Arm | Output tok/s | TTFT (ms) | TPOT (ms) | +| --- | --- | ---: | ---: | ---: | +| 128 / 32 / 8 | HF eager | 50.792 | 158.9 | 157.5 | +| | before | 14.611 | 630.7 | 544.8 | +| | **merged** | **37.724** | **273.0** | **209.9** | +| 2048 / 32 / 1 | HF eager | 6.339 | 371.2 | 150.9 | +| | before | 4.784 | 531.3 | 198.7 | +| | **merged** | **4.789** | **534.7** | **198.3** | + +A second matched run on longer/higher-concurrency workloads (same protocol, +different process, so absolute values are not cross-comparable with the table +above): + +| Input / output / concurrency | Arm | Output tok/s | TTFT (ms) | TPOT (ms) | +| --- | --- | ---: | ---: | ---: | +| 2048 / 32 / 8 | HF eager | 35.835 | 2489.5 | 150.1 | +| | before | 12.426 | 3702.4 | 545.1 | +| | **merged** | **25.007** | **3591.4** | **213.7** | +| 4096 / 32 / 1 | HF eager | 5.812 | 1091.1 | 142.4 | +| | before | 4.170 | 1532.7 | 196.3 | +| | **merged** | **4.244** | **1534.5** | **193.7** | + +The gain is in decode/overall throughput, not long-prompt TTFT: prefill is +dominated by the dense `[W, W]` attention and by host submission, and merging +calls reduces launches rather than arithmetic. Relative to HF eager, C8 +throughput moves from 0.288x to 0.743x at input 128; HF eager remains faster +in absolute terms. + +An earlier, separate A/B isolated the device-side write index alone (different +harness, older configuration, directional only): P2048/C1 TTFT 790.7 -> 531.9 ms +(-32.7%) and P128/C8 TTFT 691.7 -> 619.8 ms (-10.4%). + +### Full W1–W7 serving campaign + +`docs/unified_execution.md` defines the project's full serving benchmark: W1–W7, +three unprofiled repetitions per workload, 60 seconds of closed-loop admission +followed by full drain, median aggregate output throughput, pooled TTFT / TPOT / +ITL / E2E percentiles, fixed lengths for W1–W5, and a fixed seeded ShareGPT +selection with natural EOS and a 512-token cap for W6/W7. The merged candidate +ran as a single arm on the frozen campaign actor with no runtime monkeypatch on +NPU 5/6/7. The before column is the `d921719` baseline published in PR #168 on +2026-11-06; card assignment differs between the two runs, so this is a +same-hardware, not same-run, comparison. + +| Workload | Input / output / C | before tok/s | merged tok/s | before TTFT p95 (s) | merged TTFT p95 (s) | before TPOT p95 (ms) | merged TPOT p95 (ms) | +| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | +| W1 | 128 / 128 / 1 | 3.944 | 4.653 | 0.426 | 0.205 | 265.8 | 221.1 | +| W2 | 128 / 128 / 32 | 4.802 | 96.726 | 9.737 | 6.827 | 6642.6 | 285.9 | +| W3 | 4096 / 128 / 1 | 3.727 | 4.368 | 2.167 | 1.721 | 254.3 | 222.1 | +| W4 | 4096 / 128 / 32 | 4.201 | 24.761 | 747.440 | 137.225 | 1794.4 | 236.0 | +| W5 | 1024 / 512 / 8 | 4.575 | 30.744 | 4.676 | 2.551 | 1743.0 | 256.0 | +| W6 | ShareGPT / ≤512 / 8 | 4.660 | 21.417 | 3.572 | 1.689 | 1666.7 | 317.5 | +| W7 | ShareGPT / ≤512 / 32 | 4.760 | 27.970 | 15.765 | 7.289 | 6523.6 | 1071.7 | + +All seven workloads completed three unprofiled repetitions with zero failed +requests. Successful requests over the three runs (before → merged): W1 6→9, +W2 96→192, W3 6→9, W4 96→120, W5 24→24, W6 27→33, W7 99→105. + +The W2 ratio is dominated by the baseline's lack of concurrency scaling rather +than by a per-request speedup: on `d921719`, raising concurrency from 1 to 32 +moved aggregate throughput only from 3.944 to 4.802 tok/s (1.22×) while TPOT +p95 grew from 265.8 ms to 6642.6 ms, whereas the merged path reaches 20.79× the +C1 throughput with TPOT p95 of 285.9 ms. The same near-flat baseline scaling +appears in W4, W5, W6 and W7. Under the W1–W7 protocol a faster configuration +also covers more inputs during the same 60-second admission window (W2 +completed 32 requests per run before versus 64 after, with drain falling from +about 792 s to 25 s), so the aggregate ratio reflects both a shorter drain and +a larger completed-request count. + +The fresh campaign quality gate (GSM8K-87 three-shot strict and the paired +same-history logit vectors) has not been re-run on this change. The earlier +`kv-read-full-20261106` campaign recorded 63/87 for both HF eager and the +target, 87/87 exact sequences and 8952/8952 exact logit vectors with +`max_abs=0.0` on `d921719` and a different KV-read candidate; those figures do +not qualify this change. + +### Correctness gates + +- CPU synthetic parity in BF16: regular prefill, single-request prefill, ragged + decode, a mixed-length ragged batch with `-1` padded block tables, single + decode and an irregular grid all match the per-group path exactly + (`atol=0, rtol=0`); path-activation counters are asserted. +- Ascend selection suite on the merged candidate: `test_ascend.py`, + `test_ascend_eager.py`, `test_ascend_kv.py`, `test_ascend_serving.py`, + `test_ascend_checkpoint.py` with `--run-npu`: **31 passed, 0 skipped, + 0 failures** (66.6 s, NPU 6), including the new batched-path parity cases. +- Generated sequences against official HF: 24/24 at C8 and 3/3 at C1. +- Pre-lm-head vectors: 256/256 at C8 and 32/32 at C1, `max_abs=0.0`, + for both `merged vs HF` and `merged vs before`. +- In the timed C8 run, the batched-prefill path activated 384 times and the + ragged decode path 11904 times, with zero fallbacks. + +### Limits + +- Batched prefill requires a strict regular grid. **Chunked prefill does not + activate it**, so the documented case-203 chunk128 sequence mismatch + (27 vs 31, first divergence at the 15th output token) is **still unresolved**; + this follow-up does not change chunked-prefill behavior or claim a fix. +- Multi-row non-initial groups still use the original path. Adaptive exit, + long conversations, C16+ concurrency and models other than Ouro-1.4B are + untested. +- Three timed repetitions per arm on a single card; only 32-token outputs. +- Only the full-prefill synchronous path is qualified. Async scheduling, + graphs, speculation, PD, preemption, prefix caching and adaptive exit + remain unsupported here. diff --git a/tests/test_ascend_eager.py b/tests/test_ascend_eager.py index 37e5f6ce..d40d4aaa 100644 --- a/tests/test_ascend_eager.py +++ b/tests/test_ascend_eager.py @@ -3,7 +3,7 @@ import pytest import torch -from vllm_rlt.kernels.ascend_eager import AscendEagerAttention +from vllm_rlt.kernels.ascend_eager import AscendEagerAttention, _flat_kv_rows, _prefill_geometry @pytest.mark.parametrize("kv_heads", [1, 2, 4]) @@ -30,6 +30,39 @@ def test_eager_groups_match_independent_causal_queries(kv_heads): assert torch.count_nonzero(actual[0]) == 0 +@pytest.mark.npu +@torch.inference_mode() +def test_pooled_kv_reads_match_layer_views_across_pages_and_depths(): + from vllm_rlt.core.kv_cache_manager import KVCacheManager + from vllm_rlt.device import resolve_device + + device = resolve_device("npu:0") + torch.npu.set_device(device) + torch.manual_seed(99) + cache = KVCacheManager( + 3, 2, 64, 16, 16, 2, device=device, dtype=torch.bfloat16, backend="ascend_eager" + ) + assert cache.allocate("a", 17) + assert cache.allocate("b", 3) + batch = cache._prepare_batch(["a"] * 17 + ["b"] * 3, [0] * 17 + [1] * 3, [*range(17), 0, 1, 2]) + for layer in range(3): + k = torch.randn(20, 2, 64, device=device, dtype=torch.bfloat16) + v = torch.randn_like(k) + cache._write_prepared(layer, batch, k, v) + + q = torch.randn(20, 4, 64, device=device, dtype=torch.bfloat16) + for layer in range(3): + expected = cache.attention( + q, + cache.key_cache[:, layer], + cache.value_cache[:, layer], + batch.block_tables, + batch.context_lengths, + ) + actual = cache._attend_prepared(layer, batch, q) + torch.testing.assert_close(actual, expected, atol=0, rtol=0) + + @pytest.mark.npu @pytest.mark.parametrize("chunk_size", [4096, 128]) @torch.inference_mode() @@ -152,3 +185,108 @@ def test_bf16_probability_rounding_is_part_of_the_eager_contract(device_name): torch.testing.assert_close(actual, torch.full_like(actual, 16), atol=0, rtol=0) old_contract = torch_paged_attention(query, keys, values, table, lengths) assert not torch.equal(actual, old_contract) + + +class _PreparedBatch: + """Only the host metadata the two batched attention paths consume.""" + + def __init__(self, rows, tables, write_blocks, write_offsets): + self.rows = rows + self.block_tables = tables + self.write_blocks = write_blocks + self.write_offsets = write_offsets + + +def _batched_fixture(kind): + layers, blocks, block_size, kv_heads, head_dim = 3, 24, 4, 2, 8 + head_groups, table_width = 2, 4 + dtype = torch.bfloat16 # the Ascend eager contract is BF16 end to end + torch.manual_seed(913) + pool_k = torch.randn(blocks, layers, block_size, kv_heads, head_dim, dtype=dtype) + pool_v = torch.randn_like(pool_k) + keys, values = pool_k[:, 1], pool_v[:, 1] + + first, second, third, fourth = (object() for _ in range(4)) + if kind == "prefill_grid": + rows = [(a, 0, pos) for a in (first, second) for pos in range(4)] + elif kind == "prefill_single": + rows = [(first, 0, pos) for pos in range(4)] + elif kind == "ragged_decode": + rows = [(a, 0, 5) for a in (first, second, third, fourth)] + elif kind == "ragged_padded_tables": + # Mixed natural lengths: short rows have -1 in the high page slots, + # exactly as plan_attention_metadata pads a shorter block table. + rows = [(first, 0, 9), (second, 0, 1), (third, 0, 5), (fourth, 0, 2)] + elif kind == "single_decode": + rows = [(first, 0, 5)] + else: # irregular: a prefill group with a missing position + rows = [(first, 0, 0), (first, 0, 2), (first, 0, 3)] + + page_tables = { + id(first): [2, 5, 7, 9], + id(second): [3, 6, 8, 10], + id(third): [4, 11, 12, 13], + id(fourth): [14, 15, 16, 17], + } + if kind == "ragged_padded_tables": + page_tables = { + id(first): [2, 5, 7, 9], + id(second): [3, -1, -1, -1], + id(third): [4, 11, -1, -1], + id(fourth): [14, -1, -1, -1], + } + tables, write_blocks, write_offsets = [], [], [] + for allocation, _depth, position in rows: + table = page_tables[id(allocation)] + tables.append(table) + write_blocks.append(table[position // block_size]) + write_offsets.append(position % block_size) + batch = _PreparedBatch( + rows, + torch.tensor(tables, dtype=torch.int32), + torch.tensor(write_blocks, dtype=torch.int64), + torch.tensor(write_offsets, dtype=torch.int64), + ) + q = torch.randn(len(rows), kv_heads * head_groups, head_dim, dtype=dtype) + return q, keys, values, batch + + +@pytest.mark.parametrize( + "kind,expected_prefill,expected_ragged", + [ + ("prefill_grid", 1, 0), + ("prefill_single", 1, 0), + ("ragged_decode", 0, 1), + ("ragged_padded_tables", 0, 1), + ("single_decode", 0, 0), + ("irregular", 0, 0), + ], +) +@torch.inference_mode() +def test_batched_attention_paths_match_per_group_eager(kind, expected_prefill, expected_ragged): + """Batching may remove launches but must never change the arithmetic.""" + q, keys, values, batch = _batched_fixture(kind) + kernel = AscendEagerAttention() + groups = kernel._plan( + [(id(allocation), depth) for allocation, depth, _ in batch.rows], + [position + 1 for _, _, position in batch.rows], + q.device, + ) + expected = kernel._forward( + q, keys, values, batch.block_tables, groups, + flat_rows=_flat_kv_rows(keys, values), + ) + actual = kernel.attend_metadata(q, keys, values, batch) + torch.testing.assert_close(actual, expected, atol=0, rtol=0) + assert kernel.batched_prefill_calls == expected_prefill + assert kernel.ragged_calls == expected_ragged + + +def test_prefill_geometry_accepts_one_request_but_rejects_irregular_grids(): + first, second = object(), object() + assert _prefill_geometry([(first, 0, pos) for pos in range(4)]) == (1, 4) + assert _prefill_geometry( + [(first, 0, pos) for pos in range(4)] + [(second, 0, pos) for pos in range(4)] + ) == (2, 4) + assert _prefill_geometry([(first, 0, 0), (first, 0, 2)]) is None + assert _prefill_geometry([(first, 0, 1)]) is None diff --git a/vllm_rlt/kernels/ascend_eager.py b/vllm_rlt/kernels/ascend_eager.py index 2ab08478..815db6a5 100644 --- a/vllm_rlt/kernels/ascend_eager.py +++ b/vllm_rlt/kernels/ascend_eager.py @@ -1,5 +1,11 @@ # SPDX-License-Identifier: Apache-2.0 -"""Explicit BF16 eager attention on Ascend, matching released Ouro arithmetic.""" +"""Explicit BF16 eager attention on Ascend, matching released Ouro arithmetic. + +Two batching paths reduce host submissions without changing the arithmetic: +equal-length whole-prompt prefill becomes one attention call, and independent +single-row decode groups become one ragged batch. Everything else keeps the +original per-group path, including the layout-sensitive initial prefill. +""" import math import weakref @@ -7,12 +13,61 @@ import torch +def _flat_kv_rows(keys, values): + blocks, block_size, heads, head_dim = keys.shape + block_stride, *row_strides = keys.stride() + row_width = heads * head_dim + if ( + not blocks + or not row_width + or values.shape != keys.shape + or values.stride() != keys.stride() + or tuple(row_strides) != (row_width, head_dim, 1) + or block_stride < block_size * row_width + or block_stride % row_width + ): + return None + rows_per_block = block_stride // row_width + # The view starts at this layer's storage offset. Include the gaps between + # its pages, then gather only rows belonging to the selected layer. + shape = ((blocks - 1) * rows_per_block + block_size, heads, head_dim) + return ( + keys.as_strided(shape, row_strides), + values.as_strided(shape, row_strides), + rows_per_block, + ) + + +def _prefill_geometry(rows): + """Detect a regular grid of equal-length, complete prompt traversals.""" + if not rows or rows[0][2] != 0: + return None + width = rows[-1][2] + 1 + size, remainder = divmod(len(rows), width) + if size < 1 or width < 2 or remainder: + return None + groups = [(id(alloc), depth) for alloc, depth, _ in rows[::width]] + if len(set(groups)) != size or any( + ((id(alloc), depth), pos) != (groups[i // width], i % width) + for i, (alloc, depth, pos) in enumerate(rows) + ): + return None + return size, width + + class AscendEagerAttention: - """Batch each request/depth's queries; borrow only validated host metadata.""" + """Batch compatible queries; borrow only validated host metadata.""" def __init__(self): self._metadata = None self._groups = None + self._geometry = None + self._prefill_mask = None + self._write_slots = None + self._ragged_plan = None + self.batched_prefill_calls = 0 + self.ragged_calls = 0 + self.fallback_calls = 0 def _plan(self, keys, lengths, device): grouped = {} @@ -32,17 +87,28 @@ def _plan(self, keys, lengths, device): initial = len(rows) == maximum and [lengths[row] for row in rows] == list( range(1, maximum + 1) ) - planned.append((rows[0], indices, positions, mask, initial)) + row_lengths = torch.tensor([lengths[row] for row in rows], device=device) + planned.append((rows[0], indices, positions, mask, initial, row_lengths)) return planned - def _forward(self, q, keys, values, tables, groups): - result = torch.zeros_like(q) - block_size = keys.shape[1] - head_groups = q.shape[1] // keys.shape[2] - for first, indices, positions, mask, initial in groups: + def _forward(self, q, keys, values, tables, groups, *, flat_rows=None, result=None): + if result is None: + result = torch.zeros_like(q) + block_size, kv_heads, head_dim = keys.shape[-3:] + head_groups = q.shape[1] // kv_heads + for first, indices, positions, mask, initial, _lengths in groups: blocks = tables[first, positions // block_size] offsets = positions % block_size - k, v = keys[blocks, offsets], values[blocks, offsets] + if flat_rows is None: + k, v = keys[blocks, offsets], values[blocks, offsets] + else: + flat_keys, flat_values, rows_per_block = flat_rows + slots = blocks.to(torch.int64) * rows_per_block + offsets + # Padding table entries are -1. CANN gather rejects negative + # indices, and the additive mask already zeroes these columns, + # so clamping only changes data that cannot affect the output. + slots = slots.clamp(0, flat_keys.shape[0] - 1) + k, v = flat_keys.index_select(0, slots), flat_values.index_select(0, slots) if head_groups != 1: k = k.repeat_interleave(head_groups, dim=1) v = v.repeat_interleave(head_groups, dim=1) @@ -61,16 +127,127 @@ def _forward(self, q, keys, values, tables, groups): result.index_copy_(0, indices, output) return result + def _batched_prefill(self, q, keys, values, batch, flat_rows, geometry): + size, width = geometry + flat_keys, flat_values, rows_per_block = flat_rows + kv_heads, head_dim = keys.shape[-2:] + head_groups = q.shape[1] // kv_heads + if self._write_slots is None: + self._write_slots = ( + batch.write_blocks.to(torch.int64) * rows_per_block + + batch.write_offsets.to(torch.int64) + ).clamp(0, flat_keys.shape[0] - 1) + slots = self._write_slots + key = ( + flat_keys.index_select(0, slots) + .reshape(size, width, kv_heads, head_dim) + .transpose(1, 2) + ) + value = ( + flat_values.index_select(0, slots) + .reshape(size, width, kv_heads, head_dim) + .transpose(1, 2) + ) + if head_groups != 1: + key = key.repeat_interleave(head_groups, dim=1) + value = value.repeat_interleave(head_groups, dim=1) + query = q.reshape(size, width, q.shape[1], head_dim).transpose(1, 2).contiguous() + if self._prefill_mask is None: + positions = torch.arange(width, device=q.device) + mask = torch.zeros((width, width), dtype=q.dtype, device=q.device) + mask.masked_fill_( + positions[None, :] > positions[:, None], torch.finfo(q.dtype).min + ) + self._prefill_mask = mask + scores = torch.matmul(query, key.transpose(-2, -1)) * (1.0 / math.sqrt(head_dim)) + scores = scores + self._prefill_mask[None, None, :, :] + probabilities = torch.softmax(scores, dim=-1, dtype=torch.float32).to(q.dtype) + return torch.matmul(probabilities, value).transpose(1, 2).reshape_as(q) + + def _ragged_decode(self, q, keys, values, tables, groups, flat_rows, result): + """One attention call for independent single-row decode groups.""" + flat_keys, flat_values, rows_per_block = flat_rows + block_size, kv_heads, head_dim = keys.shape[-3:] + head_groups = q.shape[1] // kv_heads + if self._ragged_plan is None: + indices = torch.cat([group[1] for group in groups]) + lengths = torch.cat([group[5] for group in groups]) + maximum = max(group[2].numel() for group in groups) + positions = torch.arange(maximum, device=q.device) + pages = positions // block_size + offsets = positions % block_size + row_tables = tables.index_select(0, indices) + blocks = row_tables[:, pages].to(torch.int64) + slots = (blocks * rows_per_block + offsets.to(torch.int64)).clamp( + 0, flat_keys.shape[0] - 1 + ) + rows = indices.numel() + valid = positions.unsqueeze(0) < lengths.unsqueeze(1) + mask = torch.zeros((rows, maximum), dtype=torch.bfloat16, device=q.device) + mask.masked_fill_(~valid, torch.finfo(torch.bfloat16).min) + self._ragged_plan = (indices, lengths, maximum, positions, mask, slots, rows) + indices, _lengths, maximum, _positions, mask, slots, rows = self._ragged_plan + key = ( + flat_keys.index_select(0, slots.reshape(-1)) + .reshape(rows, maximum, kv_heads, head_dim) + ) + value = ( + flat_values.index_select(0, slots.reshape(-1)) + .reshape(rows, maximum, kv_heads, head_dim) + ) + if head_groups != 1: + key = key.repeat_interleave(head_groups, dim=2) + value = value.repeat_interleave(head_groups, dim=2) + query = q.index_select(0, indices).reshape(rows, q.shape[1], 1, head_dim) + key = key.permute(0, 2, 1, 3).contiguous() + value = value.permute(0, 2, 1, 3).contiguous() + scores = torch.matmul(query, key.transpose(-2, -1)) * (1.0 / math.sqrt(head_dim)) + scores = scores + mask[:, None, None, :] + probabilities = torch.softmax(scores, dim=-1, dtype=torch.float32).to(q.dtype) + output = torch.matmul(probabilities, value).squeeze(2) + result.index_copy_(0, indices, output) + return result + def __call__(self, q, keys, values, tables, lengths): groups = self._plan([tuple(row) for row in tables.tolist()], lengths.tolist(), q.device) return self._forward(q, keys, values, tables, groups) def attend_metadata(self, q, keys, values, batch): + flat_rows = _flat_kv_rows(keys, values) if self._metadata is None or self._metadata() is not batch: self._groups = self._plan( [(id(allocation), depth) for allocation, depth, _ in batch.rows], [position + 1 for _, _, position in batch.rows], q.device, ) + self._geometry = _prefill_geometry(batch.rows) + self._prefill_mask = None + self._write_slots = None + self._ragged_plan = None self._metadata = weakref.ref(batch) - return self._forward(q, keys, values, batch.block_tables, self._groups) + if flat_rows is None: + self.fallback_calls += 1 + return self._forward(q, keys, values, batch.block_tables, self._groups) + if self._geometry is not None: + self.batched_prefill_calls += 1 + return self._batched_prefill(q, keys, values, batch, flat_rows, self._geometry) + ragged, remaining = [], [] + for group in self._groups: + if not group[4] and group[1].numel() == 1: + ragged.append(group) + else: + remaining.append(group) + # A single row gains nothing from batching but pays extra plan construction. + if len(ragged) < 2: + remaining.extend(ragged) + ragged = [] + result = self._forward( + q, keys, values, batch.block_tables, remaining, flat_rows=flat_rows + ) + if ragged: + self.ragged_calls += 1 + result = self._ragged_decode( + q, keys, values, batch.block_tables, ragged, flat_rows, result + ) + return result + diff --git a/vllm_rlt/kernels/ascend_kv.py b/vllm_rlt/kernels/ascend_kv.py index fc41189c..da3c4e13 100644 --- a/vllm_rlt/kernels/ascend_kv.py +++ b/vllm_rlt/kernels/ascend_kv.py @@ -9,31 +9,35 @@ class AscendKVWriter: - """Reuse immutable traversal indices across layers without retaining the descriptor.""" + """Reuse immutable traversal indices across layers without retaining the descriptor. + + The batch already carries its write addresses as staged device tensors + (``write_blocks`` / ``write_offsets``). Building the Scatter ND index from + those tensors keeps the host out of the per-traversal hot path. + """ def __init__(self): self._metadata = None self._indices = None + self._layer_ids = None def write(self, keys, values, layer, batch, k, v): import torch_npu if self._metadata is None or self._metadata() is not batch: - owner = batch.owner - addresses = [ - ( - allocation.block_tables[owner._plane(depth)][position // owner.block_size], - position % owner.block_size, - ) - for allocation, depth, position in batch.rows - ] - self._indices = torch.tensor( - [ - [[block, index, offset] for block, offset in addresses] - for index in range(keys.shape[1]) - ], - device=keys.device, - dtype=torch.int32, + blocks = torch.as_tensor(batch.write_blocks, dtype=torch.int32, device=keys.device) + offsets = torch.as_tensor(batch.write_offsets, dtype=torch.int32, device=keys.device) + rows = len(batch.rows) + if blocks.numel() != rows or offsets.numel() != rows: + raise ValueError("KV write address count does not match the prepared rows") + layers = keys.shape[1] + if self._layer_ids is None or self._layer_ids.shape[0] != layers: + self._layer_ids = torch.arange( + layers, device=keys.device, dtype=torch.int32 + )[:, None] + layer_ids = self._layer_ids.expand(layers, rows) + self._indices = torch.stack( + (blocks.expand(layers, -1), layer_ids, offsets.expand(layers, -1)), dim=-1 ) self._metadata = weakref.ref(batch) indices = self._indices[layer] From 394f3b18e90dc3f3c0365cbd3e4bfbe074deb0dc Mon Sep 17 00:00:00 2001 From: KrystalRay Date: Tue, 17 Nov 2026 17:55:23 +0800 Subject: [PATCH 14/20] fix(ascend): preserve refill progress with coalesced preludes Keep the ordinary recurrent boundary while pending prompt or recurrent work competes under refill/mixed scheduling. Full-depth PRELUDE otherwise bypasses refill and can serialize complete generations or starve an existing request. Add bounded scheduler and single/multi-stream NPU regressions. Correct CUDA qualification wording and record all W1-W7 HF comparisons, fallback activation, latency distributions and unresolved output differences. --- docs/recipes/ouro_ascend.md | 118 ++++++++++++++++++++++++++++++++++-- tests/test_fixed_prelude.py | 109 ++++++++++++++++++++++++++++++++- vllm_rlt/core/scheduler.py | 10 +++ 3 files changed, 230 insertions(+), 7 deletions(-) diff --git a/docs/recipes/ouro_ascend.md b/docs/recipes/ouro_ascend.md index c496ba15..f850569a 100644 --- a/docs/recipes/ouro_ascend.md +++ b/docs/recipes/ouro_ascend.md @@ -263,6 +263,11 @@ CPU/CUDA/ROCm do not import or build this extension. The FP32 CPU generation of RoPE constants is an explicit NPU-only numerical correction to match HF; CUDA/ROCm retain their original device arithmetic. +With refill or mixed scheduling, pending prompts or runnable recurrent work +temporarily disable full-depth decode prelude coalescing. Keeping the ordinary +PRELUDE-to-RECURRENT boundary lets other requests make progress; stable batches +without competing work still use the coalesced path. + The November 17 refinement integrates upstream `c338aaa`, preserving its sliding windows, automatic `packed_prefill` policy and CUDA state-pool changes. Runtime source manifests, frozen inputs, pinned HF source, raw request records and @@ -270,7 +275,10 @@ reproduction scripts are in `/work/vllm-rlt-engine-20261117` inside `kelei-rlt` on `ssh npu`. Fixed and natural-input quality checks are separate from timing; they do not establish a task-accuracy score or W1–W7 qualification. -### Refinement validation +### Prior O32 validation (`028f9c6`) + +These measurements predate the refill follow-up below and use different output +lengths and token budgets from section 7. One NPU per case, CANN 9.0.0, Torch/Torch-NPU 2.10, Transformers 4.54.1, Ouro-1.4B BF16, UT4, greedy output, no graphs, one execution stream. All arms @@ -298,11 +306,111 @@ at eight clients/four resident requests: 11.68 vs 13.54 s. Raw request records, variability, latency percentiles and peak allocations are in `slim-summary.json` and `slim-results` in the experiment bundle. -The refinement passes 1,117 CPU regression tests (800 device tests skipped), +The prior refinement passed 1,117 CPU regression tests (800 device tests skipped), 81 targeted Ascend tests, three additional real-checkpoint regressions and three HTTP/SSE/disconnect regressions (Torch sync, Ascend sync and Ascend async). Separate fixed-input quality checks pass 192/192 sequences; 16 untruncated natural inputs also match HF under normal EOS with a 32-token cap (all reach the cap). -Ruff and source/wheel packaging checks pass. CUDA `ict-90` and ROCm `DCU-02` -were unreachable: unchanged shared implementations and CPU compatibility tests -are evidence of isolation, not CUDA/ROCm hardware qualification. +Ruff and source/wheel packaging checks pass. Staged CUDA compatibility runs on +`ict-90` pass 1,736 unique tests, including 619 GPU cases (181 skipped), and five +paired real-checkpoint configurations match the baseline. These runs do not +qualify unsupported SM120 paged FlashAttention or NIXL configurations. ROCm +hardware validation is deferred. + +## 7. Unified workload follow-up + +The refill follow-up preserves a RECURRENT scheduling opportunity when full-depth +PRELUDE would otherwise bypass pending prompt or recurrent work. Bounded CPU +regressions reproduce both starvation cases before the fix; the final tree passes +1,121 CPU cases (802 skipped), 51 targeted CPU/NPU cases, and full-tree Ruff checks. +The NPU-only option defaults off; its disabled CUDA/ROCm path is unchanged. + +This compares the whole enabled combination with official HF eager using the seven +workloads and measurement rules in [unified execution](../unified_execution.md#protocol-and-comparison-rules). +All 42 selected runs complete: three unprofiled 60-second closed-loop admission +windows per arm/workload, each followed by full drain. They contain 1,087 successful +requests and zero runtime failures. Model loading, compilation and full-output +cohort warmup are excluded. These are in-process actor measurements, without HTTP +or StageClient IPC. + +BF16, fixed UT4, greedy, one execution stream, graphs disabled. The engine uses +async/refill, budget/chunk 1024, contiguous KV and native host plans with fixed-depth +decode/prelude coalescing. W1–W5 ignore EOS; W6/W7 replay the frozen seed-0 selection +of 256 untruncated ShareGPT first-human messages, without a chat template, using +natural EOS and cap 512. Both arms use TASK_QUEUE_ENABLE=2, one Torch/OMP/MKL thread, +and one fixed card/CPU affinity per workload; order is native/HF, HF/native, +native/HF. The guide's CUDA FA4/graphs/PD/speculation modes are outside this NPU run. + +The 64 GiB device requires a capacity adjustment: W4 retains 32 clients with four +resident requests, W6/W7 at most eight residents. Both arms use the same limits. +Logical KV blocks are `max(256, residents * ceil((input_bound + output_cap)/16) * 4 + 16)`, +with a 4096-token input bound for natural requests. HF uses ordinary batched eager +and retains finished rows until its active cohort drains. + +Median output tok/s [three-run min–max]; throughput includes drain: + +| Workload | Input / output / clients | Residents | HF eager | Engine combination | vs HF | +| --- | --- | ---: | ---: | ---: | ---: | +| W1 | 128 / 128 / 1 | 1 | 6.152 [5.998–6.464] | 10.191 [8.951–10.274] | +65.65% | +| W2 | 128 / 128 / 32 | 32 | 192.660 [184.098–204.178] | 131.489 [113.793–148.783] | -31.75% | +| W3 | 4096 / 128 / 1 | 1 | 6.198 [6.028–6.318] | 8.379 [5.606–9.152] | +35.20% | +| W4 | 4096 / 128 / 32 | 4 | 20.449 [20.192–20.910] | 16.045 [15.719–16.325] | -21.54% | +| W5 | 1024 / 512 / 8 | 8 | 51.508 [50.409–52.387] | 40.692 [39.713–40.696] | -21.00% | +| W6 | ShareGPT / EOS cap 512 / 8 | 8 | 25.353 [23.451–25.831] | 35.410 [33.534–37.133] | +39.67% | +| W7 | ShareGPT / EOS cap 512 / 32 | 8 | 35.557 [35.320–35.800] | 35.961 [34.726–36.356] | +1.14% | + +W7's 1.14% median difference lies within the observed run ranges. W3 also has +substantial engine variability; one paired run is slower than HF. This matrix +supports case-dependent performance, not a universal speedup. + +Native layer execution is active throughout W1/W3 with zero fallback. W2/W4/W5/W6/W7 +materialize paged KV during warmup and have zero native-executor layer calls in +the timed windows; those layers use the Python eager fallback. Enabling the flag +does not establish native-path coverage for these mixed/chunked geometries. + +Complete token sequences and finish reasons match the paired HF requests as follows: + +| Workload | Identical / paired requests | Supplemental HF reference coverage | +| --- | ---: | --- | +| W1 | 10/10 | All 15 engine requests match HF | +| W2 | 221/224 | None | +| W3 | 9/9 | All 13 engine requests match HF | +| W4 | 93/108 | None | +| W5 | 18/24 | None | +| W6 | 15/27 | None | +| W7 | 45/99 | None | + +The supplemental checks cover three additional fixed inputs after all timing ends. +A bounded natural-input diagnostic also finds different HF outputs at batch 1 and +batch 8; this does not explain all engine/HF differences. Full greedy equivalence +and equal-quality performance acceptance have not passed. Natural EOS output +lengths differ: mean HF/engine output is 458.33/452.44 tokens for W6 and +457.15/466.58 for W7. Fresh GSM8K-87 task qualification has not been run. + +Latency triples are mean / p95 / p99, pooling successful requests across all three +runs. TTFT and E2E include queuing; TPOT is first-to-last arrival divided by N−1. +ITL pools adjacent token delivery intervals, with zero intervals within a delivered +chunk. Small-sample tail percentiles are descriptive. + +| Workload / arm | TTFT (ms) | TPOT (ms) | ITL (ms) | E2E (s) | +| --- | ---: | ---: | ---: | ---: | +| W1 / HF | 158.26 / 164.42 / 164.72 | 160.67 / 174.41 / 177.74 | 160.67 / 175.26 / 292.54 | 20.56 / 22.31 / 22.73 | +| W1 / engine | 210.43 / 252.72 / 313.55 | 101.54 / 119.08 / 128.71 | 101.54 / 153.61 / 161.31 | 13.11 / 15.38 / 16.66 | +| W2 / HF | 315.94 / 316.78 / 316.87 | 164.36 / 174.60 / 174.60 | 164.36 / 174.36 / 241.96 | 21.19 / 22.49 / 22.49 | +| W2 / engine | 807.75 / 1535.41 / 1569.18 | 237.03 / 330.89 / 334.54 | 237.03 / 365.28 / 448.81 | 30.91 / 42.86 / 43.11 | +| W3 / HF | 1126.36 / 1127.27 / 1127.31 | 154.25 / 159.01 / 159.36 | 154.25 / 164.21 / 166.66 | 20.72 / 21.32 / 21.37 | +| W3 / engine | 1038.28 / 1129.21 / 1134.81 | 121.93 / 172.03 / 174.34 | 121.93 / 182.16 / 190.80 | 16.52 / 22.92 / 23.25 | +| W4 / HF | 108201.82 / 184760.44 / 185035.97 | 162.69 / 185.42 / 199.48 | 162.69 / 197.72 / 215.71 | 128.86 / 204.86 / 205.16 | +| W4 / engine | 128120.96 / 228911.74 / 232203.32 | 228.93 / 241.58 / 242.65 | 228.93 / 449.12 / 640.13 | 157.19 / 258.43 / 262.65 | +| W5 / HF | 948.24 / 949.16 / 949.17 | 154.02 / 157.16 / 157.16 | 154.02 / 164.57 / 181.10 | 79.65 / 81.26 / 81.26 | +| W5 / engine | 1716.00 / 3022.56 / 3034.33 | 194.26 / 198.29 / 198.68 | 194.26 / 205.56 / 256.37 | 100.98 / 103.11 / 103.14 | +| W6 / HF | 9446.60 / 73948.53 / 74441.99 | 157.54 / 158.20 / 176.94 | 157.82 / 162.33 / 211.88 | 81.62 / 153.98 / 164.74 | +| W6 / engine | 900.40 / 1247.62 / 1247.82 | 205.59 / 217.85 / 218.08 | 205.19 / 214.00 / 358.37 | 93.53 / 112.26 / 112.26 | +| W7 / HF | 135933.80 / 259319.34 / 335009.35 | 162.34 / 165.82 / 167.64 | 162.53 / 169.73 / 218.95 | 210.07 / 343.37 / 416.61 | +| W7 / engine | 151301.80 / 320714.48 / 333012.61 | 207.59 / 222.27 / 222.30 | 207.95 / 413.73 / 441.94 | 248.12 / 425.01 / 438.91 | + +Source/protocol manifests, per-request arrivals, memory peaks, fallback counters +and the full summary are retained in +`/work/vllm-rlt-engine-20261117/unified-fairness-20261117`. The initial pre-fix +campaign is retained separately as an aborted defect diagnostic and contributes +no selected performance runs above. diff --git a/tests/test_fixed_prelude.py b/tests/test_fixed_prelude.py index 2666a2ac..28d9a798 100644 --- a/tests/test_fixed_prelude.py +++ b/tests/test_fixed_prelude.py @@ -3,6 +3,7 @@ import pytest import torch +from tests.helpers import make_cpu_cache, tiny_ouro_config from tests.test_ascend import make_npu_model from tests.test_npu_async import drain from vllm_rlt import CacheConfig, ExecutionConfig, ExitConfig, SamplingParams, SchedulerConfig @@ -37,13 +38,67 @@ def test_fixed_prelude_planning_and_submission_progress(exit_mode, min_loops, ex assert request.phase == RequestPhase.PRELUDE # Stale submissions cannot advance a reused ID. -def engine_for(model, planned, multi_stream=False): +@pytest.mark.parametrize("coalesced", [False, True]) +@pytest.mark.parametrize("late_arrival", [False, True]) +def test_fixed_prelude_preserves_refill_opportunity(coalesced, late_arrival): + scheduler = Scheduler( + SchedulerConfig(max_num_seqs=2, max_num_batched_tokens=3, prefill_chunk_size=3), + make_cpu_cache(tiny_ouro_config(), blocks=128), + total_ut_steps=4, + exit_mode="ouro_delayed", + coalesce_fixed_prelude=coalesced, + coalesce_fixed_decode=True, + ) + params = SamplingParams(max_tokens=16, min_loops=4, max_loops=4, ignore_eos=True) + first = Request("a", [1, 2, 3], params) + second = Request("b", [4, 5, 6], params) + scheduler.add_request(first) + if not late_arrival: + scheduler.add_request(second) + batch = scheduler.schedule() + assert batch.phase == RequestPhase.PREFILL + assert [item.request for item in batch.requests] == [first] + scheduler.update_after_submission(batch) + batch = scheduler.schedule() + assert batch.phase == RequestPhase.CODA + scheduler.update_after_submission(batch) + if late_arrival: + scheduler.add_request(second) + assert second.phase == (RequestPhase.WAITING if late_arrival else RequestPhase.PREFILL) + + # Every prompt consumes a complete token budget. Refill must still get a + # turn before the first generation ends, with or without prelude coalescing. + for _ in range(8): + batch = scheduler.schedule() + assert batch is not None + scheduler.update_after_submission(batch) + if any(item.request is second for item in batch.requests): + assert batch.phase == RequestPhase.PREFILL + assert first.num_scheduled_outputs < params.max_tokens + break + else: + pytest.fail("decode boundaries starved a pending prompt") + targets = {request.request_id: request.num_scheduled_outputs + 1 for request in (first, second)} + for _ in range(12): + batch = scheduler.schedule() + assert batch is not None + scheduler.update_after_submission(batch) + if all( + request.num_scheduled_outputs >= targets[request.request_id] + for request in (first, second) + ): + break + else: + pytest.fail("refilled decode starved the existing recurrent request") + + +def engine_for(model, planned, multi_stream=False, token_budget=128): return StageEngine( model, attention_backend="ascend_eager", cache_config=CacheConfig(num_blocks=128, block_size=16, contiguous_kv=True), scheduler_config=SchedulerConfig( - max_num_seqs=3, max_num_batched_tokens=128, prefill_chunk_size=128 + max_num_seqs=3, max_num_batched_tokens=token_budget, prefill_chunk_size=128 ), execution_config=ExecutionConfig( async_scheduling=True, @@ -55,6 +110,56 @@ def engine_for(model, planned, multi_stream=False): ) +@pytest.mark.npu +@torch.inference_mode() +@pytest.mark.parametrize("multi_stream", [False, True]) +def test_fixed_prelude_refills_during_generation(multi_stream): + resolve_device("npu") + torch.npu.set_device(0) + model = make_npu_model(torch.device("npu:0")) + params = SamplingParams(max_tokens=8, min_loops=4, max_loops=4, ignore_eos=True) + expected = None + for planned in [False, True]: + engine = engine_for(model, planned, multi_stream, token_budget=3) + try: + for name, prompt in [("a", [3, 7, 11]), ("b", [13, 17, 19])]: + engine.add_request(name, prompt, params) + actual, first_finished, second_started = {}, False, False + first_tokens_at_second_start = None + first_tokens, first_advanced = 0, False + for _ in range(128): + if not engine.has_unfinished_requests(): + break + for out in engine.step(): + if out.request_id == "b" and not second_started: + assert not first_finished + second_started = True + first_tokens_at_second_start = first_tokens + if out.request_id == "a": + first_tokens = len(out.token_ids) + if second_started and "b" not in actual: + first_advanced |= first_tokens > first_tokens_at_second_start + if out.finished: + actual[out.request_id] = ( + list(out.token_ids), + list(out.exit_depths), + out.finish_reason, + ) + if out.request_id == "a": + first_finished = True + else: + pytest.fail("refill did not drain in 128 steps") + assert second_started and first_advanced + assert set(actual) == {"a", "b"} + if expected is None: + expected = actual + else: + assert actual == expected + assert engine.cache_manager.num_used_blocks == 0 + finally: + engine.close() + + @pytest.mark.npu @torch.inference_mode() @pytest.mark.parametrize("multi_stream", [False, True]) diff --git a/vllm_rlt/core/scheduler.py b/vllm_rlt/core/scheduler.py index 5ac9eb8a..07c6b51f 100644 --- a/vllm_rlt/core/scheduler.py +++ b/vllm_rlt/core/scheduler.py @@ -809,6 +809,16 @@ def _planned_loop_count(self, scheduled_requests): if ( phases == {RequestPhase.PRELUDE} and self.coalesce_fixed_prelude + # Full-depth PRELUDE skips RECURRENT, where refill gets a turn. + # Keep that boundary while other prompt/core work needs a turn. + and not ( + self.config.mode in ("refill", "mixed") + and ( + self.queues[RequestPhase.WAITING] + or self.has_ready(RequestPhase.RECURRENT) + or any(self.has_ready(phase) for phase in self.policy.PREFILL_PHASES) + ) + ) and self.speculative_config is None and self.exit_mode != "trace" and all( From 7b4537f14ed6e32a87c47c92b6bac2ab3c28c058 Mon Sep 17 00:00:00 2001 From: KrystalRay Date: Tue, 17 Nov 2026 20:47:40 +0800 Subject: [PATCH 15/20] fix(ascend): account for coalesced recurrent work Reset prefill fairness accounting when a full-depth PRELUDE performs recurrent work. Preserve embedding-only and disabled-option behavior. Document the high-batch profile, separate scheduling presets, matched-output throughput and remaining latency/quality limits. --- docs/recipes/ouro_ascend.md | 131 +++++++++++++++++++++++++++++++++++- tests/test_fixed_prelude.py | 28 ++++++++ vllm_rlt/core/scheduler.py | 2 + 3 files changed, 159 insertions(+), 2 deletions(-) diff --git a/docs/recipes/ouro_ascend.md b/docs/recipes/ouro_ascend.md index f850569a..a91802d6 100644 --- a/docs/recipes/ouro_ascend.md +++ b/docs/recipes/ouro_ascend.md @@ -4,7 +4,9 @@ Run Ouro on one Ascend NPU with BF16, synchronous greedy decoding and four recurrent loops. Use the explicit `ascend_eager` attention backend. Sections 1–5 describe the synchronous path and its historical measurements. -Section 6 covers the optional asynchronous engine plan. +Section 6 covers the optional asynchronous engine plan. Sections 7–8 record +successive serving campaigns and their qualification limits. Sections 7–8 record +successive serving campaigns and their qualification limits. ## 1. Prepare the environment and model @@ -317,7 +319,7 @@ paired real-checkpoint configurations match the baseline. These runs do not qualify unsupported SM120 paged FlashAttention or NIXL configurations. ROCm hardware validation is deferred. -## 7. Unified workload follow-up +## 7. Unified workload follow-up (`394f3b1`, quota1) (`394f3b1`, quota1) The refill follow-up preserves a RECURRENT scheduling opportunity when full-depth PRELUDE would otherwise bypass pending prompt or recurrent work. Bounded CPU @@ -414,3 +416,128 @@ and the full summary are retained in `/work/vllm-rlt-engine-20261117/unified-fairness-20261117`. The initial pre-fix campaign is retained separately as an aborted defect diagnostic and contributes no selected performance runs above. + +## 8. Profile-guided high-batch scheduling + +This follow-up keeps model arithmetic, global defaults and CUDA/ROCm execution +unchanged. The runtime delta is two scheduler-accounting lines: a full-depth +coalesced PRELUDE has performed recurrent work, so it must reset the prefill +fairness counter. Embedding-only PRELUDE retains the previous counter. + +The main throughput change is an existing configuration option, +`max_prefill_batches_before_decode=32`, rather than an isolated benefit from +those two lines. At budget/chunk 1024, the default quota of one interrupts +P128/C32 prompt cohorts with decode: observed KV lengths become 130–133. +Nonuniform ranges export owned KV to paged storage and force Python layer +execution. Quota32 completes the resident prompt cohort first, preserving +uniform KV and the native eager host path. Higher quotas can postpone decode +for existing requests; the global default remains one. + +Independent P128/C32 profiles cover one complete resident UT4 decode (96 layer +calls), without model hooks. Both ops-only and full modes are retained; full +enables shapes, memory, stacks and CPU/NPU activities. Instrumented durations +are not throughput measurements. CPU trace counts and Python call counts are: + +| Decode path | Python calls | KV index_select | contiguous | linear | +| --- | ---: | ---: | ---: | ---: | +| Quota1 / paged Python fallback | 74323 | 192 | 192 | 677 | +| Quota32 / owned KV + native host | 31447 | 0 | 4 | 677 | +| HF eager | 26754 | 0 | 0 | 677 | + +Python calls fall 57.7%; the model projections remain separate. No fused +attention/norm/activation or graph replay is introduced. One HF ops-only +export lacks hardware events and is retained separately from its post-timing +recapture. Missing events are not interpreted as zero device work. + +### Unified matrix with quota32 and budget/chunk 1024 + +All 48 preselected runs complete: W1–W7 plus P128/O128/C64, each arm with +three unprofiled 60-second closed-loop admission windows followed by full +drain. Conditions, frozen inputs, complete-output cohort warmup exclusions, +capacity adjustments and latency definitions follow section 7. Each workload +uses a fixed card/CPU affinity for both arms; orders alternate native/HF, +HF/native, native/HF. Values are median [min–max] output tok/s. Output mismatch +makes affected rows diagnostic rather than equal-quality acceptance. + +| Workload | Input / output / clients | Residents | HF eager tok/s | Engine tok/s | vs HF | +| --- | --- | ---: | ---: | ---: | ---: | +| W1 | 128 / 128 / 1 | 1 | 6.112 [5.080–6.254] | 9.638 [9.317–10.071] | +57.69% | +| W2 | 128 / 128 / 32 | 32 | 198.614 [134.124–199.151] | 300.251 [283.298–305.138] | +51.17% | +| W2C64 | 128 / 128 / 64 | 64 | 370.250 [326.423–401.186] | 548.771 [536.924–557.119] | +48.22% | +| W3 | 4096 / 128 / 1 | 1 | 5.963 [5.800–6.100] | 8.913 [7.599–9.286] | +49.48% | +| W4 | 4096 / 128 / 32 | 4 | 19.987 [19.460–20.839] | 29.371 [29.081–30.408] | +46.95% | +| W5 | 1024 / 512 / 8 | 8 | 49.459 [47.994–50.664] | 77.213 [76.857–78.132] | +56.11% | +| W6 | ShareGPT / EOS cap 512 / 8 | 8 | 25.614 [25.385–26.475] | 37.633 [36.620–37.645] | +46.93% | +| W7 | ShareGPT / EOS cap 512 / 32 | 8 | 35.655 [35.381–36.094] | 34.811 [33.708–36.959] | -2.37% | + +W1–W5 and C64 retain owned KV and native host execution throughout their timed +windows, without fallback. W6/W7 materialize paged storage during warmup and +have zero native layer calls in the timed windows. W7 does not establish a +speedup; its ranges overlap and its natural outputs remain different. + +Complete-output comparison after bounded supplemental HF reference coverage: + +| Workload | Identical / engine requests with HF reference | Unverified | +| --- | ---: | ---: | +| W1 | 15 / 15 | 0 | +| W2 | 474 / 480 | 0 | +| W2C64 | 854 / 896 | 0 | +| W3 | 14 / 14 | 0 | +| W4 | 114 / 132 | 0 | +| W5 | 39 / 48 | 0 | +| W6 | 15 / 27 | 0 | +| W7 | 42 / 99 | 0 | + +### Separate full-cohort prefill preset + +The first W2 pilot divergence is input44, generated token0. A post-timing +diagnostic saves complete logits and LM-head inputs under identical prompt +history, comparing Python/native engine execution, HF batch8/batch32 and +full-cohort engine prefill. For this input: + +- Native logits equal Python logits: **True**. +- Small-budget native logits equal HF batch8: **True**. +- Full-cohort prefill logits equal HF batch32: **True**. + +Batch geometry is therefore checked explicitly, rather than treating a greedy +mismatch as acceptable rounding. This bounded diagnostic does not explain all +mixed/natural-input output differences. + +A separately declared C32/C64 matrix uses the existing settings +`max_num_batched_tokens=8192`, `prefill_chunk_size=1024` and +`max_prefill_batches_before_decode=32`. Whole-cohort prefill fits the budget; +no runtime or arithmetic change is made. Both resident counts equal client +concurrency. The logical KV block counts are 2064/4112 (16 tokens per block). +An independent 128-sequence/O128 parity gate precedes 12 fresh selected timing +runs, with the same three-run, 60-second-plus-drain protocol. They do not +replace any run in the 48-run matrix above. + +| Workload | Input / output / clients | Residents | HF eager tok/s | Engine tok/s | vs HF | +| --- | --- | ---: | ---: | ---: | ---: | +| W2 | 128 / 128 / 32 | 32 | 194.630 [194.316–201.176] | 300.901 [293.366–302.529] | +54.60% | +| W2C64 | 128 / 128 / 64 | 64 | 395.422 [389.372–414.086] | 574.780 [560.031–591.155] | +45.36% | + +Pooled TTFT p50 is 329.6/316.1 ms at C32 and 873.2/646.2 ms at C64 +(engine/HF); C32 TTFT p95 is 629.8/317.9 ms. The throughput gain does not +establish faster first output. Mean TPOT is 104.84/161.51 ms at C32 and +105.26/155.86 ms at C64. + +| Workload | Selected engine outputs equal independent HF reference | Unverified | +| --- | ---: | ---: | +| W2 | 480 / 480 | 0 | +| W2C64 | 960 / 960 | 0 | + +Fixed-input equal-quality acceptance for this separate preset: **True**. +This does not qualify natural-input task accuracy or arbitrary mixed batches. +Fresh GSM8K-87 qualification has not been run. + +The counter regression fails before the fix. The final runtime passes 1124 +CPU tests (802 skipped), 54 targeted CPU/NPU tests and full-tree Ruff checks. +The disabled coalescing option short-circuits the new accounting; no new +CUDA throughput or ROCm hardware result is claimed. + +The full report retains pooled mean/p95/p99 TTFT/TPOT/ITL/E2E, per-run memory +peaks, natural output lengths, native/fallback counters and failed/slow runs. +Raw requests, both predeclared protocols, source hashes, diagnostics and +profiles are retained in +`/work/vllm-rlt-engine-20261117/high-batch-20261117`. diff --git a/tests/test_fixed_prelude.py b/tests/test_fixed_prelude.py index 28d9a798..29e7c45b 100644 --- a/tests/test_fixed_prelude.py +++ b/tests/test_fixed_prelude.py @@ -110,6 +110,34 @@ def engine_for(model, planned, multi_stream=False, token_budget=128): ) +@pytest.mark.parametrize("coalesced,waiting", [(True, False), (False, False), (True, True)]) +def test_fixed_prelude_accounts_for_recurrent_fairness(coalesced, waiting): + scheduler = Scheduler( + SchedulerConfig(max_num_seqs=2, max_prefill_batches_before_decode=4), + make_cpu_cache(tiny_ouro_config(), blocks=128), + total_ut_steps=4, + exit_mode="ouro_delayed", + coalesce_fixed_prelude=coalesced, + ) + params = SamplingParams(max_tokens=8, min_loops=4, max_loops=4, ignore_eos=True) + scheduler.add_request(Request("a", [1, 2, 3], params)) + for expected in (RequestPhase.PREFILL, RequestPhase.CODA): + batch = scheduler.schedule() + assert batch.phase == expected + scheduler.update_after_submission(batch) + if waiting: + scheduler.add_request(Request("b", [4, 5, 6], params)) + for _ in range(scheduler.config.max_prefill_batches_before_decode): + scheduler.policy.record_batch(RequestPhase.PREFILL) + assert scheduler.policy.decode_due + batch = scheduler.schedule() + assert batch.phase == RequestPhase.PRELUDE + assert bool(batch.loop_count) == (coalesced and not waiting) + # A coalesced decode submission actually runs recurrent work. Embedding-only + # PRELUDE must retain the pending prefill budget until its core is selected. + assert scheduler.policy.decode_due == (not coalesced or waiting) + + @pytest.mark.npu @torch.inference_mode() @pytest.mark.parametrize("multi_stream", [False, True]) diff --git a/vllm_rlt/core/scheduler.py b/vllm_rlt/core/scheduler.py index 07c6b51f..16a31803 100644 --- a/vllm_rlt/core/scheduler.py +++ b/vllm_rlt/core/scheduler.py @@ -781,6 +781,8 @@ def take_phases(self, phases, *, eligible=None): ) self.policy.record_batch(recorded) loop_count = self._planned_loop_count(scheduled_requests) + if self.coalesce_fixed_prelude and selected_phases == {RequestPhase.PRELUDE} and loop_count: + self.policy.record_batch(RequestPhase.RECURRENT) scheduled_requests = [ replace( item, From fa45eee973726579955f16128be158e64b065d61 Mon Sep 17 00:00:00 2001 From: KrystalRay Date: Wed, 18 Nov 2026 00:48:42 +0800 Subject: [PATCH 16/20] docs(ascend): report complete unified workload validation Record all seven workloads with a uniform 8192-token budget, three paired repetitions and complete output-reference coverage. Preserve unresolved output differences, actual native/paged activation and latency/memory costs instead of claiming a universal speedup. --- docs/recipes/ouro_ascend.md | 218 +++++++++++++++++------------------- 1 file changed, 104 insertions(+), 114 deletions(-) diff --git a/docs/recipes/ouro_ascend.md b/docs/recipes/ouro_ascend.md index a91802d6..c4f1ffa4 100644 --- a/docs/recipes/ouro_ascend.md +++ b/docs/recipes/ouro_ascend.md @@ -419,24 +419,21 @@ no selected performance runs above. ## 8. Profile-guided high-batch scheduling -This follow-up keeps model arithmetic, global defaults and CUDA/ROCm execution -unchanged. The runtime delta is two scheduler-accounting lines: a full-depth -coalesced PRELUDE has performed recurrent work, so it must reset the prefill -fairness counter. Embedding-only PRELUDE retains the previous counter. - -The main throughput change is an existing configuration option, -`max_prefill_batches_before_decode=32`, rather than an isolated benefit from -those two lines. At budget/chunk 1024, the default quota of one interrupts -P128/C32 prompt cohorts with decode: observed KV lengths become 130–133. -Nonuniform ranges export owned KV to paged storage and force Python layer -execution. Quota32 completes the resident prompt cohort first, preserving -uniform KV and the native eager host path. Higher quotas can postpone decode -for existing requests; the global default remains one. - -Independent P128/C32 profiles cover one complete resident UT4 decode (96 layer -calls), without model hooks. Both ops-only and full modes are retained; full -enables shapes, memory, stacks and CPU/NPU activities. Instrumented durations -are not throughput measurements. CPU trace counts and Python call counts are: +The runtime follow-up is two scheduler-accounting lines: a full-depth coalesced +PRELUDE has performed recurrent work and must reset the prefill fairness counter. +Embedding-only PRELUDE retains the previous counter. The new regression fails +on `394f3b1` (one failure, two controls pass) and passes on `7b4537f`. +Global defaults and the disabled CUDA/ROCm path are unchanged. + +At budget/chunk 1024, quota 1 interleaves decode with P128/C32 prompt cohorts: +observed KV lengths become 130–133. Nonuniform ranges export owned KV to paged +storage and force Python layer execution. The existing prefill quota 32 preserves +a uniform resident cohort and the native eager host path. Higher quotas can +postpone decode for existing requests; the global default remains one. + +Independent P128/C32 ops-only/full profiles cover one resident UT4 decode +(96 layer calls), without model hooks. Full enables shapes, memory, stacks and +CPU/NPU activities. Instrumented durations are not throughput measurements. | Decode path | Python calls | KV index_select | contiguous | linear | | --- | ---: | ---: | ---: | ---: | @@ -444,100 +441,93 @@ are not throughput measurements. CPU trace counts and Python call counts are: | Quota32 / owned KV + native host | 31447 | 0 | 4 | 677 | | HF eager | 26754 | 0 | 0 | 677 | -Python calls fall 57.7%; the model projections remain separate. No fused -attention/norm/activation or graph replay is introduced. One HF ops-only -export lacks hardware events and is retained separately from its post-timing -recapture. Missing events are not interpreted as zero device work. - -### Unified matrix with quota32 and budget/chunk 1024 - -All 48 preselected runs complete: W1–W7 plus P128/O128/C64, each arm with -three unprofiled 60-second closed-loop admission windows followed by full -drain. Conditions, frozen inputs, complete-output cohort warmup exclusions, -capacity adjustments and latency definitions follow section 7. Each workload -uses a fixed card/CPU affinity for both arms; orders alternate native/HF, -HF/native, native/HF. Values are median [min–max] output tok/s. Output mismatch -makes affected rows diagnostic rather than equal-quality acceptance. - -| Workload | Input / output / clients | Residents | HF eager tok/s | Engine tok/s | vs HF | -| --- | --- | ---: | ---: | ---: | ---: | -| W1 | 128 / 128 / 1 | 1 | 6.112 [5.080–6.254] | 9.638 [9.317–10.071] | +57.69% | -| W2 | 128 / 128 / 32 | 32 | 198.614 [134.124–199.151] | 300.251 [283.298–305.138] | +51.17% | -| W2C64 | 128 / 128 / 64 | 64 | 370.250 [326.423–401.186] | 548.771 [536.924–557.119] | +48.22% | -| W3 | 4096 / 128 / 1 | 1 | 5.963 [5.800–6.100] | 8.913 [7.599–9.286] | +49.48% | -| W4 | 4096 / 128 / 32 | 4 | 19.987 [19.460–20.839] | 29.371 [29.081–30.408] | +46.95% | -| W5 | 1024 / 512 / 8 | 8 | 49.459 [47.994–50.664] | 77.213 [76.857–78.132] | +56.11% | -| W6 | ShareGPT / EOS cap 512 / 8 | 8 | 25.614 [25.385–26.475] | 37.633 [36.620–37.645] | +46.93% | -| W7 | ShareGPT / EOS cap 512 / 32 | 8 | 35.655 [35.381–36.094] | 34.811 [33.708–36.959] | -2.37% | - -W1–W5 and C64 retain owned KV and native host execution throughout their timed -windows, without fallback. W6/W7 materialize paged storage during warmup and -have zero native layer calls in the timed windows. W7 does not establish a -speedup; its ranges overlap and its natural outputs remain different. - -Complete-output comparison after bounded supplemental HF reference coverage: - -| Workload | Identical / engine requests with HF reference | Unverified | -| --- | ---: | ---: | -| W1 | 15 / 15 | 0 | -| W2 | 474 / 480 | 0 | -| W2C64 | 854 / 896 | 0 | -| W3 | 14 / 14 | 0 | -| W4 | 114 / 132 | 0 | -| W5 | 39 / 48 | 0 | -| W6 | 15 / 27 | 0 | -| W7 | 42 / 99 | 0 | - -### Separate full-cohort prefill preset - -The first W2 pilot divergence is input44, generated token0. A post-timing -diagnostic saves complete logits and LM-head inputs under identical prompt -history, comparing Python/native engine execution, HF batch8/batch32 and -full-cohort engine prefill. For this input: - -- Native logits equal Python logits: **True**. -- Small-budget native logits equal HF batch8: **True**. -- Full-cohort prefill logits equal HF batch32: **True**. - -Batch geometry is therefore checked explicitly, rather than treating a greedy -mismatch as acceptable rounding. This bounded diagnostic does not explain all -mixed/natural-input output differences. - -A separately declared C32/C64 matrix uses the existing settings -`max_num_batched_tokens=8192`, `prefill_chunk_size=1024` and -`max_prefill_batches_before_decode=32`. Whole-cohort prefill fits the budget; -no runtime or arithmetic change is made. Both resident counts equal client -concurrency. The logical KV block counts are 2064/4112 (16 tokens per block). -An independent 128-sequence/O128 parity gate precedes 12 fresh selected timing -runs, with the same three-run, 60-second-plus-drain protocol. They do not -replace any run in the 48-run matrix above. - -| Workload | Input / output / clients | Residents | HF eager tok/s | Engine tok/s | vs HF | -| --- | --- | ---: | ---: | ---: | ---: | -| W2 | 128 / 128 / 32 | 32 | 194.630 [194.316–201.176] | 300.901 [293.366–302.529] | +54.60% | -| W2C64 | 128 / 128 / 64 | 64 | 395.422 [389.372–414.086] | 574.780 [560.031–591.155] | +45.36% | - -Pooled TTFT p50 is 329.6/316.1 ms at C32 and 873.2/646.2 ms at C64 -(engine/HF); C32 TTFT p95 is 629.8/317.9 ms. The throughput gain does not -establish faster first output. Mean TPOT is 104.84/161.51 ms at C32 and -105.26/155.86 ms at C64. - -| Workload | Selected engine outputs equal independent HF reference | Unverified | +The configuration recovers the owned-KV/native-host path; this is not an isolated +speedup from the two accounting lines. No fused model operators or graph replay +are introduced. An HF ops-only export lacking hardware events is retained beside +its successful post-timing recapture. These are decode/C32 profiles; a fresh +full-request C1 before/after timeline was not collected for this increment. + +### Full unified W1-W7 validation + +**The complete matrix has not passed equal-output acceptance.** W4/W6/W7 have +unresolved sequence differences, so their throughput figures remain diagnostic. + +All seven workloads were rerun on the same final configuration from `7b4537f`: +budget 8192, chunk 1024, prefill quota 32, async/refill, owned contiguous KV, native +eager host dispatch and fixed-depth prelude/decode coalescing. No per-case tuning +or older C32/C64 result is substituted into this matrix. + +One 910B3 per arm, CANN 9, Torch/Torch-NPU 2.10, Transformers 4.54.1, Ouro-1.4B BF16, +UT4 greedy. Graphs, multi-stream, UVA, padding and static buffers are off. Both +arms set TASK_QUEUE_ENABLE=2 and Torch/OMP/MKL threads to one. Each case uses the +same card/CPU affinity with native/HF, HF/native, native/HF order. + +All **42** new selected runs complete with **1422 successful requests and zero +runtime failures**. Every process warms a complete output cohort; loading, +compilation and warmup are excluded. Each run admits closed-loop traffic for +60 seconds and fully drains. Output throughput includes drain; values are +median [min–max] tok/s. These are in-process actor timings, not HTTP/IPC latency +or a saturation sweep. This covers the guide's applicable single-stage W1–W7 +HF comparison, not its historical B300/FA4/graphs/PD/speculation mode matrix. + +W1–W5 ignore EOS. W6/W7 retain the same frozen seed-0 selection of 256 ShareGPT +first-human messages, without truncation or a chat template, using natural EOS +and cap 512. The 64GiB capacity adaptation is identical in both arms: W4 has 32 +clients/four residents; W6/W7 have 8/32 clients with at most 8 residents. + +| Workload | Input / output / clients | Residents | HF eager tok/s | Engine tok/s | vs HF | Complete output matches | +| --- | --- | ---: | ---: | ---: | ---: | ---: | +| W1 | 128 / 128 / 1 | 1 | 6.334 [5.284–6.496] | 9.464 [9.403–10.394] | +49.42% | 15/15 | +| W2 | 128 / 128 / 32 | 32 | 201.314 [200.099–204.437] | 305.496 [293.376–316.662] | +51.75% | 480/480 | +| W3 | 4096 / 128 / 1 | 1 | 6.254 [6.206–6.268] | 7.569 [6.110–9.215] | +21.03% | 12/12 | +| W4 | 4096 / 128 / 32 | 4 | 21.211 [21.012–21.580] | 31.312 [31.135–31.405] | +47.62% | 120/132 | +| W5 | 1024 / 512 / 8 | 8 | 50.885 [50.234–51.362] | 79.791 [76.843–80.666] | +56.80% | 48/48 | +| W6 | ShareGPT / EOS cap 512 / 8 | 8 | 26.343 [26.060–26.385] | 37.992 [37.906–38.879] | +44.22% | 18/27 | +| W7 | ShareGPT / EOS cap 512 / 32 | 8 | 36.341 [35.576–36.931] | 36.194 [35.808–36.679] | -0.41% | 45/99 | + +Complete tokens, finish reasons and depth 4 are checked, including supplemental +independent HF references after timing; no engine output remains unverified. +W1/W2/W3/W5 pass these covered-input checks. W4/W6/W7 do not. W7 ranges overlap +and do not establish a speedup; one W3 pair is slower than HF. This is not a +fresh GSM8K task-accuracy qualification or a universal equal-quality speedup. + +W1–W5 retain native execution and owned KV without timed-window fallback. +W6/W7 materialize paged KV during warmup and have zero native layer calls +in the timed windows. Their enabled flag does not imply fast-path coverage. + +Latency statistics pool successful requests across repetitions; values below +are mean / p95 / p99. TTFT and E2E include queuing, TPOT is first-to-last output +/(N−1), and ITL uses adjacent delivery intervals with zero intra-chunk gaps. + +| Workload / arm | TTFT (ms) | TPOT (ms) | | --- | ---: | ---: | -| W2 | 480 / 480 | 0 | -| W2C64 | 960 / 960 | 0 | - -Fixed-input equal-quality acceptance for this separate preset: **True**. -This does not qualify natural-input task accuracy or arbitrary mixed batches. -Fresh GSM8K-87 qualification has not been run. - -The counter regression fails before the fix. The final runtime passes 1124 -CPU tests (802 skipped), 54 targeted CPU/NPU tests and full-tree Ruff checks. -The disabled coalescing option short-circuits the new accounting; no new -CUDA throughput or ROCm hardware result is claimed. - -The full report retains pooled mean/p95/p99 TTFT/TPOT/ITL/E2E, per-run memory -peaks, natural output lengths, native/fallback counters and failed/slow runs. -Raw requests, both predeclared protocols, source hashes, diagnostics and -profiles are retained in -`/work/vllm-rlt-engine-20261117/high-batch-20261117`. +| W2 / HF | 314.90 / 315.98 / 316.09 | 157.04 / 160.28 / 160.28 | +| W2 / engine | 386.18 / 616.33 / 617.22 | 102.73 / 112.46 / 112.46 | +| W7 / HF | 132566.71 / 255707.79 / 326208.73 | 158.90 / 163.43 / 164.37 | +| W7 / engine | 149321.74 / 318700.31 / 321899.41 | 205.49 / 209.90 / 210.49 | + +The W2 throughput gain trades off first-output latency. At W7, mean E2E is +245.42/205.07 seconds and peak allocated memory is 30.056/19.536GiB (engine/HF). +Allocator peaks reset after warmup; these are not startup peaks or total HBM. + +### Earlier full-cohort C32/C64 check + +The separate, previously declared P128/O128 preset used the same 8192/1024/quota 32 +settings. Its 12 selected runs recorded +54.60% at C32 and +45.36% at C64, with +1440/1440 complete engine outputs matching independent HF references. Those +runs remain separate and contribute no selected result to the 42-run table. + +For the earlier input44/token0 divergence, Python/native logits match HF batch8 +exactly; full-cohort engine prefill matches HF batch32 logits and head inputs +with max_abs=0. This bounded batch-geometry finding does not explain the +unchecked W4/W6/W7 differences. They remain unresolved. + +The runtime passed 1124 CPU tests (802 skipped), 54 targeted CPU/NPU tests and +full-tree Ruff checks. The native option defaults off; this increment does not +claim a fresh CUDA/ROCm hardware run. Earlier CUDA validation retains its +original scope. + +The full report retains all latency distributions, natural length/stop counts, +memory and native/fallback counters, exact commands and source/input hashes in +`/work/vllm-rlt-engine-20261117/unified-final-8192-20261117`. Prior captures and +bounded logits diagnostics remain in the separate `high-batch-20261117` bundle. From 6e5392d6a5b4009ecd11430d573e8447480b838d Mon Sep 17 00:00:00 2001 From: KrystalRay Date: Wed, 18 Nov 2026 11:09:07 +0800 Subject: [PATCH 17/20] docs(ascend): streamline the Ouro recipe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keep the current launch configuration, execution path, and W1–W7 results. Remove superseded experiments and preserve existing links to the recipe. --- docs/recipes/ouro_ascend.md | 596 ++++++------------------------------ 1 file changed, 102 insertions(+), 494 deletions(-) diff --git a/docs/recipes/ouro_ascend.md b/docs/recipes/ouro_ascend.md index c4f1ffa4..212ec997 100644 --- a/docs/recipes/ouro_ascend.md +++ b/docs/recipes/ouro_ascend.md @@ -1,533 +1,141 @@ # Ouro on Ascend -Run Ouro on one Ascend NPU with BF16, synchronous greedy decoding and four -recurrent loops. Use the explicit `ascend_eager` attention backend. +Run Ouro-1.4B on one Ascend NPU with BF16, greedy decoding and four recurrent +loops. The `ascend_eager` backend uses eager ATen operations and HF-style +attention rounding. The example enables the optional asynchronous engine plan. -Sections 1–5 describe the synchronous path and its historical measurements. -Section 6 covers the optional asynchronous engine plan. Sections 7–8 record -successive serving campaigns and their qualification limits. Sections 7–8 record -successive serving campaigns and their qualification limits. +## Prepare the environment and model -## 1. Prepare the environment and model +Install the project dependencies in a compatible Torch, Torch-NPU and CANN +environment. Use the checkpoint's config and byte-level tokenizer. The native +host plan builds a C++ extension on first use and requires a C++17 compiler, +Ninja, PyTorch headers and a writable build cache. Set +`VLLM_RLT_NATIVE_BUILD_DIR` to select the cache location. -Use a compatible Torch, Torch-NPU and CANN environment with the project's -Python dependencies installed. Keep the vendor Torch pair; CUDA/Triton -packages are not needed for this backend. - -~~~bash +```bash source /path/to/cann/set_env.sh source /path/to/npu-environment/bin/activate python -m pip install --no-deps --no-build-isolation -e . export MODEL=/path/to/prepared/Ouro-1.4B -~~~ - -Use the checkpoint's own config and tokenizer. Model loading uses the shared -native Ouro loader, including its supported sharded checkpoint layout. +export ASCEND_RT_VISIBLE_DEVICES=0 # Replace 0 with your assigned NPU. +``` -## 2. Start the server +## Serve fixed-depth requests -Select an allocated physical NPU using `ASCEND_RT_VISIBLE_DEVICES`. +This example sizes KV for 32 resident requests with 128 input and 128 output +tokens each. Set `--num-blocks` explicitly and adjust it for the expected +sequence lengths, resident request count and available device memory. For +whole-cohort prefill, the chunk size must fit each prompt and the batched token +budget must fit the sum of prompt lengths in the cohort. -~~~bash -ASCEND_RT_VISIBLE_DEVICES= \ +```bash +TASK_QUEUE_ENABLE=2 OMP_NUM_THREADS=1 MKL_NUM_THREADS=1 MAX_JOBS=1 \ python -m vllm_rlt.entrypoints.serve \ --model "$MODEL" --tokenizer "$MODEL" --served-model-name ouro \ --device npu --dtype bfloat16 --attention-backend ascend_eager \ - --num-blocks 1024 --max-num-seqs 1 \ - --max-num-batched-tokens 4096 --prefill-chunk-size 4096 \ - --host 127.0.0.1 --port 8123 -~~~ + --mode refill --exit-mode ouro_delayed \ + --async-scheduling --single-stream --no-uva --contiguous-kv \ + --coalesce-fixed-prelude --native-eager-plan \ + --block-size 16 --num-blocks 2064 --max-num-seqs 32 \ + --max-num-batched-tokens 8192 --prefill-chunk-size 1024 \ + --max-prefill-batches-before-decode 32 --host 127.0.0.1 --port 8123 +``` -Explicit KV capacity is required. Adjust it for the model, available memory, -prompt/output lengths and concurrency. To compare with full-prefill inference, -both the chunk size and batched-token budget must fit the whole prompt. +Omit `--native-eager-plan` to use the Python eager executor without building +an extension. For synchronous eager execution, also omit `--async-scheduling`, +`--coalesce-fixed-prelude` and `--contiguous-kv`. -## 3. Send a request +Wait for `/health` to return HTTP 200, then send a completion. Add +`"stream": true` for server-sent events. -~~~bash +```bash curl --fail http://127.0.0.1:8123/health curl --fail http://127.0.0.1:8123/v1/completions \ -H 'Content-Type: application/json' \ -d '{"model":"ouro","prompt":"The capital of France is","max_tokens":8,"temperature":0,"min_loops":4,"max_loops":4,"exit_threshold":1}' -~~~ - -Wait for HTTP 200 before sending requests. Add `"stream": true` for SSE. -The `ascend_eager` backend follows HF-style BF16 attention rounding; -`--attention-backend torch` selects the existing FP32-intermediate reference. - -## 4. Run checks - -With an assigned NPU visible to the test process: - -~~~bash -pre-commit run --all-files -VLLM_RLT_OURO_CHECKPOINT="$MODEL" python -m pytest \ - tests/test_ascend.py tests/test_ascend_checkpoint.py \ - tests/test_ascend_eager.py tests/test_ascend_kv.py \ - tests/test_ascend_serving.py --run-npu -q -~~~ - -These checks cover causal attention, cross-page KV, request reuse and HTTP/SSE. -They do not establish model quality or performance; use the -[accuracy guide](../accuracy.md) and [serving benchmark guide](../serving.md) -for those evaluations. - -Graphs, speculation, PD, preemption, prefix caching and adaptive exit are -unsupported on this path. See section 6 for fixed-depth asynchronous execution. - -## 5. Attention batching and device-side KV writes - -Two follow-up changes reduce host submissions on this path without changing the -BF16 arithmetic contract: - -- `ascend_kv.py` builds the Scatter ND write index from the batch's staged - device tensors (`write_blocks` / `write_offsets`) instead of a Python loop - over `batch.rows`; a fail-closed check rejects a row/address count mismatch. -- `ascend_eager.py` merges equal-length whole-prompt prefill into one - `[size, width]` attention call, and merges independent single-row decode - groups into one ragged gather. Write slots, ragged plans and the prefill mask - are cached per batch. A single decode row keeps the original per-group path - so it does not pay for plan construction. - -The arithmetic is unchanged: BF16 score scaling, additive BF16 causal mask, -FP32 softmax cast back to BF16 probabilities, then a BF16 matmul. The default -backend is still `AscendEagerAttention`; no fused CANN attention operator is -used, because those operators do not reproduce the released HF eager rounding. - -### Measurement conditions - -One Ascend 910B3, CANN 9.0.0, Torch 2.9.0+cpu with torch_npu 2.9.0, -Ouro-1.4B BF16, fixed four recurrent loops, synchronous greedy decoding, -full prefill, frozen prompt IDs, forced 32-token output, profiler off. -Three arms share one process on the same card (NPU 6, `taskset -c 64-71`): -official HF eager as reference, the pre-change per-group eager attention, -and the merged implementation. Each arm gets one warmup and three timed -repetitions; throughput, TTFT and TPOT are medians over request samples. -P2048/C8 uses a 16384 token budget to preserve full-prefill geometry. - -### Before/after results (same hardware, different card assignment) - -| Input / output / concurrency | Arm | Output tok/s | TTFT (ms) | TPOT (ms) | -| --- | --- | ---: | ---: | ---: | -| 128 / 32 / 8 | HF eager | 50.792 | 158.9 | 157.5 | -| | before | 14.611 | 630.7 | 544.8 | -| | **merged** | **37.724** | **273.0** | **209.9** | -| 2048 / 32 / 1 | HF eager | 6.339 | 371.2 | 150.9 | -| | before | 4.784 | 531.3 | 198.7 | -| | **merged** | **4.789** | **534.7** | **198.3** | - -A second matched run on longer/higher-concurrency workloads (same protocol, -different process, so absolute values are not cross-comparable with the table -above): - -| Input / output / concurrency | Arm | Output tok/s | TTFT (ms) | TPOT (ms) | -| --- | --- | ---: | ---: | ---: | -| 2048 / 32 / 8 | HF eager | 35.835 | 2489.5 | 150.1 | -| | before | 12.426 | 3702.4 | 545.1 | -| | **merged** | **25.007** | **3591.4** | **213.7** | -| 4096 / 32 / 1 | HF eager | 5.812 | 1091.1 | 142.4 | -| | before | 4.170 | 1532.7 | 196.3 | -| | **merged** | **4.244** | **1534.5** | **193.7** | - -The gain is in decode/overall throughput, not long-prompt TTFT: prefill is -dominated by the dense `[W, W]` attention and by host submission, and merging -calls reduces launches rather than arithmetic. Relative to HF eager, C8 -throughput moves from 0.288x to 0.743x at input 128; HF eager remains faster -in absolute terms. - -An earlier, separate A/B isolated the device-side write index alone (different -harness, older configuration, directional only): P2048/C1 TTFT 790.7 -> 531.9 ms -(-32.7%) and P128/C8 TTFT 691.7 -> 619.8 ms (-10.4%). - -### Full W1–W7 serving campaign - -`docs/unified_execution.md` defines the project's full serving benchmark: W1–W7, -three unprofiled repetitions per workload, 60 seconds of closed-loop admission -followed by full drain, median aggregate output throughput, pooled TTFT / TPOT / -ITL / E2E percentiles, fixed lengths for W1–W5, and a fixed seeded ShareGPT -selection with natural EOS and a 512-token cap for W6/W7. The merged candidate -ran as a single arm on the frozen campaign actor with no runtime monkeypatch on -NPU 5/6/7. The before column is the `d921719` baseline published in PR #168 on -2026-11-06; card assignment differs between the two runs, so this is a -same-hardware, not same-run, comparison. - -| Workload | Input / output / C | before tok/s | merged tok/s | before TTFT p95 (s) | merged TTFT p95 (s) | before TPOT p95 (ms) | merged TPOT p95 (ms) | -| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | -| W1 | 128 / 128 / 1 | 3.944 | 4.653 | 0.426 | 0.205 | 265.8 | 221.1 | -| W2 | 128 / 128 / 32 | 4.802 | 96.726 | 9.737 | 6.827 | 6642.6 | 285.9 | -| W3 | 4096 / 128 / 1 | 3.727 | 4.368 | 2.167 | 1.721 | 254.3 | 222.1 | -| W4 | 4096 / 128 / 32 | 4.201 | 24.761 | 747.440 | 137.225 | 1794.4 | 236.0 | -| W5 | 1024 / 512 / 8 | 4.575 | 30.744 | 4.676 | 2.551 | 1743.0 | 256.0 | -| W6 | ShareGPT / ≤512 / 8 | 4.660 | 21.417 | 3.572 | 1.689 | 1666.7 | 317.5 | -| W7 | ShareGPT / ≤512 / 32 | 4.760 | 27.970 | 15.765 | 7.289 | 6523.6 | 1071.7 | - -All seven workloads completed three unprofiled repetitions with zero failed -requests. Successful requests over the three runs (before → merged): W1 6→9, -W2 96→192, W3 6→9, W4 96→120, W5 24→24, W6 27→33, W7 99→105. - -The W2 ratio is dominated by the baseline's lack of concurrency scaling rather -than by a per-request speedup: on `d921719`, raising concurrency from 1 to 32 -moved aggregate throughput only from 3.944 to 4.802 tok/s (1.22×) while TPOT -p95 grew from 265.8 ms to 6642.6 ms, whereas the merged path reaches 20.79× the -C1 throughput with TPOT p95 of 285.9 ms. The same near-flat baseline scaling -appears in W4, W5, W6 and W7. Under the W1–W7 protocol a faster configuration -also covers more inputs during the same 60-second admission window (W2 -completed 32 requests per run before versus 64 after, with drain falling from -about 792 s to 25 s), so the aggregate ratio reflects both a shorter drain and -a larger completed-request count. - -The fresh campaign quality gate (GSM8K-87 three-shot strict and the paired -same-history logit vectors) has not been re-run on this change. The earlier -`kv-read-full-20261106` campaign recorded 63/87 for both HF eager and the -target, 87/87 exact sequences and 8952/8952 exact logit vectors with -`max_abs=0.0` on `d921719` and a different KV-read candidate; those figures do -not qualify this change. - -### Correctness gates - -- CPU synthetic parity in BF16: regular prefill, single-request prefill, ragged - decode, a mixed-length ragged batch with `-1` padded block tables, single - decode and an irregular grid all match the per-group path exactly - (`atol=0, rtol=0`); path-activation counters are asserted. -- Ascend selection suite on the merged candidate: `test_ascend.py`, - `test_ascend_eager.py`, `test_ascend_kv.py`, `test_ascend_serving.py`, - `test_ascend_checkpoint.py` with `--run-npu`: **31 passed, 0 skipped, - 0 failures** (66.6 s, NPU 6), including the new batched-path parity cases. -- Generated sequences against official HF: 24/24 at C8 and 3/3 at C1. -- Pre-lm-head vectors: 256/256 at C8 and 32/32 at C1, `max_abs=0.0`, - for both `merged vs HF` and `merged vs before`. -- In the timed C8 run, the batched-prefill path activated 384 times and the - ragged decode path 11904 times, with zero fallbacks. - -### Limits - -- Batched prefill requires a strict regular grid. **Chunked prefill does not - activate it**, so the documented case-203 chunk128 sequence mismatch - (27 vs 31, first divergence at the 15th output token) is **still unresolved**; - this follow-up does not change chunked-prefill behavior or claim a fix. -- Multi-row non-initial groups still use the original path. Adaptive exit, - long conversations, C16+ concurrency and models other than Ouro-1.4B are - untested. -- Three timed repetitions per arm on a single card; only 32-token outputs. -- Only the full-prefill synchronous path is qualified. Async scheduling, - graphs, speculation, PD, preemption, prefix caching and adaptive exit - remain unsupported here. - - -## 6. Fixed-depth asynchronous engine plan - -This opt-in path reduces host submissions and KV gathering without replacing -model operators or replaying graphs. It reuses stable batch allocation/range -plans, combines fixed-depth decode submissions, and consumes owned KV in -`[batch, kv_head, token, head_dim]` layout. Submitted-write coverage is distinct -from device completion: stream events still protect readers, cancellation, -request-ID reuse and release. Generic cache access exports live KV into paged -storage and can temporarily require both representations. - -Use `CacheConfig(contiguous_kv=True)` and -`ExecutionConfig(async_scheduling=True, multi_stream=False, use_uva=False, -coalesce_fixed_prelude=True, native_eager_plan=True)`, with -`attention_backend="ascend_eager"`, `ExitConfig("ouro_delayed")` and greedy -`SamplingParams(min_loops=4, max_loops=4, exit_threshold=1.0, temperature=0.0)`. -Allocate explicit KV blocks and enough prefill tokens for complete prompts. -Graphs, static buffers, padding, prefix sharing, incremental allocation, -speculation, PD and preemption are unsupported in this configuration. Arbitrary -chunked/mixed-length prefill and adaptive-depth quality are not qualified. - -A checkpoint CLI smoke command is: - -```bash -ASCEND_RT_VISIBLE_DEVICES=0 TASK_QUEUE_ENABLE=2 OMP_NUM_THREADS=1 \ -python -m vllm_rlt.entrypoints.cli \ - --model /work/models/Ouro-1.4B --device npu --dtype bfloat16 \ - --attention-backend ascend_eager --prompt "Explain why the sky is blue." \ - --max-tokens 4 --min-loops 4 --max-loops 4 --exit-mode ouro_delayed \ - --async-scheduling --single-stream --no-uva --contiguous-kv \ - --coalesce-fixed-prelude --native-eager-plan --num-blocks 256 \ - --prefill-chunk-size 4096 --max-num-batched-tokens 4096 ``` -`worker/ascend.py` owns the NPU runtime, token/hidden batch reuse and batched -sampling. CUDA/ROCm retain the shared runner, runtime, sampler, workspace and -model implementation. The optional native executor in that module and -`worker/csrc/ascend_eager_plan.cpp` dispatches individual eager ATen operations -with the Python GIL released: Q/K/V and gate/up projections stay separate, with -explicit RMSNorm, RoPE, matmul and softmax. Each call borrows live weights and -returns owned outputs; unsupported module schemas, hooks/custom forwards or -paged storage use Python. Changes to Ouro arithmetic require revalidating this -executor too. The older experimental paged KV read cache has been removed. - -Native dispatch requires a C++17 compiler, Ninja, PyTorch headers and a writable -cache on first enabled startup. `VLLM_RLT_NATIVE_BUILD_DIR` selects that cache; -`MAX_JOBS=1` limits build parallelism. Its identity includes source, Torch version, -C++ ABI and Python SOABI. Disable `native_eager_plan` to avoid compilation. -CPU/CUDA/ROCm do not import or build this extension. The FP32 CPU generation of -RoPE constants is an explicit NPU-only numerical correction to match HF; -CUDA/ROCm retain their original device arithmetic. - -With refill or mixed scheduling, pending prompts or runnable recurrent work -temporarily disable full-depth decode prelude coalescing. Keeping the ordinary -PRELUDE-to-RECURRENT boundary lets other requests make progress; stable batches -without competing work still use the coalesced path. - -The November 17 refinement integrates upstream `c338aaa`, preserving its sliding -windows, automatic `packed_prefill` policy and CUDA state-pool changes. Runtime -source manifests, frozen inputs, pinned HF source, raw request records and -reproduction scripts are in `/work/vllm-rlt-engine-20261117` inside `kelei-rlt` -on `ssh npu`. Fixed and natural-input quality checks are separate from timing; -they do not establish a task-accuracy score or W1–W7 qualification. - -### Prior O32 validation (`028f9c6`) - -These measurements predate the refill follow-up below and use different output -lengths and token budgets from section 7. - -One NPU per case, CANN 9.0.0, Torch/Torch-NPU 2.10, Transformers 4.54.1, -Ouro-1.4B BF16, UT4, greedy output, no graphs, one execution stream. All arms -set `TASK_QUEUE_ENABLE=2` and OMP/MKL threads to one before import. Each has a -complete shape-matched warmup, then 60 seconds of closed-loop admission and full -drain. Rates include drain, exclude startup/compilation/warmup, and are medians of -three unprofiled runs. Orders alternate native/HF, HF/native, native/HF on the -same card and CPU affinity. These are engine-actor timings, not HTTP latency. - -| Prompt / output / client C | Resident batch | Token budget / chunk / blocks | HF tok/s | Native tok/s | vs HF | -| --- | ---: | --- | ---: | ---: | ---: | -| 128 / 32 / 8 | 8 | 1024 / 1024 / 336 | 50.732 | 80.625 | +58.92% | -| 128 / 32 / 64 | 64 | 8192 / 1024 / 2576 | 348.660 | 512.346 | +46.95% | -| 4096 / 32 / 8 | 4 | 16384 / 4096 / 4144 | 13.636 | 17.210 | +26.21% | - -Blocks hold 16 tokens. Inputs are frozen complete equal-length prompts; timing -forces 32 output tokens with EOS ignored. Native exceeds HF in all nine paired -case/repeat comparisons. All 18 runs have zero failed requests; all 3,588 native -request outputs (114,816 tokens) match the independent HF oracle, with no native -fallbacks. This is not a chunked-prefill, saturation or W1–W7 campaign. - -Throughput does not imply faster first output. Pooled TTFT p50 is 222.4 vs -157.5 ms at S8 and 634.5 vs 595.5 ms at S64 (native vs HF). L8 includes queuing -at eight clients/four resident requests: 11.68 vs 13.54 s. Raw request records, -variability, latency percentiles and peak allocations are in `slim-summary.json` -and `slim-results` in the experiment bundle. - -The prior refinement passed 1,117 CPU regression tests (800 device tests skipped), -81 targeted Ascend tests, three additional real-checkpoint regressions and three -HTTP/SSE/disconnect regressions (Torch sync, Ascend sync and Ascend async). -Separate fixed-input quality checks pass 192/192 sequences; 16 untruncated natural -inputs also match HF under normal EOS with a 32-token cap (all reach the cap). -Ruff and source/wheel packaging checks pass. Staged CUDA compatibility runs on -`ict-90` pass 1,736 unique tests, including 619 GPU cases (181 skipped), and five -paired real-checkpoint configurations match the baseline. These runs do not -qualify unsupported SM120 paged FlashAttention or NIXL configurations. ROCm -hardware validation is deferred. - -## 7. Unified workload follow-up (`394f3b1`, quota1) (`394f3b1`, quota1) - -The refill follow-up preserves a RECURRENT scheduling opportunity when full-depth -PRELUDE would otherwise bypass pending prompt or recurrent work. Bounded CPU -regressions reproduce both starvation cases before the fix; the final tree passes -1,121 CPU cases (802 skipped), 51 targeted CPU/NPU cases, and full-tree Ruff checks. -The NPU-only option defaults off; its disabled CUDA/ROCm path is unchanged. - -This compares the whole enabled combination with official HF eager using the seven -workloads and measurement rules in [unified execution](../unified_execution.md#protocol-and-comparison-rules). -All 42 selected runs complete: three unprofiled 60-second closed-loop admission -windows per arm/workload, each followed by full drain. They contain 1,087 successful -requests and zero runtime failures. Model loading, compilation and full-output -cohort warmup are excluded. These are in-process actor measurements, without HTTP -or StageClient IPC. - -BF16, fixed UT4, greedy, one execution stream, graphs disabled. The engine uses -async/refill, budget/chunk 1024, contiguous KV and native host plans with fixed-depth -decode/prelude coalescing. W1–W5 ignore EOS; W6/W7 replay the frozen seed-0 selection -of 256 untruncated ShareGPT first-human messages, without a chat template, using -natural EOS and cap 512. Both arms use TASK_QUEUE_ENABLE=2, one Torch/OMP/MKL thread, -and one fixed card/CPU affinity per workload; order is native/HF, HF/native, -native/HF. The guide's CUDA FA4/graphs/PD/speculation modes are outside this NPU run. - -The 64 GiB device requires a capacity adjustment: W4 retains 32 clients with four -resident requests, W6/W7 at most eight residents. Both arms use the same limits. -Logical KV blocks are `max(256, residents * ceil((input_bound + output_cap)/16) * 4 + 16)`, -with a 4096-token input bound for natural requests. HF uses ordinary batched eager -and retains finished rows until its active cohort drains. - -Median output tok/s [three-run min–max]; throughput includes drain: - -| Workload | Input / output / clients | Residents | HF eager | Engine combination | vs HF | -| --- | --- | ---: | ---: | ---: | ---: | -| W1 | 128 / 128 / 1 | 1 | 6.152 [5.998–6.464] | 10.191 [8.951–10.274] | +65.65% | -| W2 | 128 / 128 / 32 | 32 | 192.660 [184.098–204.178] | 131.489 [113.793–148.783] | -31.75% | -| W3 | 4096 / 128 / 1 | 1 | 6.198 [6.028–6.318] | 8.379 [5.606–9.152] | +35.20% | -| W4 | 4096 / 128 / 32 | 4 | 20.449 [20.192–20.910] | 16.045 [15.719–16.325] | -21.54% | -| W5 | 1024 / 512 / 8 | 8 | 51.508 [50.409–52.387] | 40.692 [39.713–40.696] | -21.00% | -| W6 | ShareGPT / EOS cap 512 / 8 | 8 | 25.353 [23.451–25.831] | 35.410 [33.534–37.133] | +39.67% | -| W7 | ShareGPT / EOS cap 512 / 32 | 8 | 35.557 [35.320–35.800] | 35.961 [34.726–36.356] | +1.14% | - -W7's 1.14% median difference lies within the observed run ranges. W3 also has -substantial engine variability; one paired run is slower than HF. This matrix -supports case-dependent performance, not a universal speedup. - -Native layer execution is active throughout W1/W3 with zero fallback. W2/W4/W5/W6/W7 -materialize paged KV during warmup and have zero native-executor layer calls in -the timed windows; those layers use the Python eager fallback. Enabling the flag -does not establish native-path coverage for these mixed/chunked geometries. - -Complete token sequences and finish reasons match the paired HF requests as follows: - -| Workload | Identical / paired requests | Supplemental HF reference coverage | -| --- | ---: | --- | -| W1 | 10/10 | All 15 engine requests match HF | -| W2 | 221/224 | None | -| W3 | 9/9 | All 13 engine requests match HF | -| W4 | 93/108 | None | -| W5 | 18/24 | None | -| W6 | 15/27 | None | -| W7 | 45/99 | None | - -The supplemental checks cover three additional fixed inputs after all timing ends. -A bounded natural-input diagnostic also finds different HF outputs at batch 1 and -batch 8; this does not explain all engine/HF differences. Full greedy equivalence -and equal-quality performance acceptance have not passed. Natural EOS output -lengths differ: mean HF/engine output is 458.33/452.44 tokens for W6 and -457.15/466.58 for W7. Fresh GSM8K-87 task qualification has not been run. - -Latency triples are mean / p95 / p99, pooling successful requests across all three -runs. TTFT and E2E include queuing; TPOT is first-to-last arrival divided by N−1. -ITL pools adjacent token delivery intervals, with zero intervals within a delivered -chunk. Small-sample tail percentiles are descriptive. - -| Workload / arm | TTFT (ms) | TPOT (ms) | ITL (ms) | E2E (s) | -| --- | ---: | ---: | ---: | ---: | -| W1 / HF | 158.26 / 164.42 / 164.72 | 160.67 / 174.41 / 177.74 | 160.67 / 175.26 / 292.54 | 20.56 / 22.31 / 22.73 | -| W1 / engine | 210.43 / 252.72 / 313.55 | 101.54 / 119.08 / 128.71 | 101.54 / 153.61 / 161.31 | 13.11 / 15.38 / 16.66 | -| W2 / HF | 315.94 / 316.78 / 316.87 | 164.36 / 174.60 / 174.60 | 164.36 / 174.36 / 241.96 | 21.19 / 22.49 / 22.49 | -| W2 / engine | 807.75 / 1535.41 / 1569.18 | 237.03 / 330.89 / 334.54 | 237.03 / 365.28 / 448.81 | 30.91 / 42.86 / 43.11 | -| W3 / HF | 1126.36 / 1127.27 / 1127.31 | 154.25 / 159.01 / 159.36 | 154.25 / 164.21 / 166.66 | 20.72 / 21.32 / 21.37 | -| W3 / engine | 1038.28 / 1129.21 / 1134.81 | 121.93 / 172.03 / 174.34 | 121.93 / 182.16 / 190.80 | 16.52 / 22.92 / 23.25 | -| W4 / HF | 108201.82 / 184760.44 / 185035.97 | 162.69 / 185.42 / 199.48 | 162.69 / 197.72 / 215.71 | 128.86 / 204.86 / 205.16 | -| W4 / engine | 128120.96 / 228911.74 / 232203.32 | 228.93 / 241.58 / 242.65 | 228.93 / 449.12 / 640.13 | 157.19 / 258.43 / 262.65 | -| W5 / HF | 948.24 / 949.16 / 949.17 | 154.02 / 157.16 / 157.16 | 154.02 / 164.57 / 181.10 | 79.65 / 81.26 / 81.26 | -| W5 / engine | 1716.00 / 3022.56 / 3034.33 | 194.26 / 198.29 / 198.68 | 194.26 / 205.56 / 256.37 | 100.98 / 103.11 / 103.14 | -| W6 / HF | 9446.60 / 73948.53 / 74441.99 | 157.54 / 158.20 / 176.94 | 157.82 / 162.33 / 211.88 | 81.62 / 153.98 / 164.74 | -| W6 / engine | 900.40 / 1247.62 / 1247.82 | 205.59 / 217.85 / 218.08 | 205.19 / 214.00 / 358.37 | 93.53 / 112.26 / 112.26 | -| W7 / HF | 135933.80 / 259319.34 / 335009.35 | 162.34 / 165.82 / 167.64 | 162.53 / 169.73 / 218.95 | 210.07 / 343.37 / 416.61 | -| W7 / engine | 151301.80 / 320714.48 / 333012.61 | 207.59 / 222.27 / 222.30 | 207.95 / 413.73 / 441.94 | 248.12 / 425.01 / 438.91 | - -Source/protocol manifests, per-request arrivals, memory peaks, fallback counters -and the full summary are retained in -`/work/vllm-rlt-engine-20261117/unified-fairness-20261117`. The initial pre-fix -campaign is retained separately as an aborted defect diagnostic and contributes -no selected performance runs above. - -## 8. Profile-guided high-batch scheduling - -The runtime follow-up is two scheduler-accounting lines: a full-depth coalesced -PRELUDE has performed recurrent work and must reset the prefill fairness counter. -Embedding-only PRELUDE retains the previous counter. The new regression fails -on `394f3b1` (one failure, two controls pass) and passes on `7b4537f`. -Global defaults and the disabled CUDA/ROCm path are unchanged. - -At budget/chunk 1024, quota 1 interleaves decode with P128/C32 prompt cohorts: -observed KV lengths become 130–133. Nonuniform ranges export owned KV to paged -storage and force Python layer execution. The existing prefill quota 32 preserves -a uniform resident cohort and the native eager host path. Higher quotas can -postpone decode for existing requests; the global default remains one. - -Independent P128/C32 ops-only/full profiles cover one resident UT4 decode -(96 layer calls), without model hooks. Full enables shapes, memory, stacks and -CPU/NPU activities. Instrumented durations are not throughput measurements. - -| Decode path | Python calls | KV index_select | contiguous | linear | -| --- | ---: | ---: | ---: | ---: | -| Quota1 / paged Python fallback | 74323 | 192 | 192 | 677 | -| Quota32 / owned KV + native host | 31447 | 0 | 4 | 677 | -| HF eager | 26754 | 0 | 0 | 677 | - -The configuration recovers the owned-KV/native-host path; this is not an isolated -speedup from the two accounting lines. No fused model operators or graph replay -are introduced. An HF ops-only export lacking hardware events is retained beside -its successful post-timing recapture. These are decode/C32 profiles; a fresh -full-request C1 before/after timeline was not collected for this increment. + +## Fixed-depth asynchronous engine plan + +- **Scheduling.** Stream events track completion and protect buffer and request + lifetimes. A stable batch can run decode PRELUDE and all four recurrent loops + in one submission. Competing prompt or recurrent work retains a scheduling + opportunity. A full-depth PRELUDE counts as recurrent work for refill fairness. +- **Reuse and batching.** Compatible batches reuse KV range and allocation plans, + token and hidden tensors, and attention masks. Greedy sampling, compatible + prefill and paged KV write indices are batched. Long continuation prefill can + preserve per-request PV matmul geometry for BF16 rounding. +- **KV and host dispatch.** Owned KV uses `[batch, kv_head, token, head_dim]`, + which attention reads directly for uniform ranges. The optional C++ executor + dispatches individual eager operations, including separate Q/K/V and gate/up + projections, with the Python GIL released. Incompatible KV geometry can + materialize paged KV and use the Python executor. Materialization can + temporarily hold both layouts in memory. + +The example uses private, fully reserved `last_exited` KV. Keep graphs, +static buffers, padding, prefix caching, incremental KV allocation, preemption, +speculation and prefill/decode disaggregation disabled in this configuration. +Greedy parity remains unqualified for arbitrary chunked or mixed-length inputs +and adaptive exit. + + +## Performance and correctness + +A higher prefill quota can help keep stable cohorts aligned. In a 128-input, +32-request decode profile with a 1024-token budget, changing the quota from 1 to +32 kept KV lengths uniform. KV `index_select` calls fell from 192 to 0 and Python +calls from 74,323 to 31,447 per decode iteration across the batch. The number of +`linear` calls was 677 in both profiles. A higher quota can delay decode for +existing requests. The scheduler default is 1. ### Full unified W1-W7 validation -**The complete matrix has not passed equal-output acceptance.** W4/W6/W7 have -unresolved sequence differences, so their throughput figures remain diagnostic. - -All seven workloads were rerun on the same final configuration from `7b4537f`: -budget 8192, chunk 1024, prefill quota 32, async/refill, owned contiguous KV, native -eager host dispatch and fixed-depth prelude/decode coalescing. No per-case tuning -or older C32/C64 result is substituted into this matrix. - -One 910B3 per arm, CANN 9, Torch/Torch-NPU 2.10, Transformers 4.54.1, Ouro-1.4B BF16, -UT4 greedy. Graphs, multi-stream, UVA, padding and static buffers are off. Both -arms set TASK_QUEUE_ENABLE=2 and Torch/OMP/MKL threads to one. Each case uses the -same card/CPU affinity with native/HF, HF/native, native/HF order. - -All **42** new selected runs complete with **1422 successful requests and zero -runtime failures**. Every process warms a complete output cohort; loading, -compilation and warmup are excluded. Each run admits closed-loop traffic for -60 seconds and fully drains. Output throughput includes drain; values are -median [min–max] tok/s. These are in-process actor timings, not HTTP/IPC latency -or a saturation sweep. This covers the guide's applicable single-stage W1–W7 -HF comparison, not its historical B300/FA4/graphs/PD/speculation mode matrix. - -W1–W5 ignore EOS. W6/W7 retain the same frozen seed-0 selection of 256 ShareGPT -first-human messages, without truncation or a chat template, using natural EOS -and cap 512. The 64GiB capacity adaptation is identical in both arms: W4 has 32 -clients/four residents; W6/W7 have 8/32 clients with at most 8 residents. - -| Workload | Input / output / clients | Residents | HF eager tok/s | Engine tok/s | vs HF | Complete output matches | +Runtime `7b4537f` used the same budget 8192, chunk 1024 and prefill quota 32 for +all seven [unified execution workloads](../unified_execution.md#protocol-and-comparison-rules). +The comparison ran Ouro-1.4B BF16 with four fixed loops and greedy decoding on +one 64 GiB Ascend 910B3 per run, with CANN 9, Torch/Torch-NPU 2.10 and +Transformers 4.54.1. The reference used official HF eager at revision +`574fa66cb8bf5abdc979642d01cf2b79b16bfab1` with a DynamicCache adapter. + +All 42 runs completed with 1,422 successful requests and zero runtime failures. +Each case used three unprofiled repetitions per arm on the same card and CPU +affinity. Each repetition admitted closed-loop traffic for 60 seconds and then +fully drained. Throughput includes drain and excludes startup, compilation and +warmup. Timing uses the in-process engine actor. W1–W5 ignore EOS. W6/W7 use +natural EOS with a 512-token cap and a fixed ShareGPT selection. + +| Workload | Input / output / clients | Residents per arm | HF eager tok/s | Engine tok/s | vs HF | Complete output matches | | --- | --- | ---: | ---: | ---: | ---: | ---: | | W1 | 128 / 128 / 1 | 1 | 6.334 [5.284–6.496] | 9.464 [9.403–10.394] | +49.42% | 15/15 | | W2 | 128 / 128 / 32 | 32 | 201.314 [200.099–204.437] | 305.496 [293.376–316.662] | +51.75% | 480/480 | | W3 | 4096 / 128 / 1 | 1 | 6.254 [6.206–6.268] | 7.569 [6.110–9.215] | +21.03% | 12/12 | | W4 | 4096 / 128 / 32 | 4 | 21.211 [21.012–21.580] | 31.312 [31.135–31.405] | +47.62% | 120/132 | | W5 | 1024 / 512 / 8 | 8 | 50.885 [50.234–51.362] | 79.791 [76.843–80.666] | +56.80% | 48/48 | -| W6 | ShareGPT / EOS cap 512 / 8 | 8 | 26.343 [26.060–26.385] | 37.992 [37.906–38.879] | +44.22% | 18/27 | -| W7 | ShareGPT / EOS cap 512 / 32 | 8 | 36.341 [35.576–36.931] | 36.194 [35.808–36.679] | -0.41% | 45/99 | - -Complete tokens, finish reasons and depth 4 are checked, including supplemental -independent HF references after timing; no engine output remains unverified. -W1/W2/W3/W5 pass these covered-input checks. W4/W6/W7 do not. W7 ranges overlap -and do not establish a speedup; one W3 pair is slower than HF. This is not a -fresh GSM8K task-accuracy qualification or a universal equal-quality speedup. - -W1–W5 retain native execution and owned KV without timed-window fallback. -W6/W7 materialize paged KV during warmup and have zero native layer calls -in the timed windows. Their enabled flag does not imply fast-path coverage. +| W6 | ShareGPT / ≤512 / 8 | 8 | 26.343 [26.060–26.385] | 37.992 [37.906–38.879] | +44.22% | 18/27 | +| W7 | ShareGPT / ≤512 / 32 | 8 | 36.341 [35.576–36.931] | 36.194 [35.808–36.679] | -0.41% | 45/99 | -Latency statistics pool successful requests across repetitions; values below -are mean / p95 / p99. TTFT and E2E include queuing, TPOT is first-to-last output -/(N−1), and ITL uses adjacent delivery intervals with zero intra-chunk gaps. +Rates are medians [min–max]. Resident limits reflect single-device memory +capacity and apply equally to both arms. Complete output checks compare token IDs +and finish reasons against timed or supplemental HF references. -| Workload / arm | TTFT (ms) | TPOT (ms) | -| --- | ---: | ---: | -| W2 / HF | 314.90 / 315.98 / 316.09 | 157.04 / 160.28 / 160.28 | -| W2 / engine | 386.18 / 616.33 / 617.22 | 102.73 / 112.46 / 112.46 | -| W7 / HF | 132566.71 / 255707.79 / 326208.73 | 158.90 / 163.43 / 164.37 | -| W7 / engine | 149321.74 / 318700.31 / 321899.41 | 205.49 / 209.90 / 210.49 | +**W1, W2, W3 and W5 passed the covered-input output checks.** W4, W6 and W7 +still differ from HF, so their throughput figures are diagnostic. W1–W5 used +owned KV and native dispatch throughout timing. W6/W7 used paged KV and Python +execution. W7 throughput ranges overlap, and its peak allocated memory was +30.06 GiB for the engine versus 19.54 GiB for HF. At W2, mean TPOT fell from +157.04 to 102.73 ms while TTFT p95 rose from 315.98 to 616.33 ms. -The W2 throughput gain trades off first-output latency. At W7, mean E2E is -245.42/205.07 seconds and peak allocated memory is 30.056/19.536GiB (engine/HF). -Allocator peaks reset after warmup; these are not startup peaks or total HBM. +## Run regression checks -### Earlier full-cohort C32/C64 check +With an assigned NPU visible and `MODEL` set as above: -The separate, previously declared P128/O128 preset used the same 8192/1024/quota 32 -settings. Its 12 selected runs recorded +54.60% at C32 and +45.36% at C64, with -1440/1440 complete engine outputs matching independent HF references. Those -runs remain separate and contribute no selected result to the 42-run table. - -For the earlier input44/token0 divergence, Python/native logits match HF batch8 -exactly; full-cohort engine prefill matches HF batch32 logits and head inputs -with max_abs=0. This bounded batch-geometry finding does not explain the -unchecked W4/W6/W7 differences. They remain unresolved. - -The runtime passed 1124 CPU tests (802 skipped), 54 targeted CPU/NPU tests and -full-tree Ruff checks. The native option defaults off; this increment does not -claim a fresh CUDA/ROCm hardware run. Earlier CUDA validation retains its -original scope. +```bash +VLLM_RLT_OURO_CHECKPOINT="$MODEL" python -m pytest -q --run-npu \ + tests/test_ascend_eager.py tests/test_ascend_kv.py \ + tests/test_fixed_prelude.py tests/test_native_ouro_plan.py \ + tests/test_ascend_checkpoint.py tests/test_ascend_serving.py +``` -The full report retains all latency distributions, natural length/stop counts, -memory and native/fallback counters, exact commands and source/input hashes in -`/work/vllm-rlt-engine-20261117/unified-final-8192-20261117`. Prior captures and -bounded logits diagnostics remain in the separate `high-batch-20261117` bundle. +For broader evaluation, see [accuracy](../accuracy.md) and +[serving benchmarks](../serving.md). From 1a8f080c460c8c7698fb7fa9fafe512b991e93ab Mon Sep 17 00:00:00 2001 From: bjf-frz Date: Wed, 18 Nov 2026 10:25:00 +0000 Subject: [PATCH 18/20] refactor(ascend): use shared Ouro runtime and paged KV Route Ascend attention through Torch SDPA and keep Ouro recurrent execution in the shared Python model. Retain paged KV metadata reuse, batched sampling, and fixed-depth async scheduling. --- MANIFEST.in | 2 - README.md | 2 +- docs/recipes/ouro_ascend.md | 54 +- pyproject.toml | 3 - tests/test_ascend_compatibility.py | 13 +- tests/test_ascend_eager.py | 54 +- tests/test_ascend_metadata.py | 8 +- tests/test_cohort_coverage.py | 37 -- tests/test_contiguous_kv.py | 210 -------- tests/test_fast_execution_plan.py | 67 +-- tests/test_fixed_prelude.py | 2 +- tests/test_native_ouro_plan.py | 143 ------ tests/test_npu_async.py | 4 +- tests/test_npu_batch_decode.py | 2 +- tests/test_resident_metadata.py | 71 --- tests/test_sequence_plans.py | 6 +- vllm_rlt/attention/backend.py | 19 +- vllm_rlt/config.py | 13 +- vllm_rlt/core/kv_cache_manager.py | 283 +--------- vllm_rlt/engine/stage_engine.py | 35 +- vllm_rlt/entrypoints/cli.py | 10 +- vllm_rlt/entrypoints/runtime_args.py | 12 - vllm_rlt/entrypoints/serve.py | 7 +- .../{ascend_eager.py => ascend_attention.py} | 238 +++------ vllm_rlt/kernels/contiguous_kv.py | 230 --------- vllm_rlt/models/ouro.py | 50 +- vllm_rlt/worker/ascend.py | 483 ------------------ vllm_rlt/worker/ascend_model_runner.py | 220 ++++++++ vllm_rlt/worker/csrc/ascend_eager_plan.cpp | 92 ---- vllm_rlt/worker/executor.py | 12 +- vllm_rlt/worker/metadata.py | 175 +------ vllm_rlt/worker/model_runner.py | 19 +- 32 files changed, 469 insertions(+), 2107 deletions(-) delete mode 100644 tests/test_cohort_coverage.py delete mode 100644 tests/test_contiguous_kv.py delete mode 100644 tests/test_native_ouro_plan.py delete mode 100644 tests/test_resident_metadata.py rename vllm_rlt/kernels/{ascend_eager.py => ascend_attention.py} (59%) delete mode 100644 vllm_rlt/kernels/contiguous_kv.py delete mode 100644 vllm_rlt/worker/ascend.py create mode 100644 vllm_rlt/worker/ascend_model_runner.py delete mode 100644 vllm_rlt/worker/csrc/ascend_eager_plan.cpp diff --git a/MANIFEST.in b/MANIFEST.in index 9471f7fd..2a4ebb76 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1 @@ recursive-include csrc *.h *.cuh *.cpp *.cu - -recursive-include vllm_rlt/worker/csrc *.cpp diff --git a/README.md b/README.md index f40a7ca3..ee15564d 100644 --- a/README.md +++ b/README.md @@ -138,7 +138,7 @@ independently by `use_uva`; see the [selection rules](docs/runtime.md#packed-pre | Checkpoint | Implementation | Documented validation scope | Guide | | --- | --- | --- | --- | -| Ouro-1.4B | Native Ouro | Ascend BF16 full prefill; opt-in fixed-depth async scheduling, owned KV and native host dispatch using unchanged eager operators; arbitrary chunked prefill unresolved | [Runtime](docs/runtime.md), [Ascend recipe](docs/recipes/ouro_ascend.md#6-fixed-depth-asynchronous-engine-plan) | +| Ouro-1.4B | Native Ouro | Ascend BF16 full prefill with eager Python execution, paged KV and optional fixed-depth async scheduling. Arbitrary chunked prefill remains unresolved. | [Runtime](docs/runtime.md), [Ascend recipe](docs/recipes/ouro_ascend.md#6-fixed-depth-asynchronous-engine-plan) | | Ouro-2.6B | Shared Ouro | Shares the Ouro implementation; not separately qualified, and 1.4B quality/performance results are not inherited | [Scope notes](docs/recipes/ouro_thinking.md#6-evidence-and-limitations) | | Ouro-1.4B-Thinking | Reuses Ouro | Bounded BF16, fixed-four-loop synchronous basic integration | [Shared recipe](docs/recipes/ouro_thinking.md) | | Ouro-2.6B-Thinking | Reuses Ouro | Bounded BF16, fixed-four-loop synchronous basic integration | [Shared recipe](docs/recipes/ouro_thinking.md) | diff --git a/docs/recipes/ouro_ascend.md b/docs/recipes/ouro_ascend.md index 212ec997..6c5a9234 100644 --- a/docs/recipes/ouro_ascend.md +++ b/docs/recipes/ouro_ascend.md @@ -1,16 +1,15 @@ # Ouro on Ascend Run Ouro-1.4B on one Ascend NPU with BF16, greedy decoding and four recurrent -loops. The `ascend_eager` backend uses eager ATen operations and HF-style -attention rounding. The example enables the optional asynchronous engine plan. +loops. The Python runtime uses Torch's `scaled_dot_product_attention` through +the `ascend_eager` backend, matching HF's SDPA entry point. Torch-NPU selects +the device kernel according to the input shape, mask and hardware. +The example enables the optional asynchronous engine plan. ## Prepare the environment and model Install the project dependencies in a compatible Torch, Torch-NPU and CANN -environment. Use the checkpoint's config and byte-level tokenizer. The native -host plan builds a C++ extension on first use and requires a C++17 compiler, -Ninja, PyTorch headers and a writable build cache. Set -`VLLM_RLT_NATIVE_BUILD_DIR` to select the cache location. +environment. Use the checkpoint's config and byte-level tokenizer. ```bash source /path/to/cann/set_env.sh @@ -29,21 +28,20 @@ whole-cohort prefill, the chunk size must fit each prompt and the batched token budget must fit the sum of prompt lengths in the cohort. ```bash -TASK_QUEUE_ENABLE=2 OMP_NUM_THREADS=1 MKL_NUM_THREADS=1 MAX_JOBS=1 \ +TASK_QUEUE_ENABLE=2 OMP_NUM_THREADS=1 MKL_NUM_THREADS=1 \ python -m vllm_rlt.entrypoints.serve \ --model "$MODEL" --tokenizer "$MODEL" --served-model-name ouro \ --device npu --dtype bfloat16 --attention-backend ascend_eager \ --mode refill --exit-mode ouro_delayed \ - --async-scheduling --single-stream --no-uva --contiguous-kv \ - --coalesce-fixed-prelude --native-eager-plan \ + --async-scheduling --single-stream --no-uva \ + --coalesce-fixed-prelude \ --block-size 16 --num-blocks 2064 --max-num-seqs 32 \ --max-num-batched-tokens 8192 --prefill-chunk-size 1024 \ --max-prefill-batches-before-decode 32 --host 127.0.0.1 --port 8123 ``` -Omit `--native-eager-plan` to use the Python eager executor without building -an extension. For synchronous eager execution, also omit `--async-scheduling`, -`--coalesce-fixed-prelude` and `--contiguous-kv`. +For synchronous eager execution, omit `--async-scheduling` and +`--coalesce-fixed-prelude`. Wait for `/health` to return HTTP 200, then send a completion. Add `"stream": true` for server-sent events. @@ -63,15 +61,12 @@ curl --fail http://127.0.0.1:8123/v1/completions \ in one submission. Competing prompt or recurrent work retains a scheduling opportunity. A full-depth PRELUDE counts as recurrent work for refill fairness. - **Reuse and batching.** Compatible batches reuse KV range and allocation plans, - token and hidden tensors, and attention masks. Greedy sampling, compatible - prefill and paged KV write indices are batched. Long continuation prefill can - preserve per-request PV matmul geometry for BF16 rounding. -- **KV and host dispatch.** Owned KV uses `[batch, kv_head, token, head_dim]`, - which attention reads directly for uniform ranges. The optional C++ executor - dispatches individual eager operations, including separate Q/K/V and gate/up - projections, with the Python GIL released. Incompatible KV geometry can - materialize paged KV and use the Python executor. Materialization can - temporarily hold both layouts in memory. + hidden tensors, and attention masks. Greedy sampling, compatible + prefill and paged KV write indices are batched. Full prefill uses causal SDPA. + Cached chunks and ragged decode pass boolean masks to preserve each row's + visible KV range. +- **Paged KV.** Each loop writes to the page pool using its own block table. + The Ascend backend gathers the visible KV rows for Torch SDPA. The example uses private, fully reserved `last_exited` KV. Keep graphs, static buffers, padding, prefix caching, incremental KV allocation, preemption, @@ -82,15 +77,18 @@ and adaptive exit. ## Performance and correctness -A higher prefill quota can help keep stable cohorts aligned. In a 128-input, -32-request decode profile with a 1024-token budget, changing the quota from 1 to -32 kept KV lengths uniform. KV `index_select` calls fell from 192 to 0 and Python -calls from 74,323 to 31,447 per decode iteration across the batch. The number of -`linear` calls was 677 in both profiles. A higher quota can delay decode for -existing requests. The scheduler default is 1. +A higher prefill quota lets more requests reach decode together and can delay +decode for existing requests. The scheduler default is 1. ### Full unified W1-W7 validation +This historical run used the C++ host executor and explicit eager attention +on both sides of the comparison. For the SDPA configuration above, compare +against HF SDPA with the same weights, input IDs, output lengths and concurrency. +Check the actual prefill and decode kernels on both sides with a short profile. +Measure throughput, TTFT and TPOT with the profiler disabled. Separate +synchronous and asynchronous vllm-rlt runs to measure the effect of scheduling. + Runtime `7b4537f` used the same budget 8192, chunk 1024 and prefill quota 32 for all seven [unified execution workloads](../unified_execution.md#protocol-and-comparison-rules). The comparison ran Ouro-1.4B BF16 with four fixed loops and greedy decoding on @@ -133,7 +131,7 @@ With an assigned NPU visible and `MODEL` set as above: ```bash VLLM_RLT_OURO_CHECKPOINT="$MODEL" python -m pytest -q --run-npu \ tests/test_ascend_eager.py tests/test_ascend_kv.py \ - tests/test_fixed_prelude.py tests/test_native_ouro_plan.py \ + tests/test_fixed_prelude.py \ tests/test_ascend_checkpoint.py tests/test_ascend_serving.py ``` diff --git a/pyproject.toml b/pyproject.toml index 1758a90c..b9e8980a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,6 +53,3 @@ line-length = 100 [tool.ruff.lint] select = ["E", "F", "I", "UP"] - -[tool.setuptools.package-data] -vllm_rlt = ["worker/csrc/*.cpp"] diff --git a/tests/test_ascend_compatibility.py b/tests/test_ascend_compatibility.py index 71af8fee..422baee6 100644 --- a/tests/test_ascend_compatibility.py +++ b/tests/test_ascend_compatibility.py @@ -28,7 +28,7 @@ def test_backend_capability_positional_arguments_keep_their_meaning(): "sliding_window", ) caps = BackendCapabilities(False, False, False, False, False, True) - assert caps.sliding_window and not caps.sequence_ranges and not caps.owned_contiguous_kv + assert caps.sliding_window and not caps.sequence_ranges @pytest.mark.parametrize("hip", [None, "6.4"]) @@ -37,7 +37,7 @@ def test_cuda_and_rocm_stream_selection_never_loads_npu(monkeypatch, hip): original = builtins.__import__ def guarded(name, *args, **kwargs): - if name == "torch_npu" or name.startswith("vllm_rlt.worker.ascend"): + if name == "torch_npu" or name.startswith("vllm_rlt.worker.ascend_model_runner"): pytest.fail("CUDA/ROCm selected an Ascend implementation") return original(name, *args, **kwargs) @@ -68,7 +68,7 @@ def test_default_engine_keeps_upstream_runtime_and_sampler(monkeypatch): original = builtins.__import__ def guarded(name, *args, **kwargs): - if name == "torch_npu" or name.startswith("vllm_rlt.worker.ascend"): + if name == "torch_npu" or name.startswith("vllm_rlt.worker.ascend_model_runner"): pytest.fail("default engine loaded Ascend worker code") return original(name, *args, **kwargs) @@ -86,9 +86,8 @@ def guarded(name, *args, **kwargs): engine.close() -@pytest.mark.parametrize("flag", ["contiguous_kv", "coalesce_fixed_prelude", "native_eager_plan"]) -def test_ascend_options_fail_before_allocating_on_other_devices(flag): - cache = CacheConfig(num_blocks=128, **({flag: True} if flag == "contiguous_kv" else {})) - execution = ExecutionConfig(**({flag: True} if flag != "contiguous_kv" else {})) +def test_ascend_options_fail_before_allocating_on_other_devices(): + cache = CacheConfig(num_blocks=128) + execution = ExecutionConfig(coalesce_fixed_prelude=True) with pytest.raises(ValueError, match="Ascend execution options require an NPU"): StageEngine(tiny_ouro_model(), cache_config=cache, execution_config=execution) diff --git a/tests/test_ascend_eager.py b/tests/test_ascend_eager.py index 0bddb798..30f95021 100644 --- a/tests/test_ascend_eager.py +++ b/tests/test_ascend_eager.py @@ -1,9 +1,9 @@ -"""Causal/GQA semantics for the separately selected BF16 eager contract.""" +"""Causal/GQA semantics for Ascend SDPA over paged KV.""" import pytest import torch -from vllm_rlt.kernels.ascend_eager import AscendEagerAttention, _flat_kv_rows, _prefill_geometry +from vllm_rlt.kernels.ascend_attention import AscendAttention, _flat_kv_rows, _prefill_geometry @pytest.mark.parametrize("kv_heads", [1, 2, 4]) @@ -14,7 +14,7 @@ def test_eager_groups_match_independent_causal_queries(kv_heads): q = torch.randn(7, 4, 64, dtype=torch.bfloat16) lengths = torch.tensor([0, 1, 15, 16, 17, 3, 31], dtype=torch.int32) tables = torch.tensor([[4, 1], [4, 1], [4, 1], [4, 1], [4, 1], [7, 2], [7, 2]]) - actual = AscendEagerAttention()(q, keys, values, tables, lengths) + actual = AscendAttention()(q, keys, values, tables, lengths) expected = torch.zeros_like(q) for row, length in enumerate(lengths.tolist()): if not length: @@ -162,29 +162,37 @@ def test_engine_matches_the_same_manual_prefill_chunks(chunk_size): @pytest.mark.parametrize("device_name", ["cpu", pytest.param("npu", marks=pytest.mark.npu)]) @torch.inference_mode() -def test_bf16_probability_rounding_is_part_of_the_eager_contract(device_name): +def test_cached_continuation_passes_bool_mask_to_sdpa(device_name, monkeypatch): from vllm_rlt.device import resolve_device - from vllm_rlt.kernels.paged_attention import torch_paged_attention device = resolve_device(device_name) if device_name == "npu": torch.npu.set_device(0) - keys = torch.zeros(1, 16, 1, 64, dtype=torch.bfloat16, device=device) + keys = torch.zeros(1, 16, 2, 64, dtype=torch.bfloat16, device=device) values = torch.zeros_like(keys) - keys[0, 0].fill_(1.03125) - keys[0, 1].fill_(1) - values[0, 0].fill_(128) - values[0, 1].fill_(-128) - query = torch.ones(1, 1, 64, dtype=torch.bfloat16, device=device) - table = torch.tensor([[0]], dtype=torch.int32, device=device) - lengths = torch.tensor([2], dtype=torch.int32, device=device) - # QK/sqrt(64) is [8.25,8]. BF16 probabilities are [.5625,.4375], - # so their weighted value is exactly16. FP32 probabilities retain - # [.5621765,.4378235] and round the final value to15.9375 instead. - actual = AscendEagerAttention()(query, keys, values, table, lengths) - torch.testing.assert_close(actual, torch.full_like(actual, 16), atol=0, rtol=0) - old_contract = torch_paged_attention(query, keys, values, table, lengths) - assert not torch.equal(actual, old_contract) + for pos, value in enumerate((1, 3, 8)): + values[0, pos].fill_(value) + query = torch.ones(2, 2, 64, dtype=torch.bfloat16, device=device) + table = torch.tensor([[0], [0]], dtype=torch.int32, device=device) + lengths = torch.tensor([2, 3], dtype=torch.int32, device=device) + sdpa = torch.nn.functional.scaled_dot_product_attention + calls = [] + + def check_sdpa(q, k, v, *, attn_mask, is_causal): + calls.append(attn_mask) + assert not is_causal + assert q.shape == (1, 2, 2, 64) and k.shape == v.shape == (1, 2, 3, 64) + assert attn_mask.dtype == torch.bool + assert attn_mask.tolist() == [[True, True, False], [True, True, True]] + return sdpa(q, k, v, attn_mask=attn_mask, is_causal=is_causal) + + monkeypatch.setattr(torch.nn.functional, "scaled_dot_product_attention", check_sdpa) + actual = AscendAttention()(query, keys, values, table, lengths) + # Equal scores average the visible values. The first query sees the cached + # prefix and itself, while only the second query can see the last value. + expected = torch.tensor([2, 4], device=device, dtype=query.dtype)[:, None, None] + torch.testing.assert_close(actual, expected.expand_as(actual), atol=0.016, rtol=0) + assert len(calls) == 1 class _PreparedBatch: @@ -265,10 +273,10 @@ def _batched_fixture(kind): ], ) @torch.inference_mode() -def test_batched_attention_paths_match_per_group_eager(kind, expected_prefill, expected_ragged): - """Batching may remove launches but must never change the arithmetic.""" +def test_batched_attention_paths_match_per_group_sdpa(kind, expected_prefill, expected_ragged): + """Keep query/result ordering and masks consistent when batching SDPA calls.""" q, keys, values, batch = _batched_fixture(kind) - kernel = AscendEagerAttention() + kernel = AscendAttention() groups = kernel._plan( [(id(allocation), depth) for allocation, depth, _ in batch.rows], [position + 1 for _, _, position in batch.rows], diff --git a/tests/test_ascend_metadata.py b/tests/test_ascend_metadata.py index 5192fd15..5d8ec1c3 100644 --- a/tests/test_ascend_metadata.py +++ b/tests/test_ascend_metadata.py @@ -7,7 +7,7 @@ import torch from vllm_rlt.core.kv_cache_manager import KVCacheManager, _sequence_coverage, _WrittenPositions -from vllm_rlt.kernels.ascend_eager import AscendEagerAttention, _flat_kv_rows +from vllm_rlt.kernels.ascend_attention import AscendAttention, _flat_kv_rows @pytest.mark.parametrize( @@ -89,7 +89,7 @@ def test_unwritten_nan_padding_is_never_consumed(padded): q = torch.randn(4 if padded else 3, 4, 128, device=cache.device, dtype=cache.dtype)[:, :, ::2] for layer in range(2): keys, values = cache.key_cache[:, layer], cache.value_cache[:, layer] - expected = _reference(AscendEagerAttention(), q, keys, values, batch) + expected = _reference(AscendAttention(), q, keys, values, batch) actual = cache._attend_prepared(layer, batch, q) assert torch.isfinite(actual).all() torch.testing.assert_close(actual, expected, rtol=0, atol=0) @@ -111,7 +111,7 @@ def test_layout_reuse_does_not_reuse_addresses_or_outputs(): ["0", "1", "2"], [depth] * 3, [n - 1 for n in lengths], for_write=False ) expected = _reference( - AscendEagerAttention(), q, cache.key_cache[:, 0], cache.value_cache[:, 0], batch + AscendAttention(), q, cache.key_cache[:, 0], cache.value_cache[:, 0], batch ) actual = cache._attend_prepared(0, batch, q) torch.testing.assert_close(actual, expected, rtol=0, atol=0) @@ -197,7 +197,7 @@ def test_failed_plan_preparation_invalidates_the_previous_descriptor(kind, monke from tests.test_ascend_eager import _batched_fixture q, keys, values, first = _batched_fixture(kind) - kernel = AscendEagerAttention() + kernel = AscendAttention() expected = kernel.attend_metadata(q, keys, values, first).clone() second = copy(first) target = "_prepare_decode" if kind == "ragged_decode" else "_plan" diff --git a/tests/test_cohort_coverage.py b/tests/test_cohort_coverage.py deleted file mode 100644 index 05a62813..00000000 --- a/tests/test_cohort_coverage.py +++ /dev/null @@ -1,37 +0,0 @@ -"""Shared prefix proofs must detach under individual mutation or snapshots.""" - -from copy import deepcopy -from types import SimpleNamespace - -from vllm_rlt.core.kv_cache_manager import _CohortWrittenPositions, _WrittenPositions - - -def test_cohort_frontier_mutation_and_snapshot_isolation(): - frontier = SimpleNamespace(end=3) - left = _CohortWrittenPositions(_WrittenPositions(prefix=2), frontier) - right = _CohortWrittenPositions(_WrittenPositions(prefix=2), frontier) - snapshot = deepcopy([left, right]) - assert left.prefix == right.prefix == 3 - left.add(7) - frontier.end = 4 - assert left.prefix == 3 and left.pending == {7} - assert right.prefix == 4 - assert [x.prefix for x in snapshot] == [3, 3] - right.prefix = 1 - frontier.end = 8 - assert right.prefix == 1 and left.prefix == 3 - left.add_range(3, 7) - assert left.prefix == 8 and not left.pending - - -def test_rebinding_keeps_previous_cohort_rows_independent(): - old = SimpleNamespace(end=5) - moved = _CohortWrittenPositions(_WrittenPositions(), old) - survivor = _CohortWrittenPositions(_WrittenPositions(), old) - new = SimpleNamespace(end=6) - moved = _CohortWrittenPositions(moved, new) - old.end = 7 - assert moved.prefix == 6 and survivor.prefix == 7 - moved.add_range(6, 8) - new.end = 20 - assert moved.prefix == 8 and survivor.prefix == 7 diff --git a/tests/test_contiguous_kv.py b/tests/test_contiguous_kv.py deleted file mode 100644 index 24b7ca3a..00000000 --- a/tests/test_contiguous_kv.py +++ /dev/null @@ -1,210 +0,0 @@ -"""The eager cohort store is authoritative until an ordered paged materialization.""" - -from dataclasses import replace - -import pytest -import torch - -from vllm_rlt.attention.backend import AttentionBackend, BackendCapabilities -from vllm_rlt.core.kv_cache_manager import KVCacheManager -from vllm_rlt.kernels.ascend_eager import AscendEagerAttention -from vllm_rlt.worker.metadata import stage_range_metadata - - -def cache_for(device, enabled=True): - device = torch.device(device) - attention = None - if device.type == "cpu": - attention = AttentionBackend( - AscendEagerAttention(), - BackendCapabilities(sequence_ranges=True, owned_contiguous_kv=True), - dict(selected_backend="torch", backend="torch"), - ) - cache = KVCacheManager( - 2, - 2, - 32, - 64, - 4, - 2, - device=device, - dtype=torch.bfloat16, - backend="ascend_eager" if device.type == "npu" else "torch", - attention=attention, - contiguous_kv=enabled, - ) - for rid in ["a", "b"]: - assert cache.allocate(rid, 16) - return cache - - -def metadata(cache, ids, start, count, depth=0): - layouts = cache.metadata_builder.build_ranges( - ids, [start] * len(ids), [count] * len(ids), [[depth] * len(ids)] - ) - batch = stage_range_metadata(layouts, cache._stage)[0] - return replace(batch, block_tables=batch.table_plan.bind(cache.device)) - - -@pytest.mark.parametrize("device", ["cpu", pytest.param("npu:0", marks=pytest.mark.npu)]) -@torch.inference_mode() -def test_cohort_kv_matches_paged_and_materializes_all_layers(device): - if device.startswith("npu"): - import torch_npu # noqa: F401 - - torch.npu.set_device(device) - torch.manual_seed(41) - dense, paged = cache_for(device), cache_for(device, False) - held = snapshot = None - for start, count in [(0, 5), (5, 3), (8, 1)]: - for depth in range(2): - a, b = ( - metadata(dense, ["a", "b"], start, count, depth), - metadata(paged, ["a", "b"], start, count, depth), - ) - for layer in range(2): - k = torch.randn(2 * count, 2, 32, device=device, dtype=torch.bfloat16) - v, q = torch.randn_like(k), torch.randn_like(k) - dense._write_prepared(layer, a, k, v) - paged._write_prepared(layer, b, k, v) - k.fill_(float("nan")) - v.fill_(float("nan")) - out = dense._attend_prepared(layer, a, q) - expected = paged._attend_prepared(layer, b, q) - torch.testing.assert_close(out, expected, rtol=0, atol=0) - if held is not None: - torch.testing.assert_close(held, snapshot, rtol=0, atol=0) - held, snapshot = out, out.clone() - assert dense._key_cache is None - for depth in range(2): - for layer in range(2): - for rid in ["a", "b"]: - for x, y in zip(dense.read(layer, rid, depth, 9), paged.read(layer, rid, depth, 9)): - torch.testing.assert_close(x, y, rtol=0, atol=0) - assert dense._contiguous is None - - -@torch.inference_mode() -def test_cancel_regroup_and_id_reuse_preserve_survivor_history(): - cache = cache_for("cpu") - histories = {} - - def write(ids, start, count): - b = metadata(cache, ids, start, count) - k = torch.randn(len(ids) * count, 2, 32, dtype=torch.bfloat16) - v = torch.randn_like(k) - for i, rid in enumerate(ids): - incoming = ( - k[i * count : (i + 1) * count].clone(), - v[i * count : (i + 1) * count].clone(), - ) - histories[rid] = ( - incoming - if start == 0 - else tuple(torch.cat((old, new)) for old, new in zip(histories[rid], incoming)) - ) - cache._write_prepared(0, b, k, v) - return b - - stale = write(["a", "b"], 0, 5) - cache.free("a") - assert cache.allocate("a", 16) - with pytest.raises(RuntimeError, match="stale"): - cache._attend_prepared(0, stale, torch.zeros(10, 2, 32, dtype=torch.bfloat16)) - write(["a"], 0, 5) - write(["b", "a"], 5, 2) - assert cache._key_cache is None - for rid in ["a", "b"]: - for actual, expected in zip(cache.read(0, rid, 0, 7), histories[rid]): - torch.testing.assert_close(actual, expected, rtol=0, atol=0) - cache.free("a") - cache.free("b") - assert cache.num_used_blocks == 0 - - -@torch.inference_mode() -def test_hole_and_overwrite_escape_to_paged_preserves_untouched_suffix(): - cache = cache_for("cpu") - b = metadata(cache, ["a"], 0, 5) - k = torch.randn(5, 2, 32, dtype=torch.bfloat16) - cache._write_prepared(0, b, k, k) - overwrite = metadata(cache, ["a"], 0, 1) - patch = torch.randn(1, 2, 32, dtype=torch.bfloat16) - cache._write_prepared(0, overwrite, patch, patch) - assert cache._contiguous is None - expected = k.clone() - expected[:1] = patch - torch.testing.assert_close(cache.read(0, "a", 0, 5)[0], expected, rtol=0, atol=0) - gap = metadata(cache, ["a"], 7, 1) - cache._write_prepared(0, gap, patch, patch) - with pytest.raises(RuntimeError, match="uninitialized"): - cache._attend_prepared(0, gap, patch) - - -@pytest.mark.npu -@torch.inference_mode() -def test_cross_stream_escape_orders_producer_and_keeps_output_owned(): - import torch_npu # noqa: F401 - - torch.npu.set_device(0) - cache = cache_for("npu:0") - producer, consumer = torch.npu.Stream(), torch.npu.Stream() - with torch.npu.stream(producer): - b = metadata(cache, ["a", "b"], 0, 5) - k = torch.randn(10, 2, 32, device="npu:0", dtype=torch.bfloat16) - expected = k.clone() - cache._write_prepared(0, b, k, k) - out = cache._attend_prepared(0, b, k) - snapshot = out.clone() - with torch.npu.stream(consumer): - actual, _ = cache.read(0, "a", 0, 5) - assert cache._contiguous is None - torch.testing.assert_close(actual, expected[:5], rtol=0, atol=0) - torch.npu.synchronize() - torch.testing.assert_close(out, snapshot, rtol=0, atol=0) - - -@torch.inference_mode() -def test_stranded_group_compaction_exports_correct_survivor_rows(): - cache = cache_for("cpu") - assert cache.allocate("c", 16) - initial = metadata(cache, ["a", "b", "c"], 0, 5) - k = torch.randn(15, 2, 32, dtype=torch.bfloat16) - cache._write_prepared(0, initial, k, k) - advance = metadata(cache, ["a"], 5, 1) - tail = torch.randn(1, 2, 32, dtype=torch.bfloat16) - cache._write_prepared(0, advance, tail, tail) - assert cache._contiguous.compactions == 1 - assert cache._key_cache is None - for i, rid in enumerate(["a", "b", "c"]): - expected = k[i * 5 : (i + 1) * 5] - if rid == "a": - expected = torch.cat((expected, tail)) - actual, _ = cache.read(0, rid, 0, len(expected)) - torch.testing.assert_close(actual, expected, rtol=0, atol=0) - with pytest.raises(RuntimeError, match="uninitialized"): - cache.read(1, "a", 0, 1) - - -def test_contiguous_configuration_rejects_shared_and_unreserved_modes(): - from vllm_rlt import CacheConfig - - for kwargs in [ - dict(layout="shared"), - dict(incremental_allocation=True), - dict(enable_prefix_caching=True), - ]: - with pytest.raises(ValueError, match="contiguous KV"): - CacheConfig(contiguous_kv=True, **kwargs) - with pytest.raises(ValueError, match="sequence-range"): - KVCacheManager(1, 1, 32, 8, 4, 1, contiguous_kv=True) - - -def test_sequence_metadata_capability_does_not_imply_owned_contiguous_kv(): - attention = AttentionBackend( - AscendEagerAttention(), - BackendCapabilities(sequence_ranges=True), - dict(selected_backend="torch", backend="torch"), - ) - with pytest.raises(ValueError, match="owned contiguous attention"): - KVCacheManager(2, 2, 32, 16, 4, 2, device="cpu", attention=attention, contiguous_kv=True) diff --git a/tests/test_fast_execution_plan.py b/tests/test_fast_execution_plan.py index 8a963676..78b127ef 100644 --- a/tests/test_fast_execution_plan.py +++ b/tests/test_fast_execution_plan.py @@ -1,4 +1,4 @@ -"""Ownership and failure boundaries of the immutable Ascend decode plan.""" +"""Ownership and failure boundaries of Ascend recurrent execution.""" from dataclasses import replace @@ -6,13 +6,9 @@ import torch from tests.test_ascend import make_npu_model -from tests.test_fixed_prelude import engine_for -from tests.test_npu_async import drain -from vllm_rlt import SamplingParams from vllm_rlt.core.kv_cache_manager import KVCacheManager from vllm_rlt.device import resolve_device from vllm_rlt.request import Operation -from vllm_rlt.worker.ascend import AscendOuroRuntime from vllm_rlt.worker.executor import ForwardBatch, ModelRuntime from vllm_rlt.worker.metadata import stage_range_metadata @@ -31,7 +27,6 @@ def cache_for(model): device="npu:0", dtype=torch.bfloat16, backend="ascend_eager", - contiguous_kv=True, ) assert cache.allocate("a", 8) return cache @@ -53,7 +48,9 @@ def test_position_plan_reuses_depths_and_expires_between_tokens(): expected = None for reuse in [False, True]: cache = cache_for(model) - runtime = (AscendOuroRuntime if reuse else ModelRuntime)(model, cache) + runtime = ModelRuntime( + model, cache, prepare_recurrent=model.prepare_recurrent if reuse else None + ) observed = [] handle = model.model.rotary_emb.register_forward_hook(lambda *args: observed.append(1)) actual = [] @@ -85,6 +82,7 @@ def test_failed_traversal_keeps_partial_layer_coverage_visible(): torch.npu.set_device(0) model = make_npu_model(torch.device("npu:0")) cache = cache_for(model) + runtime = ModelRuntime(model, cache, prepare_recurrent=model.prepare_recurrent) batch = metadata(cache, 0, 3)[0] def fail(*args): @@ -94,9 +92,7 @@ def fail(*args): try: hidden = model.prelude(torch.tensor([3, 7, 11], device="npu:0")) with pytest.raises(RuntimeError, match="injected layer failure"): - AscendOuroRuntime(model, cache).forward( - ForwardBatch(Operation.RECURRENT, hidden=hidden, metadata=(batch,)) - ) + runtime.forward(ForwardBatch(Operation.RECURRENT, hidden=hidden, metadata=(batch,))) finally: handle.remove() with pytest.raises(RuntimeError, match="every layer"): @@ -111,13 +107,10 @@ def fail(*args): assert torch.isfinite(cache._attend_prepared(0, batch, q)).all() with pytest.raises(RuntimeError, match="uninitialized KV history"): cache._attend_prepared(1, batch, q) - # Retrying the write after an interrupted traversal must escape to validated - # paged overwrite semantics, rather than retaining a stale fast lease. - actual = ( - AscendOuroRuntime(model, cache) - .forward(ForwardBatch(Operation.RECURRENT, hidden=hidden, metadata=(batch,))) - .hidden - ) + # Rerun all layers for the same page slots after the interrupted write. + actual = runtime.forward( + ForwardBatch(Operation.RECURRENT, hidden=hidden, metadata=(batch,)) + ).hidden reference = cache_for(model) expected = model.recurrent_prepared(hidden, metadata(reference, 0, 3)[0], reference)[0] torch.testing.assert_close(actual, expected, atol=0, rtol=0) @@ -126,49 +119,15 @@ def fail(*args): @torch.inference_mode() -def test_device_token_batch_is_borrowed_and_remains_owned(): - resolve_device("npu") - torch.npu.set_device(0) - model = make_npu_model(torch.device("npu:0")) - engine = engine_for(model, True) - try: - engine.add_request( - "a", [3, 7, 11], SamplingParams(max_tokens=4, min_loops=4, max_loops=4, ignore_eos=True) - ) - engine.step() - engine.step() - original = engine.model_runner.token_batch.tokens - snapshot = original.clone() - observed = [] - handle = model.model.embed_tokens.register_forward_pre_hook( - lambda module, args: observed.append(args[0]) - ) - try: - engine.step() - finally: - handle.remove() - assert len(observed) == 1 - assert observed[0].data_ptr() == original.data_ptr() - drain(engine) - torch.testing.assert_close(original, snapshot, atol=0, rtol=0) - finally: - engine.close() - - -@torch.inference_mode() -def test_completion_proof_does_not_survive_truncation_or_restore(): +def test_paged_coverage_after_truncation_and_restore(): resolve_device("npu") torch.npu.set_device(0) model = make_npu_model(torch.device("npu:0")) cache = cache_for(model) + runtime = ModelRuntime(model, cache, prepare_recurrent=model.prepare_recurrent) hidden = model.prelude(torch.tensor([3, 7, 11], device="npu:0")) for batch in metadata(cache, 0, 3): - AscendOuroRuntime(model, cache).forward( - ForwardBatch(Operation.RECURRENT, hidden=hidden, metadata=(batch,)) - ) - allocation = cache._get_allocation("a") - proof = allocation.dense_submitted[cache._plane(3)] - assert proof.all_layers_end == 3 + runtime.forward(ForwardBatch(Operation.RECURRENT, hidden=hidden, metadata=(batch,))) cache.finalize_token("a", 2, 3) snapshot = cache.snapshot("a") cache.truncate_suffix("a", 1) diff --git a/tests/test_fixed_prelude.py b/tests/test_fixed_prelude.py index 29e7c45b..e403c708 100644 --- a/tests/test_fixed_prelude.py +++ b/tests/test_fixed_prelude.py @@ -96,7 +96,7 @@ def engine_for(model, planned, multi_stream=False, token_budget=128): return StageEngine( model, attention_backend="ascend_eager", - cache_config=CacheConfig(num_blocks=128, block_size=16, contiguous_kv=True), + cache_config=CacheConfig(num_blocks=128, block_size=16), scheduler_config=SchedulerConfig( max_num_seqs=3, max_num_batched_tokens=token_budget, prefill_chunk_size=128 ), diff --git a/tests/test_native_ouro_plan.py b/tests/test_native_ouro_plan.py deleted file mode 100644 index 17b3fe78..00000000 --- a/tests/test_native_ouro_plan.py +++ /dev/null @@ -1,143 +0,0 @@ -"""Native host dispatch keeps eager states, live weights and callback semantics.""" - -import pytest -import torch - -from tests.test_ascend import make_npu_model -from tests.test_fast_execution_plan import cache_for, metadata -from tests.test_npu_async import drain -from vllm_rlt import CacheConfig, ExecutionConfig, ExitConfig, SamplingParams, SchedulerConfig -from vllm_rlt.device import resolve_device -from vllm_rlt.engine.stage_engine import StageEngine -from vllm_rlt.request import Operation -from vllm_rlt.worker.ascend import AscendOuroRuntime -from vllm_rlt.worker.executor import ForwardBatch - -pytestmark = pytest.mark.npu - - -@torch.inference_mode() -def test_native_plan_matches_hidden_and_all_kv_layers(): - resolve_device("npu") - torch.npu.set_device(0) - model = make_npu_model(torch.device("npu:0")) - expected = None - for native in [False, True]: - cache = cache_for(model) - runtime = AscendOuroRuntime(model, cache, native=native) - actual = [] - try: - for start, ids in [(0, [3, 7, 11]), (3, [13])]: - forward = ForwardBatch( - Operation.RECURRENT, - hidden=model.prelude(torch.tensor(ids, device="npu:0")), - metadata=metadata(cache, start, len(ids)), - ) - output = runtime.forward(forward) - actual.append(output.hidden.clone()) - for depth in range(4): - for layer in range(model.config.num_hidden_layers): - actual.extend(x.clone() for x in cache._contiguous.groups[depth, layer].pair) - if expected is None: - expected = actual - else: - for a, b in zip(actual, expected): - torch.testing.assert_close(a, b, atol=0, rtol=0) - assert runtime.layer_executor.native_layers == 8 * model.config.num_hidden_layers - assert not runtime.layer_executor.fallbacks - finally: - cache.free("a") - - -def engine_for(model, native, *, multi_stream=False): - return StageEngine( - model, - attention_backend="ascend_eager", - cache_config=CacheConfig(num_blocks=128, block_size=16, contiguous_kv=True), - scheduler_config=SchedulerConfig( - max_num_seqs=2, max_num_batched_tokens=128, prefill_chunk_size=128 - ), - execution_config=ExecutionConfig( - async_scheduling=True, - multi_stream=multi_stream, - use_uva=False, - coalesce_fixed_prelude=True, - native_eager_plan=native, - ), - exit_config=ExitConfig("ouro_delayed"), - ) - - -@torch.inference_mode() -def test_native_plan_uses_assigned_weights_and_preserves_child_hooks(): - resolve_device("npu") - torch.npu.set_device(0) - model = make_npu_model(torch.device("npu:0")) - params = SamplingParams(max_tokens=4, min_loops=4, max_loops=4, ignore_eos=True) - native = engine_for(model, True) - - def once(engine): - engine.add_request("a", [3, 7, 11], params) - return drain(engine) - - try: - once(native) - state = {k: v.clone() for k, v in model.state_dict().items()} - state["model.layers.0.self_attn.q_proj.weight"].mul_(0.5) - model.load_state_dict(state, assign=True) - handle = model.model.layers[0].input_layernorm.register_forward_pre_hook( - lambda m, args: (args[0] + 0.125,) - ) - reference = engine_for(model, False) - try: - assert once(native) == once(reference) - assert ( - native.model_runner.runtime.layer_executor.fallbacks["child_hooks_or_forward"] > 0 - ) - finally: - handle.remove() - reference.close() - reference = engine_for(model, False) - try: - assert once(native) == once(reference) - finally: - reference.close() - assert native.model_runner.runtime.layer_executor.native_layers > 0 - finally: - native.close() - - -@pytest.mark.parametrize("multi_stream", [False, True]) -@torch.inference_mode() -def test_native_plan_cancel_reuse_and_owned_outputs(multi_stream): - resolve_device("npu") - torch.npu.set_device(0) - model = make_npu_model(torch.device("npu:0")) - params = SamplingParams(max_tokens=4, min_loops=4, max_loops=4, ignore_eos=True) - reference = engine_for(model, False, multi_stream=multi_stream) - for i, p in enumerate([[3, 7, 11], [13, 17, 19]]): - reference.add_request(str(i), p, params) - expected = drain(reference) - reference.close() - engine = engine_for(model, True, multi_stream=multi_stream) - held = [] - handle = model.model.layers[0].register_forward_hook( - lambda m, args, output: held.append(output) - ) - try: - for i, p in enumerate([[3, 7, 11], [13, 17, 19]]): - engine.add_request(str(i), p, params) - engine.step() - # A consumer on another stream must first wait for the producer. - torch.npu.synchronize() - snapshot = held[0].clone() - engine.step() - engine.step() - engine.abort_request("0") - engine.add_request("0", [3, 7, 11], params) - assert drain(engine) == expected - torch.testing.assert_close(held[0], snapshot, atol=0, rtol=0) - assert engine.model_runner.runtime.layer_executor.native_layers > 0 - finally: - handle.remove() - engine.close() diff --git a/tests/test_npu_async.py b/tests/test_npu_async.py index cf268b6d..dc98e30c 100644 --- a/tests/test_npu_async.py +++ b/tests/test_npu_async.py @@ -171,7 +171,7 @@ def test_npu_cancel_reuse_rejects_late_coda_and_eos_stops(): @pytest.mark.npu def test_npu_batched_greedy_preserves_ties_and_padding(): - from vllm_rlt.worker.ascend import AscendSampler + from vllm_rlt.worker.ascend_model_runner import AscendSampler device = resolve_device("npu:0") logits = torch.zeros((3, 49152), dtype=torch.bfloat16) @@ -193,7 +193,7 @@ def test_npu_batched_greedy_preserves_ties_and_padding(): def test_npu_hidden_gather_keeps_cross_stream_sources_alive(): from types import SimpleNamespace - from vllm_rlt.worker.ascend import AscendModelRunner + from vllm_rlt.worker.ascend_model_runner import AscendModelRunner device = resolve_device("npu:0") torch.npu.set_device(device) diff --git a/tests/test_npu_batch_decode.py b/tests/test_npu_batch_decode.py index e7b45e98..0850ee79 100644 --- a/tests/test_npu_batch_decode.py +++ b/tests/test_npu_batch_decode.py @@ -8,7 +8,7 @@ from tests.helpers import make_cpu_cache, tiny_ouro_model from vllm_rlt import ExecutionConfig, SamplingParams, SchedulerConfig from vllm_rlt.request import Request -from vllm_rlt.worker.ascend import AscendModelRunner +from vllm_rlt.worker.ascend_model_runner import AscendModelRunner from vllm_rlt.worker.model_runner import ModelRunner diff --git a/tests/test_resident_metadata.py b/tests/test_resident_metadata.py deleted file mode 100644 index 03343f3a..00000000 --- a/tests/test_resident_metadata.py +++ /dev/null @@ -1,71 +0,0 @@ -"""Resident positions remain immutable; paged escape binds deferred addresses.""" - -import pytest -import torch - -from tests.test_ascend import make_npu_model -from tests.test_fast_execution_plan import cache_for, metadata -from vllm_rlt.device import resolve_device -from vllm_rlt.worker.metadata import MetadataBuilder, stage_range_metadata - -pytestmark = pytest.mark.npu - - -def deferred(cache, start, width): - layouts = cache.metadata_builder.build_ranges(["a"], [start], [width], [(0,)]) - - def reject_upload(*args): - pytest.fail("owned metadata unexpectedly staged paged addresses") - - host = stage_range_metadata(layouts, reject_upload, defer_addresses=True)[0] - assert host.position_ids is None - return MetadataBuilder.bind(host, {}) - - -@torch.inference_mode() -def test_owned_inputs_materialize_on_paged_escape(): - resolve_device("npu") - torch.npu.set_device(0) - model = make_npu_model(torch.device("npu:0")) - cache = cache_for(model) - batch = deferred(cache, 0, 3) - hidden = model.prelude(torch.tensor([3, 7, 11], device="npu:0")) - model.recurrent_prepared(hidden, batch, cache) - assert cache._key_cache is None and batch.write_blocks is None - _ = cache.key_cache # Ordered export before a legacy overwrite. - assert cache._contiguous is None - actual = model.recurrent_prepared(hidden, batch, cache)[0] - full = cache.metadata_builder.materialize_paged(batch) - assert full.write_blocks.device.type == "npu" and full.context_lengths is not None - assert full.position_ids is batch.position_ids and batch.write_blocks is None - reference = cache_for(model) - expected = model.recurrent_prepared(hidden, metadata(reference, 0, 3)[0], reference)[0] - torch.testing.assert_close(actual, expected, atol=0, rtol=0) - original = torch.npu.current_stream() - foreign = torch.npu.Stream() - with torch.npu.stream(foreign): - foreign.wait_stream(original) - other = cache.metadata_builder.materialize_paged(batch) - assert other.write_blocks.data_ptr() != full.write_blocks.data_ptr() - torch.testing.assert_close(other.write_blocks, full.write_blocks, atol=0, rtol=0) - foreign.synchronize() - cache.free("a") - reference.free("a") - - -@torch.inference_mode() -def test_resident_position_views_survive_growth_and_eviction(): - resolve_device("npu") - torch.npu.set_device(0) - model = make_npu_model(torch.device("npu:0")) - cache = cache_for(model) - batch = deferred(cache, 0, 3) - original, snapshot = batch.position_ids, batch.position_ids.clone() - for start, width in [(0, x) for x in range(1, 9)] + [(1, 2), (2, 2)]: - current = deferred(cache, start, width) - torch.testing.assert_close( - current.position_ids.cpu(), torch.arange(start, start + width), atol=0, rtol=0 - ) - assert len(cache.metadata_builder._position_layouts) <= 8 - torch.testing.assert_close(original, snapshot, atol=0, rtol=0) - cache.free("a") diff --git a/tests/test_sequence_plans.py b/tests/test_sequence_plans.py index 72bc970d..54568f80 100644 --- a/tests/test_sequence_plans.py +++ b/tests/test_sequence_plans.py @@ -5,7 +5,7 @@ from vllm_rlt.attention.backend import AttentionBackend, BackendCapabilities from vllm_rlt.core.kv_cache_manager import KVCacheManager -from vllm_rlt.kernels.ascend_eager import AscendEagerAttention +from vllm_rlt.kernels.ascend_attention import AscendAttention from vllm_rlt.worker.metadata import MetadataBuilder, stage_range_metadata @@ -13,7 +13,7 @@ def cache_for(device="cpu", *, incremental=False): device = torch.device(device) attention = ( AttentionBackend( - AscendEagerAttention(), + AscendAttention(), BackendCapabilities(sequence_ranges=True), dict(selected_backend="torch", backend="torch"), ) @@ -183,7 +183,7 @@ class Device: type = "npu" device = Device() - assert AscendEagerAttention._stream_key(device) == 73 + assert AscendAttention._stream_key(device) == 73 plan = RangeTablePlan([[1]]) sentinel = object() plan.tensors[device, 73] = sentinel diff --git a/vllm_rlt/attention/backend.py b/vllm_rlt/attention/backend.py index 75ea63a5..ff39d8c0 100644 --- a/vllm_rlt/attention/backend.py +++ b/vllm_rlt/attention/backend.py @@ -31,8 +31,6 @@ class BackendCapabilities: sliding_window: bool = False packed_sliding_window: bool = False sequence_ranges: bool = False - # Consume private, owned KV pairs in [batch, kv_head, token, head_dim] layout. - owned_contiguous_kv: bool = False @dataclass @@ -75,9 +73,7 @@ def generation(self): _TORCH_CAPABILITIES = BackendCapabilities(sliding_window=True) -_ASCEND_CAPABILITIES = BackendCapabilities( - async_scheduling=True, sequence_ranges=True, owned_contiguous_kv=True -) +_ASCEND_CAPABILITIES = BackendCapabilities(async_scheduling=True, sequence_ranges=True) _TRITON_CAPABILITIES = BackendCapabilities( cuda_graphs=True, async_scheduling=True, device_context_lengths=True, sliding_window=True ) @@ -99,7 +95,11 @@ def backend_capabilities(implementation) -> BackendCapabilities: return implementation.capabilities -def validate_backend_name(backend: str) -> None: +def validate_backend_name(backend: str, *, device_type: str | None = None) -> None: + if device_type == "npu" and backend not in ("torch", "ascend_eager"): + raise ValueError( + "NPU execution requires explicit attention_backend='torch' or 'ascend_eager'" + ) if backend not in {"auto", "torch", "triton", "ascend_eager", *FLASH_BACKENDS}: raise ValueError("unknown attention backend") @@ -108,9 +108,9 @@ def _create_explicit(backend, device, dtype, head_dim, block_size, *, packed, pa if backend == "ascend_eager": if device.type != "npu" or dtype != torch.bfloat16: raise ValueError("ascend_eager requires an NPU and BF16") - from vllm_rlt.kernels.ascend_eager import AscendEagerAttention + from vllm_rlt.kernels.ascend_attention import AscendAttention - return AttentionBackend(AscendEagerAttention(), _ASCEND_CAPABILITIES, {"backend": backend}) + return AttentionBackend(AscendAttention(), _ASCEND_CAPABILITIES, {"backend": backend}) if backend == "triton": if device.type != "cuda": raise ValueError("the Triton attention backend requires a CUDA or ROCm device") @@ -166,7 +166,7 @@ def create_backend( reordering candidates; required packed prefill still rejects unsupported ones. A packed window shape opts into tile selection and keeps row attention available. """ - validate_backend_name(backend) + validate_backend_name(backend, device_type=device.type) candidates = _AUTO_PRIORITY if backend == "auto" else (backend,) skipped = [] if backend == "auto" and device.type != "cuda": @@ -191,7 +191,6 @@ def create_backend( "async_scheduling", "device_context_lengths", "sequence_ranges", - "owned_contiguous_kv", "sliding_window", "packed_sliding_window", ) diff --git a/vllm_rlt/config.py b/vllm_rlt/config.py index aaf7fa49..c6d5eeb2 100644 --- a/vllm_rlt/config.py +++ b/vllm_rlt/config.py @@ -20,23 +20,15 @@ class CacheConfig: incremental_allocation: bool = False # Admission headroom for incremental allocation; otherwise effective watermark is zero. watermark_ratio: float = 0.0 - # Experimental private, uniform eager cohorts. Generic accesses materialize paged KV. - contiguous_kv: bool = False def __post_init__(self): - for name in ("enable_prefix_caching", "incremental_allocation", "contiguous_kv"): + for name in ("enable_prefix_caching", "incremental_allocation"): if type(getattr(self, name)) is not bool: raise ValueError(f"{name} must be a boolean") if not 0 <= self.watermark_ratio < 1: raise ValueError("watermark_ratio must be in [0, 1)") if self.enable_prefix_caching and self.layout != "last_exited": raise ValueError("prefix caching requires last_exited KV") - if self.contiguous_kv and ( - self.layout != "last_exited" - or self.enable_prefix_caching - or self.incremental_allocation - ): - raise ValueError("contiguous KV requires private, fully reserved last_exited storage") if self.num_blocks is not None: _positive("num_blocks", self.num_blocks) _positive("block_size", self.block_size) @@ -165,8 +157,6 @@ class ExecutionConfig: packed_prefill: bool | None = None # Run embedding and the fixed full recurrent depth in one decode submission. coalesce_fixed_prelude: bool = False - # Execute the same eager ATen layer operations through a native host plan. - native_eager_plan: bool = False def __post_init__(self): if self.packed_prefill is not None and type(self.packed_prefill) is not bool: @@ -179,7 +169,6 @@ def __post_init__(self): "pad_to_power_of_two", "cuda_graphs", "coalesce_fixed_prelude", - "native_eager_plan", ): if type(getattr(self, name)) is not bool: raise ValueError(f"{name} must be a boolean") diff --git a/vllm_rlt/core/kv_cache_manager.py b/vllm_rlt/core/kv_cache_manager.py index 7ca5370c..caeb7566 100644 --- a/vllm_rlt/core/kv_cache_manager.py +++ b/vllm_rlt/core/kv_cache_manager.py @@ -13,7 +13,6 @@ from collections.abc import Callable, Sequence from copy import deepcopy from dataclasses import dataclass, field -from functools import cached_property from numbers import Integral import torch @@ -67,41 +66,6 @@ def __contains__(self, position: int) -> bool: return position < self.prefix or position in self.pending -@dataclass -class _DenseFrontier: - # Submission coverage only. Device completion still belongs to stream events. - end: int - all_layers_end: int = 0 - - -class _CohortWrittenPositions(_WrittenPositions): - """Share an append-only cohort frontier; detach on an individual mutation.""" - - def __init__(self, source, frontier): - self._prefix = source.prefix - self.pending = source.pending.copy() - self._frontier = frontier - - @property - def prefix(self): - return self._prefix if self._frontier is None else self._frontier.end - - @prefix.setter - def prefix(self, value): - self._prefix, self._frontier = value, None - - def _detach(self): - self.prefix = self.prefix - - def add(self, position): - self._detach() - super().add(position) - - def add_range(self, start, end): - self._detach() - super().add_range(start, end) - - @dataclass class _Allocation: max_tokens: int @@ -109,7 +73,6 @@ class _Allocation: written: list[list[_WrittenPositions]] transfer_leases: set[str] = field(default_factory=set) release_requested: bool = False - dense_submitted: dict = field(default_factory=dict) @dataclass(frozen=True) @@ -185,118 +148,6 @@ class KVSnapshot: values: torch.Tensor -@dataclass(frozen=True) -class EagerLayerInputs: - history: tuple | None - mask: torch.Tensor - batch_size: int - token_start: int - query_tokens: int - head_dim: int - kv_heads: int - append_plan: object - - -class _KVTraversal: - """One validated dense traversal inside an inference-only model call.""" - - def __init__(self, manager, stream, batch, storage, context): - self.manager, self.stream, self.batch = manager, stream, batch - self.storage, self.context = storage, context - self.epoch = manager._allocation_epoch - self.pair = self.layer = None - self.next_layer = 0 - self.ordered = True - - def _active(self, batch): - return ( - batch is self.batch - and self.manager._contiguous is self.storage - and self.manager._allocation_epoch == self.epoch - ) - - def _write_prepared(self, layer, batch, k, v): - manager = self.manager - if not self._active(batch): - return manager._write_prepared(layer, batch, k, v) - manager._validate_layer(layer) - manager._validate_tensor(k, len(batch.position_ids), "k") - manager._validate_tensor(v, len(batch.position_ids), "v") - if not self.storage._append(layer, batch, k, v, self.context): - self.pair = self.layer = None - return manager._write_prepared(layer, batch, k, v, _stream=self.stream) - self.pair = self.storage.groups[self.context[-1], layer].pair - self.ordered = self.ordered and layer == self.next_layer - self.next_layer += 1 - self.layer = layer - - def prepare_native(self, layer, hidden): - if not self._active(self.batch): - return None - self.manager._validate_layer(layer) - size, start, count, _ = self.context - if ( - hidden.ndim != 2 - or len(hidden) != size * count - or hidden.device != self.manager.device - or hidden.dtype != self.manager.dtype - ): - raise ValueError("native eager input must match the validated cache schema") - plan = self.storage.prepare_append(layer, self.batch, self.context) - if plan is None: - return None - _, mask = self.manager.attention.kernel.prepare_eager_attention( - hidden, self.batch, stream=self.stream - ) - return EagerLayerInputs( - plan.history, - mask, - size, - start, - count, - self.manager.head_dim, - self.manager.num_kv_heads, - plan, - ) - - def commit_native(self, layer, inputs, pair): - if not self._active(self.batch): - raise RuntimeError("stale native eager traversal") - expected = ( - inputs.batch_size, - inputs.kv_heads, - inputs.token_start + inputs.query_tokens, - inputs.head_dim, - ) - if any( - tuple(x.shape) != expected - or x.device != self.manager.device - or x.dtype != self.manager.dtype - for x in pair - ): - raise ValueError("native eager KV output does not match the cache schema") - self.storage.commit_append(layer, self.batch, self.context, inputs.append_plan, pair) - self.pair = pair - self.ordered = self.ordered and layer == self.next_layer - self.next_layer += 1 - self.layer = layer - - def finish(self): - if self._active(self.batch) and self.ordered and self.next_layer == self.manager.num_layers: - group = self.storage.groups.get((self.context[-1], 0)) - if group is not None and group.allocations is self.batch.allocations: - group.frontier.all_layers_end = self.context[1] + self.context[2] - - def _attend_prepared(self, layer, batch, q, *, window=None): - manager = self.manager - if window is not None: - return manager._attend_prepared(layer, batch, q, window=window, _stream=self.stream) - if not self._active(batch) or layer != self.layer or self.pair is None: - return manager._attend_prepared(layer, batch, q) - manager._validate_tensor(q, len(batch.position_ids), "q", query=True) - return manager.attention.kernel.attend_contiguous(q, self.pair, batch, stream=self.stream) - - class KVCacheManager: """Own a fixed physical page pool shared by request/depth allocations. @@ -323,7 +174,6 @@ def __init__( recurrent_layers: Sequence[int] | None = None, *, attention: AttentionBackend | None = None, - contiguous_kv: bool = False, ): for name, value in ( ("num_layers", num_layers), @@ -362,33 +212,9 @@ def __init__( self.attention_info = self.attention.info self.backend = self.attention_info["selected_backend"] shape = (num_blocks, num_layers, block_size, num_kv_heads, head_dim) - if contiguous_kv and ( - not self.attention_capabilities.sequence_ranges - or not self.attention_capabilities.owned_contiguous_kv - or layout != "last_exited" - or enable_prefix_caching - or incremental_allocation - ): - raise ValueError( - "contiguous KV requires private reserved sequence-range storage " - "and owned contiguous attention" - ) - self._paged_shape = shape - self._contiguous = None - self._key_cache = self._value_cache = None - if contiguous_kv: - # Only owned NPU KV needs device resolution without a paged allocation. - self.device = torch.empty(0, device=self.device, dtype=dtype).device - from vllm_rlt.kernels.contiguous_kv import ContiguousKVStorage - - self._contiguous = ContiguousKVStorage( - num_kv_heads, head_dim, self.device, publish_coverage=self._publish_dense_coverage - ) - else: - self._key_cache = torch.empty(shape, device=self.device, dtype=dtype) - self._value_cache = torch.empty_like(self._key_cache) - self.device = self._key_cache.device - self.key_cache, self.value_cache = self._key_cache, self._value_cache + self.key_cache = torch.empty(shape, device=self.device, dtype=dtype) + self.value_cache = torch.empty_like(self.key_cache) + self.device = self.key_cache.device self._free_blocks = list(reversed(range(num_blocks))) self._allocations: dict[str, _Allocation] = {} self._allocation_epoch = 0 @@ -568,7 +394,6 @@ def allocate(self, request_id: str, max_tokens: int, *, initial_tokens=None, pre ], ) self._allocation_epoch += 1 - self._invalidate_contiguous(preserve_contiguous=True) self.prefix_queries += 1 self.prefix_hits += length return True @@ -589,7 +414,6 @@ def ensure_capacity(self, request_id, tokens): for d, t in enumerate(allocation.block_tables) ) self._allocation_epoch += 1 - self._invalidate_contiguous(preserve_contiguous=True) return True def truncate_suffix(self, request_id: str, frontier: int) -> None: @@ -607,8 +431,6 @@ def truncate_suffix(self, request_id: str, frontier: int) -> None: for table in allocation.block_tables: if any(self._refs[b] > 1 for b in table[frontier // self.block_size :]): raise ValueError("cannot truncate shared prefix pages") - self._invalidate_contiguous() - allocation.dense_submitted.clear() for plane in allocation.written: for written in plane: written.prefix = min(written.prefix, frontier) @@ -616,19 +438,15 @@ def truncate_suffix(self, request_id: str, frontier: int) -> None: def free(self, request_id: str) -> None: allocation = self._allocations.get(request_id) - self._invalidate_contiguous(preserve_contiguous=True) if allocation is not None and allocation.transfer_leases: allocation.release_requested = True return - if self._contiguous is not None and allocation is not None: - self._contiguous.release(allocation, self.num_layers, self.storage_depths) allocation = self._allocations.pop(request_id, None) if allocation is not None: self._allocation_epoch += 1 self._drop_refs(b for table in allocation.block_tables for b in reversed(table)) def pin_transfer(self, request_id: str, transfer_id: str): - self._invalidate_contiguous() allocation = self._get_allocation(request_id) if allocation.release_requested: raise RuntimeError("cannot transfer KV scheduled for release") @@ -649,7 +467,6 @@ def mark_imported_prefix(self, request_id: str, length: int, *, start=0): raise RuntimeError("cannot activate cancelled KV") if any(w.prefix != start or w.pending for plane in allocation.written for w in plane): raise RuntimeError("KV prefix was already initialized") - allocation.dense_submitted.clear() for plane in allocation.written: for written in plane: written.prefix = length @@ -727,9 +544,6 @@ def _require_exit_written(self, request_id, position, exit_depth) -> _Allocation allocation = self._get_allocation(request_id) self._validate_depth(exit_depth) self._validate_position(allocation, position) - proof = allocation.dense_submitted.get(self._plane(exit_depth)) - if proof is not None and position < proof.all_layers_end: - return allocation if any( position not in allocation.written[self._plane(exit_depth)][layer] for layer in self.recurrent_layers @@ -784,12 +598,10 @@ def restore(self, request_id: str, snapshot: KVSnapshot) -> None: raise ValueError("snapshot page count does not match the allocation") if any(w.prefix or w.pending for plane in allocation.written for w in plane): raise RuntimeError("cannot restore into an allocation that already holds KV") - self._invalidate_contiguous() blocks = [b for table in allocation.block_tables for b in table] for row, block in enumerate(blocks): self.key_cache[block].copy_(snapshot.keys[row]) self.value_cache[block].copy_(snapshot.values[row]) - allocation.dense_submitted.clear() allocation.written = deepcopy(snapshot.written) def _plane(self, depth: int) -> int: @@ -952,43 +764,6 @@ def _validate_tensor(self, tensor, batch_size, name, *, query=False): if tensor.device != self.device or tensor.dtype != self.dtype: raise ValueError(f"{name} must use device {self.device} and dtype {self.dtype}") - @staticmethod - def _publish_dense_coverage(spans, depth, layer, end, frontier): - """Record submitted copies after storage has enqueued both K and V. - - Stable cohorts advance one shared proof. Individual mutation detaches a - row, and an interrupted traversal never publishes all-layer coverage. - """ - if frontier is not None: - frontier.end = end - return frontier - frontier = _DenseFrontier(end) - for span in spans: - previous = span.allocation.written[depth][layer] - span.allocation.written[depth][layer] = _CohortWrittenPositions(previous, frontier) - if layer == 0: - span.allocation.dense_submitted[depth] = frontier - return frontier - - def _begin_traversal(self, batch): - if ( - self._contiguous is None - or not isinstance(batch, KVMetadata) - or self.device.type != "npu" - or not torch.is_inference_mode_enabled() - ): - return self - if not batch.writable: - return self - coverage = self._require_live_batch(batch) - if coverage is None: - return self - stream = torch.npu.current_stream(self.device) - context = self._contiguous._context(batch, stream) - if context is None: - return self - return _KVTraversal(self, stream, batch, self._contiguous, context) - @torch.no_grad() def write( self, @@ -1017,13 +792,9 @@ def _write_prepared( batch: KVMetadata | DeviceKVMetadata, k: torch.Tensor, v: torch.Tensor, - *, - _stream=None, ) -> None: self._validate_layer(layer) coverage = self._require_live_batch(batch) - if not isinstance(batch, KVMetadata) or batch.sequence_ranges is None: - self._invalidate_contiguous() if isinstance(batch, DeviceKVMetadata): self._validate_tensor(k, len(batch.position_ids), "k") self._validate_tensor(v, len(batch.position_ids), "v") @@ -1041,16 +812,7 @@ def _write_prepared( self._validate_tensor(v, len(batch.position_ids), "v") if not batch.rows: return - direct = self._contiguous is not None and self._contiguous.append( - layer, batch, k, v, stream=_stream - ) - if direct: - # The store advances a shared layer frontier only after both KV - # writes enqueue. Public coverage readers still observe each row. - return - elif self.device.type == "npu" and self.backend == "ascend_eager": - if batch.deferred_addresses: - batch = self.metadata_builder.materialize_paged(batch) + if self.device.type == "npu" and self.backend == "ascend_eager": if not hasattr(self, "_npu_writer"): from vllm_rlt.kernels.ascend_kv import AscendKVWriter @@ -1110,7 +872,6 @@ def _attend_prepared( q: torch.Tensor, *, window: int | None = None, - _stream=None, ) -> torch.Tensor: self._validate_layer(layer) coverage = self._require_live_batch(batch) @@ -1136,13 +897,6 @@ def _attend_prepared( ) if not batch.rows: return torch.empty_like(q) - if self._contiguous is not None: - pair = self._contiguous.get(layer, batch, stream=_stream) - if pair is not None: - # The authoritative append-only store proves an initialized - # prefix for every live row. Holes, overwrite, truncate and - # generic writes escape to the paged checks below. - return self.attention.kernel.attend_contiguous(q, pair, batch, stream=_stream) if coverage is None: for allocation, depth, position in batch.rows: self._require_prefix(allocation, layer, depth, position + 1) @@ -1151,8 +905,6 @@ def _attend_prepared( self._require_prefix( sequence.allocation, layer, sequence.plane, sequence.ranges[-1][1] ) - if batch.deferred_addresses: - batch = self.metadata_builder.materialize_paged(batch) if batch.cu_seqlens_q is not None: return self.attention.prefill( q, @@ -1213,8 +965,6 @@ def finalize_token(self, request_id: str, position: int, exit_depth: int) -> Non allocation.block_tables[self._plane(depth)][logical_page] for depth in range(exit_depth + 1, self.max_loops) ] - if destinations: - self._invalidate_contiguous() # Basic-index copies avoid a blocking host->device index tensor on the # boundary stream. That transfer would serialize final core and routing. for destination in destinations: @@ -1231,31 +981,6 @@ def finalize_token(self, request_id: str, position: int, exit_depth: int) -> Non ) self._record_finalized(allocation, position, exit_depth) - @cached_property - def key_cache(self): - self._materialize_paged() - return self._key_cache - - @cached_property - def value_cache(self): - self._materialize_paged() - return self._value_cache - - def _materialize_paged(self): - if self._key_cache is None: - key = torch.empty(self._paged_shape, device=self.device, dtype=self.dtype) - value = torch.empty_like(key) - self._key_cache, self._value_cache = key, value - if self._contiguous is not None: - self._contiguous.export(self._key_cache, self._value_cache, self.block_size) - self._contiguous = None - # Preserve direct Tensor attribute access for the default paged backends. - self.key_cache, self.value_cache = self._key_cache, self._value_cache - - def _invalidate_contiguous(self, *, preserve_contiguous=False): - if not preserve_contiguous and self._contiguous is not None and self._contiguous.rows: - self._materialize_paged() - @torch.no_grad() def read(self, layer: int, request_id: str, depth: int, length: int | None = None): """Materialize a fully initialized prefix as [token, KV head, dimension].""" diff --git a/vllm_rlt/engine/stage_engine.py b/vllm_rlt/engine/stage_engine.py index dbbc0c15..c6296dae 100644 --- a/vllm_rlt/engine/stage_engine.py +++ b/vllm_rlt/engine/stage_engine.py @@ -75,17 +75,9 @@ def __init__( self.execution_config = resolve_cuda_graph_rows( execution_config or ExecutionConfig(), scheduler_config, speculative_config ) - if parameter.device.type != "npu" and ( - cache_config.contiguous_kv - or self.execution_config.coalesce_fixed_prelude - or self.execution_config.native_eager_plan - ): + if parameter.device.type != "npu" and self.execution_config.coalesce_fixed_prelude: raise ValueError("Ascend execution options require an NPU") if parameter.device.type == "npu": - if attention_backend not in ("torch", "ascend_eager"): - raise ValueError( - "NPU execution requires explicit attention_backend='torch' or 'ascend_eager'" - ) if config.to_dict()["model_type"] != "ouro" or config.total_ut_steps != 4: raise ValueError("NPU currently supports Ouro with four loops") if parameter.dtype != torch.bfloat16: @@ -243,26 +235,11 @@ def __init__( and self.execution_config.async_scheduling ), device_context_lengths=self.async_speculative, - sequence_ranges=cache_config.contiguous_kv, - owned_contiguous_kv=cache_config.contiguous_kv, sliding_window=sliding_window, ), prefer_packed_prefill=prefer_packed and not sliding_window, packed_window_shape=packed_window_shape, ) - if self.execution_config.native_eager_plan and ( - parameter.device.type != "npu" - or not cache_config.contiguous_kv - or attention.info["selected_backend"] != "ascend_eager" - or self.execution_config.cuda_graphs - or self.execution_config.static_buffers - or getattr(model, "requires_boundary_kv", False) - or speculative_config is not None - ): - raise ValueError( - "native eager plan requires Ascend eager owned KV " - "without graphs, static buffers or speculation" - ) if self.execution_config.packed_prefill is None: supported = attention.capabilities.packed_prefill and ( not sliding_window or attention.capabilities.packed_sliding_window @@ -293,7 +270,6 @@ def __init__( attention=attention, enable_prefix_caching=cache_config.enable_prefix_caching, incremental_allocation=cache_config.incremental_allocation, - contiguous_kv=cache_config.contiguous_kv, watermark_ratio=cache_config.watermark_ratio, recurrent_layers=getattr(model, "recurrent_kv_layers", None), ) @@ -327,7 +303,7 @@ def __init__( self.rng = RngRegistry() runner_type = ModelRunner if parameter.device.type == "npu": - from vllm_rlt.worker.ascend import AscendModelRunner + from vllm_rlt.worker.ascend_model_runner import AscendModelRunner runner_type = AscendModelRunner self.model_runner = runner_type( @@ -996,12 +972,7 @@ def _step_async(self): request_events=self.model_runner.events, signals=signals, ) - for row in update.next_prelude: - batch.requests[row].request.input_token_tensor = ticket.device_values[row] - if self.model_runner.device.type == "npu": - self.model_runner._retain_device_tokens( - batch, ticket.device_values, update.next_prelude - ) + self.model_runner.bind_sampled_tokens(batch, ticket.device_values, update.next_prelude) for row in update.retain_signal: self._pending_exit_signals[batch.requests[row].request.request_id] = SignalHandle( ticket, row diff --git a/vllm_rlt/entrypoints/cli.py b/vllm_rlt/entrypoints/cli.py index 29a96295..f05313c1 100644 --- a/vllm_rlt/entrypoints/cli.py +++ b/vllm_rlt/entrypoints/cli.py @@ -7,6 +7,7 @@ import torch from vllm_rlt import LLM, SamplingParams, SchedulerConfig +from vllm_rlt.attention import validate_backend_name from vllm_rlt.entrypoints.runtime_args import ( add_runtime_args, profile_config_from_args, @@ -101,11 +102,10 @@ def main(): parser.error("--toy uses built-in token ID prompts; omit --prompt") if not args.toy and not args.prompt: parser.error("provide --prompt or use --toy for a CPU smoke test") - if args.device.split(":", 1)[0] == "npu" and args.attention_backend not in ( - "torch", - "ascend_eager", - ): - parser.error("NPU execution requires explicit --attention-backend torch or ascend_eager") + try: + validate_backend_name(args.attention_backend, device_type=args.device.split(":", 1)[0]) + except ValueError as exc: + parser.error(str(exc)) profiling = profile_config_from_args(args) if profiling is not None: profiling.resolve_activities(args.device) diff --git a/vllm_rlt/entrypoints/runtime_args.py b/vllm_rlt/entrypoints/runtime_args.py index 23017724..bb73cdae 100644 --- a/vllm_rlt/entrypoints/runtime_args.py +++ b/vllm_rlt/entrypoints/runtime_args.py @@ -21,11 +21,6 @@ def add_runtime_args(parser): parser.add_argument("--target-loops", type=int, default=4) parser.add_argument("--enable-prefix-caching", action="store_true") parser.add_argument("--incremental-kv", action="store_true") - parser.add_argument( - "--contiguous-kv", - action="store_true", - help="Use owned eager KV cohorts (Ascend private reserved cache)", - ) parser.add_argument("--kv-watermark", type=float, default=0.0) parser.add_argument("--scheduling-policy", choices=["fcfs", "priority"], default="fcfs") parser.add_argument("--enable-preemption", action="store_true") @@ -90,11 +85,6 @@ def add_runtime_args(parser): action="store_true", help="Disable multi-stream CUDA execution independently of async scheduling", ) - parser.add_argument( - "--native-eager-plan", - action="store_true", - help="Dispatch unchanged Ouro eager operations from a native host plan (NPU)", - ) parser.add_argument("--static-buffers", action="store_true") parser.add_argument("--pad-to-power-of-two", action="store_true") parser.add_argument("--prefill-chunk-size", type=int, default=128) @@ -126,7 +116,6 @@ def runtime_configs(args): cache_config=CacheConfig( enable_prefix_caching=args.enable_prefix_caching, incremental_allocation=args.incremental_kv, - contiguous_kv=args.contiguous_kv, watermark_ratio=args.kv_watermark, num_blocks=args.num_blocks, block_size=args.block_size, @@ -151,7 +140,6 @@ def runtime_configs(args): cuda_graph_memory_reserve_bytes=args.cuda_graph_memory_reserve_bytes, async_scheduling=args.async_scheduling, coalesce_fixed_prelude=args.coalesce_fixed_prelude, - native_eager_plan=args.native_eager_plan, multi_stream=not args.single_stream, static_buffers=args.static_buffers, pad_to_power_of_two=args.pad_to_power_of_two, diff --git a/vllm_rlt/entrypoints/serve.py b/vllm_rlt/entrypoints/serve.py index e718a975..c7fc3c84 100644 --- a/vllm_rlt/entrypoints/serve.py +++ b/vllm_rlt/entrypoints/serve.py @@ -7,6 +7,7 @@ import torch +from vllm_rlt.attention import validate_backend_name from vllm_rlt.config import SchedulerConfig from vllm_rlt.engine.llm_engine import StageClient from vllm_rlt.entrypoints.runtime_args import ( @@ -118,8 +119,10 @@ def main(): limits = ServingLimits(**{name: getattr(args, name) for name in defaults}) if args.device == "cpu" and args.attention_backend not in ("auto", "torch"): parser.error("CPU execution requires --attention-backend torch or auto") - if args.device == "npu" and args.attention_backend not in ("torch", "ascend_eager"): - parser.error("NPU execution requires explicit --attention-backend torch or ascend_eager") + try: + validate_backend_name(args.attention_backend, device_type=args.device) + except ValueError as exc: + parser.error(str(exc)) torch.set_num_threads(args.cpu_threads) logging.basicConfig(level=logging.INFO) app = create_app( diff --git a/vllm_rlt/kernels/ascend_eager.py b/vllm_rlt/kernels/ascend_attention.py similarity index 59% rename from vllm_rlt/kernels/ascend_eager.py rename to vllm_rlt/kernels/ascend_attention.py index ca8c9bd5..d014b119 100644 --- a/vllm_rlt/kernels/ascend_eager.py +++ b/vllm_rlt/kernels/ascend_attention.py @@ -1,19 +1,27 @@ # SPDX-License-Identifier: Apache-2.0 -"""Explicit BF16 eager attention on Ascend, matching released Ouro arithmetic. +"""Use Torch SDPA on Ascend with the runtime's paged KV cache.""" -Two batching paths reduce host submissions without changing the arithmetic: -equal-length whole-prompt prefill becomes one attention call, and independent -single-row decode groups become one ragged batch. Everything else keeps the -original per-group path, including the layout-sensitive initial prefill. -""" - -import math import weakref from collections import OrderedDict import torch - -from vllm_rlt.worker.metadata import UniformSequenceRanges +from torch.nn import functional as F + + +def _attend(query, key, value, mask=None, *, is_causal=False): + # Match HF's BNSD inputs. A bool mask lets Torch-NPU dispatch fused SDPA + # for cached chunks and variable-length decode on supported hardware. + groups = query.shape[1] // key.shape[1] + if groups != 1: + key = key.repeat_interleave(groups, dim=1) + value = value.repeat_interleave(groups, dim=1) + return F.scaled_dot_product_attention( + query.contiguous(), + key.contiguous(), + value.contiguous(), + attn_mask=mask, + is_causal=is_causal, + ) def _flat_kv_rows(keys, values): @@ -58,14 +66,13 @@ def _prefill_geometry(rows): return size, width -class AscendEagerAttention: +class AscendAttention: """Batch compatible queries; borrow only validated host metadata.""" def __init__(self): self._metadata = None self._groups = None self._geometry = None - self._prefill_mask = None self._write_slots = None self._ragged_plan = None self._signature = None @@ -80,89 +87,46 @@ def __init__(self): self.batched_prefill_calls = 0 self.ragged_calls = 0 self.fallback_calls = 0 - self.batch_continuations = True self.continuation_calls = 0 self._range_metadata = None self._range_signature = None - self._range_mask = self._range_slots = self._range_groups = None + self._range_mask = self._range_slots = None - def _prepare_ranges(self, q, batch, stream=None): + def _prepare_ranges(self, q, batch): ranges = batch.sequence_ranges rows = len(ranges) - if isinstance(ranges, UniformSequenceRanges): - width, start = ranges.query_count, ranges.start - else: - width, start = ranges[0].count, ranges[0].start - signature = ( - q.device, - q.dtype, - q.shape, - stream.stream_id if stream is not None else self._stream_key(q.device), - ) + width, start = ranges[0].count, ranges[0].start + signature = (q.device, q.dtype, q.shape, self._stream_key(q.device)) if ( self._range_metadata is None or self._range_metadata() is not batch or self._range_signature != signature ): self._range_metadata = self._range_signature = None - self._range_slots = self._range_groups = None + self._range_slots = self._range_mask = None def build(): positions = torch.arange(start + width, device=q.device) - query_positions = positions[start:] - mask = torch.zeros((width, start + width), dtype=q.dtype, device=q.device) - mask.masked_fill_( - positions[None, :] > query_positions[:, None], torch.finfo(q.dtype).min - ) - return (mask,) + return (positions[None, :] <= positions[start:, None],) - self._range_mask = self._layout( - ("range", q.device, signature[-1], q.dtype, start, width), build - )[0] + if start and width > 1: + self._range_mask = self._layout( + ("range", q.device, signature[-1], start, width), build + )[0] self._range_metadata = weakref.ref(batch) self._range_signature = signature return rows, width, start - def _range_fallback(self, q, keys, values, batch, flat_rows): - # Preserve per-sequence behavior for the independently disabled - # continuation-batching ablation. Tables have one row per sequence. - if self._range_groups is None: - groups = self._plan( - [(id(a), d) for a, d, _ in batch.rows], - [p + 1 for _, _, p in batch.rows], - q.device, - ) - width = batch.sequence_ranges[0].count - self._range_groups = [(first // width, *rest) for first, *rest in groups] - return self._forward( - q, keys, values, batch.block_tables, self._range_groups, flat_rows=flat_rows - ) - - def _range_attention(self, q, pair, batch, geometry): + def _range_attention(self, q, pair, geometry): rows, width, start = geometry key, value = pair head_dim = q.shape[-1] - groups = q.shape[1] // key.shape[1] - if groups != 1: - key = key.repeat_interleave(groups, dim=1) - value = value.repeat_interleave(groups, dim=1) query = ( - q.contiguous().reshape(rows, q.shape[1], 1, head_dim) + q.reshape(rows, q.shape[1], 1, head_dim) if width == 1 - else q.reshape(rows, width, q.shape[1], head_dim).transpose(1, 2).contiguous() + else q.reshape(rows, width, q.shape[1], head_dim).transpose(1, 2) ) - scores = torch.matmul(query, key.transpose(-2, -1)) * (1.0 / math.sqrt(head_dim)) - scores = scores + self._range_mask[None, None, :, :] - probabilities = torch.softmax(scores, dim=-1, dtype=torch.float32).to(q.dtype) - if start and width > 1 and rows > 1: - # CANN changes PV's reduction/rounding with the batch geometry at - # long contexts. Keep its original per-request geometry; QK, mask, - # softmax and KV reads still share one batched submission each. - output = torch.cat( - [torch.matmul(probabilities[i : i + 1], value[i : i + 1]) for i in range(rows)] - ) - else: - output = torch.matmul(probabilities, value) + output = _attend(query, key, value, self._range_mask, is_causal=start == 0 and width > 1) if width == 1: self.ragged_calls += 1 return output.squeeze(2) @@ -171,41 +135,12 @@ def _range_attention(self, q, pair, batch, geometry): self.continuation_calls += 1 return output.transpose(1, 2).reshape_as(q) - def prepare_eager_attention(self, hidden, batch, *, stream=None): - """Borrow the same read-only mask used by the Python eager path.""" - geometry = self._prepare_ranges(hidden, batch, stream) - return geometry, self._range_mask - - def attend_contiguous(self, q, pair, batch, *, stream=None): - geometry = self._prepare_ranges(q, batch, stream) - if geometry[2] and geometry[1] > 1 and not self.batch_continuations: - # An ablation may retain contiguous reads but disable batching. - output = torch.empty_like(q) - rows, width, _ = geometry - for row in range(rows): - query = q[row * width : (row + 1) * width] - k, v = (x[row : row + 1] for x in pair) - heads = q.shape[1] // k.shape[1] - if heads != 1: - k, v = k.repeat_interleave(heads, 1), v.repeat_interleave(heads, 1) - query = query.transpose(0, 1).unsqueeze(0).contiguous() - scores = torch.matmul(query, k.transpose(-2, -1)) * (1.0 / math.sqrt(q.shape[-1])) - scores = scores + self._range_mask[None, None, :, :] - probabilities = torch.softmax(scores, -1, dtype=torch.float32).to(q.dtype) - output[row * width : (row + 1) * width].copy_( - torch.matmul(probabilities, v).squeeze(0).transpose(0, 1) - ) - return output - return self._range_attention(q, pair, batch, geometry) - def _attend_ranges(self, q, keys, values, batch): geometry = self._prepare_ranges(q, batch) rows, width, start = geometry flat_rows = _flat_kv_rows(keys, values) if flat_rows is None: raise ValueError("compact Ascend ranges require regular paged KV storage") - if start and width > 1 and not self.batch_continuations: - return self._range_fallback(q, keys, values, batch, flat_rows) flat_keys, flat_values, rows_per_block = flat_rows if self._range_slots is None: positions = torch.arange(start + width, device=q.device) @@ -215,9 +150,7 @@ def _attend_ranges(self, q, keys, values, batch): shape = (rows, start + width, keys.shape[-2], keys.shape[-1]) key = flat_keys.index_select(0, self._range_slots).reshape(shape).transpose(1, 2) value = flat_values.index_select(0, self._range_slots).reshape(shape).transpose(1, 2) - if start: - key, value = key.contiguous(), value.contiguous() - return self._range_attention(q, (key, value), batch, geometry) + return self._range_attention(q, (key, value), geometry) @staticmethod def _stream_key(device): @@ -256,14 +189,12 @@ def _prepare_decode(self, q, keys, batch, flat_rows, stream): def build(): positions = torch.arange(maximum, device=q.device) - mask = torch.zeros((len(lengths), maximum), dtype=torch.bfloat16, device=q.device) valid = None if min(lengths) != maximum: valid = positions[None, :] < torch.tensor(lengths, device=q.device)[:, None] - mask.masked_fill_(~valid, torch.finfo(torch.bfloat16).min) - return positions // block_size, positions % block_size, mask, valid + return positions // block_size, positions % block_size, valid - pages, offsets, mask, valid = self._layout( + pages, offsets, valid = self._layout( ("decode", q.device, stream, block_size, lengths), build ) rows = len(lengths) @@ -274,10 +205,10 @@ def build(): flat_keys, _, rows_per_block = flat_rows slots = (blocks * rows_per_block + offsets).clamp(0, flat_keys.shape[0] - 1) if valid is not None: - # Masked columns must not gather unwritten bytes: NaN KV would - # survive an additive mask (and 0 * NaN in PV). Position zero is + # Avoid reading unwritten bytes in masked columns. Position zero is # initialized whenever the cache's prefix check admits this row. slots = torch.where(valid, slots, slots[:, :1]) + mask = valid[:, None, None, :] if valid is not None else None self._decode_plan = (slots.reshape(-1), mask, rows, maximum) def _decode(self, q, keys, values, flat_rows): @@ -286,20 +217,11 @@ def _decode(self, q, keys, values, flat_rows): kv_heads, head_dim = keys.shape[-2:] key = flat_keys.index_select(0, slots).reshape(rows, maximum, kv_heads, head_dim) value = flat_values.index_select(0, slots).reshape(rows, maximum, kv_heads, head_dim) - head_groups = q.shape[1] // kv_heads - if head_groups != 1: - key = key.repeat_interleave(head_groups, dim=2) - value = value.repeat_interleave(head_groups, dim=2) # Real rows form an identity prefix. Avoid gathering Q and scattering - # the result on every layer; keep the same contiguous matmul layouts. + # the result on every layer. query = q if len(q) == rows else q[:rows] - query = query.contiguous().reshape(rows, q.shape[1], 1, head_dim) - key = key.permute(0, 2, 1, 3).contiguous() - value = value.permute(0, 2, 1, 3).contiguous() - scores = torch.matmul(query, key.transpose(-2, -1)) * (1.0 / math.sqrt(head_dim)) - scores = scores + mask[:, None, None, :] - probabilities = torch.softmax(scores, dim=-1, dtype=torch.float32).to(q.dtype) - output = torch.matmul(probabilities, value).squeeze(2) + query = query.reshape(rows, q.shape[1], 1, head_dim) + output = _attend(query, key.transpose(1, 2), value.transpose(1, 2), mask).squeeze(2) if len(q) == rows: return output result = torch.zeros_like(q) @@ -316,23 +238,19 @@ def _plan(self, keys, lengths, device): maximum = max(lengths[row] for row in rows) indices = torch.tensor(rows, dtype=torch.long, device=device) positions = torch.arange(maximum, device=device) - valid = positions.unsqueeze(0) < torch.tensor( - [lengths[row] for row in rows], device=device - ).unsqueeze(1) - mask = torch.zeros((len(rows), maximum), dtype=torch.bfloat16, device=device) - mask.masked_fill_(~valid, torch.finfo(torch.bfloat16).min) - initial = len(rows) == maximum and [lengths[row] for row in rows] == list( - range(1, maximum + 1) + group_lengths = [lengths[row] for row in rows] + initial = group_lengths == list(range(1, maximum + 1)) + row_lengths = torch.tensor(group_lengths, device=device) + mask = ( + positions[None, :] < row_lengths[:, None] if not initial and len(rows) > 1 else None ) - row_lengths = torch.tensor([lengths[row] for row in rows], device=device) planned.append((rows[0], indices, positions, mask, initial, row_lengths)) return planned def _forward(self, q, keys, values, tables, groups, *, flat_rows=None, result=None): if result is None: result = torch.zeros_like(q) - block_size, kv_heads, head_dim = keys.shape[-3:] - head_groups = q.shape[1] // kv_heads + block_size = keys.shape[-3] for first, indices, positions, mask, initial, _lengths in groups: blocks = tables[first, positions // block_size] offsets = positions % block_size @@ -341,26 +259,16 @@ def _forward(self, q, keys, values, tables, groups, *, flat_rows=None, result=No else: flat_keys, flat_values, rows_per_block = flat_rows slots = blocks.to(torch.int64) * rows_per_block + offsets - # Padding table entries are -1. CANN gather rejects negative - # indices, and the additive mask already zeroes these columns, - # so clamping only changes data that cannot affect the output. slots = slots.clamp(0, flat_keys.shape[0] - 1) k, v = flat_keys.index_select(0, slots), flat_values.index_select(0, slots) - if head_groups != 1: - k = k.repeat_interleave(head_groups, dim=1) - v = v.repeat_interleave(head_groups, dim=1) - query = q.index_select(0, indices).transpose(0, 1).unsqueeze(0).contiguous() + query = q.index_select(0, indices).transpose(0, 1).unsqueeze(0) key = k.transpose(0, 1).unsqueeze(0) value = v.transpose(0, 1).unsqueeze(0) - # HF's initial V is a projection view; subsequent cache concatenation - # makes K/V contiguous. Preserve that operation layout on this path. - if not initial: - key, value = key.contiguous(), value.contiguous() - scores = torch.matmul(query, key.transpose(-2, -1)) * (1.0 / math.sqrt(q.shape[-1])) - # Keep the BF16 score/scaling and probability rounding boundaries. - scores = scores + mask[None, None, :, :] - probabilities = torch.softmax(scores, dim=-1, dtype=torch.float32).to(q.dtype) - output = torch.matmul(probabilities, value).squeeze(0).transpose(0, 1) + output = ( + _attend(query, key, value, mask, is_causal=initial and indices.numel() > 1) + .squeeze(0) + .transpose(0, 1) + ) result.index_copy_(0, indices, output) return result @@ -368,7 +276,6 @@ def _batched_prefill(self, q, keys, values, batch, flat_rows, geometry): size, width = geometry flat_keys, flat_values, rows_per_block = flat_rows kv_heads, head_dim = keys.shape[-2:] - head_groups = q.shape[1] // kv_heads if self._write_slots is None: self._write_slots = ( batch.write_blocks.to(torch.int64) * rows_per_block @@ -385,31 +292,13 @@ def _batched_prefill(self, q, keys, values, batch, flat_rows, geometry): .reshape(size, width, kv_heads, head_dim) .transpose(1, 2) ) - if head_groups != 1: - key = key.repeat_interleave(head_groups, dim=1) - value = value.repeat_interleave(head_groups, dim=1) - query = q.reshape(size, width, q.shape[1], head_dim).transpose(1, 2).contiguous() - if self._prefill_mask is None: - - def build(): - positions = torch.arange(width, device=q.device) - mask = torch.zeros((width, width), dtype=q.dtype, device=q.device) - mask.masked_fill_(positions[None, :] > positions[:, None], torch.finfo(q.dtype).min) - return (mask,) - - self._prefill_mask = self._layout( - ("prefill", q.device, self._stream_key(q.device), q.dtype, width), build - )[0] - scores = torch.matmul(query, key.transpose(-2, -1)) * (1.0 / math.sqrt(head_dim)) - scores = scores + self._prefill_mask[None, None, :, :] - probabilities = torch.softmax(scores, dim=-1, dtype=torch.float32).to(q.dtype) - return torch.matmul(probabilities, value).transpose(1, 2).reshape_as(q) + query = q.reshape(size, width, q.shape[1], head_dim).transpose(1, 2) + return _attend(query, key, value, is_causal=True).transpose(1, 2).reshape_as(q) def _ragged_decode(self, q, keys, values, tables, groups, flat_rows, result): """One attention call for independent single-row decode groups.""" flat_keys, flat_values, rows_per_block = flat_rows block_size, kv_heads, head_dim = keys.shape[-3:] - head_groups = q.shape[1] // kv_heads if self._ragged_plan is None: indices = torch.cat([group[1] for group in groups]) lengths = torch.cat([group[5] for group in groups]) @@ -424,27 +313,17 @@ def _ragged_decode(self, q, keys, values, tables, groups, flat_rows, result): ) rows = indices.numel() valid = positions.unsqueeze(0) < lengths.unsqueeze(1) - mask = torch.zeros((rows, maximum), dtype=torch.bfloat16, device=q.device) - mask.masked_fill_(~valid, torch.finfo(torch.bfloat16).min) slots = torch.where(valid, slots, slots[:, :1]) - self._ragged_plan = (indices, lengths, maximum, positions, mask, slots, rows) - indices, _lengths, maximum, _positions, mask, slots, rows = self._ragged_plan + self._ragged_plan = (indices, maximum, valid[:, None, None, :], slots, rows) + indices, maximum, mask, slots, rows = self._ragged_plan key = flat_keys.index_select(0, slots.reshape(-1)).reshape( rows, maximum, kv_heads, head_dim ) value = flat_values.index_select(0, slots.reshape(-1)).reshape( rows, maximum, kv_heads, head_dim ) - if head_groups != 1: - key = key.repeat_interleave(head_groups, dim=2) - value = value.repeat_interleave(head_groups, dim=2) query = q.index_select(0, indices).reshape(rows, q.shape[1], 1, head_dim) - key = key.permute(0, 2, 1, 3).contiguous() - value = value.permute(0, 2, 1, 3).contiguous() - scores = torch.matmul(query, key.transpose(-2, -1)) * (1.0 / math.sqrt(head_dim)) - scores = scores + mask[:, None, None, :] - probabilities = torch.softmax(scores, dim=-1, dtype=torch.float32).to(q.dtype) - output = torch.matmul(probabilities, value).squeeze(2) + output = _attend(query, key.transpose(1, 2), value.transpose(1, 2), mask).squeeze(2) result.index_copy_(0, indices, output) return result @@ -464,7 +343,6 @@ def attend_metadata(self, q, keys, values, batch): self._metadata = None self._signature = None self._geometry = _prefill_geometry(batch.rows) - self._prefill_mask = None self._write_slots = None self._ragged_plan = None self._decode_plan = None diff --git a/vllm_rlt/kernels/contiguous_kv.py b/vllm_rlt/kernels/contiguous_kv.py deleted file mode 100644 index b98438a3..00000000 --- a/vllm_rlt/kernels/contiguous_kv.py +++ /dev/null @@ -1,230 +0,0 @@ -"""Owned eager KV in attention layout, with cohort reuse and lossless paged escape. - -Only append-only, uniform private traversals enter this storage. Logical pages -and written coverage remain with KVCacheManager. A generic storage access exports -all live rows before the manager permanently switches back to paged storage. -""" - -import weakref -from dataclasses import dataclass - -import torch - -from vllm_rlt.worker.metadata import UniformSequenceRanges - - -@dataclass(eq=False) -class _Group: - allocations: tuple - pair: tuple - live: int - frontier: object = None - - -@dataclass(frozen=True) -class _AppendPlan: - history: tuple | None - refs: list | None - group: _Group | None - previous_pair: tuple | None - same: bool - epoch: int - - -class ContiguousKVStorage: - def __init__(self, heads, dim, device, *, publish_coverage=None): - self.heads, self.dim, self.device = heads, dim, device - self.publish_coverage = publish_coverage - self.rows = {} - self.groups = {} - self.stream = None - self._metadata = None - self._epoch = None - self._geometry = None - self.writes = self.regroups = self.compactions = 0 - - def _on_owner_stream(self, stream=None): - if self.device.type != "npu": - return True - stream = stream if stream is not None else torch.npu.current_stream(self.device) - if self.stream is None: - self.stream = stream - return stream.stream_id == self.stream.stream_id - - def _context(self, batch, stream=None): - if not self._on_owner_stream(stream): - return None - epoch = batch.owner._allocation_epoch - if self._metadata is not None and self._metadata() is batch and self._epoch == epoch: - return self._geometry - self._metadata = weakref.ref(batch) - self._epoch = epoch - spans = batch.sequence_ranges - self._geometry = None - if isinstance(spans, UniformSequenceRanges): - self._geometry = len(spans), spans.start, spans.query_count, spans.depth - elif spans: - start, count, depth = spans[0].start, spans[0].count, spans[0].depth - if all((s.start, s.count, s.depth) == (start, count, depth) for s in spans): - self._geometry = (len(spans), start, count, depth) - return self._geometry - - def append(self, layer, batch, k, v, *, stream=None): - context = self._context(batch, stream) - if context is None: - return False - return self._append(layer, batch, k, v, context) - - def prepare_append(self, layer, batch, context): - size, start, count, depth = context - spans = batch.sequence_ranges - slot = depth, layer - group = self.groups.get(slot) - same = group is not None and group.allocations is batch.allocations - history = None - refs = None - if same: - if group.pair[0].shape[2] != start: - return None # Overwrite/hole semantics use paged storage. - history = group.pair - else: - refs = [self.rows.get((id(s.allocation), depth, layer)) for s in spans] - if start == 0: - if any(r is not None for r in refs): - return None - if any(s.allocation.written[depth][layer].prefix for s in spans): - return None - else: - if any(r is None or r[0].pair[0].shape[2] != start for r in refs): - return None - # Rebatch only the requested live rows. The usual stable cohort - # takes the identity fast path above and needs no row lookups. - history = tuple( - torch.cat([r[0].pair[i][r[1] : r[1] + 1] for r in refs], dim=0) - for i in range(2) - ) - if self.publish_coverage is not None and not same: - if any( - s.allocation.written[depth][layer].prefix != start - or s.allocation.written[depth][layer].pending - for s in spans - ): - return None - return _AppendPlan( - history, refs, group, group.pair if group else None, same, batch.owner._allocation_epoch - ) - - def _append(self, layer, batch, k, v, context): - plan = self.prepare_append(layer, batch, context) - if plan is None: - return False - size, _, count, _ = context - incoming = tuple( - x.reshape(size, count, self.heads, self.dim).transpose(1, 2) for x in (k, v) - ) - pair = ( - tuple(x.clone() for x in incoming) - if plan.history is None - else tuple(torch.cat((old, new), dim=2) for old, new in zip(plan.history, incoming)) - ) - self.commit_append(layer, batch, context, plan, pair) - return True - - def commit_append(self, layer, batch, context, plan, pair): - size, start, count, depth = context - slot = depth, layer - group = self.groups.get(slot) - if ( - batch.owner._allocation_epoch != plan.epoch - or group is not plan.group - or (group is not None and group.pair is not plan.previous_pair) - ): - raise RuntimeError("stale prepared contiguous KV append") - same, refs = plan.same, plan.refs - spans = batch.sequence_ranges - # Publish after BOTH copies enqueue. Public write inputs may be mutated, - # and returned attention outputs never alias these storage tensors. - if same: - group.pair = pair - else: - group = _Group(batch.allocations, pair, size) - displaced = set() - for index, (span, previous) in enumerate(zip(spans, refs)): - if previous is not None: - previous[0].live -= 1 - displaced.add(previous[0]) - self.rows[id(span.allocation), depth, layer] = group, index - self.groups[slot] = group - for old in displaced: - self._compact(old, depth, layer) - self.regroups += 1 - if self.publish_coverage is not None: - group.frontier = self.publish_coverage( - spans, depth, layer, start + count, group.frontier if same else None - ) - self.writes += 1 - - def _compact(self, group, depth, layer): - if not group.live or group.live == group.pair[0].shape[0]: - return - refs = [] - for _, allocation in group.allocations: - ref = self.rows.get((id(allocation), depth, layer)) - if ref is not None and ref[0] is group: - refs.append((allocation, ref[1])) - indices = torch.tensor([i for _, i in refs], device=self.device, dtype=torch.long) - group.pair = tuple(x.index_select(0, indices) for x in group.pair) - for i, (allocation, _) in enumerate(refs): - self.rows[id(allocation), depth, layer] = group, i - self.compactions += 1 - - def get(self, layer, batch, *, stream=None): - context = self._context(batch, stream) - if context is None: - return None - _, start, count, depth = context - group = self.groups.get((depth, layer)) - if group is None or group.allocations is not batch.allocations: - return None - if group.pair[0].shape[2] != start + count: - return None - return group.pair - - def release(self, allocation, layers, planes): - for depth in range(planes): - for layer in range(layers): - ref = self.rows.pop((id(allocation), depth, layer), None) - if ref is not None: - group = ref[0] - group.live -= 1 - if not group.live and self.groups.get((depth, layer)) is group: - del self.groups[depth, layer] - - def export(self, keys, values, block_size): - # Materialization may be requested on a different stream by a generic - # caller. Order after the producer and protect allocator reuse there. - stream = None - if self.device.type == "npu" and self.stream is not None: - stream = torch.npu.current_stream(self.device) - if stream.stream_id != self.stream.stream_id: - stream.wait_stream(self.stream) - allocations = { - id(a): a - for group in {ref[0] for ref in self.rows.values()} - for _, a in group.allocations - } - addresses = {} - for (identity, depth, layer), (group, row) in self.rows.items(): - length = group.pair[0].shape[2] - address_key = identity, depth, length - if address_key not in addresses: - positions = torch.arange(length, device=self.device) - table = torch.tensor(allocations[identity].block_tables[depth], device=self.device) - addresses[address_key] = table[positions // block_size], positions % block_size - blocks, offsets = addresses[address_key] - for destination, source in zip((keys, values), group.pair): - if stream is not None and stream.stream_id != self.stream.stream_id: - source.record_stream(stream) - destination[blocks, layer, offsets] = source[row].transpose(0, 1) - self.rows.clear() - self.groups.clear() diff --git a/vllm_rlt/models/ouro.py b/vllm_rlt/models/ouro.py index 5041fdf3..7b929c54 100644 --- a/vllm_rlt/models/ouro.py +++ b/vllm_rlt/models/ouro.py @@ -281,9 +281,55 @@ def recurrent( batch = cache._prepare_batch(request_ids, depths, positions) return self.recurrent_prepared(hidden, batch, cache, compute_gate=compute_gate) - def recurrent_prepared(self, hidden, batch, cache, *, compute_gate=True): + @property + def recurrent_returns_owned_hidden(self): + if type(self) is not OuroForCausalLM or "recurrent_prepared" in self.__dict__: + return False + norm = self.model.norm + return ( + type(norm) is RMSNorm + and not norm._forward_hooks + and not norm._forward_pre_hooks + and "forward" not in norm.__dict__ + ) + + def prepare_recurrent(self): + """Keep RoPE reuse local to one forward, across loops with the same position view.""" + recurrent = self.recurrent_prepared + if type(self) is not OuroForCausalLM or "recurrent_prepared" in self.__dict__: + return recurrent + position_key = position_embeddings = None + + def run(hidden, batch, cache, *, compute_gate): + nonlocal position_key, position_embeddings + positions = batch.position_ids + key = ( + positions.data_ptr(), + positions.shape, + positions.stride(), + positions.dtype, + positions.device, + hidden.dtype, + ) + if key != position_key: + position_embeddings = self.model.rotary_emb(hidden, positions) + position_key = key + return recurrent( + hidden, + batch, + cache, + compute_gate=compute_gate, + position_embeddings=position_embeddings, + ) + + return run + + def recurrent_prepared( + self, hidden, batch, cache, *, compute_gate=True, position_embeddings=None + ): """Run core with runner-owned metadata, including inactive padding rows.""" - position_embeddings = self.model.rotary_emb(hidden, batch.position_ids) + if position_embeddings is None: + position_embeddings = self.model.rotary_emb(hidden, batch.position_ids) for layer in self.model.layers: hidden = layer(hidden, position_embeddings, batch, cache) # Norm is inside the recurrence in Ouro; this normalized state is the diff --git a/vllm_rlt/worker/ascend.py b/vllm_rlt/worker/ascend.py deleted file mode 100644 index 0ef46d8e..00000000 --- a/vllm_rlt/worker/ascend.py +++ /dev/null @@ -1,483 +0,0 @@ -"""Optional host executor; every model operation remains ordinary eager ATen.""" - -import hashlib -import os -import sysconfig -import weakref -from collections import Counter -from collections.abc import Generator -from dataclasses import dataclass -from functools import lru_cache -from pathlib import Path - -import torch -from torch.nn.modules import module as module_hooks - -from vllm_rlt.layers.normalization import RMSNorm -from vllm_rlt.request import Operation -from vllm_rlt.worker.executor import ForwardResult, ModelRuntime -from vllm_rlt.worker.model_runner import ModelRunner -from vllm_rlt.worker.sampler import Sampler - - -@lru_cache(maxsize=1) -def load_native_plan(): - from torch.utils.cpp_extension import load - - source = Path(__file__).parent / "csrc" / "ascend_eager_plan.cpp" - identity = ( - source.read_bytes() - + str(torch.__version__).encode() - + str(torch._C._GLIBCXX_USE_CXX11_ABI).encode() - + str(sysconfig.get_config_var("SOABI")).encode() - ) - digest = hashlib.sha256(identity).hexdigest()[:12] - root = os.environ.get( - "VLLM_RLT_NATIVE_BUILD_DIR", str(Path.home() / ".cache/vllm-rlt/native-eager") - ) - build = Path(root) / digest - build.mkdir(parents=True, exist_ok=True) - return load( - name="rlt_ascend_eager_" + digest, - sources=[str(source)], - build_directory=str(build), - extra_cflags=["-O3", "-g0"], - with_cuda=False, - verbose=True, - ) - - -class NativeOuroExecutor: - """Eager layer calls with current weights and conservative Python fallback. - - The extension stores no model tensors. Each call borrows the module's live - parameters, and returns owned output/KV tensors. KV is published through its - traversal owner only after a complete layer call has enqueued successfully. - """ - - def __init__(self): - self.extension = load_native_plan() - self.native_layers = 0 - self.fallbacks = Counter() - - def _fallback(self, reason): - self.fallbacks[reason] += 1 - return None - - def layer(self, layer, hidden, position_embeddings, batch, traversal): - from vllm_rlt.models.ouro import OuroAttention, OuroDecoderLayer, OuroMLP - - if ( - type(layer) is not OuroDecoderLayer - or type(layer.self_attn) is not OuroAttention - or type(layer.mlp) is not OuroMLP - ): - return self._fallback("module_schema") - if type(hidden) is not torch.Tensor or torch._C._get_tracing_state(): - return self._fallback("tensor_or_tracing") - if any( - getattr(module_hooks, name, {}) - for name in ( - "_global_forward_hooks", - "_global_forward_pre_hooks", - "_global_backward_hooks", - "_global_backward_pre_hooks", - ) - ): - return self._fallback("global_hooks") - if ( - layer._forward_hooks - or layer._forward_pre_hooks - or layer._backward_hooks - or layer._backward_pre_hooks - or "forward" in layer.__dict__ - or getattr(layer, "_compiled_call_impl", None) is not None - ): - return self._fallback("layer_hooks_or_forward") - # Hooked layers run through their ordinary nn.Module call. Child callbacks, - # custom forwards and compiled children retain the Python implementation. - for child in layer.modules(): - if child is layer: - continue - if ( - child._forward_hooks - or child._forward_pre_hooks - or child._backward_hooks - or child._backward_pre_hooks - or "forward" in child.__dict__ - or getattr(child, "_compiled_call_impl", None) is not None - ): - return self._fallback("child_hooks_or_forward") - a, m = layer.self_attn, layer.mlp - norms = ( - layer.input_layernorm, - layer.input_layernorm_2, - layer.post_attention_layernorm, - layer.post_attention_layernorm_2, - ) - linears = (a.q_proj, a.k_proj, a.v_proj, a.o_proj, m.gate_proj, m.up_proj, m.down_proj) - if any(type(x) is not RMSNorm for x in norms) or any( - type(x) is not torch.nn.Linear for x in linears - ): - return self._fallback("leaf_schema") - if any(x.bias is not None for x in linears): - return self._fallback("linear_bias") - if any(x.variance_epsilon != norms[0].variance_epsilon for x in norms): - return self._fallback("norm_epsilon") - weights = [x.weight for x in (*norms, *linears)] - if any(type(x) is not torch.nn.Parameter for x in weights): - return self._fallback("parameter_schema") - prepare = getattr(traversal, "prepare_native", None) - if prepare is None: - return self._fallback("paged_storage") - inputs = prepare(a.layer_idx, hidden) - if inputs is None: - return self._fallback("append_geometry") - if a.config.head_dim != inputs.head_dim: - return self._fallback("head_geometry") - output, key, value = self.extension.layer_forward( - hidden, - *position_embeddings, - weights, - [] if inputs.history is None else inputs.history, - inputs.mask, - inputs.batch_size, - inputs.query_tokens, - inputs.token_start, - inputs.head_dim, - inputs.kv_heads, - norms[0].variance_epsilon, - ) - traversal.commit_native(a.layer_idx, inputs, (key, value)) - self.native_layers += 1 - return output - - -class AscendSampler(Sampler): - """Keep batched greedy dispatch out of the CUDA/ROCm sampling path.""" - - def sample_batch(self, logits, params, generators): - if getattr(self.sample, "__func__", None) is Sampler.sample and all( - item.temperature == 0 for item in params - ): - return logits[: len(params)].argmax(dim=-1) - return torch.stack( - [ - self.sample(row, item, generator) - for row, item, generator in zip(logits, params, generators) - ] - ) - - -# The shared runtime and model keep their upstream implementation. Only an NPU -# runner creates this adapter; it owns preparation reused within one forward. -class AscendOuroRuntime(ModelRuntime): - def __init__(self, model, cache, *, native=False): - super().__init__(model, cache) - self.layer_executor = NativeOuroExecutor() if native else None - - @property - def owns_hidden(self): - from vllm_rlt.models.ouro import OuroForCausalLM - - if type(self.model) is not OuroForCausalLM: - return False - norm = self.model.model.norm - return ( - "recurrent_prepared" not in self.model.__dict__ - and type(norm) is RMSNorm - and not norm._forward_hooks - and not norm._forward_pre_hooks - and "forward" not in norm.__dict__ - ) - - def forward(self, batch): - from vllm_rlt.models.ouro import OuroForCausalLM - - if ( - type(self.model) is not OuroForCausalLM - or "recurrent_prepared" in self.model.__dict__ - or batch.graphs is not None - or batch.operation not in (Operation.PRELUDE, Operation.RECURRENT) - ): - return super().forward(batch) - hidden, gate = batch.hidden, None - try: - if batch.operation == Operation.PRELUDE: - if batch.tokens is None: - raise ValueError("PRELUDE requires token IDs") - hidden = self.model.prelude(batch.tokens) - if hidden is None: - raise ValueError("RECURRENT requires hidden state") - position_key = position_embeddings = None - for kv in batch.metadata: - positions = kv.position_ids - key = ( - positions.data_ptr(), - positions.shape, - positions.stride(), - positions.dtype, - positions.device, - hidden.dtype, - ) - if key != position_key: - position_embeddings = self.model.model.rotary_emb(hidden, positions) - position_key = key - traversal = self.cache._begin_traversal(kv) - for layer in self.model.model.layers: - output = ( - None - if self.layer_executor is None - else self.layer_executor.layer( - layer, hidden, position_embeddings, kv, traversal - ) - ) - hidden = ( - layer(hidden, position_embeddings, kv, traversal) - if output is None - else output - ) - hidden = self.model.model.norm(hidden) - gate = ( - self.model.model.early_exit_gate(hidden).squeeze(-1) - if batch.compute_gate - else None - ) - if traversal is not self.cache: - traversal.finish() - return ForwardResult(hidden, gate) - finally: - if isinstance(batch.metadata, Generator): - batch.metadata.close() - - -@dataclass(frozen=True) -class HiddenBatch: - """Immutable eager output; Request.hidden_state remains authoritative. - - Row views retain their owning allocation. Weak request identities and exact - view identities prevent reuse after cancellation, replacement or regrouping. - Only the latest batch is retained; this is not a second request-state store. - """ - - hidden: torch.Tensor - requests: tuple[weakref.ReferenceType, ...] - rows: tuple[torch.Tensor, ...] - - def matches(self, requests): - return len(requests) == len(self.rows) and all( - owner() is request and request.hidden_state is row - for owner, request, row in zip(self.requests, requests, self.rows) - ) - - -@dataclass(frozen=True) -class TokenBatch: - tokens: torch.Tensor - requests: tuple - rows: tuple - - def matches(self, requests, sources): - return len(requests) == len(self.rows) and all( - owner() is request and source is row - for owner, request, source, row in zip(self.requests, requests, sources, self.rows) - ) - - -class AscendModelRunner(ModelRunner): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - if self.device.type != "npu": - raise ValueError("AscendModelRunner requires an NPU") - self.runtime = AscendOuroRuntime( - self.model, self.cache_manager, native=self.execution_config.native_eager_plan - ) - self.sampler = AscendSampler() - self.hidden_batch = self.token_batch = None - self.retain_hidden_batch = ( - self.execution_config.async_scheduling - and self.cache_manager.backend == "ascend_eager" - and not self.execution_config.static_buffers - and not self.execution_config.cuda_graphs - ) - - def _save(self, request, state): - self.hidden_batch = None - return super()._save(request, state) - - def _save_batch(self, requests, hidden, routing): - if ( - self.retain_hidden_batch - and self.runtime.owns_hidden - and routing is None - and len(hidden) == len(requests) - ): - rows = tuple(hidden.unbind(0)) - for request, state in zip(requests, rows): - request.hidden_state = state - self.hidden_batch = HiddenBatch( - hidden, tuple(weakref.ref(request) for request in requests), rows - ) - return - self.hidden_batch = None - if routing is None: - for request, state in zip(requests, hidden): - self._save(request, state) - return - routing.scatter(hidden) - for request in requests: - request.hidden_state = self.states[self.state_slots[request.request_id]] - - def _gather(self, requests, workspace, size): - saved = self.hidden_batch - if ( - saved is not None - and workspace is None - and size == len(requests) - and saved.matches(requests) - ): - if self.execution_config.multi_stream: - # The next submission can replace all Request row views before - # this stream has consumed the previous allocation. - saved.hidden.record_stream(self.device_ops.current_stream(self.device)) - return saved.hidden - states = [request.hidden_state for request in requests] - if workspace is None: - hidden = torch.stack(states) - else: - hidden = workspace.hidden[:size] - hidden.zero_() - for row, state in enumerate(states): - hidden[row].copy_(state) - if hidden.device.type == "npu" and self.execution_config.multi_stream: - # Request states can be replaced before this stream consumes them. - # NPU uses these tensors instead of CUDA's persistent state slots. - stream = self.device_ops.current_stream(hidden.device) - for state in states: - state.record_stream(stream) - return hidden - - def _prepare_tokens(self, prepared, size): - """Materialize snapshotted token sources on the current compute stream.""" - sources = prepared.token_sources - workspace = prepared.workspace - borrowed = prepared.token_ids - stream = ( - self.device_ops.current_stream(self.device) if self.device_ops is not None else None - ) - if borrowed is not None and prepared.routing is None and size == len(borrowed): - tokens = borrowed - tokens.record_stream(stream) - if self.token_batch is not None and self.token_batch.tokens is tokens: - self.token_batch = None - elif prepared.routing is not None: - tokens = prepared.routing.gather(tokens=True) - elif all(isinstance(source, torch.Tensor) for source in sources): - tokens = torch.stack(sources) - elif all(isinstance(source, tuple) and len(source) == 1 for source in sources): - ids = [source[0] for source in sources] - tokens = ( - workspace.tokens(ids, size) - if workspace is not None - else torch.tensor(ids, device=self.device, dtype=torch.long) - ) - else: - tokens = torch.cat( - [ - source.reshape(-1) - if isinstance(source, torch.Tensor) - else torch.tensor(source, device=self.device, dtype=torch.long) - for source in sources - ] - ) - if prepared.routing is None and size > len(tokens): - tokens = torch.cat((tokens, tokens.new_zeros(size - len(tokens)))) - for span, source in zip(prepared.batch.spans, sources): - if ( - tokens is not borrowed - and isinstance(source, torch.Tensor) - and source.device.type in ("cuda", "npu") - ): - source.record_stream(stream) - if span.request.input_token_tensor is source: - span.request.input_token_tensor = None - return tokens - - def _wait_for_requests(self, batch, stream): - # A previous batch often gives many requests the same completion event. - dependencies = set() - saved = self.hidden_batch - retained = saved is not None and saved.matches([span.request for span in batch.spans]) - if retained: - saved.hidden.record_stream(stream) - for span in batch.spans: - event = self.events.get(span.request.request_id) - if event is not None and event not in dependencies: - stream.wait_event(event) - dependencies.add(event) - if not retained and span.request.hidden_state is not None: - span.request.hidden_state.record_stream(stream) - - def _retain_device_tokens(self, batch, values, rows): - self.token_batch = None - if ( - self.device.type != "npu" - or self.cache_manager._contiguous is None - or not self.retain_hidden_batch - or len(rows) != len(batch.requests) - or list(rows) != list(range(len(batch.requests))) - ): - return - requests = [item.request for item in batch.requests] - self.token_batch = TokenBatch( - values, - tuple(weakref.ref(r) for r in requests), - tuple(r.input_token_tensor for r in requests), - ) - - def _finalize_npu_batch(self, requests): - cache = self.cache_manager - copies = [] - for request in requests: - depth = request.loops_done - 1 - if cache.layout == "shared" or depth + 1 == cache.max_loops: - # This still verifies every required layer. With no destination - # there is no device work: the existing core event is sufficient. - cache.finalize_token(request.request_id, request.position, depth) - else: - copies.append(request) - if not copies: - return - stream = self.boundary_stream - with self.device_ops.stream(stream): - try: - dependencies = { - self.events[request.request_id] - for request in copies - if request.request_id in self.events - } - for event in dependencies: - stream.wait_event(event) - for request in copies: - cache.finalize_token( - request.request_id, request.position, request.loops_done - 1 - ) - done = self.device_ops.Event() - done.record(stream) - except Exception: - # A later request can fail after earlier copies were enqueued. - # Finish those readers before cleanup may recycle their pages. - stream.synchronize() - raise - for request in copies: - self.events[request.request_id] = done - - def finalize_many(self, requests): - if self.execution_config.async_scheduling and self.cache_manager.backend == "ascend_eager": - self._finalize_npu_batch(requests) - else: - super().finalize_many(requests) - - def _free_runner_slot(self, request_id): - self.hidden_batch = self.token_batch = None - return super()._free_runner_slot(request_id) diff --git a/vllm_rlt/worker/ascend_model_runner.py b/vllm_rlt/worker/ascend_model_runner.py new file mode 100644 index 00000000..097b5981 --- /dev/null +++ b/vllm_rlt/worker/ascend_model_runner.py @@ -0,0 +1,220 @@ +"""Ascend execution with device-resident state, batched sampling and KV finalization.""" + +import weakref +from dataclasses import dataclass + +import torch + +from vllm_rlt.worker.executor import ModelRuntime +from vllm_rlt.worker.model_runner import ModelRunner +from vllm_rlt.worker.sampler import Sampler + + +class AscendSampler(Sampler): + """Keep batched greedy dispatch out of the CUDA/ROCm sampling path.""" + + def sample_batch(self, logits, params, generators): + if getattr(self.sample, "__func__", None) is Sampler.sample and all( + item.temperature == 0 for item in params + ): + return logits[: len(params)].argmax(dim=-1) + return torch.stack( + [ + self.sample(row, item, generator) + for row, item, generator in zip(logits, params, generators) + ] + ) + + +@dataclass(frozen=True) +class HiddenBatch: + """Immutable eager output; Request.hidden_state remains authoritative. + + Row views retain their owning allocation. Weak request identities and exact + view identities prevent reuse after cancellation, replacement or regrouping. + Only the latest batch is retained; this is not a second request-state store. + """ + + hidden: torch.Tensor + requests: tuple[weakref.ReferenceType, ...] + rows: tuple[torch.Tensor, ...] + + def matches(self, requests): + return len(requests) == len(self.rows) and all( + owner() is request and request.hidden_state is row + for owner, request, row in zip(self.requests, requests, self.rows) + ) + + +class AscendModelRunner(ModelRunner): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + if self.device.type != "npu": + raise ValueError("AscendModelRunner requires an NPU") + self.runtime = ModelRuntime( + self.model, + self.cache_manager, + prepare_recurrent=getattr(self.model, "prepare_recurrent", None), + ) + self.sampler = AscendSampler() + self.hidden_batch = None + self.retain_hidden_batch = ( + self.execution_config.async_scheduling + and self.cache_manager.backend == "ascend_eager" + and not self.execution_config.static_buffers + and not self.execution_config.cuda_graphs + ) + + def _save(self, request, state): + self.hidden_batch = None + return super()._save(request, state) + + def _save_batch(self, requests, hidden, routing): + if ( + self.retain_hidden_batch + and getattr(self.model, "recurrent_returns_owned_hidden", False) + and routing is None + and len(hidden) == len(requests) + ): + rows = tuple(hidden.unbind(0)) + for request, state in zip(requests, rows): + request.hidden_state = state + self.hidden_batch = HiddenBatch( + hidden, tuple(weakref.ref(request) for request in requests), rows + ) + return + self.hidden_batch = None + if routing is None: + for request, state in zip(requests, hidden): + self._save(request, state) + return + routing.scatter(hidden) + for request in requests: + request.hidden_state = self.states[self.state_slots[request.request_id]] + + def _gather(self, requests, workspace, size): + saved = self.hidden_batch + if ( + saved is not None + and workspace is None + and size == len(requests) + and saved.matches(requests) + ): + if self.execution_config.multi_stream: + # The next submission can replace all Request row views before + # this stream has consumed the previous allocation. + saved.hidden.record_stream(self.device_ops.current_stream(self.device)) + return saved.hidden + states = [request.hidden_state for request in requests] + if workspace is None: + hidden = torch.stack(states) + else: + hidden = workspace.hidden[:size] + hidden.zero_() + for row, state in enumerate(states): + hidden[row].copy_(state) + if hidden.device.type == "npu" and self.execution_config.multi_stream: + # Request states can be replaced before this stream consumes them. + # NPU uses these tensors instead of CUDA's persistent state slots. + stream = self.device_ops.current_stream(hidden.device) + for state in states: + state.record_stream(stream) + return hidden + + def _prepare_tokens(self, prepared, size): + """Materialize snapshotted token sources on the current compute stream.""" + sources = prepared.token_sources + workspace = prepared.workspace + stream = ( + self.device_ops.current_stream(self.device) if self.device_ops is not None else None + ) + if prepared.routing is not None: + tokens = prepared.routing.gather(tokens=True) + elif all(isinstance(source, torch.Tensor) for source in sources): + tokens = torch.stack(sources) + elif all(isinstance(source, tuple) and len(source) == 1 for source in sources): + ids = [source[0] for source in sources] + tokens = ( + workspace.tokens(ids, size) + if workspace is not None + else torch.tensor(ids, device=self.device, dtype=torch.long) + ) + else: + tokens = torch.cat( + [ + source.reshape(-1) + if isinstance(source, torch.Tensor) + else torch.tensor(source, device=self.device, dtype=torch.long) + for source in sources + ] + ) + if prepared.routing is None and size > len(tokens): + tokens = torch.cat((tokens, tokens.new_zeros(size - len(tokens)))) + for span, source in zip(prepared.batch.spans, sources): + if isinstance(source, torch.Tensor) and source.device.type in ("cuda", "npu"): + source.record_stream(stream) + if span.request.input_token_tensor is source: + span.request.input_token_tensor = None + return tokens + + def _wait_for_requests(self, batch, stream): + # A previous batch often gives many requests the same completion event. + dependencies = set() + saved = self.hidden_batch + retained = saved is not None and saved.matches([span.request for span in batch.spans]) + if retained: + saved.hidden.record_stream(stream) + for span in batch.spans: + event = self.events.get(span.request.request_id) + if event is not None and event not in dependencies: + stream.wait_event(event) + dependencies.add(event) + if not retained and span.request.hidden_state is not None: + span.request.hidden_state.record_stream(stream) + + def _finalize_npu_batch(self, requests): + cache = self.cache_manager + copies = [] + for request in requests: + depth = request.loops_done - 1 + if cache.layout == "shared" or depth + 1 == cache.max_loops: + # This still verifies every required layer. With no destination + # there is no device work: the existing core event is sufficient. + cache.finalize_token(request.request_id, request.position, depth) + else: + copies.append(request) + if not copies: + return + stream = self.boundary_stream + with self.device_ops.stream(stream): + try: + dependencies = { + self.events[request.request_id] + for request in copies + if request.request_id in self.events + } + for event in dependencies: + stream.wait_event(event) + for request in copies: + cache.finalize_token( + request.request_id, request.position, request.loops_done - 1 + ) + done = self.device_ops.Event() + done.record(stream) + except Exception: + # A later request can fail after earlier copies were enqueued. + # Finish those readers before cleanup may recycle their pages. + stream.synchronize() + raise + for request in copies: + self.events[request.request_id] = done + + def finalize_many(self, requests): + if self.execution_config.async_scheduling and self.cache_manager.backend == "ascend_eager": + self._finalize_npu_batch(requests) + else: + super().finalize_many(requests) + + def _free_runner_slot(self, request_id): + self.hidden_batch = None + return super()._free_runner_slot(request_id) diff --git a/vllm_rlt/worker/csrc/ascend_eager_plan.cpp b/vllm_rlt/worker/csrc/ascend_eager_plan.cpp deleted file mode 100644 index 8a84c89c..00000000 --- a/vllm_rlt/worker/csrc/ascend_eager_plan.cpp +++ /dev/null @@ -1,92 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// Host-side eager execution only: every operation below dispatches separately -// through ATen to the existing Torch-NPU implementation. No graph or fusion. -#include -#include -#include -#include - -using at::Tensor; - -static Tensor rms(const Tensor& x, const Tensor& weight, double eps) { - auto value = x.to(at::kFloat); - auto variance = value.pow(2).mean(-1, true); - value = value * at::rsqrt(variance + eps); - return weight * value.to(x.scalar_type()); -} - -static Tensor rotate_half(const Tensor& x) { - auto halves = x.chunk(2, -1); - return at::cat({-halves[1], halves[0]}, -1); -} - -std::vector layer_forward( - const Tensor& hidden, const Tensor& cosine, const Tensor& sine, - const std::vector& weights, const std::vector& history, - const Tensor& mask, int64_t rows, int64_t width, int64_t start, - int64_t head_dim, int64_t kv_heads, double eps) { - c10::InferenceMode guard(true); - TORCH_CHECK(weights.size() == 11, "Ouro layer plan requires eleven weights"); - TORCH_CHECK(rows > 0 && width > 0 && start >= 0 && hidden.size(0) == rows * width, - "invalid eager traversal geometry"); - TORCH_CHECK(history.size() == (start ? 2 : 0), "invalid eager KV history"); - auto normalized = rms(hidden, weights[0], eps); - auto q = at::linear(normalized, weights[4]).view({rows * width, -1, head_dim}); - auto k = at::linear(normalized, weights[5]).view({rows * width, -1, head_dim}); - auto v = at::linear(normalized, weights[6]).view({rows * width, -1, head_dim}); - q = q * cosine + rotate_half(q) * sine; - k = k * cosine + rotate_half(k) * sine; - auto incoming_k = k.reshape({rows, width, kv_heads, head_dim}).transpose(1, 2); - auto incoming_v = v.reshape({rows, width, kv_heads, head_dim}).transpose(1, 2); - Tensor key, value; - if (!start) { - key = incoming_k.clone(); - value = incoming_v.clone(); - } else { - key = at::cat({history[0], incoming_k}, 2); - value = at::cat({history[1], incoming_v}, 2); - } - // Published cache tensors own their bytes; attention's head expansion is - // temporary and must not replace the original KV-head representation. - auto stored_key = key; - auto stored_value = value; - auto groups = q.size(1) / kv_heads; - TORCH_CHECK(groups > 0 && q.size(1) % kv_heads == 0, "invalid grouped heads"); - if (groups != 1) { - key = at::repeat_interleave(key, groups, 1); - value = at::repeat_interleave(value, groups, 1); - } - auto query = width == 1 - ? q.contiguous().reshape({rows, q.size(1), 1, head_dim}) - : q.reshape({rows, width, q.size(1), head_dim}).transpose(1, 2).contiguous(); - auto scores = at::matmul(query, key.transpose(-2, -1)) * (1.0 / std::sqrt(double(head_dim))); - scores = scores + mask.unsqueeze(0).unsqueeze(0); - auto probabilities = at::softmax(scores, -1, at::kFloat).to(q.scalar_type()); - Tensor attention; - if (start && width > 1 && rows > 1) { - std::vector outputs; - outputs.reserve(rows); - for (int64_t i = 0; i < rows; ++i) { - outputs.push_back(at::matmul(probabilities.slice(0, i, i + 1), - value.slice(0, i, i + 1))); - } - attention = at::cat(outputs, 0); - } else { - attention = at::matmul(probabilities, value); - } - attention = width == 1 ? attention.squeeze(2) - : attention.transpose(1, 2).reshape({rows * width, q.size(1), head_dim}); - attention = at::linear(attention.reshape({rows * width, -1}), weights[7]); - auto state = hidden + rms(attention, weights[1], eps); - auto mlp_input = rms(state, weights[2], eps); - auto gate = at::linear(mlp_input, weights[8]); - auto up = at::linear(mlp_input, weights[9]); - auto mlp = at::linear(at::silu(gate) * up, weights[10]); - auto output = state + rms(mlp, weights[3], eps); - return {output, stored_key, stored_value}; -} - -PYBIND11_MODULE(TORCH_EXTENSION_NAME, module) { - module.def("layer_forward", &layer_forward, - pybind11::call_guard()); -} diff --git a/vllm_rlt/worker/executor.py b/vllm_rlt/worker/executor.py index 70404371..24cce2c0 100644 --- a/vllm_rlt/worker/executor.py +++ b/vllm_rlt/worker/executor.py @@ -139,9 +139,10 @@ class ForwardResult: class ModelRuntime: - def __init__(self, model, cache): + def __init__(self, model, cache, *, prepare_recurrent=None): self.model = model self.cache = cache + self.prepare_recurrent = prepare_recurrent def prepare(self, batch: ForwardBatch) -> ForwardBatch: """Choose graph/fallback for every call before execution. @@ -209,6 +210,7 @@ def forward(self, batch: ForwardBatch) -> ForwardResult: raise ValueError(f"unsupported operation {batch.operation}") if hidden is None: raise ValueError("RECURRENT requires hidden state") + recurrent = None for index, kv in enumerate(batch.metadata): if batch.graphs is not None: call = batch.graph_calls[index] @@ -216,7 +218,13 @@ def forward(self, batch: ForwardBatch) -> ForwardResult: batch.graphs.prepare_inputs(call, hidden) hidden, gate = batch.graphs.execute(call, hidden) else: - hidden, gate = self.model.recurrent_prepared( + if recurrent is None: + recurrent = ( + self.prepare_recurrent() + if self.prepare_recurrent is not None + else self.model.recurrent_prepared + ) + hidden, gate = recurrent( hidden, kv, self.cache, compute_gate=batch.compute_gate ) return ForwardResult(hidden, gate) diff --git a/vllm_rlt/worker/metadata.py b/vllm_rlt/worker/metadata.py index 70dfdf69..7c65cde3 100644 --- a/vllm_rlt/worker/metadata.py +++ b/vllm_rlt/worker/metadata.py @@ -6,10 +6,8 @@ from __future__ import annotations -import weakref -from collections import OrderedDict from collections.abc import Sequence -from dataclasses import dataclass, field, replace +from dataclasses import dataclass, replace from typing import TYPE_CHECKING import numpy as np @@ -44,9 +42,8 @@ class KVMetadata: max_seqlen_q: int = 1 # Compact plans use one table per sequence, not one duplicated table per # query. Only a backend declaring sequence_ranges may consume this form. - sequence_ranges: tuple[SequenceRange, ...] | UniformSequenceRanges | None = None + sequence_ranges: tuple[SequenceRange, ...] | None = None table_plan: RangeTablePlan | None = None - deferred_addresses: bool = False @dataclass(frozen=True) @@ -61,50 +58,12 @@ def end(self): return self.start + self.count -@dataclass(frozen=True, eq=False) -class UniformSequenceRanges(Sequence): - """One owned cohort and scalar geometry; expand row objects only on demand.""" - - allocations: tuple - depth: int - start: int - query_count: int - _rows: object = field(default=None, init=False, compare=False, repr=False) - - def __len__(self): - return len(self.allocations) - - def __getitem__(self, index): - if isinstance(index, slice): - return tuple(self)[index] - if self._rows is not None: - return self._rows[index] - _, allocation = self.allocations[index] - return SequenceRange(allocation, self.depth, self.start, self.query_count) - - def __iter__(self): - if self._rows is None: - object.__setattr__( - self, - "_rows", - tuple( - SequenceRange(a, self.depth, self.start, self.query_count) - for _, a in self.allocations - ), - ) - return iter(self._rows) - - class RangeRows(Sequence): """Immutable lazy compatibility view; fast consumers use the ranges.""" def __init__(self, ranges): - self.ranges = ranges if isinstance(ranges, UniformSequenceRanges) else tuple(ranges) - self.width = ( - ranges.query_count - if isinstance(ranges, UniformSequenceRanges) - else self.ranges[0].count - ) + self.ranges = ranges + self.width = ranges[0].count def __len__(self): return len(self.ranges) * self.width @@ -128,8 +87,7 @@ class RangeTablePlan: another stream. Position/write tensors remain submission-owned snapshots. """ - def __init__(self, tables, *, max_context_tokens=None): - self.max_context_tokens = max_context_tokens + def __init__(self, tables): width = max(map(len, tables)) self.host = np.full((len(tables), width), -1, dtype=np.int32) for i, table in enumerate(tables): @@ -152,7 +110,7 @@ def bind(self, device): @dataclass(frozen=True) class RangeLayout: owner: KVCacheManager - ranges: tuple[SequenceRange, ...] | UniformSequenceRanges + ranges: tuple[SequenceRange, ...] allocations: tuple positions: np.ndarray offsets: np.ndarray @@ -161,29 +119,9 @@ class RangeLayout: writable: bool -def stage_range_metadata(layouts, stage, *, defer_addresses=False): +def stage_range_metadata(layouts, stage): """Snapshot changing indices; stable page tables stay with their plan.""" first = layouts[0] - if defer_addresses: - if not all(isinstance(x.ranges, UniformSequenceRanges) for x in layouts): - raise ValueError("deferred addresses require uniform owned KV ranges") - return tuple( - KVMetadata( - owner=x.owner, - rows=RangeRows(x.ranges), - allocations=x.allocations, - position_ids=None, - write_blocks=None, - write_offsets=None, - block_tables=None, - context_lengths=None, - writable=x.writable, - sequence_ranges=x.ranges, - table_plan=x.tables, - deferred_addresses=True, - ) - for x in layouts - ) n = len(first.positions) wide = stage( np.concatenate([first.positions, first.offsets, *(x.blocks for x in layouts)]), @@ -264,9 +202,6 @@ def __init__(self, cache): self._relative_positions = None self._range_bound_allocation = None self._range_min_pages = None - self._position_layouts = OrderedDict() - self._position_bytes = 0 - self._paged_bindings = weakref.WeakKeyDictionary() self.range_plan_builds = 0 self.range_plan_reuses = 0 @@ -303,8 +238,7 @@ def build_ranges(self, request_ids, starts, counts, depth_sets, *, for_write=Tru cache._validate_depth(depth) tables.append( RangeTablePlan( - [a.block_tables[cache._plane(d)] for (_, a), d in zip(allocations, depths)], - max_context_tokens=min(a.max_tokens for _, a in allocations), + [a.block_tables[cache._plane(d)] for (_, a), d in zip(allocations, depths)] ) ) # Publish only after preparation succeeds. @@ -333,13 +267,9 @@ def build_ranges(self, request_ids, starts, counts, depth_sets, *, for_write=Tru pages = (positions // cache.block_size).reshape(size, width) layouts = [] for depths, table in zip(depths_key, self._range_tables): - ranges = ( - UniformSequenceRanges(allocations, depths[0], starts[0], width) - if cache._contiguous is not None and len(set(depths)) == 1 - else tuple( - SequenceRange(a, depth, start, width) - for (_, a), depth, start in zip(allocations, depths, starts) - ) + ranges = tuple( + SequenceRange(allocation, depth, start, width) + for (_, allocation), depth, start in zip(allocations, depths, starts) ) blocks = np.take_along_axis(table.host, pages, axis=1).reshape(-1) layouts.append( @@ -349,86 +279,6 @@ def build_ranges(self, request_ids, starts, counts, depth_sets, *, for_write=Tru ) return tuple(layouts) - @staticmethod - def _stream_key(device): - if device.type == "npu": - return torch.npu.current_stream(device).stream_id - if device.type == "cuda": - return torch.cuda.current_stream(device).cuda_stream - return None - - def _position_input(self, metadata): - ranges = metadata.sequence_ranges - rows, width, start = len(ranges), ranges.query_count, ranges.start - device = self.cache.device - stream = self._stream_key(device) - key = (device, stream, rows, "decode" if width == 1 else (start, width)) - tensor = self._position_layouts.get(key) - required = max(start + width, metadata.table_plan.max_context_tokens or 0) - if tensor is None or (width == 1 and len(tensor) < required): - if tensor is not None: - self._position_bytes -= tensor.numel() * tensor.element_size() - del self._position_layouts[key] - if width == 1: - capacity = 1 << (required - 1).bit_length() - tensor = ( - torch.arange(capacity, dtype=torch.int64, device=device)[:, None] - .expand(capacity, rows) - .contiguous() - ) - else: - tensor = torch.arange( - start, start + width, dtype=torch.int64, device=device - ).repeat(rows) - size = tensor.numel() * tensor.element_size() - if size <= 64 * 1024 * 1024: - while self._position_layouts and ( - len(self._position_layouts) >= 8 - or self._position_bytes + size > 64 * 1024 * 1024 - ): - _, old = self._position_layouts.popitem(last=False) - self._position_bytes -= old.numel() * old.element_size() - self._position_layouts[key] = tensor - self._position_bytes += size - elif key in self._position_layouts: - self._position_layouts.move_to_end(key) - return tensor[start] if width == 1 else tensor - - def _bind_owned(self, metadata): - if metadata.owner is not self.cache: - raise ValueError("owned metadata belongs to a different cache") - return replace(metadata, position_ids=self._position_input(metadata)) - - def materialize_paged(self, metadata): - """Bind deferred addresses on the consuming stream before paged escape.""" - if not metadata.deferred_addresses: - return metadata - stream = self._stream_key(self.cache.device) - bindings = self._paged_bindings.get(metadata) - if bindings is not None and stream in bindings: - return bindings[stream] - ranges = metadata.sequence_ranges - rows, width, start = len(ranges), ranges.query_count, ranges.start - positions = np.tile(np.arange(start, start + width, dtype=np.int64), rows) - pages = (positions // self.cache.block_size).reshape(rows, width) - blocks = np.take_along_axis(metadata.table_plan.host, pages, axis=1).reshape(-1) - device = self.cache.device - result = replace( - metadata, - write_blocks=torch.tensor(blocks, device=device, dtype=torch.int64), - write_offsets=torch.tensor( - positions % self.cache.block_size, device=device, dtype=torch.int64 - ), - block_tables=metadata.table_plan.bind(device), - context_lengths=torch.tensor(positions + 1, device=device, dtype=torch.int32), - deferred_addresses=False, - ) - if bindings is None: - bindings = {} - self._paged_bindings[metadata] = bindings - bindings[stream] = result - return result - def prepare_device( self, *, @@ -617,9 +467,6 @@ def select_rows(layout, rows, *, packed=False): def bind(metadata, storage): """Bind a host-planned layout to its uploaded typed storage.""" - if metadata.deferred_addresses: - return metadata.owner.metadata_builder._bind_owned(metadata) - def view(tensor): return storage[tensor.dtype].as_strided( tensor.shape, tensor.stride(), tensor.storage_offset() diff --git a/vllm_rlt/worker/model_runner.py b/vllm_rlt/worker/model_runner.py index 58591738..c67e1eb8 100644 --- a/vllm_rlt/worker/model_runner.py +++ b/vllm_rlt/worker/model_runner.py @@ -19,7 +19,7 @@ ) from vllm_rlt.worker.cuda_graph import CodaGraphs, PreludeGraphs, RecurrentGraphs from vllm_rlt.worker.executor import ForwardBatch, ModelRuntime -from vllm_rlt.worker.metadata import UniformSequenceRanges, stage_range_metadata +from vllm_rlt.worker.metadata import stage_range_metadata from vllm_rlt.worker.output import ModelRunnerOutput from vllm_rlt.worker.sampler import RngRegistry, Sampler from vllm_rlt.worker.submission_queue import MAX_CONCURRENT_SUBMISSIONS, SubmissionQueue @@ -615,6 +615,11 @@ def _save_batch(self, requests, hidden, routing): for request in requests: request.hidden_state = self.states[self.state_slots[request.request_id]] + def bind_sampled_tokens(self, batch, values, rows): + """Bind the scheduler-selected sample rows as the next prelude inputs.""" + for row in rows: + batch.requests[row].request.input_token_tensor = values[row] + def _gather(self, requests, workspace, size): if workspace is None: return torch.stack([r.hidden_state for r in requests]) @@ -825,12 +830,7 @@ def _stage_range_plan(self, batch): [span.token_count for span in spans], [[span.depth + i for span in spans] for i in range(batch.loop_count)], ) - deferred = ( - self.device.type == "npu" - and self.cache_manager._contiguous is not None - and all(isinstance(x.ranges, UniformSequenceRanges) for x in layouts) - ) - return stage_range_metadata(layouts, self._append_input, defer_addresses=deferred) + return stage_range_metadata(layouts, self._append_input) def _append_input(self, values, dtype): return self.input_buffers[dtype].append(values) @@ -988,11 +988,6 @@ def _prepare_cpu_inputs(self, batch: ExecutionBatch): return replace( prepared, token_sources=sources, - token_ids=( - self.token_batch.tokens - if self.token_batch is not None and self.token_batch.matches(requests, sources) - else None - ), # Preserve the fixed decode path's gate work; its output has # no consumer and is not copied back to the host. forwards=( From 756f7bf5d1b4ff2f9ac9e760d572de1c25bbcfe6 Mon Sep 17 00:00:00 2001 From: bjf-frz Date: Thu, 19 Nov 2026 02:20:49 +0000 Subject: [PATCH 19/20] refactor(ascend): reuse packed prefill and per-loop scheduling Consume the shared packed KV descriptor on Ascend. Remove the separate range metadata and batch-stability fast paths so decode retains its per-loop scheduler boundaries. --- docs/recipes/ouro_ascend.md | 18 +- tests/test_ascend_compatibility.py | 12 +- ...de.py => test_ascend_decode_scheduling.py} | 190 +++++++--------- tests/test_ascend_eager.py | 3 +- tests/test_ascend_packed_prefill.py | 56 +++++ tests/test_fast_execution_plan.py | 51 +---- tests/test_npu_async.py | 38 ++-- tests/test_npu_batch_decode.py | 84 +------ tests/test_sequence_plans.py | 190 ---------------- vllm_rlt/attention/backend.py | 23 +- vllm_rlt/config.py | 3 - vllm_rlt/core/kv_cache_manager.py | 64 +----- vllm_rlt/core/scheduler.py | 66 +----- vllm_rlt/engine/packed_prefill.py | 7 +- vllm_rlt/engine/stage_engine.py | 17 -- vllm_rlt/entrypoints/runtime_args.py | 6 - vllm_rlt/kernels/ascend_attention.py | 87 ++------ vllm_rlt/models/ouro.py | 50 +---- vllm_rlt/worker/ascend_model_runner.py | 111 ---------- vllm_rlt/worker/executor.py | 12 +- vllm_rlt/worker/metadata.py | 209 +----------------- vllm_rlt/worker/model_runner.py | 97 +------- 22 files changed, 233 insertions(+), 1161 deletions(-) rename tests/{test_fixed_prelude.py => test_ascend_decode_scheduling.py} (51%) create mode 100644 tests/test_ascend_packed_prefill.py delete mode 100644 tests/test_sequence_plans.py diff --git a/docs/recipes/ouro_ascend.md b/docs/recipes/ouro_ascend.md index 6c5a9234..01826bfe 100644 --- a/docs/recipes/ouro_ascend.md +++ b/docs/recipes/ouro_ascend.md @@ -34,14 +34,12 @@ python -m vllm_rlt.entrypoints.serve \ --device npu --dtype bfloat16 --attention-backend ascend_eager \ --mode refill --exit-mode ouro_delayed \ --async-scheduling --single-stream --no-uva \ - --coalesce-fixed-prelude \ --block-size 16 --num-blocks 2064 --max-num-seqs 32 \ --max-num-batched-tokens 8192 --prefill-chunk-size 1024 \ --max-prefill-batches-before-decode 32 --host 127.0.0.1 --port 8123 ``` -For synchronous eager execution, omit `--async-scheduling` and -`--coalesce-fixed-prelude`. +For synchronous eager execution, omit `--async-scheduling`. Wait for `/health` to return HTTP 200, then send a completion. Add `"stream": true` for server-sent events. @@ -57,12 +55,12 @@ curl --fail http://127.0.0.1:8123/v1/completions \ ## Fixed-depth asynchronous engine plan - **Scheduling.** Stream events track completion and protect buffer and request - lifetimes. A stable batch can run decode PRELUDE and all four recurrent loops - in one submission. Competing prompt or recurrent work retains a scheduling - opportunity. A full-depth PRELUDE counts as recurrent work for refill fairness. -- **Reuse and batching.** Compatible batches reuse KV range and allocation plans, - hidden tensors, and attention masks. Greedy sampling, compatible - prefill and paged KV write indices are batched. Full prefill uses causal SDPA. + lifetimes. Decode PRELUDE and each recurrent loop are separate submissions, + so the scheduler can select a new batch at each boundary. +- **Reuse and batching.** Ascend uses the shared packed-prefill metadata by + default with `ascend_eager`. Use `--no-packed-prefill` to select the per-query + layout. Greedy sampling, compatible prefill and paged KV write indices are + batched. Full prefill uses causal SDPA. Cached chunks and ragged decode pass boolean masks to preserve each row's visible KV range. - **Paged KV.** Each loop writes to the page pool using its own block table. @@ -131,7 +129,7 @@ With an assigned NPU visible and `MODEL` set as above: ```bash VLLM_RLT_OURO_CHECKPOINT="$MODEL" python -m pytest -q --run-npu \ tests/test_ascend_eager.py tests/test_ascend_kv.py \ - tests/test_fixed_prelude.py \ + tests/test_ascend_decode_scheduling.py \ tests/test_ascend_checkpoint.py tests/test_ascend_serving.py ``` diff --git a/tests/test_ascend_compatibility.py b/tests/test_ascend_compatibility.py index 6ddff5ad..9f264189 100644 --- a/tests/test_ascend_compatibility.py +++ b/tests/test_ascend_compatibility.py @@ -7,7 +7,7 @@ import torch from tests.helpers import tiny_ouro_model -from vllm_rlt import CacheConfig, ExecutionConfig, SchedulerConfig +from vllm_rlt import CacheConfig, SchedulerConfig from vllm_rlt.attention.backend import BackendCapabilities from vllm_rlt.device import stream_module from vllm_rlt.engine.stage_engine import StageEngine @@ -29,10 +29,9 @@ def test_backend_capability_positional_arguments_keep_their_meaning(): "device_context_lengths", "sliding_window", "packed_sliding_window", - "sequence_ranges", ) caps = BackendCapabilities(False, False, False, False, False, True, False, True) - assert caps.sliding_window and caps.fp8_kv_static and not caps.sequence_ranges + assert caps.sliding_window and caps.fp8_kv_static @pytest.mark.parametrize("hip", [None, "6.4"]) @@ -88,10 +87,3 @@ def guarded(name, *args, **kwargs): assert type(engine.model_runner.sampler) is Sampler finally: engine.close() - - -def test_ascend_options_fail_before_allocating_on_other_devices(): - cache = CacheConfig(num_blocks=128) - execution = ExecutionConfig(coalesce_fixed_prelude=True) - with pytest.raises(ValueError, match="Ascend execution options require an NPU"): - StageEngine(tiny_ouro_model(), cache_config=cache, execution_config=execution) diff --git a/tests/test_fixed_prelude.py b/tests/test_ascend_decode_scheduling.py similarity index 51% rename from tests/test_fixed_prelude.py rename to tests/test_ascend_decode_scheduling.py index e403c708..c66aea06 100644 --- a/tests/test_fixed_prelude.py +++ b/tests/test_ascend_decode_scheduling.py @@ -1,4 +1,4 @@ -"""Fixed decode plans keep device inputs and preserve result retirement boundaries.""" +"""Ascend decode scheduling, refill and asynchronous result delivery.""" import pytest import torch @@ -13,41 +13,31 @@ from vllm_rlt.request import Request, RequestPhase -@pytest.mark.parametrize( - "exit_mode,min_loops,expected", - [("ouro_delayed", 4, 4), ("ouro_delayed", 1, 0), ("trace", 4, 0)], -) -def test_fixed_prelude_planning_and_submission_progress(exit_mode, min_loops, expected): - scheduler = Scheduler( - SchedulerConfig(), None, total_ut_steps=4, exit_mode=exit_mode, coalesce_fixed_prelude=True - ) - request = Request("a", [1], SamplingParams(min_loops=min_loops, max_loops=4)) +def test_decode_prelude_leaves_recurrent_work_to_scheduler(): + scheduler = Scheduler(SchedulerConfig(), None, total_ut_steps=4, exit_mode="ouro_delayed") + request = Request("a", [1], SamplingParams(min_loops=4, max_loops=4)) request.phase = RequestPhase.PRELUDE scheduler.requests["a"] = request item = ScheduledRequest(request, phase=RequestPhase.PRELUDE) - assert scheduler._planned_loop_count([item]) == expected - update = scheduler.update_after_submission( - SchedulerOutput(RequestPhase.PRELUDE, [item], expected) - ) - assert request.loops_done == expected - assert request.phase == (RequestPhase.CODA if expected else RequestPhase.RECURRENT) - assert update.exited == ([request] if expected else []) + assert scheduler._planned_loop_count([item]) == 0 + batch = SchedulerOutput(RequestPhase.PRELUDE, [item], 0) + update = scheduler.update_after_submission(batch) + assert request.loops_done == 0 + assert request.phase == RequestPhase.RECURRENT + assert update.exited == [] request.phase = RequestPhase.PRELUDE scheduler.requests.pop("a") - scheduler.update_after_submission(SchedulerOutput(RequestPhase.PRELUDE, [item], expected)) - assert request.phase == RequestPhase.PRELUDE # Stale submissions cannot advance a reused ID. + scheduler.update_after_submission(batch) + assert request.phase == RequestPhase.PRELUDE # A stale submission cannot advance a request. -@pytest.mark.parametrize("coalesced", [False, True]) @pytest.mark.parametrize("late_arrival", [False, True]) -def test_fixed_prelude_preserves_refill_opportunity(coalesced, late_arrival): +def test_decode_preserves_refill_opportunity(late_arrival): scheduler = Scheduler( SchedulerConfig(max_num_seqs=2, max_num_batched_tokens=3, prefill_chunk_size=3), make_cpu_cache(tiny_ouro_config(), blocks=128), total_ut_steps=4, exit_mode="ouro_delayed", - coalesce_fixed_prelude=coalesced, - coalesce_fixed_decode=True, ) params = SamplingParams(max_tokens=16, min_loops=4, max_loops=4, ignore_eos=True) first = Request("a", [1, 2, 3], params) @@ -67,7 +57,7 @@ def test_fixed_prelude_preserves_refill_opportunity(coalesced, late_arrival): assert second.phase == (RequestPhase.WAITING if late_arrival else RequestPhase.PREFILL) # Every prompt consumes a complete token budget. Refill must still get a - # turn before the first generation ends, with or without prelude coalescing. + # turn before the first generation ends. for _ in range(8): batch = scheduler.schedule() assert batch is not None @@ -92,7 +82,7 @@ def test_fixed_prelude_preserves_refill_opportunity(coalesced, late_arrival): pytest.fail("refilled decode starved the existing recurrent request") -def engine_for(model, planned, multi_stream=False, token_budget=128): +def engine_for(model, multi_stream=False, token_budget=128): return StageEngine( model, attention_backend="ascend_eager", @@ -104,20 +94,18 @@ def engine_for(model, planned, multi_stream=False, token_budget=128): async_scheduling=True, multi_stream=multi_stream, use_uva=False, - coalesce_fixed_prelude=planned, ), exit_config=ExitConfig("ouro_delayed"), ) -@pytest.mark.parametrize("coalesced,waiting", [(True, False), (False, False), (True, True)]) -def test_fixed_prelude_accounts_for_recurrent_fairness(coalesced, waiting): +@pytest.mark.parametrize("waiting", [False, True]) +def test_decode_prelude_keeps_recurrent_due(waiting): scheduler = Scheduler( SchedulerConfig(max_num_seqs=2, max_prefill_batches_before_decode=4), make_cpu_cache(tiny_ouro_config(), blocks=128), total_ut_steps=4, exit_mode="ouro_delayed", - coalesce_fixed_prelude=coalesced, ) params = SamplingParams(max_tokens=8, min_loops=4, max_loops=4, ignore_eos=True) scheduler.add_request(Request("a", [1, 2, 3], params)) @@ -132,109 +120,95 @@ def test_fixed_prelude_accounts_for_recurrent_fairness(coalesced, waiting): assert scheduler.policy.decode_due batch = scheduler.schedule() assert batch.phase == RequestPhase.PRELUDE - assert bool(batch.loop_count) == (coalesced and not waiting) - # A coalesced decode submission actually runs recurrent work. Embedding-only - # PRELUDE must retain the pending prefill budget until its core is selected. - assert scheduler.policy.decode_due == (not coalesced or waiting) + assert batch.loop_count == 0 + assert scheduler.policy.decode_due @pytest.mark.npu @torch.inference_mode() @pytest.mark.parametrize("multi_stream", [False, True]) -def test_fixed_prelude_refills_during_generation(multi_stream): +def test_ascend_decode_refills_during_generation(multi_stream): resolve_device("npu") torch.npu.set_device(0) model = make_npu_model(torch.device("npu:0")) params = SamplingParams(max_tokens=8, min_loops=4, max_loops=4, ignore_eos=True) - expected = None - for planned in [False, True]: - engine = engine_for(model, planned, multi_stream, token_budget=3) - try: - for name, prompt in [("a", [3, 7, 11]), ("b", [13, 17, 19])]: - engine.add_request(name, prompt, params) - actual, first_finished, second_started = {}, False, False - first_tokens_at_second_start = None - first_tokens, first_advanced = 0, False - for _ in range(128): - if not engine.has_unfinished_requests(): - break - for out in engine.step(): - if out.request_id == "b" and not second_started: - assert not first_finished - second_started = True - first_tokens_at_second_start = first_tokens + engine = engine_for(model, multi_stream, token_budget=3) + try: + for name, prompt in [("a", [3, 7, 11]), ("b", [13, 17, 19])]: + engine.add_request(name, prompt, params) + actual, first_finished, second_started = {}, False, False + first_tokens_at_second_start = None + first_tokens, first_advanced = 0, False + for _ in range(128): + if not engine.has_unfinished_requests(): + break + for out in engine.step(): + if out.request_id == "b" and not second_started: + assert not first_finished + second_started = True + first_tokens_at_second_start = first_tokens + if out.request_id == "a": + first_tokens = len(out.token_ids) + if second_started and "b" not in actual: + first_advanced |= first_tokens > first_tokens_at_second_start + if out.finished: + actual[out.request_id] = ( + list(out.token_ids), + list(out.exit_depths), + out.finish_reason, + ) if out.request_id == "a": - first_tokens = len(out.token_ids) - if second_started and "b" not in actual: - first_advanced |= first_tokens > first_tokens_at_second_start - if out.finished: - actual[out.request_id] = ( - list(out.token_ids), - list(out.exit_depths), - out.finish_reason, - ) - if out.request_id == "a": - first_finished = True - else: - pytest.fail("refill did not drain in 128 steps") - assert second_started and first_advanced - assert set(actual) == {"a", "b"} - if expected is None: - expected = actual - else: - assert actual == expected - assert engine.cache_manager.num_used_blocks == 0 - finally: - engine.close() + first_finished = True + else: + pytest.fail("refill did not drain in 128 steps") + assert second_started and first_advanced + assert set(actual) == {"a", "b"} + assert engine.cache_manager.num_used_blocks == 0 + finally: + engine.close() @pytest.mark.npu @torch.inference_mode() @pytest.mark.parametrize("multi_stream", [False, True]) -def test_fixed_prelude_preserves_generation_cancel_reuse_and_eos(multi_stream): +def test_ascend_decode_preserves_cancel_reuse_and_eos(multi_stream): resolve_device("npu") torch.npu.set_device(0) model = make_npu_model(torch.device("npu:0")) params = SamplingParams(max_tokens=4, min_loops=4, max_loops=4, ignore_eos=True) - expected = None - for planned in [False, True]: - engine = engine_for(model, planned, multi_stream) - try: - for i, prompt in enumerate([[3, 7, 11], [13, 17, 19]]): - engine.add_request(str(i), prompt, params) - actual = drain(engine) - if expected is None: - expected = actual - else: - assert actual == expected - assert engine.cache_manager._key_cache is None - engine.add_request("0", [2, 5, 9], params) - engine.add_request("1", [13, 17, 19], params) - engine.step() - engine.step() - engine.step() - engine.abort_request("0") - engine.add_request("0", [3, 7, 11], params) - assert drain(engine) == expected - eos = model.config.eos_token_id - engine.model_runner.sampler.sample_batch = lambda logits, options, generators: ( - logits.new_full((len(options),), eos, dtype=torch.long) - ) - engine.add_request( - "eos", [3, 7, 11], SamplingParams(max_tokens=4, min_loops=4, max_loops=4) - ) - assert drain(engine)["eos"] == ([eos], [4], "stop") - finally: - engine.close() + engine = engine_for(model, multi_stream) + try: + for i, prompt in enumerate([[3, 7, 11], [13, 17, 19]]): + engine.add_request(str(i), prompt, params) + expected = drain(engine) + assert engine.cache_manager.num_used_blocks == 0 + engine.add_request("0", [2, 5, 9], params) + engine.add_request("1", [13, 17, 19], params) + engine.step() + engine.step() + engine.step() + engine.abort_request("0") + engine.add_request("0", [3, 7, 11], params) + assert drain(engine) == expected + eos = model.config.eos_token_id + engine.model_runner.sampler.sample_batch = lambda logits, options, generators: ( + logits.new_full((len(options),), eos, dtype=torch.long) + ) + engine.add_request( + "eos", [3, 7, 11], SamplingParams(max_tokens=4, min_loops=4, max_loops=4) + ) + assert drain(engine)["eos"] == ([eos], [4], "stop") + finally: + engine.close() @pytest.mark.npu @torch.inference_mode() -def test_fixed_prelude_uses_device_sample_before_coda_delivery(): +def test_ascend_prelude_uses_device_sample_before_coda_delivery(): resolve_device("npu") torch.npu.set_device(0) model = make_npu_model(torch.device("npu:0")) - engine = engine_for(model, True) + engine = engine_for(model) class HeldEvent: def query(self): @@ -255,9 +229,9 @@ def synchronize(self): try: engine.step() assert engine.last_schedule.phase == RequestPhase.PRELUDE - assert engine.last_schedule.loop_count == 4 + assert engine.last_schedule.loop_count == 0 request = engine.scheduler.requests["a"] - assert request.loops_done == 4 and request.generated_token_ids == [] + assert request.loops_done == 0 and request.generated_token_ids == [] finally: ticket.event = actual_event assert drain(engine)["a"][1] == [4, 4, 4] diff --git a/tests/test_ascend_eager.py b/tests/test_ascend_eager.py index 30f95021..f985a6d0 100644 --- a/tests/test_ascend_eager.py +++ b/tests/test_ascend_eager.py @@ -138,6 +138,7 @@ def test_engine_matches_the_same_manual_prefill_chunks(chunk_size): ), ) try: + assert engine.execution_config.packed_prefill engine.add_request( "engine", tokens, @@ -196,7 +197,7 @@ def check_sdpa(q, k, v, *, attn_mask, is_causal): class _PreparedBatch: - sequence_ranges = None + cu_seqlens_q = None """Only the host metadata the two batched attention paths consume.""" diff --git a/tests/test_ascend_packed_prefill.py b/tests/test_ascend_packed_prefill.py new file mode 100644 index 00000000..12ddcc5e --- /dev/null +++ b/tests/test_ascend_packed_prefill.py @@ -0,0 +1,56 @@ +"""Ascend attention consumes the common packed KV descriptor.""" + +import pytest +import torch + +from vllm_rlt.attention.backend import AttentionBackend, BackendCapabilities +from vllm_rlt.core.kv_cache_manager import KVCacheManager +from vllm_rlt.device import resolve_device +from vllm_rlt.kernels.ascend_attention import AscendAttention + + +@pytest.mark.parametrize("device_name", ["cpu", pytest.param("npu:0", marks=pytest.mark.npu)]) +@torch.inference_mode() +def test_packed_prefill_mixes_cached_chunks_and_single_queries(device_name): + device = resolve_device(device_name) + if device.type == "npu": + torch.npu.set_device(device) + attention = AttentionBackend( + AscendAttention(), + BackendCapabilities(packed_prefill=True), + {"backend": "ascend_eager", "selected_backend": "ascend_eager"}, + ) + cache = KVCacheManager( + 1, 2, 64, 16, 16, 2, device=device, dtype=torch.bfloat16, attention=attention + ) + for request in ("a", "b", "c"): + assert cache.allocate(request, 3) + + def values(items): + return ( + torch.tensor(items, device=device, dtype=cache.dtype) + .view(-1, 1, 1) + .expand(-1, cache.num_kv_heads, cache.head_dim) + .contiguous() + ) + + history = values([1, 10, 20, 100]) + cache.write( + 0, ["a", "b", "b", "c"], [0, 1, 1, 0], [0, 0, 1, 0], torch.zeros_like(history), history + ) + ids, depths, positions = ["a", "a", "b", "c"], [0, 0, 1, 0], [1, 2, 2, 1] + packed = cache._prepare_batch(ids, depths, positions, packed_prefill=True) + assert packed.cu_seqlens_q.tolist() == [0, 2, 3, 4] + assert packed.context_lengths.tolist() == [3, 3, 2] + assert packed.block_tables.shape[0] == 3 + + new = values([3, 8, 30, 120]) + query = torch.zeros_like(new) + cache._write_prepared(0, packed, torch.zeros_like(new), new) + actual = cache._attend_prepared(0, packed, query) + expected = values([2, 4, 20, 110]) + torch.testing.assert_close(actual, expected, atol=0, rtol=0) + + per_query = cache._prepare_batch(ids, depths, positions, for_write=False) + reference = cache._attend_prepared(0, per_query, query) + torch.testing.assert_close(actual, reference, atol=0, rtol=0) diff --git a/tests/test_fast_execution_plan.py b/tests/test_fast_execution_plan.py index 78b127ef..85bd0528 100644 --- a/tests/test_fast_execution_plan.py +++ b/tests/test_fast_execution_plan.py @@ -1,7 +1,5 @@ """Ownership and failure boundaries of Ascend recurrent execution.""" -from dataclasses import replace - import pytest import torch @@ -10,7 +8,6 @@ from vllm_rlt.device import resolve_device from vllm_rlt.request import Operation from vllm_rlt.worker.executor import ForwardBatch, ModelRuntime -from vllm_rlt.worker.metadata import stage_range_metadata pytestmark = pytest.mark.npu @@ -33,47 +30,11 @@ def cache_for(model): def metadata(cache, start, count): - layouts = cache.metadata_builder.build_ranges(["a"], [start], [count], [(d,) for d in range(4)]) - batches = stage_range_metadata( - layouts, lambda values, dtype: torch.as_tensor(values, dtype=dtype, device="npu:0") + return cache._prepare_batches( + ["a"] * count, + [[depth] * count for depth in range(4)], + list(range(start, start + count)), ) - return tuple(replace(b, block_tables=b.table_plan.bind(torch.device("npu:0"))) for b in batches) - - -@torch.inference_mode() -def test_position_plan_reuses_depths_and_expires_between_tokens(): - resolve_device("npu") - torch.npu.set_device(0) - model = make_npu_model(torch.device("npu:0")) - expected = None - for reuse in [False, True]: - cache = cache_for(model) - runtime = ModelRuntime( - model, cache, prepare_recurrent=model.prepare_recurrent if reuse else None - ) - observed = [] - handle = model.model.rotary_emb.register_forward_hook(lambda *args: observed.append(1)) - actual = [] - try: - for start, ids in [(0, [3, 7, 11]), (3, [13])]: - hidden = model.prelude(torch.tensor(ids, device="npu:0")) - result = runtime.forward( - ForwardBatch( - Operation.RECURRENT, - hidden=hidden, - metadata=metadata(cache, start, len(ids)), - ) - ) - actual.append(result.hidden.clone()) - assert len(observed) == (2 if reuse else 8) - if expected is None: - expected = actual - else: - for a, b in zip(actual, expected): - torch.testing.assert_close(a, b, atol=0, rtol=0) - finally: - handle.remove() - cache.free("a") @torch.inference_mode() @@ -82,7 +43,7 @@ def test_failed_traversal_keeps_partial_layer_coverage_visible(): torch.npu.set_device(0) model = make_npu_model(torch.device("npu:0")) cache = cache_for(model) - runtime = ModelRuntime(model, cache, prepare_recurrent=model.prepare_recurrent) + runtime = ModelRuntime(model, cache) batch = metadata(cache, 0, 3)[0] def fail(*args): @@ -124,7 +85,7 @@ def test_paged_coverage_after_truncation_and_restore(): torch.npu.set_device(0) model = make_npu_model(torch.device("npu:0")) cache = cache_for(model) - runtime = ModelRuntime(model, cache, prepare_recurrent=model.prepare_recurrent) + runtime = ModelRuntime(model, cache) hidden = model.prelude(torch.tensor([3, 7, 11], device="npu:0")) for batch in metadata(cache, 0, 3): runtime.forward(ForwardBatch(Operation.RECURRENT, hidden=hidden, metadata=(batch,))) diff --git a/tests/test_npu_async.py b/tests/test_npu_async.py index dc98e30c..52ba27de 100644 --- a/tests/test_npu_async.py +++ b/tests/test_npu_async.py @@ -11,23 +11,18 @@ from vllm_rlt.request import Request, RequestPhase -def test_coalesced_submission_records_all_loops_and_preserves_trace(): - scheduler = Scheduler( - SchedulerConfig(), - None, - total_ut_steps=4, - exit_mode="ouro_delayed", - coalesce_fixed_decode=True, - ) +def test_fixed_depth_decode_keeps_each_loop_schedulable(): + scheduler = Scheduler(SchedulerConfig(), None, total_ut_steps=4, exit_mode="ouro_delayed") request = Request("a", [1], SamplingParams(min_loops=1, max_loops=4)) request.phase = RequestPhase.RECURRENT scheduler.requests["a"] = request - item = ScheduledRequest(request, depth=0, phase=RequestPhase.RECURRENT) - assert scheduler._planned_loop_count([item]) == 4 - scheduler.update_after_submission(SchedulerOutput(RequestPhase.RECURRENT, [item], 4)) - assert request.loops_done == 4 and request.phase == RequestPhase.CODA - scheduler.exit_mode = "trace" - assert scheduler._planned_loop_count([item]) == 1 + for depth in range(4): + assert request.phase == RequestPhase.RECURRENT + item = ScheduledRequest(request, depth=depth, phase=RequestPhase.RECURRENT) + assert scheduler._planned_loop_count([item]) == 1 + scheduler.update_after_submission(SchedulerOutput(RequestPhase.RECURRENT, [item], 1)) + assert request.loops_done == depth + 1 + assert request.phase == RequestPhase.CODA def make_engine(model, asynchronous, multi_stream): @@ -193,7 +188,7 @@ def test_npu_batched_greedy_preserves_ties_and_padding(): def test_npu_hidden_gather_keeps_cross_stream_sources_alive(): from types import SimpleNamespace - from vllm_rlt.worker.ascend_model_runner import AscendModelRunner + from vllm_rlt.worker.model_runner import ModelRunner device = resolve_device("npu:0") torch.npu.set_device(device) @@ -208,7 +203,8 @@ def test_npu_hidden_gather_keeps_cross_stream_sources_alive(): consumer.synchronize() del warm pointer = source.data_ptr() - request = SimpleNamespace(hidden_state=source) + request = SimpleNamespace(request_id="a", hidden_state=source) + batch = SimpleNamespace(spans=[SimpleNamespace(request=request)]) matrix = torch.ones((16384, 16384), dtype=torch.bfloat16, device=device) product = torch.empty_like(matrix) blocker.wait_stream(torch.npu.current_stream(device)) @@ -218,15 +214,11 @@ def test_npu_hidden_gather_keeps_cross_stream_sources_alive(): gate = torch.npu.Event() gate.record(blocker) consumer.wait_event(gate) - runner = SimpleNamespace( - hidden_batch=None, - device=device, - device_ops=torch.npu, - execution_config=SimpleNamespace(multi_stream=True), - ) + runner = SimpleNamespace(events={}) try: with torch.npu.stream(consumer): - actual = AscendModelRunner._gather(runner, [request], None, 1) + ModelRunner._wait_for_requests(runner, batch, consumer) + actual = ModelRunner._gather(runner, [request], None, 1) request.hidden_state = None del source poisoned = [] diff --git a/tests/test_npu_batch_decode.py b/tests/test_npu_batch_decode.py index 0850ee79..6c5dac9e 100644 --- a/tests/test_npu_batch_decode.py +++ b/tests/test_npu_batch_decode.py @@ -1,4 +1,4 @@ -"""Owned batch state and completion events preserve request lifetimes.""" +"""Decode scheduling and KV completion events preserve request lifetimes.""" from contextlib import nullcontext @@ -40,7 +40,6 @@ def test_cpu_keeps_copy_semantics(): expected = reqs[0].hidden_state.clone() hidden.fill_(99) assert torch.equal(reqs[0].hidden_state, expected) - assert not hasattr(runner, "hidden_batch") class Event: @@ -155,9 +154,8 @@ def test_partial_finalize_failure_drains_copies_before_cleanup(): @pytest.mark.npu @pytest.mark.parametrize("multi_stream", [False, True]) -@pytest.mark.parametrize("coalesce", [False, True]) @torch.inference_mode() -def test_npu_owned_states_survive_future_tokens(multi_stream, coalesce): +def test_npu_async_decode_matches_sync_when_reusing_request_ids(multi_stream): from tests.test_ascend import make_npu_model from tests.test_npu_async import drain, make_engine from vllm_rlt.device import resolve_device @@ -173,93 +171,15 @@ def test_npu_owned_states_survive_future_tokens(multi_stream, coalesce): expected = drain(baseline) baseline.close() engine = make_engine(model, True, multi_stream) - engine.scheduler.coalesce_fixed_decode = coalesce - runner = engine.model_runner - save = runner._save_batch - gather = runner._gather - held = [] - reuses = [] - - def capture(reqs, hidden, routing): - save(reqs, hidden, routing) - if runner.hidden_batch is not None: - held.append((reqs[0].hidden_state, reqs[0].hidden_state.clone())) - - def load(reqs, workspace, size): - saved = runner.hidden_batch - result = gather(reqs, workspace, size) - if saved is not None and result is saved.hidden: - reuses.append(True) - return result - - runner._save_batch = capture - runner._gather = load try: for repeat in range(2): for i, p in enumerate(prompts): engine.add_request(str(i), p, params) assert drain(engine) == expected - assert held and reuses - for actual, reference in held: - assert torch.equal(actual, reference) - assert runner.hidden_batch is None finally: engine.close() -@pytest.mark.npu -@torch.inference_mode() -def test_npu_retained_batch_records_delayed_consumer(): - from tests.test_ascend import make_npu_model - from tests.test_npu_async import make_engine - from vllm_rlt.device import resolve_device - - device = resolve_device("npu:0") - torch.npu.set_device(device) - engine = make_engine(make_npu_model(device), True, True) - runner = engine.model_runner - producer = torch.npu.Stream(device=device) - consumer = torch.npu.Stream(device=device) - blocker = torch.npu.Stream(device=device) - reqs = requests() - with torch.npu.stream(producer): - source = torch.full((2, 256), 7.0, device=device, dtype=torch.bfloat16) - producer.synchronize() - pointer = source.data_ptr() - runner._save_batch(reqs, source, None) - matrix = torch.ones((16384, 16384), device=device, dtype=torch.bfloat16) - product = torch.empty_like(matrix) - blocker.wait_stream(torch.npu.current_stream(device)) - with torch.npu.stream(blocker): - for _ in range(4): - torch.mm(matrix, matrix, out=product) - gate = torch.npu.Event() - gate.record(blocker) - consumer.wait_event(gate) - try: - with torch.npu.stream(consumer): - gathered = runner._gather(reqs, None, 2) - assert gathered is source - actual = gathered + 0 - runner.hidden_batch = None - for r in reqs: - r.hidden_state = None - del source, gathered - with torch.npu.stream(producer): - poison = [ - torch.full((2, 256), 99.0, device=device, dtype=torch.bfloat16) for _ in range(32) - ] - assert not gate.query(), "consumer delay expired before allocator probe" - assert all(t.data_ptr() != pointer for t in poison) - consumer.synchronize() - assert actual.cpu().tolist() == [[7.0] * 256] * 2 - finally: - producer.synchronize() - consumer.synchronize() - blocker.synchronize() - engine.close() - - @pytest.mark.npu @pytest.mark.parametrize("multi_stream", [False, True]) @torch.inference_mode() diff --git a/tests/test_sequence_plans.py b/tests/test_sequence_plans.py deleted file mode 100644 index 54568f80..00000000 --- a/tests/test_sequence_plans.py +++ /dev/null @@ -1,190 +0,0 @@ -"""Range plans and contiguous reads preserve ownership and paged KV truth.""" - -import pytest -import torch - -from vllm_rlt.attention.backend import AttentionBackend, BackendCapabilities -from vllm_rlt.core.kv_cache_manager import KVCacheManager -from vllm_rlt.kernels.ascend_attention import AscendAttention -from vllm_rlt.worker.metadata import MetadataBuilder, stage_range_metadata - - -def cache_for(device="cpu", *, incremental=False): - device = torch.device(device) - attention = ( - AttentionBackend( - AscendAttention(), - BackendCapabilities(sequence_ranges=True), - dict(selected_backend="torch", backend="torch"), - ) - if device.type == "cpu" - else None - ) - cache = KVCacheManager( - 2, - 2, - 64, - 64, - 16, - 2, - device=device, - dtype=torch.bfloat16, - backend="ascend_eager" if device.type == "npu" else "torch", - attention=attention, - incremental_allocation=incremental, - ) - cache.key_cache.fill_(float("nan")) - cache.value_cache.fill_(float("nan")) - for rid in ("a", "b"): - assert cache.allocate(rid, 35, initial_tokens=1 if incremental else None) - return cache, MetadataBuilder(cache) - - -def compact(builder, start, count, depth=0): - layouts = builder.build_ranges(["a", "b"], [start] * 2, [count] * 2, [[depth] * 2]) - return stage_range_metadata(layouts, builder.cache._stage)[0] - - -def test_plan_reuses_tables_without_reusing_mutable_positions(): - cache, builder = cache_for() - first = compact(builder, 0, 17) - later = compact(builder, 17, 1) - assert first.table_plan is later.table_plan - assert first.table_plan.bind(cache.device) is later.table_plan.bind(cache.device) - assert first.table_plan.host.shape == (2, 3) - assert first.position_ids.tolist() == list(range(17)) * 2 - assert later.position_ids.tolist() == [17, 17] - legacy = cache._prepare_batch(["a"] * 17 + ["b"] * 17, [0] * 34, list(range(17)) * 2) - assert tuple(first.rows) == legacy.rows - assert torch.equal(first.write_blocks, legacy.write_blocks) - assert torch.equal(first.write_offsets, legacy.write_offsets) - cache.free("a") - assert cache.allocate("a", 35) - with pytest.raises(RuntimeError, match="stale"): - cache._require_live_batch(first) - assert compact(builder, 0, 1).table_plan is not first.table_plan - - -@pytest.mark.parametrize( - "starts,counts,ids", - [ - ([0, 1], [2, 2], ["a", "b"]), - ([0, 0], [2, 3], ["a", "b"]), - ([0, 0], [2, 2], ["a", "a"]), - ([0, 0], [0, 0], ["a", "b"]), - ([34, 34], [2, 2], ["a", "b"]), - ], -) -def test_invalid_ranges_rejected(starts, counts, ids): - _, builder = cache_for() - with pytest.raises(ValueError): - builder.build_ranges(ids, starts, counts, [[0, 0]]) - - -def test_incremental_growth_invalidates_table_plan(): - cache, builder = cache_for(incremental=True) - before = compact(builder, 0, 1) - with pytest.raises(ValueError, match="physical"): - compact(builder, 16, 1) - assert cache.ensure_capacity("a", 17) - assert cache.ensure_capacity("b", 17) - after = compact(builder, 16, 1) - assert before.table_plan is not after.table_plan - - -@pytest.mark.parametrize("device", ["cpu", pytest.param("npu:0", marks=pytest.mark.npu)]) -@torch.inference_mode() -def test_range_attention_matches_paged_queries_and_preserves_outputs(device): - if device.startswith("npu"): - import torch_npu # noqa: F401 - - torch.npu.set_device(device) - torch.manual_seed(83) - cache, builder = cache_for(device) - held = snapshot = None - for start, count in ((0, 17), (17, 9), (26, 1)): - batch = compact(builder, start, count) - # Direct tests have no runner upload; bind the owned sequence table. - from dataclasses import replace - - batch = replace(batch, block_tables=batch.table_plan.bind(cache.device)) - ids = ["a"] * count + ["b"] * count - positions = list(range(start, start + count)) * 2 - legacy = cache._prepare_batch(ids, [0] * len(ids), positions) - k = torch.randn(len(ids), 2, 64, device=cache.device, dtype=cache.dtype) - v = torch.randn_like(k) - q = torch.randn(len(ids), 4, 128, device=cache.device, dtype=cache.dtype)[:, :, ::2] - for layer in range(2): - cache._write_prepared(layer, batch, k, v) - actual = cache._attend_prepared(layer, batch, q) - expected = cache.attention.kernel.attend_metadata( - q, cache.key_cache[:, layer], cache.value_cache[:, layer], legacy - ) - torch.testing.assert_close(actual, expected, atol=0, rtol=0) - assert torch.isfinite(actual).all() - if held is None: - held, snapshot = actual, actual.clone() - k.zero_() - torch.testing.assert_close(held, snapshot, atol=0, rtol=0) - - -@pytest.mark.npu -@torch.inference_mode() -def test_sequence_tables_do_not_cross_stream_ownership(): - from dataclasses import replace - - import torch_npu # noqa: F401 - - torch.npu.set_device(0) - cache, builder = cache_for("npu:0") - batch = compact(builder, 0, 17) - k = torch.randn(34, 2, 64, device=cache.device, dtype=cache.dtype) - q = torch.randn(34, 4, 64, device=cache.device, dtype=cache.dtype) - producer, consumer = torch.npu.Stream(), torch.npu.Stream() - producer.wait_stream(torch.npu.current_stream()) - with torch.npu.stream(producer): - first = replace(batch, block_tables=batch.table_plan.bind(cache.device)) - cache._write_prepared(0, first, k, k) - held = cache._attend_prepared(0, first, q) - snapshot = held.clone() - ready = torch.npu.Event() - ready.record() - consumer.wait_event(ready) - with torch.npu.stream(consumer): - second = replace(batch, block_tables=batch.table_plan.bind(cache.device)) - assert first.block_tables.data_ptr() != second.block_tables.data_ptr() - actual = cache._attend_prepared(0, second, q) - torch.npu.synchronize() - torch.testing.assert_close(actual, snapshot, atol=0, rtol=0) - torch.testing.assert_close(held, snapshot, atol=0, rtol=0) - cache.free("a") - cache.free("b") - - -def test_npu_cache_identity_never_resolves_raw_stream_handle(monkeypatch): - """Identity lookups must not drain Torch-NPU's asynchronous task queue.""" - from types import SimpleNamespace - - from vllm_rlt.worker.metadata import RangeTablePlan - - class Stream: - stream_id = 73 - - @property - def npu_stream(self): - raise AssertionError("raw handle access drains the NPU submission queue") - - # No physical NPU required: both cache-hit paths only need identity. - monkeypatch.setattr( - torch, "npu", SimpleNamespace(current_stream=lambda device: Stream()), raising=False - ) - - class Device: - type = "npu" - - device = Device() - assert AscendAttention._stream_key(device) == 73 - plan = RangeTablePlan([[1]]) - sentinel = object() - plan.tensors[device, 73] = sentinel - assert plan.bind(device) is sentinel diff --git a/vllm_rlt/attention/backend.py b/vllm_rlt/attention/backend.py index a6fd023b..c5ebdbfa 100644 --- a/vllm_rlt/attention/backend.py +++ b/vllm_rlt/attention/backend.py @@ -34,7 +34,6 @@ class BackendCapabilities: # Mask keys outside a per-layer sliding window (not storage reclamation). sliding_window: bool = False packed_sliding_window: bool = False - sequence_ranges: bool = False @dataclass @@ -49,14 +48,21 @@ def __call__(self, *args, **kwargs): return self.kernel(*args, **kwargs) def attend_metadata(self, q, keys, values, batch, **kwargs): - """Borrow validated host metadata when supported; retain the legacy tensor API. - - Metadata-aware kernels can reuse host planning across layers without - reading device tables or lengths back to the host. - """ + """Use the prepared batch for packed or per-query attention.""" method = getattr(self.kernel, "attend_metadata", None) if method is not None: return method(q, keys, values, batch, **kwargs) + if batch.cu_seqlens_q is not None: + return self.prefill( + q, + keys, + values, + batch.block_tables, + batch.context_lengths, + batch.cu_seqlens_q, + batch.max_seqlen_q, + **kwargs, + ) return self(q, keys, values, batch.block_tables, batch.context_lengths, **kwargs) def prefill(self, *args, **kwargs): @@ -77,7 +83,6 @@ def generation(self): _TORCH_CAPABILITIES = BackendCapabilities(fp8_kv=True, fp8_kv_static=True, sliding_window=True) -_ASCEND_CAPABILITIES = BackendCapabilities(async_scheduling=True, sequence_ranges=True) _TRITON_CAPABILITIES = BackendCapabilities( cuda_graphs=True, async_scheduling=True, @@ -127,7 +132,8 @@ def _create_explicit(backend, device, dtype, head_dim, block_size, *, packed, pa raise ValueError("ascend_eager requires an NPU and BF16") from vllm_rlt.kernels.ascend_attention import AscendAttention - return AttentionBackend(AscendAttention(), _ASCEND_CAPABILITIES, {"backend": backend}) + capabilities = BackendCapabilities(packed_prefill=packed, async_scheduling=True) + return AttentionBackend(AscendAttention(), capabilities, {"backend": backend}) if backend == "triton": if device.type != "cuda": raise ValueError("the Triton attention backend requires a CUDA or ROCm device") @@ -232,7 +238,6 @@ def create_backend( "device_context_lengths", "fp8_kv", "fp8_kv_static", - "sequence_ranges", "sliding_window", "packed_sliding_window", ) diff --git a/vllm_rlt/config.py b/vllm_rlt/config.py index 025fbdc8..7213adb0 100644 --- a/vllm_rlt/config.py +++ b/vllm_rlt/config.py @@ -160,8 +160,6 @@ class ExecutionConfig: # ordinary (non-staged) prefill. Sliding windows additionally require packed-window # support and compatible synchronous eager execution; off otherwise. packed_prefill: bool | None = None - # Run embedding and the fixed full recurrent depth in one decode submission. - coalesce_fixed_prelude: bool = False def __post_init__(self): if self.packed_prefill is not None and type(self.packed_prefill) is not bool: @@ -173,7 +171,6 @@ def __post_init__(self): "static_buffers", "pad_to_power_of_two", "cuda_graphs", - "coalesce_fixed_prelude", ): if type(getattr(self, name)) is not bool: raise ValueError(f"{name} must be a boolean") diff --git a/vllm_rlt/core/kv_cache_manager.py b/vllm_rlt/core/kv_cache_manager.py index efd34dbc..2b1464ed 100644 --- a/vllm_rlt/core/kv_cache_manager.py +++ b/vllm_rlt/core/kv_cache_manager.py @@ -123,27 +123,6 @@ class _PreparedCoverage: sequences: Sequence[_SequenceCoverage] -class _LazyRangeCoverage(Sequence): - """Materialize row coverage only when the generic paged path needs it.""" - - def __init__(self, ranges, shared): - self.ranges, self.shared, self.sequences = ranges, shared, None - - def __len__(self): - return len(self.ranges) - - def __getitem__(self, index): - return tuple(self)[index] - - def __iter__(self): - if self.sequences is None: - self.sequences = tuple( - _SequenceCoverage(r.allocation, 0 if self.shared else r.depth, ((r.start, r.end),)) - for r in self.ranges - ) - return iter(self.sequences) - - def _sequence_coverage(rows, plane): """Collapse query rows once, leaving written state owned by the allocation.""" grouped = {} @@ -309,7 +288,6 @@ def __init__( if self.device.type == "npu" and self.backend == "ascend_eager" else None ) - self._validated_range_tables = weakref.WeakKeyDictionary() self.enable_prefix_caching = enable_prefix_caching self.incremental_allocation = incremental_allocation # Full allocation already reserves each request's maximum KV capacity. @@ -816,12 +794,6 @@ def _stage(self, values, dtype): def _require_live_batch(self, batch: KVMetadata | DeviceKVMetadata): if isinstance(batch, KVMetadata) and batch.owner is not self: raise ValueError("prepared KV batch belongs to a different cache manager") - if ( - isinstance(batch, KVMetadata) - and batch.sequence_ranges is not None - and not self.attention_capabilities.sequence_ranges - ): - raise ValueError("attention backend does not support compact sequence ranges") plan = None reusable = self._prepared_coverage is not None and isinstance(batch, KVMetadata) epoch = self._allocation_epoch @@ -829,25 +801,14 @@ def _require_live_batch(self, batch: KVMetadata | DeviceKVMetadata): plan = self._prepared_coverage.get(batch) if plan is not None and plan.allocation_epoch == epoch: return plan - table_plan = batch.table_plan if reusable else None - validated = self._validated_range_tables.get(table_plan) if table_plan is not None else None - if validated is None or validated[0] != epoch or validated[1] is not batch.allocations: - for request_id, allocation in batch.allocations: - if self._allocations.get(request_id) is not allocation: - raise RuntimeError(f"stale prepared KV batch for request {request_id!r}") - if table_plan is not None: - self._validated_range_tables[table_plan] = epoch, batch.allocations + for request_id, allocation in batch.allocations: + if self._allocations.get(request_id) is not allocation: + raise RuntimeError(f"stale prepared KV batch for request {request_id!r}") if reusable: - plan = _PreparedCoverage( - epoch, - ( - _LazyRangeCoverage(batch.sequence_ranges, self.layout == "shared") - if batch.sequence_ranges is not None - else _sequence_coverage(batch.rows, self._plane) - ) - if plan is None - else plan.sequences, + sequences = ( + _sequence_coverage(batch.rows, self._plane) if plan is None else plan.sequences ) + plan = _PreparedCoverage(epoch, sequences) self._prepared_coverage[batch] = plan return plan @@ -1061,24 +1022,13 @@ def _attend_prepared( self._require_prefix( sequence.allocation, layer, sequence.plane, sequence.ranges[-1][1] ) - if batch.cu_seqlens_q is not None: - return self.attention.prefill( - q, - self.key_cache[:, layer], - self.value_cache[:, layer], - batch.block_tables, - batch.context_lengths, - batch.cu_seqlens_q, - batch.max_seqlen_q, - **self._layer_scales(layer), - **({} if window is None else {"window": window}), - ) return self.attention.attend_metadata( q, self.key_cache[:, layer], self.value_cache[:, layer], batch, **self._layer_scales(layer), + **({} if window is None else {"window": window}), ) @torch.no_grad() diff --git a/vllm_rlt/core/scheduler.py b/vllm_rlt/core/scheduler.py index 16a31803..14d95950 100644 --- a/vllm_rlt/core/scheduler.py +++ b/vllm_rlt/core/scheduler.py @@ -140,8 +140,6 @@ def __init__( total_ut_steps: int | None = None, eos_token_id=(), exit_mode: str = "ouro", - coalesce_fixed_decode: bool = False, - coalesce_fixed_prelude: bool = False, ): self.config = config self.cache_manager = cache_manager @@ -156,8 +154,6 @@ def __init__( else: self.eos_token_ids = (eos_token_id,) self.exit_mode = exit_mode - self.coalesce_fixed_decode = coalesce_fixed_decode - self.coalesce_fixed_prelude = coalesce_fixed_prelude self.requests: dict[str, Request] = {} self.queues: dict[RequestPhase, deque[str]] = {s: deque() for s in RequestPhase} # Bound by Engine when preemption is enabled. These callbacks can copy @@ -781,8 +777,6 @@ def take_phases(self, phases, *, eligible=None): ) self.policy.record_batch(recorded) loop_count = self._planned_loop_count(scheduled_requests) - if self.coalesce_fixed_prelude and selected_phases == {RequestPhase.PRELUDE} and loop_count: - self.policy.record_batch(RequestPhase.RECURRENT) scheduled_requests = [ replace( item, @@ -808,29 +802,6 @@ def _planned_loop_count(self, scheduled_requests): if RequestPhase.SPECULATIVE in phases or RequestPhase.CODA in phases: return 0 if RequestPhase.PRELUDE in phases: - if ( - phases == {RequestPhase.PRELUDE} - and self.coalesce_fixed_prelude - # Full-depth PRELUDE skips RECURRENT, where refill gets a turn. - # Keep that boundary while other prompt/core work needs a turn. - and not ( - self.config.mode in ("refill", "mixed") - and ( - self.queues[RequestPhase.WAITING] - or self.has_ready(RequestPhase.RECURRENT) - or any(self.has_ready(phase) for phase in self.policy.PREFILL_PHASES) - ) - ) - and self.speculative_config is None - and self.exit_mode != "trace" - and all( - item.request.sampling_params.min_loops == self.total_ut_steps - and (item.request.sampling_params.max_loops or self.total_ut_steps) - == self.total_ut_steps - for item in scheduled_requests - ) - ): - return self.total_ut_steps return 0 if self.wavefront_chunked_prefill: return 1 if RequestPhase.PREFILLING_RECURRENT in phases else 0 @@ -848,25 +819,6 @@ def _planned_loop_count(self, scheduled_requests): return 1 return min(self._max_contiguous_prefill_loops(item) for item in scheduled_requests) if RequestPhase.RECURRENT in phases: - # Fixed-depth requests have no intervening host exit decision. - # Trace and adaptive policies retain their per-loop boundaries. - depths = {item.depth for item in scheduled_requests} - if ( - self.coalesce_fixed_decode - and self.exit_mode != "trace" - and len(depths) == 1 - and all( - item.request.sampling_params.exit_threshold == 1.0 - for item in scheduled_requests - ) - ): - depth = next(iter(depths)) or 0 - room = min( - (item.request.sampling_params.max_loops or self.total_ut_steps) - depth - for item in scheduled_requests - ) - if room > 0: - return room return 1 raise ValueError(f"cannot derive recurrent boundary for phases: {phases}") @@ -1362,15 +1314,12 @@ def update_from_output( ): continue if phase == RequestPhase.PRELUDE: - request.loops_done = loop_count if self.coalesce_fixed_prelude else 0 + request.loops_done = 0 request.remaining_probability = 1.0 request.pending_exit_depth = None - complete = self.coalesce_fixed_prelude and loop_count == self.total_ut_steps - if complete: - update.exited.append(request) - self.enqueue(request, RequestPhase.CODA if complete else RequestPhase.RECURRENT) + self.enqueue(request, RequestPhase.RECURRENT) elif phase == RequestPhase.RECURRENT: - request.loops_done += max(1, loop_count) if self.coalesce_fixed_decode else 1 + request.loops_done += 1 if self.exit_mode == "trace": should_exit = self._trace_exit(request) else: @@ -1482,13 +1431,10 @@ def update_after_submission( ): continue if phase == RequestPhase.PRELUDE: - request.loops_done = loop_count if self.coalesce_fixed_prelude else 0 + request.loops_done = 0 request.remaining_probability = 1.0 request.pending_exit_depth = None - complete = self.coalesce_fixed_prelude and loop_count == self.total_ut_steps - if complete: - update.exited.append(request) - self.enqueue(request, RequestPhase.CODA if complete else RequestPhase.RECURRENT) + self.enqueue(request, RequestPhase.RECURRENT) elif phase == RequestPhase.CODA: request.num_output_placeholders += 1 if request.num_scheduled_outputs < request.sampling_params.max_tokens: @@ -1508,7 +1454,7 @@ def update_after_submission( and request.sampling_params.exit_threshold < 1 and self.needs_exit_signal(request) ) - request.loops_done += max(1, loop_count) if self.coalesce_fixed_decode else 1 + request.loops_done += 1 should_exit = ( self._trace_exit(request) if self.exit_mode == "trace" diff --git a/vllm_rlt/engine/packed_prefill.py b/vllm_rlt/engine/packed_prefill.py index 82eec246..dfb5a486 100644 --- a/vllm_rlt/engine/packed_prefill.py +++ b/vllm_rlt/engine/packed_prefill.py @@ -34,8 +34,9 @@ def packed_prefill_eligible( "with ordinary prefill, without speculation or preemption" ) return False - if device_type != "cuda" or layout != "last_exited": + if device_type not in ("cuda", "npu") or layout != "last_exited": if requested is True: - raise ValueError("packed_prefill requires CUDA packed prefill with last_exited KV") + raise ValueError("packed_prefill requires CUDA or Ascend with last_exited KV") return False - return requested is True or not staged_prefill + # Ascend prepares both ordinary and staged packed metadata on the host. + return requested is True or device_type == "npu" or not staged_prefill diff --git a/vllm_rlt/engine/stage_engine.py b/vllm_rlt/engine/stage_engine.py index 2d429a79..3e3866af 100644 --- a/vllm_rlt/engine/stage_engine.py +++ b/vllm_rlt/engine/stage_engine.py @@ -75,8 +75,6 @@ def __init__( self.execution_config = resolve_cuda_graph_rows( execution_config or ExecutionConfig(), scheduler_config, speculative_config ) - if parameter.device.type != "npu" and self.execution_config.coalesce_fixed_prelude: - raise ValueError("Ascend execution options require an NPU") if parameter.device.type == "npu": if config.to_dict()["model_type"] != "ouro" or config.total_ut_steps != 4: raise ValueError("NPU currently supports Ouro with four loops") @@ -87,7 +85,6 @@ def __init__( if ( self.execution_config.cuda_graphs or (self.execution_config.async_scheduling and self.execution_config.static_buffers) - or self.execution_config.packed_prefill or speculative_config is not None or self.stage_config.role != "prefill_decode" or scheduler_config.enable_preemption @@ -290,16 +287,6 @@ def __init__( kv_cache_dtype=cache_config.kv_cache_dtype, fp8_scale=fp8_scale, ) - if self.execution_config.coalesce_fixed_prelude and ( - parameter.device.type != "npu" - or not self.execution_config.async_scheduling - or self.execution_config.static_buffers - or self.execution_config.cuda_graphs - or getattr(model, "requires_boundary_kv", False) - or attention.info["selected_backend"] != "ascend_eager" - or speculative_config is not None - ): - raise ValueError("fixed decode prelude requires asynchronous Ascend eager execution") self.scheduler = Scheduler( scheduler_config, self.cache_manager, @@ -308,10 +295,6 @@ def __init__( total_ut_steps=config.total_ut_steps, eos_token_id=config.eos_token_id, exit_mode=self.exit_config.mode, - coalesce_fixed_decode=( - parameter.device.type == "npu" and self.execution_config.async_scheduling - ), - coalesce_fixed_prelude=self.execution_config.coalesce_fixed_prelude, ) self.connector = connector # One RNG registry per engine, shared by both runners: the CODA and diff --git a/vllm_rlt/entrypoints/runtime_args.py b/vllm_rlt/entrypoints/runtime_args.py index 947edde1..679a198c 100644 --- a/vllm_rlt/entrypoints/runtime_args.py +++ b/vllm_rlt/entrypoints/runtime_args.py @@ -63,11 +63,6 @@ def add_runtime_args(parser): parser.add_argument("--exit-trace", help="JSON file with depths_by_request") parser.add_argument("--lookahead-seed", type=int, default=0) parser.add_argument("--async-scheduling", action="store_true") - parser.add_argument( - "--coalesce-fixed-prelude", - action="store_true", - help="Run fixed full-depth decode prelude and loops together (NPU eager)", - ) parser.add_argument( "--cuda-graphs", action="store_true", @@ -146,7 +141,6 @@ def runtime_configs(args): cuda_graph_max_graphs=args.cuda_graph_max_graphs, cuda_graph_memory_reserve_bytes=args.cuda_graph_memory_reserve_bytes, async_scheduling=args.async_scheduling, - coalesce_fixed_prelude=args.coalesce_fixed_prelude, multi_stream=not args.single_stream, static_buffers=args.static_buffers, pad_to_power_of_two=args.pad_to_power_of_two, diff --git a/vllm_rlt/kernels/ascend_attention.py b/vllm_rlt/kernels/ascend_attention.py index d014b119..12940520 100644 --- a/vllm_rlt/kernels/ascend_attention.py +++ b/vllm_rlt/kernels/ascend_attention.py @@ -88,69 +88,6 @@ def __init__(self): self.ragged_calls = 0 self.fallback_calls = 0 self.continuation_calls = 0 - self._range_metadata = None - self._range_signature = None - self._range_mask = self._range_slots = None - - def _prepare_ranges(self, q, batch): - ranges = batch.sequence_ranges - rows = len(ranges) - width, start = ranges[0].count, ranges[0].start - signature = (q.device, q.dtype, q.shape, self._stream_key(q.device)) - if ( - self._range_metadata is None - or self._range_metadata() is not batch - or self._range_signature != signature - ): - self._range_metadata = self._range_signature = None - self._range_slots = self._range_mask = None - - def build(): - positions = torch.arange(start + width, device=q.device) - return (positions[None, :] <= positions[start:, None],) - - if start and width > 1: - self._range_mask = self._layout( - ("range", q.device, signature[-1], start, width), build - )[0] - self._range_metadata = weakref.ref(batch) - self._range_signature = signature - return rows, width, start - - def _range_attention(self, q, pair, geometry): - rows, width, start = geometry - key, value = pair - head_dim = q.shape[-1] - query = ( - q.reshape(rows, q.shape[1], 1, head_dim) - if width == 1 - else q.reshape(rows, width, q.shape[1], head_dim).transpose(1, 2) - ) - output = _attend(query, key, value, self._range_mask, is_causal=start == 0 and width > 1) - if width == 1: - self.ragged_calls += 1 - return output.squeeze(2) - self.batched_prefill_calls += 1 - if start: - self.continuation_calls += 1 - return output.transpose(1, 2).reshape_as(q) - - def _attend_ranges(self, q, keys, values, batch): - geometry = self._prepare_ranges(q, batch) - rows, width, start = geometry - flat_rows = _flat_kv_rows(keys, values) - if flat_rows is None: - raise ValueError("compact Ascend ranges require regular paged KV storage") - flat_keys, flat_values, rows_per_block = flat_rows - if self._range_slots is None: - positions = torch.arange(start + width, device=q.device) - pages = positions // keys.shape[-3] - blocks = batch.block_tables[:, pages].to(torch.int64) - self._range_slots = (blocks * rows_per_block + positions % keys.shape[-3]).reshape(-1) - shape = (rows, start + width, keys.shape[-2], keys.shape[-1]) - key = flat_keys.index_select(0, self._range_slots).reshape(shape).transpose(1, 2) - value = flat_values.index_select(0, self._range_slots).reshape(shape).transpose(1, 2) - return self._range_attention(q, (key, value), geometry) @staticmethod def _stream_key(device): @@ -228,13 +165,13 @@ def _decode(self, q, keys, values, flat_rows): result[:rows].copy_(output) return result - def _plan(self, keys, lengths, device): + def _plan(self, sequence_keys, lengths, device, *, packed=False): grouped = {} - for row, (key, length) in enumerate(zip(keys, lengths, strict=True)): + for row, (key, length) in enumerate(zip(sequence_keys, lengths, strict=True)): if length: grouped.setdefault(key, []).append(row) planned = [] - for rows in grouped.values(): + for sequence, rows in enumerate(grouped.values()): maximum = max(lengths[row] for row in rows) indices = torch.tensor(rows, dtype=torch.long, device=device) positions = torch.arange(maximum, device=device) @@ -244,15 +181,18 @@ def _plan(self, keys, lengths, device): mask = ( positions[None, :] < row_lengths[:, None] if not initial and len(rows) > 1 else None ) - planned.append((rows[0], indices, positions, mask, initial, row_lengths)) + # Packed table rows follow sequence order. Query indices still + # address the flattened Q and output tensors. + table_row = sequence if packed else rows[0] + planned.append((table_row, indices, positions, mask, initial, row_lengths)) return planned def _forward(self, q, keys, values, tables, groups, *, flat_rows=None, result=None): if result is None: result = torch.zeros_like(q) block_size = keys.shape[-3] - for first, indices, positions, mask, initial, _lengths in groups: - blocks = tables[first, positions // block_size] + for table_row, indices, positions, mask, initial, _lengths in groups: + blocks = tables[table_row, positions // block_size] offsets = positions % block_size if flat_rows is None: k, v = keys[blocks, offsets], values[blocks, offsets] @@ -306,7 +246,8 @@ def _ragged_decode(self, q, keys, values, tables, groups, flat_rows, result): positions = torch.arange(maximum, device=q.device) pages = positions // block_size offsets = positions % block_size - row_tables = tables.index_select(0, indices) + table_rows = torch.tensor([group[0] for group in groups], device=q.device) + row_tables = tables.index_select(0, table_rows) blocks = row_tables[:, pages].to(torch.int64) slots = (blocks * rows_per_block + offsets.to(torch.int64)).clamp( 0, flat_keys.shape[0] - 1 @@ -332,8 +273,7 @@ def __call__(self, q, keys, values, tables, lengths): return self._forward(q, keys, values, tables, groups) def attend_metadata(self, q, keys, values, batch): - if batch.sequence_ranges is not None: - return self._attend_ranges(q, keys, values, batch) + packed = batch.cu_seqlens_q is not None flat_rows = _flat_kv_rows(keys, values) stream = self._stream_key(q.device) signature = (q.device, q.dtype, q.shape, keys.shape, keys.stride(), values.stride(), stream) @@ -342,7 +282,7 @@ def attend_metadata(self, q, keys, values, batch): # pointing at partially replaced groups/addresses. self._metadata = None self._signature = None - self._geometry = _prefill_geometry(batch.rows) + self._geometry = _prefill_geometry(batch.rows) if len(q) == len(batch.rows) else None self._write_slots = None self._ragged_plan = None self._decode_plan = None @@ -360,6 +300,7 @@ def attend_metadata(self, q, keys, values, batch): [(id(allocation), depth) for allocation, depth, _ in batch.rows], [position + 1 for _, _, position in batch.rows], q.device, + packed=packed, ) self._metadata = weakref.ref(batch) self._signature = signature diff --git a/vllm_rlt/models/ouro.py b/vllm_rlt/models/ouro.py index 7b929c54..5041fdf3 100644 --- a/vllm_rlt/models/ouro.py +++ b/vllm_rlt/models/ouro.py @@ -281,55 +281,9 @@ def recurrent( batch = cache._prepare_batch(request_ids, depths, positions) return self.recurrent_prepared(hidden, batch, cache, compute_gate=compute_gate) - @property - def recurrent_returns_owned_hidden(self): - if type(self) is not OuroForCausalLM or "recurrent_prepared" in self.__dict__: - return False - norm = self.model.norm - return ( - type(norm) is RMSNorm - and not norm._forward_hooks - and not norm._forward_pre_hooks - and "forward" not in norm.__dict__ - ) - - def prepare_recurrent(self): - """Keep RoPE reuse local to one forward, across loops with the same position view.""" - recurrent = self.recurrent_prepared - if type(self) is not OuroForCausalLM or "recurrent_prepared" in self.__dict__: - return recurrent - position_key = position_embeddings = None - - def run(hidden, batch, cache, *, compute_gate): - nonlocal position_key, position_embeddings - positions = batch.position_ids - key = ( - positions.data_ptr(), - positions.shape, - positions.stride(), - positions.dtype, - positions.device, - hidden.dtype, - ) - if key != position_key: - position_embeddings = self.model.rotary_emb(hidden, positions) - position_key = key - return recurrent( - hidden, - batch, - cache, - compute_gate=compute_gate, - position_embeddings=position_embeddings, - ) - - return run - - def recurrent_prepared( - self, hidden, batch, cache, *, compute_gate=True, position_embeddings=None - ): + def recurrent_prepared(self, hidden, batch, cache, *, compute_gate=True): """Run core with runner-owned metadata, including inactive padding rows.""" - if position_embeddings is None: - position_embeddings = self.model.rotary_emb(hidden, batch.position_ids) + position_embeddings = self.model.rotary_emb(hidden, batch.position_ids) for layer in self.model.layers: hidden = layer(hidden, position_embeddings, batch, cache) # Norm is inside the recurrence in Ouro; this normalized state is the diff --git a/vllm_rlt/worker/ascend_model_runner.py b/vllm_rlt/worker/ascend_model_runner.py index 097b5981..efd1c74d 100644 --- a/vllm_rlt/worker/ascend_model_runner.py +++ b/vllm_rlt/worker/ascend_model_runner.py @@ -1,11 +1,7 @@ """Ascend execution with device-resident state, batched sampling and KV finalization.""" -import weakref -from dataclasses import dataclass - import torch -from vllm_rlt.worker.executor import ModelRuntime from vllm_rlt.worker.model_runner import ModelRunner from vllm_rlt.worker.sampler import Sampler @@ -26,100 +22,12 @@ def sample_batch(self, logits, params, generators): ) -@dataclass(frozen=True) -class HiddenBatch: - """Immutable eager output; Request.hidden_state remains authoritative. - - Row views retain their owning allocation. Weak request identities and exact - view identities prevent reuse after cancellation, replacement or regrouping. - Only the latest batch is retained; this is not a second request-state store. - """ - - hidden: torch.Tensor - requests: tuple[weakref.ReferenceType, ...] - rows: tuple[torch.Tensor, ...] - - def matches(self, requests): - return len(requests) == len(self.rows) and all( - owner() is request and request.hidden_state is row - for owner, request, row in zip(self.requests, requests, self.rows) - ) - - class AscendModelRunner(ModelRunner): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) if self.device.type != "npu": raise ValueError("AscendModelRunner requires an NPU") - self.runtime = ModelRuntime( - self.model, - self.cache_manager, - prepare_recurrent=getattr(self.model, "prepare_recurrent", None), - ) self.sampler = AscendSampler() - self.hidden_batch = None - self.retain_hidden_batch = ( - self.execution_config.async_scheduling - and self.cache_manager.backend == "ascend_eager" - and not self.execution_config.static_buffers - and not self.execution_config.cuda_graphs - ) - - def _save(self, request, state): - self.hidden_batch = None - return super()._save(request, state) - - def _save_batch(self, requests, hidden, routing): - if ( - self.retain_hidden_batch - and getattr(self.model, "recurrent_returns_owned_hidden", False) - and routing is None - and len(hidden) == len(requests) - ): - rows = tuple(hidden.unbind(0)) - for request, state in zip(requests, rows): - request.hidden_state = state - self.hidden_batch = HiddenBatch( - hidden, tuple(weakref.ref(request) for request in requests), rows - ) - return - self.hidden_batch = None - if routing is None: - for request, state in zip(requests, hidden): - self._save(request, state) - return - routing.scatter(hidden) - for request in requests: - request.hidden_state = self.states[self.state_slots[request.request_id]] - - def _gather(self, requests, workspace, size): - saved = self.hidden_batch - if ( - saved is not None - and workspace is None - and size == len(requests) - and saved.matches(requests) - ): - if self.execution_config.multi_stream: - # The next submission can replace all Request row views before - # this stream has consumed the previous allocation. - saved.hidden.record_stream(self.device_ops.current_stream(self.device)) - return saved.hidden - states = [request.hidden_state for request in requests] - if workspace is None: - hidden = torch.stack(states) - else: - hidden = workspace.hidden[:size] - hidden.zero_() - for row, state in enumerate(states): - hidden[row].copy_(state) - if hidden.device.type == "npu" and self.execution_config.multi_stream: - # Request states can be replaced before this stream consumes them. - # NPU uses these tensors instead of CUDA's persistent state slots. - stream = self.device_ops.current_stream(hidden.device) - for state in states: - state.record_stream(stream) - return hidden def _prepare_tokens(self, prepared, size): """Materialize snapshotted token sources on the current compute stream.""" @@ -157,21 +65,6 @@ def _prepare_tokens(self, prepared, size): span.request.input_token_tensor = None return tokens - def _wait_for_requests(self, batch, stream): - # A previous batch often gives many requests the same completion event. - dependencies = set() - saved = self.hidden_batch - retained = saved is not None and saved.matches([span.request for span in batch.spans]) - if retained: - saved.hidden.record_stream(stream) - for span in batch.spans: - event = self.events.get(span.request.request_id) - if event is not None and event not in dependencies: - stream.wait_event(event) - dependencies.add(event) - if not retained and span.request.hidden_state is not None: - span.request.hidden_state.record_stream(stream) - def _finalize_npu_batch(self, requests): cache = self.cache_manager copies = [] @@ -214,7 +107,3 @@ def finalize_many(self, requests): self._finalize_npu_batch(requests) else: super().finalize_many(requests) - - def _free_runner_slot(self, request_id): - self.hidden_batch = None - return super()._free_runner_slot(request_id) diff --git a/vllm_rlt/worker/executor.py b/vllm_rlt/worker/executor.py index 24cce2c0..70404371 100644 --- a/vllm_rlt/worker/executor.py +++ b/vllm_rlt/worker/executor.py @@ -139,10 +139,9 @@ class ForwardResult: class ModelRuntime: - def __init__(self, model, cache, *, prepare_recurrent=None): + def __init__(self, model, cache): self.model = model self.cache = cache - self.prepare_recurrent = prepare_recurrent def prepare(self, batch: ForwardBatch) -> ForwardBatch: """Choose graph/fallback for every call before execution. @@ -210,7 +209,6 @@ def forward(self, batch: ForwardBatch) -> ForwardResult: raise ValueError(f"unsupported operation {batch.operation}") if hidden is None: raise ValueError("RECURRENT requires hidden state") - recurrent = None for index, kv in enumerate(batch.metadata): if batch.graphs is not None: call = batch.graph_calls[index] @@ -218,13 +216,7 @@ def forward(self, batch: ForwardBatch) -> ForwardResult: batch.graphs.prepare_inputs(call, hidden) hidden, gate = batch.graphs.execute(call, hidden) else: - if recurrent is None: - recurrent = ( - self.prepare_recurrent() - if self.prepare_recurrent is not None - else self.model.recurrent_prepared - ) - hidden, gate = recurrent( + hidden, gate = self.model.recurrent_prepared( hidden, kv, self.cache, compute_gate=batch.compute_gate ) return ForwardResult(hidden, gate) diff --git a/vllm_rlt/worker/metadata.py b/vllm_rlt/worker/metadata.py index 7c65cde3..95fde7bb 100644 --- a/vllm_rlt/worker/metadata.py +++ b/vllm_rlt/worker/metadata.py @@ -10,7 +10,6 @@ from dataclasses import dataclass, replace from typing import TYPE_CHECKING -import numpy as np import torch from vllm_rlt.attention import AttentionMetadataValues, AttentionRows, plan_attention_metadata @@ -30,120 +29,16 @@ class KVMetadata: """ owner: KVCacheManager - rows: Sequence[tuple[_Allocation, int, int]] + rows: tuple[tuple[_Allocation, int, int], ...] allocations: tuple[tuple[str, _Allocation], ...] - position_ids: torch.Tensor | None - write_blocks: torch.Tensor | None - write_offsets: torch.Tensor | None - block_tables: torch.Tensor | None - context_lengths: torch.Tensor | None + position_ids: torch.Tensor + write_blocks: torch.Tensor + write_offsets: torch.Tensor + block_tables: torch.Tensor + context_lengths: torch.Tensor writable: bool cu_seqlens_q: torch.Tensor | None = None max_seqlen_q: int = 1 - # Compact plans use one table per sequence, not one duplicated table per - # query. Only a backend declaring sequence_ranges may consume this form. - sequence_ranges: tuple[SequenceRange, ...] | None = None - table_plan: RangeTablePlan | None = None - - -@dataclass(frozen=True) -class SequenceRange: - allocation: _Allocation - depth: int - start: int - count: int - - @property - def end(self): - return self.start + self.count - - -class RangeRows(Sequence): - """Immutable lazy compatibility view; fast consumers use the ranges.""" - - def __init__(self, ranges): - self.ranges = ranges - self.width = ranges[0].count - - def __len__(self): - return len(self.ranges) * self.width - - def __getitem__(self, index): - if isinstance(index, slice): - return tuple(self)[index] - if index < 0: - index += len(self) - if not 0 <= index < len(self): - raise IndexError(index) - row, offset = divmod(index, self.width) - span = self.ranges[row] - return span.allocation, span.depth, span.start + offset - - -class RangeTablePlan: - """Owned, immutable page tables; tensors are created during GPU binding. - - Each stream has its own upload, so a first upload never races a reader on - another stream. Position/write tensors remain submission-owned snapshots. - """ - - def __init__(self, tables): - width = max(map(len, tables)) - self.host = np.full((len(tables), width), -1, dtype=np.int32) - for i, table in enumerate(tables): - self.host[i, : len(table)] = table - self.host.flags.writeable = False - self.tensors = {} - - def bind(self, device): - stream = None - if device.type == "npu": - stream = torch.npu.current_stream(device).stream_id - elif device.type == "cuda": - stream = torch.cuda.current_stream(device).cuda_stream - key = device, stream - if key not in self.tensors: - self.tensors[key] = torch.tensor(self.host, device=device, dtype=torch.int32) - return self.tensors[key] - - -@dataclass(frozen=True) -class RangeLayout: - owner: KVCacheManager - ranges: tuple[SequenceRange, ...] - allocations: tuple - positions: np.ndarray - offsets: np.ndarray - blocks: np.ndarray - tables: RangeTablePlan - writable: bool - - -def stage_range_metadata(layouts, stage): - """Snapshot changing indices; stable page tables stay with their plan.""" - first = layouts[0] - n = len(first.positions) - wide = stage( - np.concatenate([first.positions, first.offsets, *(x.blocks for x in layouts)]), - torch.int64, - ) - lengths = stage((first.positions + 1).astype(np.int32), torch.int32) - return tuple( - KVMetadata( - owner=x.owner, - rows=RangeRows(x.ranges), - allocations=x.allocations, - position_ids=wide[:n], - write_blocks=wide[(i + 2) * n : (i + 3) * n], - write_offsets=wide[n : 2 * n], - block_tables=None, - context_lengths=lengths, - writable=x.writable, - sequence_ranges=x.ranges, - table_plan=x.tables, - ) - for i, x in enumerate(layouts) - ) @dataclass(frozen=True, eq=False) @@ -194,90 +89,6 @@ class KVLayout: class MetadataBuilder: def __init__(self, cache): self.cache = cache - self.range_plans_enabled = True - self._range_key = None - self._range_tables = () - self._range_allocations = () - self._range_shape = None - self._relative_positions = None - self._range_bound_allocation = None - self._range_min_pages = None - self.range_plan_builds = 0 - self.range_plan_reuses = 0 - - def build_ranges(self, request_ids, starts, counts, depth_sets, *, for_write=True): - """Resolve one contiguous range per private sequence, across depths. - - Scalar bounds/identity checks stay per sequence. Every traversal still - gets immutable ranges and freshly snapshotted write positions. - """ - cache = self.cache - size = len(request_ids) - if ( - not size - or len(set(request_ids)) != size - or len(starts) != size - or len(counts) != size - or len(set(starts)) != 1 - or len(set(counts)) != 1 - or any(type(x) is not int or x <= 0 for x in counts) - or any(type(x) is not int or x < 0 for x in starts) - or not depth_sets - or any(len(depths) != size for depths in depth_sets) - or cache.layout != "last_exited" - or cache.enable_prefix_caching - ): - raise ValueError("compact KV plans require unique private uniform sequence ranges") - depths_key = tuple(tuple(depths) for depths in depth_sets) - key = cache._allocation_epoch, tuple(request_ids), depths_key - if key != self._range_key: - allocations = tuple((rid, cache._get_allocation(rid)) for rid in request_ids) - tables = [] - for depths in depths_key: - for depth in depths: - cache._validate_depth(depth) - tables.append( - RangeTablePlan( - [a.block_tables[cache._plane(d)] for (_, a), d in zip(allocations, depths)] - ) - ) - # Publish only after preparation succeeds. - self._range_bound_allocation = min( - (a for _, a in allocations), key=lambda a: a.max_tokens - ) - self._range_min_pages = min(len(a.block_tables[0]) for _, a in allocations) - self._range_key = key - self._range_allocations = allocations - self._range_tables = tuple(tables) - self.range_plan_builds += 1 - else: - self.range_plan_reuses += 1 - allocations = self._range_allocations - width = counts[0] - end_position = starts[0] + width - 1 - cache._validate_position(self._range_bound_allocation, end_position) - if end_position // cache.block_size >= self._range_min_pages: - raise ValueError("compact range exceeds reserved physical KV pages") - shape = size, width - if shape != self._range_shape: - self._relative_positions = np.tile(np.arange(width, dtype=np.int64), size) - self._range_shape = shape - positions = self._relative_positions + np.repeat(np.asarray(starts), width) - offsets = positions % cache.block_size - pages = (positions // cache.block_size).reshape(size, width) - layouts = [] - for depths, table in zip(depths_key, self._range_tables): - ranges = tuple( - SequenceRange(allocation, depth, start, width) - for (_, allocation), depth, start in zip(allocations, depths, starts) - ) - blocks = np.take_along_axis(table.host, pages, axis=1).reshape(-1) - layouts.append( - RangeLayout( - cache, ranges, allocations, positions, offsets, blocks, table, for_write - ) - ) - return tuple(layouts) def prepare_device( self, @@ -477,11 +288,7 @@ def view(tensor): position_ids=view(metadata.position_ids), write_blocks=view(metadata.write_blocks), write_offsets=view(metadata.write_offsets), - block_tables=( - metadata.table_plan.bind(storage[metadata.position_ids.dtype].device) - if metadata.table_plan is not None - else view(metadata.block_tables) - ), + block_tables=view(metadata.block_tables), context_lengths=view(metadata.context_lengths), cu_seqlens_q=(None if metadata.cu_seqlens_q is None else view(metadata.cu_seqlens_q)), ) @@ -489,8 +296,6 @@ def view(tensor): @staticmethod def slice_rows(metadata, start, end): """Select contiguous single-query rows, e.g. a dependent coda subset.""" - if metadata.sequence_ranges is not None: - raise ValueError("compact sequence ranges must be rebuilt before slicing") if metadata.cu_seqlens_q is not None: raise ValueError("packed query boundaries must be rebuilt when selecting rows") return replace( diff --git a/vllm_rlt/worker/model_runner.py b/vllm_rlt/worker/model_runner.py index a5ce3c73..d7312d92 100644 --- a/vllm_rlt/worker/model_runner.py +++ b/vllm_rlt/worker/model_runner.py @@ -19,7 +19,6 @@ ) from vllm_rlt.worker.cuda_graph import CodaGraphs, PreludeGraphs, RecurrentGraphs from vllm_rlt.worker.executor import ForwardBatch, ModelRuntime -from vllm_rlt.worker.metadata import stage_range_metadata from vllm_rlt.worker.output import ModelRunnerOutput from vllm_rlt.worker.sampler import RngRegistry, Sampler from vllm_rlt.worker.submission_queue import MAX_CONCURRENT_SUBMISSIONS, SubmissionQueue @@ -728,7 +727,9 @@ def _prepare_shared_loop_inputs(self, batch): def _uses_packed_loop_inputs(self, batch): capabilities = self.cache_manager.attention_capabilities return ( - batch.operation == Operation.PRELUDE + # PackedMetadataBuffer expands write addresses with a CUDA kernel. + self.device.type == "cuda" + and batch.operation == Operation.PRELUDE and batch.loop_count and self.cache_manager.layout != "shared" and capabilities.packed_prefill @@ -739,23 +740,6 @@ def _uses_packed_loop_inputs(self, batch): def _prepare_loop_inputs(self, batch, packed_metadata=None): cache = self.cache_manager - if self._uses_range_plan(batch): - spans = batch.spans - metadata = self._stage_range_plan(batch) - tokens = [token for span in spans for token in span.input_tokens] - tensor = self.input_buffers[torch.int64].append(tokens) - end = 0 - outputs = [] - for span in spans: - end += span.token_count - outputs.append((end - 1, span.request)) - return ( - PreparedForward( - ForwardBatch(Operation.PRELUDE, tokens=tensor, metadata=metadata), - outputs=tuple(outputs), - requests=tuple(span.request for span in spans), - ), - ) if cache.layout == "shared": return self._prepare_shared_loop_inputs(batch) ids, positions, tokens, requests, outputs, depths = [], [], [], [], [], [] @@ -802,36 +786,6 @@ def _stage_metadata(self, ids, depth_sets, positions, *, packed=False, padded_ro layouts = self.metadata.build(ids, depth_sets, positions, packed=packed) return stage_metadata(layouts, self._append_input, padded_rows=padded_rows) - def _uses_range_plan(self, batch): - cache = self.cache_manager - spans = batch.spans - return ( - self.device.type == "npu" - and self.metadata.range_plans_enabled - and cache.attention_capabilities.sequence_ranges - and cache.layout == "last_exited" - and not cache.enable_prefix_caching - and not self.execution_config.static_buffers - and not self.execution_config.cuda_graphs - and not self.execution_config.pad_to_power_of_two - and not getattr(self.model, "requires_boundary_kv", False) - and bool(spans) - and len({span.request.request_id for span in spans}) == len(spans) - and len({span.token_start for span in spans}) == 1 - and len({span.token_count for span in spans}) == 1 - and (not batch.has_chunks or batch.operation == Operation.PRELUDE) - ) - - def _stage_range_plan(self, batch): - spans = batch.spans - layouts = self.metadata.build_ranges( - [span.request.request_id for span in spans], - [span.token_start for span in spans], - [span.token_count for span in spans], - [[span.depth + i for span in spans] for i in range(batch.loop_count)], - ) - return stage_range_metadata(layouts, self._append_input) - def _append_input(self, values, dtype): return self.input_buffers[dtype].append(values) @@ -864,11 +818,7 @@ def input_view(tensor): if prepared.packed_metadata is not None: tokens, metadata = prepared.packed_metadata.upload() else: - tokens = ( - self._prepare_tokens(prepared, prepared.batch.num_rows) - if self.device.type == "npu" and forward.tokens is None - else input_view(forward.tokens) - ) + tokens = input_view(forward.tokens) metadata = tuple(self.metadata.bind(kv, uploaded) for kv in forward.metadata) kv_metadata = None hidden_buffer = None @@ -908,10 +858,6 @@ def prepare_on_cpu(self, batch: ExecutionBatch): metadata, boundary_metadata = (), None if batch.operation == Operation.RECURRENT: start = 1 if prepared.kv is not None else 0 - if start == 0 and self._uses_range_plan(batch): - metadata = self._stage_range_plan(batch) - forward = ForwardBatch(batch.operation, metadata=metadata) - return replace(prepared, forwards=(PreparedForward(forward),)) if start < batch.loop_count: ids, depths, positions = batch.addresses() metadata = self._stage_metadata( @@ -961,41 +907,6 @@ def _prepare_cpu_inputs(self, batch: ExecutionBatch): ) if batch.operation == Operation.CODA: prepared = replace(prepared, sampling_params=tuple(r.sampling_params for r in requests)) - if ( - self.device.type == "npu" - and batch.operation == Operation.PRELUDE - and batch.loop_count - and not batch.has_chunks - and all(span.input_tokens is None for span in batch.spans) - ): - # Decode sources can be preceding device samples not yet delivered - # to the CPU. Snapshot their identities before advancing requests. - if any(span.token_count != 1 for span in batch.spans): - raise ValueError("coalesced decode prelude requires one token per request") - sources = tuple( - r.input_token_tensor if r.input_token_tensor is not None else (r.input_token_id,) - for r in requests - ) - if self._uses_range_plan(batch): - metadata = self._stage_range_plan(batch) - else: - ids, depths, positions = batch.addresses() - metadata = self._stage_metadata( - ids, - [[depth + offset for depth in depths] for offset in range(batch.loop_count)], - positions, - ) - return replace( - prepared, - token_sources=sources, - # Preserve the fixed decode path's gate work; its output has - # no consumer and is not copied back to the host. - forwards=( - PreparedForward( - ForwardBatch(Operation.PRELUDE, metadata=metadata, compute_gate=True) - ), - ), - ) if batch.operation == Operation.PRELUDE and batch.loop_count: packed_metadata = ( self.packed_metadata_buffers[self.submissions.slot] From c611c100d88295e2ba7bfa7fa4646040d97ee073 Mon Sep 17 00:00:00 2001 From: bjf-frz Date: Thu, 19 Nov 2026 02:38:37 +0000 Subject: [PATCH 20/20] refactor(ascend): use the shared paged KV writer Use ordinary indexed assignment for BF16 KV and let Torch-NPU dispatch it. Remove the Scatter ND writer and keep paged write coverage for both attention backends. --- docs/recipes/ouro_ascend.md | 7 ++--- tests/test_ascend_kv.py | 15 ++--------- vllm_rlt/core/kv_cache_manager.py | 16 +---------- vllm_rlt/kernels/ascend_kv.py | 45 ------------------------------- 4 files changed, 7 insertions(+), 76 deletions(-) delete mode 100644 vllm_rlt/kernels/ascend_kv.py diff --git a/docs/recipes/ouro_ascend.md b/docs/recipes/ouro_ascend.md index 01826bfe..82c3887a 100644 --- a/docs/recipes/ouro_ascend.md +++ b/docs/recipes/ouro_ascend.md @@ -59,11 +59,12 @@ curl --fail http://127.0.0.1:8123/v1/completions \ so the scheduler can select a new batch at each boundary. - **Reuse and batching.** Ascend uses the shared packed-prefill metadata by default with `ascend_eager`. Use `--no-packed-prefill` to select the per-query - layout. Greedy sampling, compatible prefill and paged KV write indices are - batched. Full prefill uses causal SDPA. + layout. Greedy sampling and compatible prefill are batched. Full prefill + uses causal SDPA. Cached chunks and ragged decode pass boolean masks to preserve each row's visible KV range. -- **Paged KV.** Each loop writes to the page pool using its own block table. +- **Paged KV.** Each loop writes to the page pool through PyTorch indexed + assignment using its own block table. The Ascend backend gathers the visible KV rows for Torch SDPA. The example uses private, fully reserved `last_exited` KV. Keep graphs, diff --git a/tests/test_ascend_kv.py b/tests/test_ascend_kv.py index 298d0c1a..7d53d071 100644 --- a/tests/test_ascend_kv.py +++ b/tests/test_ascend_kv.py @@ -1,4 +1,4 @@ -"""Assigned-device regressions for native KV copies and descriptor lifetime.""" +"""Paged KV writes and descriptor lifetime on an assigned Ascend device.""" import pytest import torch @@ -12,19 +12,9 @@ @pytest.mark.parametrize("rows", [1, 3]) @pytest.mark.parametrize("backend", ["torch", "ascend_eager"]) @torch.inference_mode() -def test_native_scatter_preserves_inputs_other_layers_and_reused_ids(rows, backend, monkeypatch): +def test_paged_kv_writes_preserve_inputs_layers_and_reused_ids(rows, backend): device = resolve_device("npu:0") torch.npu.set_device(device) - import torch_npu - - native = torch_npu.npu_scatter_nd_update_ - calls = [] - - def observed(*args, **kwargs): - calls.append(True) - return native(*args, **kwargs) - - monkeypatch.setattr(torch_npu, "npu_scatter_nd_update_", observed) cache = KVCacheManager( 2, 2, 64, 32, 16, 4, device=device, dtype=torch.bfloat16, backend=backend ) @@ -62,4 +52,3 @@ def observed(*args, **kwargs): cache.free("b") assert cache.num_used_blocks == 0 torch.npu.synchronize() - assert len(calls) == (6 if backend == "ascend_eager" else 0) diff --git a/vllm_rlt/core/kv_cache_manager.py b/vllm_rlt/core/kv_cache_manager.py index 2b1464ed..c6187d77 100644 --- a/vllm_rlt/core/kv_cache_manager.py +++ b/vllm_rlt/core/kv_cache_manager.py @@ -876,21 +876,7 @@ def _write_prepared( rows = len(batch.rows) if not rows: return - if self.device.type == "npu" and self.backend == "ascend_eager": - if not hasattr(self, "_npu_writer"): - from vllm_rlt.kernels.ascend_kv import AscendKVWriter - - self._npu_writer = AscendKVWriter() - self._npu_writer.write( - self.key_cache, - self.value_cache, - layer, - batch, - k if len(k) == rows else k[:rows], - v if len(v) == rows else v[:rows], - ) - else: - self._store_layer(layer, batch.write_blocks, batch.write_offsets, k[:rows], v[:rows]) + self._store_layer(layer, batch.write_blocks, batch.write_offsets, k[:rows], v[:rows]) if coverage is None: for allocation, depth, position in batch.rows: allocation.written[self._plane(depth)][layer].add(position) diff --git a/vllm_rlt/kernels/ascend_kv.py b/vllm_rlt/kernels/ascend_kv.py deleted file mode 100644 index 9b77eb69..00000000 --- a/vllm_rlt/kernels/ascend_kv.py +++ /dev/null @@ -1,45 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -"""Exact KV copies through native Scatter ND on Ascend.""" - -from __future__ import annotations - -import weakref - -import torch - - -class AscendKVWriter: - """Reuse immutable traversal indices across layers without retaining the descriptor. - - The batch already carries its write addresses as staged device tensors - (``write_blocks`` / ``write_offsets``). Building the Scatter ND index from - those tensors keeps the host out of the per-traversal hot path. - """ - - def __init__(self): - self._metadata = None - self._indices = None - self._layer_ids = None - - def write(self, keys, values, layer, batch, k, v): - import torch_npu - - if self._metadata is None or self._metadata() is not batch: - blocks = torch.as_tensor(batch.write_blocks, dtype=torch.int32, device=keys.device) - offsets = torch.as_tensor(batch.write_offsets, dtype=torch.int32, device=keys.device) - rows = len(batch.rows) - if blocks.numel() != rows or offsets.numel() != rows: - raise ValueError("KV write address count does not match the prepared rows") - layers = keys.shape[1] - if self._layer_ids is None or self._layer_ids.shape[0] != layers: - self._layer_ids = torch.arange(layers, device=keys.device, dtype=torch.int32)[ - :, None - ] - layer_ids = self._layer_ids.expand(layers, rows) - self._indices = torch.stack( - (blocks.expand(layers, -1), layer_ids, offsets.expand(layers, -1)), dim=-1 - ) - self._metadata = weakref.ref(batch) - indices = self._indices[layer] - torch_npu.npu_scatter_nd_update_(keys, indices, k) - torch_npu.npu_scatter_nd_update_(values, indices, v)