From c8868f03ee04f5858bb7f8be73a02bfb359f319d Mon Sep 17 00:00:00 2001 From: cecilsd <127472276+cecilsd@users.noreply.github.com> Date: Wed, 18 Nov 2026 19:04:29 +0800 Subject: [PATCH 1/3] feat(runtime): prepare runners and CUDA Graphs before engine readiness Prepare the serving runners after connector initialization and before worker readiness. Direct CUDA engines also prepare during initialization. - Reuse serving execution and current KV allocations for bounded prefill, decode, sampling, and configured speculative warmup - Capture planned Graph shapes and verify representative replays while retaining lazy capture and reserves for uncovered shapes - Drain device work before releasing scratch requests, KV, and async state; preserve serving RNG and business counters - Add idempotent preparation and phased coverage, Graph, and memory reports - Keep estimated KV sizing and make CPU preparation workload-free - Add lifecycle, failure, BF16/FP8, and request-reuse regressions and docs Real-checkpoint validation preserves outputs within each KV dtype. Preparation reduces first-request setup at the cost of slower startup; it does not establish a steady-state throughput improvement or implement the rest of RFC #192. Refs #192 --- README.md | 1 + docs/runtime.md | 48 ++ tests/test_cdb_runtime.py | 7 + tests/test_cuda_graph.py | 4 +- tests/test_input_preparation.py | 2 + tests/test_runtime_startup.py | 711 +++++++++++++++++++++++++++ tests/test_split_prefill.py | 5 +- vllm_rlt/engine/stage_engine.py | 69 +++ vllm_rlt/engine/worker.py | 24 +- vllm_rlt/worker/async_speculative.py | 3 - vllm_rlt/worker/buffers.py | 6 + vllm_rlt/worker/cuda_graph.py | 9 +- vllm_rlt/worker/model_runner.py | 1 + vllm_rlt/worker/warmup.py | 361 ++++++++++++++ 14 files changed, 1238 insertions(+), 13 deletions(-) create mode 100644 tests/test_runtime_startup.py create mode 100644 vllm_rlt/worker/warmup.py diff --git a/README.md b/README.md index 17502f55..7568799a 100644 --- a/README.md +++ b/README.md @@ -126,6 +126,7 @@ No PyPI publication is implied. gateway; the server has no built-in authentication or TLS. - [Supported models](#model-support) and [Ouro-Thinking recipe](docs/recipes/ouro_thinking.md) - [Configuration and compatibility](docs/runtime.md) +- [CUDA startup preparation and its latency tradeoff](docs/runtime.md#startup-execution) - [Native CUDA build](docs/launching.md#automatic-native-cuda-build) - [Migration from v0.1.0](CHANGELOG.md) diff --git a/docs/runtime.md b/docs/runtime.md index 571ecb59..fe08839a 100644 --- a/docs/runtime.md +++ b/docs/runtime.md @@ -76,6 +76,54 @@ OMP_NUM_THREADS=1 python -m vllm_rlt.entrypoints.cli --toy --dtype float32 \ For a task-assigned GPU, set `CUDA_VISIBLE_DEVICES` and add `--device cuda --attention-backend triton`. Use `--single-stream` for the single-stream pipeline. `--num-blocks` overrides automatic sizing, or specify `--kv-cache-memory-bytes`; these two overrides are mutually exclusive. CPU auto mode keeps a 256-block default. `StageEngine.memory_plan` records the worker-local sizing decision; it is not an attribute of the public `LLM.engine` client. +## Startup execution + +CUDA workers prepare their actual runners after connector initialization and +before sending `ready`. Preparation runs bounded prefill, decode and configured +speculative work with valid scratch KV allocations. It captures the graph shapes +encountered by those workloads and replays representative paths. Scratch requests +never enter the scheduler or publish prefix-cache entries. Streams and submission +queues are drained before request state and KV leases are released; the serving +RNG and request counters are preserved. CPU workers do not execute this workload. + +Preparation increases startup time and retains serving workspaces earlier. It +reduces setup work in the first request; steady-state throughput and total cold +process latency require separate measurement for each model and workload. + +`StageEngine.prepare_runtime()` is idempotent. Its worker-local `startup_report` +records the covered token lengths and batch sizes, graph keys and replay/fallback +counts, and remaining coverage gaps. The `initialized`, `captured`, `replayed` +and `cleaned` phase snapshots record elapsed time, driver-free memory and PyTorch +allocated/reserved memory, including peaks since the preceding snapshot. The +allocator cache is retained consistently at these boundaries. Times are cumulative +seconds from the start of preparation; memory values are bytes. The first peak +includes earlier engine initialization; subsequent peaks cover the interval since +the preceding snapshot. Driver-free values are boundary samples, not a continuous +device high-water mark. The report is published only after successful cleanup. + +Single-request history covers the configured prefill limit and crosses a sliding +window when committed KV permits. Decode batch buckets also cross a KV page. +Greedy and full-vocabulary top-p sampling use scratch request generators. +Async multi-stream preparation additionally submits an independent prompt core +and coda before collection, followed by a decode core/coda pair, within the +request/token/KV limits. Its +`concurrent_prepared` and `concurrent` snapshots separate initial shape preparation +from repeated concurrent execution. `concurrent_batches` lists coda row counts; +each is submitted alongside one independent core request. Actual overlap depends +on device execution and host submission cost. These phases drain the serving streams. + +This preparation does not replace automatic KV sizing. The report describes the +existing allocated KV pool, including FP8 scale storage when configured, and +sizing still uses the current estimates and reserves. The graph and workspace +allocations remain resident; the KV pool itself is not a transient workspace. +Future storage integration must preserve captured pointers, views and strides, +and validate replay after physical commitment before publishing readiness. Small explicit pools can reduce warmup coverage. Unseen exact graph +shapes retain bounded lazy capture. Longer contexts and mixed speculative/prompt +overlap retain estimated transient allowances. Pipeline preparation reaches +target verification before releasing its scratch state. The report is a lifecycle +boundary for subsequent memory-planning integration, not a guarantee that every +serving shape or peak allocation has been measured. + ## Packed prefill `ExecutionConfig.packed_prefill` controls packed prefill attention and metadata diff --git a/tests/test_cdb_runtime.py b/tests/test_cdb_runtime.py index b6e224f6..3aeb0ae6 100644 --- a/tests/test_cdb_runtime.py +++ b/tests/test_cdb_runtime.py @@ -690,11 +690,17 @@ def test_cuda_graph_reserve_reduces_budget_with_fixed_memory_inputs(monkeypatch, def test_cuda_memory_plan_reclaims_allocator_residue_on_recreation(): import gc + # Earlier tests can leave unreachable engines whose streams retain buffers. + gc.collect() + torch.cuda.synchronize() + torch.cuda.empty_cache() base = model().to(device="cuda", dtype=torch.bfloat16) options = dict( cache_config=CacheConfig(block_size=2, memory_reserve_bytes=0), scheduler_config=SchedulerConfig(max_num_seqs=2, max_num_batched_tokens=3), attention_backend="triton", + # Isolate planning from per-stream library workspaces created by warmup. + _defer_runtime_preparation=True, ) first = LLMEngine(base, **options) budget = first.memory_plan["kv_budget_bytes"] @@ -709,5 +715,6 @@ def test_cuda_memory_plan_reclaims_allocator_residue_on_recreation(): second = LLMEngine(base, **options) assert second.cache_manager.num_blocks == blocks assert abs(second.memory_plan["kv_budget_bytes"] - budget) < 16 * 1024 * 1024 + second.prepare_runtime() second.add_request("recreated", [1, 2], SamplingParams(max_tokens=2, ignore_eos=True)) assert len(drain(second)["recreated"].token_ids) == 2 diff --git a/tests/test_cuda_graph.py b/tests/test_cuda_graph.py index a0f08c51..5d786fc9 100644 --- a/tests/test_cuda_graph.py +++ b/tests/test_cuda_graph.py @@ -506,7 +506,9 @@ def __torch_dispatch__(self, func, types, args=(), kwargs=None): keys = torch.full((5, 2, 4, 3, 9), -1, dtype=dtype, device="cuda") values = torch.full_like(keys, -2) - proxy = _DeviceCache(SimpleNamespace(key_cache=keys, value_cache=values, backend="triton")) + proxy = _DeviceCache( + SimpleNamespace(key_cache=keys, value_cache=values, key_scale=None, backend="triton") + ) batch = SimpleNamespace( active_rows=None, write_mask=None, diff --git a/tests/test_input_preparation.py b/tests/test_input_preparation.py index 23e315ab..505b166d 100644 --- a/tests/test_input_preparation.py +++ b/tests/test_input_preparation.py @@ -48,6 +48,8 @@ def test_loop_preparation_reuses_growing_storage_and_only_saves_final_hidden( exit_config=ExitConfig("ouro"), scheduler_config=SchedulerConfig(max_num_seqs=2, max_num_batched_tokens=8), attention_backend=backend, + # Exercise lazy buffer growth independently of startup preallocation. + _defer_runtime_preparation=True, ) runner = engine.model_runner cpu_prepare, gpu_prepare = runner.prepare_on_cpu, runner.prepare_on_gpu diff --git a/tests/test_runtime_startup.py b/tests/test_runtime_startup.py new file mode 100644 index 00000000..19583dab --- /dev/null +++ b/tests/test_runtime_startup.py @@ -0,0 +1,711 @@ +"""Startup execution must use valid KV and leave no serving request state.""" + +import pytest +import torch + +from tests.helpers import tiny_ouro_config +from vllm_rlt import CacheConfig, ExecutionConfig, ExitConfig, SamplingParams, SchedulerConfig +from vllm_rlt.engine.stage_engine import StageEngine +from vllm_rlt.models import OuroForCausalLM + + +def make_engine(*, blocks=64, staged=False, shared=False): + return StageEngine( + OuroForCausalLM(tiny_ouro_config()), + cache_config=CacheConfig( + blocks, + 2, + layout="shared" if shared else "last_exited", + enable_prefix_caching=not shared, + ), + scheduler_config=SchedulerConfig( + max_num_seqs=3, + max_num_batched_tokens=8, + prefill_chunk_size=4, + wavefront_chunked_prefill=staged, + ), + execution_config=ExecutionConfig(static_buffers=True), + ) + + +@pytest.mark.parametrize("staged,shared", [(False, False), (True, False), (False, True)]) +def test_warmup_executes_valid_history_and_releases_all_request_state(staged, shared): + from vllm_rlt.worker.warmup import prepare_runner + + engine = make_engine(staged=staged, shared=shared) + rng = torch.get_rng_state().clone() + result = prepare_runner(engine.model_runner, engine.scheduler.config) + cache, runner = engine.cache_manager, engine.model_runner + assert result["prefill_tokens"] == (1, 4) + assert result["decode_batches"] == (1, 2, 3) + assert result["history_tokens"] > cache.block_size + assert cache.num_used_blocks == 0 + assert not cache._allocations and not cache._prefixes and not cache._pending_prefixes + assert cache.prefix_queries == cache.prefix_hits == 0 + assert not engine.rng.generators + assert not runner.state_slots and not runner.prompt_token_owners and not runner.events + assert not engine.scheduler.requests + assert all(not getattr(buffer, "allocations", ()) for buffer in runner.packed_metadata_buffers) + torch.testing.assert_close(torch.get_rng_state(), rng) + engine.add_request("__warmup_0", [1, 2, 3], SamplingParams(max_tokens=2, ignore_eos=True)) + outputs = [] + while engine.has_unfinished_requests(): + outputs.extend(engine.step()) + assert len(outputs[-1].token_ids) == 2 + + +def test_warmup_failure_releases_scratch_before_propagating(monkeypatch): + from vllm_rlt.worker.warmup import prepare_runner + + engine = make_engine(staged=True) + runner = engine.model_runner + execute = runner._execute + calls = 0 + + def fail(*args): + nonlocal calls + output = execute(*args) + calls += 1 + if calls == 3: + raise RuntimeError("startup execution failed") + return output + + monkeypatch.setattr(runner, "_execute", fail) + with pytest.raises(RuntimeError, match="startup execution failed"): + prepare_runner(runner, engine.scheduler.config) + assert engine.cache_manager.num_used_blocks == 0 + assert engine.cache_manager.prefix_queries == 0 + assert not runner.state_slots and not runner.events and not runner.prompt_token_owners + assert not engine.rng.generators + + +def test_small_kv_capacity_reports_reduced_coverage(): + from vllm_rlt.worker.warmup import prepare_runner + + engine = make_engine(blocks=8) + result = prepare_runner(engine.model_runner, engine.scheduler.config) + assert result["decode_batches"] == (1,) + assert result["coverage_gaps"] + assert engine.cache_manager.num_used_blocks == 0 + + +def test_warmup_populates_history_across_sliding_window(monkeypatch): + from tests.test_dory import tiny_dory_config + from vllm_rlt.models import DoryForCausalLM + from vllm_rlt.worker.warmup import prepare_runner + + engine = StageEngine( + DoryForCausalLM(tiny_dory_config(sliding_window=8)), + cache_config=CacheConfig(128, 2), + scheduler_config=SchedulerConfig( + max_num_seqs=3, + max_num_batched_tokens=8, + prefill_chunk_size=4, + ), + ) + positions = [] + submit = engine.model_runner.submit + + def observe(*args, batch): + positions.extend(span.token_start for span in batch.spans) + return submit(*args, batch=batch) + + monkeypatch.setattr(engine.model_runner, "submit", observe) + result = prepare_runner(engine.model_runner, engine.scheduler.config) + assert result["history_tokens"] == 9 + assert max(positions) >= 9 + assert not engine.cache_manager._allocations and not engine.rng.generators + + +def test_warmup_reports_capacity_below_sliding_window(): + from tests.test_dory import tiny_dory_config + from vllm_rlt.models import DoryForCausalLM + from vllm_rlt.worker.warmup import prepare_runner + + engine = StageEngine( + DoryForCausalLM(tiny_dory_config(sliding_window=64)), + cache_config=CacheConfig(16, 2), + scheduler_config=SchedulerConfig( + max_num_seqs=1, max_num_batched_tokens=4, prefill_chunk_size=4 + ), + ) + report = prepare_runner(engine.model_runner, engine.scheduler.config) + assert report["history_tokens"] > engine.cache_manager.block_size + assert any("limits prefill/history coverage" in gap for gap in report["coverage_gaps"]) + assert not engine.cache_manager._allocations + + +def test_warmup_exercises_stochastic_sampling_without_rng_leak(monkeypatch): + from vllm_rlt.worker.warmup import prepare_runner + + engine = make_engine() + sample = engine.model_runner.sampler.sample + params_seen = [] + state = torch.get_rng_state().clone() + + def observe(logits, params, generator): + params_seen.append(params) + return sample(logits, params, generator) + + monkeypatch.setattr(engine.model_runner.sampler, "sample", observe) + prepare_runner(engine.model_runner, engine.scheduler.config) + assert any(params.temperature > 0 for params in params_seen) + assert any(params.top_k == engine.model.config.vocab_size for params in params_seen) + assert not engine.rng.generators + torch.testing.assert_close(torch.get_rng_state(), state) + + +@pytest.mark.parametrize("staged", [False, True]) +def test_warmup_uses_serving_core_loop_boundary(staged, monkeypatch): + from vllm_rlt.request import Operation + from vllm_rlt.worker.warmup import prepare_runner + + engine = make_engine(staged=staged) + loops = [] + submit = engine.model_runner.submit + + def observe(*args, batch): + if batch.operation == Operation.RECURRENT: + loops.append(batch.loop_count) + return submit(*args, batch=batch) + + monkeypatch.setattr(engine.model_runner, "submit", observe) + prepare_runner(engine.model_runner, engine.scheduler.config) + assert loops and all(count == 1 for count in loops) + + +def test_warmup_covers_decode_batch_above_graph_limit(monkeypatch): + from dataclasses import replace + + from vllm_rlt.request import Operation + from vllm_rlt.worker.warmup import prepare_runner + + engine = make_engine() + runner = engine.model_runner + runner.execution_config = replace(runner.execution_config, cuda_graph_max_batch_size=1) + submit = runner.submit + rows = [] + + def observe(*args, batch): + if batch.operation == Operation.RECURRENT: + rows.append(len(batch.spans)) + return submit(*args, batch=batch) + + monkeypatch.setattr(runner, "submit", observe) + report = prepare_runner(runner, engine.scheduler.config) + assert max(rows) == 3 + assert report["decode_batches"] == (1, 3) + + +@pytest.mark.gpu +@pytest.mark.parametrize("staged", [False, True]) +def test_async_startup_submits_independent_streams_before_collect(staged, monkeypatch): + from vllm_rlt.request import Operation + from vllm_rlt.worker.model_runner import Submission + + engine = StageEngine( + OuroForCausalLM(tiny_ouro_config(head_dim=64)).to("cuda", torch.bfloat16), + attention_backend="triton", + cache_config=CacheConfig(128, 16), + scheduler_config=SchedulerConfig( + max_num_seqs=3, + max_num_batched_tokens=8, + prefill_chunk_size=4, + wavefront_chunked_prefill=staged, + ), + execution_config=ExecutionConfig(async_scheduling=True, cuda_graphs=True), + exit_config=ExitConfig("ouro_delayed"), + _defer_runtime_preparation=True, + ) + pending = [] + pairs = [] + decode_pairs = [] + submit, collect = engine.model_runner.submit, Submission.collect + + def submitted(*args, batch): + ids = {span.request.request_id for span in batch.spans} + ticket = submit(*args, batch=batch) + if batch.operation == Operation.CODA: + pairs.extend( + (other, ticket) + for other, other_ids, loops, _ in pending + if loops and ids.isdisjoint(other_ids) + ) + decode_pairs.extend( + (other, ticket) + for other, other_ids, _, operation in pending + if operation == Operation.RECURRENT and ids.isdisjoint(other_ids) + ) + if batch.loop_count: + pairs.extend( + (other, ticket) + for other, other_ids, loops, _ in pending + if not loops and ids.isdisjoint(other_ids) + ) + pending.append((ticket, ids, batch.loop_count, batch.operation)) + return ticket + + def collected(ticket): + pending[:] = [item for item in pending if item[0] is not ticket] + return collect(ticket) + + monkeypatch.setattr(engine.model_runner, "submit", submitted) + monkeypatch.setattr(Submission, "collect", collected) + try: + report = engine.prepare_runtime() + assert pairs and decode_pairs + assert report["concurrent_batches"] == (1, 2) + assert "concurrent" in [phase["name"] for phase in report["phases"]] + assert not engine.cache_manager._allocations and not engine.rng.generators + assert not engine.model_runner.events + finally: + engine.close() + + +@pytest.mark.gpu +@pytest.mark.parametrize("limit", ["requests", "tokens", "kv", "single_stream"]) +def test_startup_reports_when_independent_overlap_is_unavailable(limit): + engine = StageEngine( + OuroForCausalLM(tiny_ouro_config(head_dim=64)).to("cuda", torch.bfloat16), + attention_backend="triton", + cache_config=CacheConfig(4 if limit == "kv" else 128, 16), + scheduler_config=SchedulerConfig( + max_num_seqs=1 if limit == "requests" else 3, + max_num_batched_tokens=1 if limit == "tokens" else 8, + prefill_chunk_size=4, + ), + execution_config=ExecutionConfig( + async_scheduling=True, multi_stream=limit != "single_stream" + ), + exit_config=ExitConfig("ouro_delayed"), + ) + try: + report = engine.startup_report + assert report["concurrent_batches"] == () + if limit != "single_stream": + assert any( + "prevent independent stream overlap" in gap for gap in report["coverage_gaps"] + ) + assert not engine.cache_manager._allocations and not engine.rng.generators + finally: + engine.close() + + +@pytest.mark.gpu +def test_concurrent_startup_failure_drains_before_releasing_requests(monkeypatch): + from vllm_rlt.request import Operation + from vllm_rlt.worker.model_runner import Submission + + engine = StageEngine( + OuroForCausalLM(tiny_ouro_config(head_dim=64)).to("cuda", torch.bfloat16), + attention_backend="triton", + cache_config=CacheConfig(128, 16), + scheduler_config=SchedulerConfig( + max_num_seqs=3, max_num_batched_tokens=8, prefill_chunk_size=4 + ), + execution_config=ExecutionConfig(async_scheduling=True), + exit_config=ExitConfig("ouro_delayed"), + _defer_runtime_preparation=True, + ) + runner = engine.model_runner + submit, collect = runner.submit, Submission.collect + pending_core = [] + + def submitted(*args, batch): + ticket = submit(*args, batch=batch) + if batch.operation == Operation.RECURRENT: + pending_core.append(ticket) + if pending_core and batch.operation == Operation.CODA: + raise RuntimeError("concurrent preparation failed") + return ticket + + def collected(ticket): + pending_core[:] = [item for item in pending_core if item is not ticket] + return collect(ticket) + + try: + with monkeypatch.context() as patch: + patch.setattr(runner, "submit", submitted) + patch.setattr(Submission, "collect", collected) + with pytest.raises(RuntimeError, match="concurrent preparation failed"): + engine.prepare_runtime() + assert engine.startup_report is None + assert not engine.cache_manager._allocations and not engine.rng.generators + assert not runner.events and not runner.state_slots + assert engine.prepare_runtime()["status"] == "prepared" + finally: + engine.close() + + +@pytest.mark.parametrize("draft_loops", [1, 2]) +@pytest.mark.parametrize( + "device,asynchronous", + [ + ("cpu", False), + pytest.param("cuda", False, marks=pytest.mark.gpu), + pytest.param("cuda", True, marks=pytest.mark.gpu), + ], +) +def test_pipeline_warmup_reaches_target_verification(draft_loops, device, asynchronous): + from vllm_rlt import SpeculativeConfig + from vllm_rlt.worker.warmup import prepare_runner + + engine = StageEngine( + OuroForCausalLM(tiny_ouro_config(head_dim=64)).to(device), + attention_backend="triton" if device == "cuda" else "torch", + cache_config=CacheConfig(128, 2), + scheduler_config=SchedulerConfig( + max_num_seqs=3, max_num_batched_tokens=8, prefill_chunk_size=4 + ), + speculative_config=SpeculativeConfig(2, draft_loops=draft_loops, mode="pipeline"), + execution_config=ExecutionConfig(async_scheduling=asynchronous), + _defer_runtime_preparation=True, + ) + verified = [] + try: + prepare_runner( + engine.model_runner, + engine.scheduler.config, + engine.speculative_runner, + phase=lambda name: verified.append(engine.speculative_runner.stats.verified_rows), + ) + assert verified and all(count > 0 for count in verified) + states = "states" if asynchronous else "pipeline_states" + assert not getattr(engine.speculative_runner, states) + assert engine.speculative_runner.stats.verified_rows == 0 + assert not engine.cache_manager._allocations + finally: + engine.close() + + +@pytest.mark.parametrize("kind", ["dory", "huginn"]) +def test_boundary_warmup_keeps_rng_and_valid_prefixes(kind): + from tests.test_dory import tiny_dory_config + from tests.test_huginn import tiny_huginn_config + from vllm_rlt.models import DoryForCausalLM, HuginnForCausalLM + from vllm_rlt.worker.warmup import prepare_runner + + model = ( + DoryForCausalLM(tiny_dory_config()) + if kind == "dory" + else HuginnForCausalLM(tiny_huginn_config()) + ) + engine = StageEngine( + model, + cache_config=CacheConfig(128, 2), + scheduler_config=SchedulerConfig( + max_num_seqs=3, max_num_batched_tokens=8, prefill_chunk_size=4 + ), + ) + state = torch.get_rng_state().clone() + prepare_runner(engine.model_runner, engine.scheduler.config) + torch.testing.assert_close(torch.get_rng_state(), state) + assert not engine.rng.generators and not engine.cache_manager._allocations + + +@pytest.mark.gpu +def test_startup_keeps_direct_gpu_preparation_usable(): + from vllm_rlt.request import ExecutionBatch, Operation, Request, StateSpan + + engine = StageEngine( + OuroForCausalLM(tiny_ouro_config()).to("cuda"), + attention_backend="triton", + cache_config=CacheConfig(64, 2), + scheduler_config=SchedulerConfig( + mode="mixed", max_num_seqs=1, max_num_batched_tokens=4, prefill_chunk_size=2 + ), + ) + runner = engine.model_runner + request = Request("direct", [1, 2], SamplingParams()) + assert engine.cache_manager.allocate("direct", 2) + try: + batch = ExecutionBatch( + Operation.RECURRENT, (StateSpan(request, 0, 2, 0, chunk_start=0, retain_chunk=True),) + ) + runner.prepare_on_gpu(runner.prepare_on_cpu(batch)) + finally: + runner.synchronize() + runner.release("direct") + engine.cache_manager.free("direct") + engine.close() + + +def test_cpu_preparation_is_idempotent_and_does_not_execute(monkeypatch): + engine = make_engine() + + def forbidden(*args, **kwargs): + raise AssertionError("CPU startup must not run a synthetic workload") + + monkeypatch.setattr(engine.model_runner, "submit", forbidden) + report = engine.prepare_runtime() + assert report is engine.prepare_runtime() + assert report["status"] == "cpu" + assert report["capacity_source"] == "blocks" + assert not engine.scheduler.requests + + +def test_worker_prepares_after_connector_and_before_ready(monkeypatch): + from types import SimpleNamespace + + from vllm_rlt.engine.stage_config import StageConfig + from vllm_rlt.engine.worker import StageWorker + from vllm_rlt.models import ModelConfig + + events = [] + prepare = StageEngine.prepare_runtime + + def prepared(engine): + assert engine.connector is connector + events.append("prepare") + return prepare(engine) + + def factory(engine, **kwargs): + assert engine.startup_report is None + events.append("connector") + return connector + + connector = SimpleNamespace(startup_info=lambda: {}) + channel = SimpleNamespace(send=lambda message: events.append(message["kind"])) + monkeypatch.setattr(StageEngine, "prepare_runtime", prepared) + StageWorker( + StageConfig(), + "cpu", + channel, + tiny_ouro_config(), + dict( + seed=123, + dtype="float32", + revision=None, + model_config=ModelConfig(tiny_ouro_config()), + engine={}, + ), + connector_factory=factory, + ) + assert events == ["connector", "prepare", "ready"] + + +@pytest.mark.parametrize("failure", ["connector", "prepare"]) +def test_worker_startup_failure_closes_resources_without_ready(monkeypatch, failure): + from types import SimpleNamespace + + from vllm_rlt.engine.stage_config import StageConfig + from vllm_rlt.engine.worker import StageWorker + from vllm_rlt.models import ModelConfig + + events = [] + close = StageEngine.close + + def closed(engine): + close(engine) + events.append("engine closed") + + def prepared(engine): + raise RuntimeError("prepare failed") + + def factory(engine, **kwargs): + if failure == "connector": + raise RuntimeError("connector failed") + return SimpleNamespace(close=lambda: events.append("connector closed")) + + monkeypatch.setattr(StageEngine, "close", closed) + monkeypatch.setattr(StageEngine, "prepare_runtime", prepared) + with pytest.raises(RuntimeError, match=f"{failure} failed"): + StageWorker( + StageConfig(), + "cpu", + SimpleNamespace(send=lambda message: events.append(message["kind"])), + tiny_ouro_config(), + dict( + seed=123, + dtype="float32", + revision=None, + model_config=ModelConfig(tiny_ouro_config()), + engine={}, + ), + connector_factory=factory, + ) + assert events == ( + ["engine closed"] if failure == "connector" else ["engine closed", "connector closed"] + ) + + +@pytest.mark.gpu +@pytest.mark.parametrize( + "model_kind,asynchronous,speculative,staged", + [ + ("ouro", False, False, False), + ("ouro", True, False, False), + ("ouro", True, False, True), + ("ouro", False, True, False), + ("ouro", True, True, False), + ("dory", False, False, False), + ("dory", True, False, False), + ("huginn", False, False, False), + ("nanbeige", False, False, False), + ("nanbeige", True, False, False), + ("nanbeige", True, False, True), + ], +) +@pytest.mark.parametrize("temperature", [0.0, 0.8]) +@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8_e4m3"]) +def test_cuda_startup_replay_and_generation_match_unprepared( + model_kind, asynchronous, speculative, staged, temperature, kv_cache_dtype +): + from tests.helpers import tiny_nanbeige_config + from tests.test_dory import tiny_dory_config + from tests.test_huginn import tiny_huginn_config + from vllm_rlt import ExitConfig, SpeculativeConfig + from vllm_rlt.models import DoryForCausalLM, HuginnForCausalLM, NanbeigeForCausalLM + + torch.manual_seed(123) + if model_kind == "dory": + model = DoryForCausalLM(tiny_dory_config(head_dim=64, d_model=256)) + elif model_kind == "huginn": + model = HuginnForCausalLM(tiny_huginn_config(n_embd=256, intermediate_size=512)) + elif model_kind == "nanbeige": + model = NanbeigeForCausalLM(tiny_nanbeige_config(head_dim=64)) + else: + model = OuroForCausalLM(tiny_ouro_config(head_dim=64)) + model = model.to("cuda", torch.bfloat16) + results = [] + for prepared in (False, True): + rng = torch.cuda.get_rng_state().clone() + engine = StageEngine( + model, + attention_backend="triton", + cache_config=CacheConfig( + 128, + 16, + enable_prefix_caching=model_kind == "ouro", + incremental_allocation=True, + kv_cache_dtype=kv_cache_dtype, + ), + scheduler_config=SchedulerConfig( + max_num_seqs=3, + max_num_batched_tokens=8, + prefill_chunk_size=4, + wavefront_chunked_prefill=staged, + ), + execution_config=ExecutionConfig( + cuda_graphs=True, + static_buffers=True, + async_scheduling=asynchronous, + ), + exit_config=ExitConfig("ouro" if speculative else "ouro_delayed"), + speculative_config=SpeculativeConfig(2) if speculative else None, + _defer_runtime_preparation=not prepared, + ) + try: + if prepared: + report = engine.startup_report + assert report["status"] == "prepared" + cache = engine.cache_manager + assert report["kv_bytes"] == cache.num_blocks * cache.bytes_per_block + assert [p["name"] for p in report["phases"]] == [ + "initialized", + "captured", + "replayed", + *(["concurrent_prepared", "concurrent"] if asynchronous else []), + "cleaned", + ] + assert all(p["allocated_bytes"] > 0 for p in report["phases"]) + assert engine.cache_manager.num_used_blocks == 0 + assert engine.cache_manager.prefix_queries == 0 + assert not engine.rng.generators + assert not engine.model_runner.state_slots + from vllm_rlt.worker.warmup import graph_owners + + assert all( + owner.captures > 0 + for owner in graph_owners(engine.model_runner, engine.speculative_runner) + ) + coverage = report["graph_coverage"] + assert "runner.graphs" in coverage + assert all(entry["keys"] and entry["replays"] > 0 for entry in coverage.values()) + assert all(entry["remaining_entries"] >= 0 for entry in coverage.values()) + torch.testing.assert_close(torch.cuda.get_rng_state(), rng) + rounds = [] + for _ in range(2): + for i in range(3): + engine.add_request( + str(i), + [2 + i, 3, 4] * (i + 1), + SamplingParams( + max_tokens=5, + ignore_eos=True, + seed=42 + i, + temperature=temperature, + top_k=7, + top_p=0.8, + ), + ) + outputs = {} + while engine.has_unfinished_requests(): + for output in engine.step(): + if output.finished: + outputs[output.request_id] = (output.token_ids, output.exit_depths) + assert len(outputs) == 3 + rounds.append(outputs) + results.append(rounds) + finally: + engine.close() + assert results[0] == results[1] + + +@pytest.mark.gpu +def test_cuda_failed_preparation_clears_partial_graphs_and_can_retry(monkeypatch): + from vllm_rlt.worker.warmup import graph_owners + + engine = StageEngine( + OuroForCausalLM(tiny_ouro_config(head_dim=64)).to("cuda", torch.bfloat16), + attention_backend="triton", + cache_config=CacheConfig(64, 16), + scheduler_config=SchedulerConfig( + max_num_seqs=2, max_num_batched_tokens=4, prefill_chunk_size=2 + ), + execution_config=ExecutionConfig(cuda_graphs=True), + _defer_runtime_preparation=True, + ) + runner = engine.model_runner + execute = runner._execute + + def failed(*args): + output = execute(*args) + if runner.graphs.captures: + raise RuntimeError("failure after capture") + return output + + try: + with monkeypatch.context() as patch: + patch.setattr(runner, "_execute", failed) + with pytest.raises(RuntimeError, match="failure after capture"): + engine.prepare_runtime() + assert engine.startup_report is None + assert not engine.cache_manager._allocations and not runner.state_slots + assert not engine.rng.generators + assert all(not owner.entries for owner in graph_owners(runner)) + assert engine.prepare_runtime()["status"] == "prepared" + assert engine.prepare_runtime() is engine.startup_report + finally: + engine.close() + + +@pytest.mark.gpu +def test_startup_captures_with_graph_row_limit_below_scheduler_limit(): + engine = StageEngine( + OuroForCausalLM(tiny_ouro_config(head_dim=64)).to("cuda", torch.bfloat16), + attention_backend="triton", + cache_config=CacheConfig(64, 16), + scheduler_config=SchedulerConfig( + max_num_seqs=3, max_num_batched_tokens=4, prefill_chunk_size=2 + ), + execution_config=ExecutionConfig( + cuda_graphs=True, cuda_graph_max_batch_size=2, cuda_graph_max_graphs=1 + ), + ) + try: + coverage = engine.startup_report["graph_coverage"]["runner.graphs"] + assert coverage["keys"] and coverage["replays"] > 0 + assert all(key[0] <= 2 for key in coverage["keys"]) + finally: + engine.close() diff --git a/tests/test_split_prefill.py b/tests/test_split_prefill.py index 1080eaef..4d7a906f 100644 --- a/tests/test_split_prefill.py +++ b/tests/test_split_prefill.py @@ -1501,8 +1501,9 @@ def run(cuda_graphs): assert eager[1]["static_buffer_bytes"] == eager[2] > 0 assert graphed[1]["static_buffer_bytes"] == graphed[2] > 0 before_resume, after_resume = graphed[3] - assert before_resume == (1, 2, 0) - assert after_resume == (1, 4, 0) + assert before_resume[0] > 0 # Planned decode entries already exist at startup. + assert before_resume[1:] == (2, 0) + assert after_resume == (before_resume[0], 4, 0) @pytest.mark.gpu diff --git a/vllm_rlt/engine/stage_engine.py b/vllm_rlt/engine/stage_engine.py index ec40c771..4ccde2b6 100644 --- a/vllm_rlt/engine/stage_engine.py +++ b/vllm_rlt/engine/stage_engine.py @@ -1,7 +1,10 @@ import logging +import time from dataclasses import dataclass, replace from typing import TYPE_CHECKING, cast +import torch + from vllm_rlt.attention import BackendCapabilities, create_backend from vllm_rlt.config import ( CacheConfig, @@ -55,6 +58,7 @@ def __init__( speculative_config=None, stage_config=None, connector: StageConnector | None = None, + _defer_runtime_preparation=False, ): self.model = model self.stage_config = stage_config or StageConfig() @@ -330,6 +334,71 @@ def __init__( if scheduler_config.enable_preemption: self.scheduler.suspend_callback = self.preemption.suspend self.scheduler.resume_callback = self.preemption.resume + self.startup_report = None + if not _defer_runtime_preparation: + self.prepare_runtime() + + def prepare_runtime(self): + """Prepare serving execution before readiness; KV sizing remains estimate-based. + + Workers defer this boundary until connector-owned buffers are initialized. + The report is a handoff for measured sizing, not a new capacity estimate. + """ + if self.startup_report is not None: + return self.startup_report + from vllm_rlt.worker.warmup import drain, prepare_runner + + runner = self.model_runner + report = dict(capacity_source=self.memory_plan["source"], phases=[]) + if runner.device.type != "cuda": + report["status"] = "cpu" + else: + started = time.perf_counter() + + def phase(name): + drain(runner, self.speculative_runner) + free, total = torch.cuda.mem_get_info(runner.device) + report["phases"].append( + dict( + name=name, + elapsed_seconds=time.perf_counter() - started, + driver_free_bytes=free, + device_total_bytes=total, + allocated_bytes=torch.cuda.memory_allocated(runner.device), + reserved_bytes=torch.cuda.memory_reserved(runner.device), + peak_allocated_bytes=torch.cuda.max_memory_allocated(runner.device), + peak_reserved_bytes=torch.cuda.max_memory_reserved(runner.device), + ) + ) + torch.cuda.reset_peak_memory_stats(runner.device) + + phase("initialized") + try: + report.update( + prepare_runner( + runner, self.scheduler.config, self.speculative_runner, phase=phase + ) + ) + except BaseException: + drain(runner, self.speculative_runner) + # Partial entries can retain pool outputs after a failed capture. + from vllm_rlt.worker.warmup import graph_owners + + for owner in graph_owners(runner, self.speculative_runner): + owner.entries.clear() + owner.last_event = None + raise + phase("cleaned") + report.update( + status="prepared", + sizing="estimated", + allocator_cache_policy="retain", + kv_bytes=self.cache_manager.num_blocks * self.memory_plan["bytes_per_block"], + num_blocks=self.cache_manager.num_blocks, + ) + logger.info("Runtime startup preparation: %s", report) + self.startup_report = report + return report @property def connector(self): diff --git a/vllm_rlt/engine/worker.py b/vllm_rlt/engine/worker.py index 9a2b2805..1dab1a8b 100644 --- a/vllm_rlt/engine/worker.py +++ b/vllm_rlt/engine/worker.py @@ -59,13 +59,25 @@ def __init__( dtype=dtype, model_config=model_config, ) - self.engine = StageEngine(self.model, stage_config=stage_config, **options["engine"]) - self.connector = ( - connector_factory(self.engine, notify=self.send) - if connector_factory is not None - else None + self.engine = StageEngine( + self.model, + stage_config=stage_config, + _defer_runtime_preparation=True, + **options["engine"], ) - self.engine.connector = self.connector + self.connector = None + try: + if connector_factory is not None: + self.connector = connector_factory(self.engine, notify=self.send) + self.engine.connector = self.connector + self.engine.prepare_runtime() + except BaseException: + try: + self.engine.close() + finally: + if self.connector is not None: + self.connector.close() + raise self.running = True self.draining = False cache = self.engine.cache_manager diff --git a/vllm_rlt/worker/async_speculative.py b/vllm_rlt/worker/async_speculative.py index 8b18dfa2..7d6efc9c 100644 --- a/vllm_rlt/worker/async_speculative.py +++ b/vllm_rlt/worker/async_speculative.py @@ -373,9 +373,6 @@ def __init__( if execution.cuda_graphs else None ) - if self.graphs is not None: - self.graphs.prepare() - self.coda_graphs.prepare() self.compute_stream = compute_stream self.copy_stream = copy_stream self.compute_stream.wait_stream(torch.cuda.current_stream(self.device)) diff --git a/vllm_rlt/worker/buffers.py b/vllm_rlt/worker/buffers.py index 621e1715..a6e665a1 100644 --- a/vllm_rlt/worker/buffers.py +++ b/vllm_rlt/worker/buffers.py @@ -286,6 +286,12 @@ def store(self, metadata, tokens=()): ) self.storage.np[: len(values)] = values + def clear(self): + """Release host request bindings after the slot's consumers have drained.""" + self.layouts = () + self.owner = None + self.allocations = () + def upload(self): from vllm_rlt.kernels.metadata import expand_kv_addresses diff --git a/vllm_rlt/worker/cuda_graph.py b/vllm_rlt/worker/cuda_graph.py index d3cd05c0..9c3c8afd 100644 --- a/vllm_rlt/worker/cuda_graph.py +++ b/vllm_rlt/worker/cuda_graph.py @@ -247,6 +247,7 @@ def __init__(self, model, cache, execution, compute_gate): self.last_event = None self.captures = self.replays = self.fallbacks = 0 self.warned_cache_full = False + self.startup = False def select(self, hidden, batch): count = _query_count(batch) @@ -323,6 +324,7 @@ def prepare_inputs(self, call, hidden): lambda: self.model.recurrent_prepared( entry.hidden, entry.metadata, proxy, compute_gate=self.compute_gate ), + startup=self.startup, ) self.captures += 1 call.inputs_ready = True @@ -437,6 +439,7 @@ def __init__(self, model, execution, cache=None, *, bucketed=False, capture_size self.last_event = None self.captures = self.replays = self.fallbacks = 0 self.warned_cache_full = False + self.startup = False @torch.inference_mode() def prepare(self): @@ -498,7 +501,11 @@ def prepare_inputs(self, call, hidden, *, startup=False): entry.hidden[len(hidden) :].zero_() if entry.graph is None: entry.graph, entry.output = _capture( - entry.hidden, self.stream, self.pool, lambda: self._run(entry), startup=startup + entry.hidden, + self.stream, + self.pool, + lambda: self._run(entry), + startup=startup or self.startup, ) self.captures += 1 call.inputs_ready = True diff --git a/vllm_rlt/worker/model_runner.py b/vllm_rlt/worker/model_runner.py index 70c251ae..e71ff275 100644 --- a/vllm_rlt/worker/model_runner.py +++ b/vllm_rlt/worker/model_runner.py @@ -960,6 +960,7 @@ def _prepare_cpu_inputs(self, batch: ExecutionBatch): prepared = replace(prepared, kv=kv) return prepared + @torch.inference_mode() def prepare_on_gpu(self, prepared): """Prepare all model inputs before forward on the selected execution streams.""" batch = prepared.batch diff --git a/vllm_rlt/worker/warmup.py b/vllm_rlt/worker/warmup.py new file mode 100644 index 00000000..783f0292 --- /dev/null +++ b/vllm_rlt/worker/warmup.py @@ -0,0 +1,361 @@ +"""Bounded startup execution using the serving runners and scratch KV leases.""" + +from dataclasses import replace + +import torch + +from vllm_rlt.config import cuda_graph_rows +from vllm_rlt.core.scheduler import ScheduledRequest, SchedulerOutput +from vllm_rlt.request import ExecutionBatch, Operation, Request, RequestPhase, StateSpan +from vllm_rlt.sampling_params import SamplingParams +from vllm_rlt.worker.cuda_graph import graph_capture_sizes + + +def graph_owners(runner, speculation=None): + return tuple(_graph_owners(runner, speculation).values()) + + +def _graph_owners(runner, speculation): + return { + f"{label}.{name}": owner + for label, component in (("runner", runner), ("speculation", speculation)) + if component is not None + for name in ("graphs", "prelude_graphs", "coda_graphs") + if (owner := getattr(component, name, None)) is not None + } + + +def drain(runner, speculation=None): + runner.synchronize() + runner.submissions.drain() + if speculation is not None: + if hasattr(speculation, "synchronize"): + speculation.synchronize() + elif speculation.compute_stream is not None: + speculation.compute_stream.synchronize() + speculation.submissions.drain() + for owner in graph_owners(runner, speculation): + stream = getattr(owner, "capture_stream", None) or getattr(owner, "stream", None) + stream.synchronize() + + +@torch.inference_mode() +def prepare_runner(runner, scheduler, speculation=None, *, phase=None): + """Warm valid prefill/decode paths without admitting or publishing requests. + + Coverage is bounded by configured limits and committed KV. Exact graph keys + not encountered here retain the existing bounded lazy-capture allowance. + """ + cache, config = runner.cache_manager, runner.model.config + if cache._allocations: + raise RuntimeError("runtime preparation requires an idle KV pool") + named_owners = _graph_owners(runner, speculation) + owners = tuple(named_owners.values()) + queries, hits = cache.prefix_queries, cache.prefix_hits + devices = [runner.device.index] if runner.device.type == "cuda" else [] + gaps = [] + extra = 2 + (2 * (speculation.config.num_speculative_tokens + 1) if speculation else 0) + capacity = min( + config.max_position_embeddings, cache.num_blocks // cache.storage_depths * cache.block_size + ) + if capacity < 1: + raise ValueError("KV capacity cannot back one startup token at every depth") + if capacity <= extra: + extra = min(extra, max(0, capacity - 1)) + gaps.append("KV capacity limits decode/speculative coverage") + prefill_limit = min(scheduler.prefill_chunk_size, scheduler.max_num_batched_tokens) + prefill = tuple(sorted({1, min(prefill_limit, capacity - extra)})) + batch_history = min(cache.block_size + 1, capacity - extra) + history_limit = max( + cache.block_size + 1, prefill_limit, (getattr(config, "sliding_window", 0) or 0) + 1 + ) + history = min(history_limit, capacity - extra) + max_batch = min( + scheduler.max_num_seqs, + scheduler.max_num_batched_tokens, + cache.num_blocks // cache.required_blocks(batch_history + extra), + ) + graph_batch = min(max_batch, cuda_graph_rows(runner.execution_config)) + execution = replace(runner.execution_config, cuda_graph_max_batch_size=graph_batch) + sizes = tuple(sorted({*graph_capture_sizes(execution, (graph_batch,)), max_batch})) + if max_batch < min(scheduler.max_num_seqs, scheduler.max_num_batched_tokens): + gaps.append("KV capacity limits decode batch coverage") + if prefill[-1] < prefill_limit or history < history_limit: + gaps.append("KV capacity limits prefill/history coverage") + captured = tuple((owner, owner.replays, owner.fallbacks) for owner in owners) + counters = (runner.split_prefill_submissions, runner.split_prefill_submission_waits) + speculative_stats = replace(speculation.stats) if speculation is not None else None + speculative_counters = { + name: getattr(speculation, name) + for name in ("submitted_before_collect", "discarded_rounds", "readback_waits") + if speculation is not None and hasattr(speculation, name) + } + + def execute(operation, spans, loops=0): + ticket = runner.submit(batch=ExecutionBatch(operation, tuple(spans), loops)) + output = ticket.collect() + if operation == Operation.CODA and ticket.device_values is not None: + for row, span in enumerate(spans): + span.request.input_token_tensor = ticket.device_values[row] + return output + + def run_case(count, length, decode, *, concurrent=False, stochastic=False): + requests = [] + tickets = [] + try: + for index in range(count): + request = Request( + f"__warmup_{index}", + [0] * length, + SamplingParams( + max_tokens=max(1, extra), + ignore_eos=True, + seed=index, + temperature=0.8 if stochastic else 0.0, + top_k=config.vocab_size if stochastic else -1, + top_p=0.9, + ), + ) + requests.append(request) + if not cache.allocate(request.request_id, length + extra): + raise RuntimeError("insufficient KV blocks for startup preparation") + chunk_size = min( + prefill_limit, scheduler.max_num_batched_tokens - (count - 1 if concurrent else 0) + ) + for start in range(0, length, chunk_size): + end = min(length, start + chunk_size) + spans = [ + StateSpan( + request, + start, + end - start, + 0, + chunk_start=start if runner.staged_prefill else None, + retain_chunk=runner.staged_prefill, + input_tokens=tuple(request.prompt_token_ids[start:end]), + ) + for request in requests + ] + # One request at a time respects the prefill token budget. + for index, span in enumerate(spans): + pending = concurrent and end == length and index == count - 1 + if runner.staged_prefill: + execute(Operation.PRELUDE, (span,)) + operation = Operation.RECURRENT + else: + operation = Operation.PRELUDE + loops = 1 if runner.staged_prefill else config.total_ut_steps + if runner.staged_prefill: + # Wavefront/mixed work yields at each existing depth boundary. + for depth in range(config.total_ut_steps - 1): + execute(operation, (replace(span, depth=depth),), loops) + span = replace(span, depth=config.total_ut_steps - 1) + if pending: + # Serving also submits independent boundary work before core. + for request in requests[:-1]: + request.loops_done = config.total_ut_steps + tickets.append( + runner.submit( + batch=ExecutionBatch( + Operation.CODA, + tuple(StateSpan(r, length - 1, 1, 0) for r in requests[:-1]), + ) + ) + ) + tickets.append( + runner.submit(batch=ExecutionBatch(operation, (span,), loops)) + ) + else: + execute(operation, (span,), loops) + for request in requests: + request.loops_done = config.total_ut_steps + spans = [StateSpan(request, length - 1, 1, 0) for request in requests] + if concurrent: + # Independent coda on boundary can overlap the final prompt core. + # Keep both tickets alive until their device readers have retired. + for ticket in tickets: + ticket.collect() + output = execute(Operation.CODA, spans[-1:]) + request = requests[-1] + request.generated_token_ids.extend(output.token_ids) + if extra: + span = StateSpan(request, length, 1, 0) + execute(Operation.PRELUDE, (span,)) + for depth in range(config.total_ut_steps - 1): + execute(Operation.RECURRENT, (replace(span, depth=depth, emit_gate=True),)) + # Decode core graphs avoid the host launch cost of fused prefill. + tickets.append( + runner.submit( + batch=ExecutionBatch( + Operation.RECURRENT, + (replace(span, depth=config.total_ut_steps - 1, emit_gate=True),), + ) + ) + ) + execute(Operation.CODA, spans[:-1]) + tickets[-1].collect() + return + output = execute(Operation.CODA, spans) + for request, token in zip(requests, output.token_ids): + request.generated_token_ids.append(token) + if not decode or not extra: + return + ordinary_steps = min(extra, 2) + for offset in range(ordinary_steps): + position = length + offset + execute( + Operation.PRELUDE, + [StateSpan(r, position, 1, 0) for r in requests], + ) + for depth in range(config.total_ut_steps): + execute( + Operation.RECURRENT, + [StateSpan(r, position, 1, depth, emit_gate=True) for r in requests], + ) + output = execute(Operation.CODA, [StateSpan(r, position, 1, 0) for r in requests]) + for request, token in zip(requests, output.token_ids): + request.generated_token_ids.append(token) + if speculation is not None: + width = min( + speculation.config.num_speculative_tokens + 1, + (extra - ordinary_steps) // 2, + scheduler.max_num_batched_tokens // count, + ) + if width < 1: + gaps.append("KV capacity cannot cover a speculative round") + return + for request in requests: + request.phase = RequestPhase.SPECULATIVE + pipeline = speculation.config.mode == "pipeline" + rounds = 2 if hasattr(speculation, "buffers") else 1 + if pipeline: + rounds = speculation.config.target_loops // speculation.config.draft_loops + for offset in range(rounds): + batch = SchedulerOutput( + RequestPhase.SPECULATIVE, + [ + ScheduledRequest( + request, + request.position + if pipeline + else length + ordinary_steps + offset * width, + width, + ) + for request in requests + ], + ) + ticket = speculation.submit(batch) + tickets.append(ticket) + if pipeline: + output = ticket.collect() + for request, result in zip(requests, output.speculative): + request.generated_token_ids.extend(result.token_ids) + if hasattr(ticket, "retire"): + ticket.retire() + for ticket in tickets: + ticket.collect() + finally: + drain(runner, speculation) + for ticket in tickets: + if hasattr(ticket, "retire"): + ticket.retire(discard=True) + if speculation is not None and hasattr(speculation, "discard_buffers"): + speculation.discard_buffers() + for request in requests: + if speculation is not None: + speculation.release(request) + runner.release(request.request_id) + cache.free(request.request_id) + + try: + with torch.random.fork_rng(devices=devices): + for owner in owners: + owner.startup = True + if speculation is not None and hasattr(speculation.graphs, "capture_sizes"): + if not speculation.graphs.entries: + speculation.graphs.prepare() + speculation.coda_graphs.prepare() + for length in prefill: + run_case(1, length, False) + for count in sizes: + run_case(count, batch_history, True) + if history != batch_history: + run_case(1, history, True) + run_case(sizes[-1], batch_history, False, stochastic=True) + if phase is not None: + phase("captured") + run_case(1, prefill[-1], False) + run_case(sizes[-1], batch_history, True) + phase("replayed") + concurrent_batches = () + if runner.execution_config.async_scheduling and runner.execution_config.multi_stream: + concurrent_limit = ( + min( + scheduler.max_num_seqs, + scheduler.max_num_batched_tokens, + cache.num_blocks // cache.required_blocks(prefill[-1] + extra), + ) + - 1 + ) + if concurrent_limit > 0: + concurrent_batches = graph_capture_sizes( + replace(execution, cuda_graph_max_batch_size=concurrent_limit), + (concurrent_limit,), + ) + # Prepare exact shapes, then isolate concurrent replay peaks. + for count in concurrent_batches: + run_case(count + 1, prefill[-1], False, concurrent=True, stochastic=True) + if phase is not None: + phase("concurrent_prepared") + for count in concurrent_batches: + run_case(count + 1, prefill[-1], False, concurrent=True, stochastic=True) + if phase is not None: + phase("concurrent") + else: + gaps.append("request/token/KV limits prevent independent stream overlap") + finally: + drain(runner, speculation) + cache.prefix_queries, cache.prefix_hits = queries, hits + runner.split_prefill_submissions, runner.split_prefill_submission_waits = counters + if speculation is not None: + speculation.stats = speculative_stats + for name, value in speculative_counters.items(): + setattr(speculation, name, value) + coverage = { + name: dict( + keys=tuple(key for key, entry in owner.entries.items() if entry.graph is not None), + replays=owner.replays - replays, + fallbacks=owner.fallbacks - fallbacks, + remaining_entries=max( + 0, owner.execution.cuda_graph_max_graphs - len(owner.entries) + ), + ) + for name, (owner, replays, fallbacks) in zip(named_owners, captured) + } + for owner, replays, fallbacks in captured: + owner.startup = False + owner.replays, owner.fallbacks = replays, fallbacks + for buffers in runner.batch_input_buffers: + buffers.reset() + for buffers in runner.packed_metadata_buffers: + buffers.clear() + if runner.async_state is not None: + state = runner.async_state + state.submissions.drain() + state.finalize_submissions.drain() + for buffers in (*state.routing_buffers, *state.finalize_buffers): + buffers.imports.clear() + if owners: + gaps.append("unplanned exact graph shapes retain bounded lazy capture and graph reserve") + gaps.append("contexts beyond the reported history retain the estimated transient reserve") + if runner.execution_config.async_scheduling and speculation is not None: + gaps.append("mixed speculative/prompt overlap retains the estimated transient reserve") + return dict( + prefill_tokens=prefill, + decode_batches=sizes, + history_tokens=history, + decode_history_tokens=batch_history, + concurrent_batches=concurrent_batches, + graph_coverage=coverage, + coverage_gaps=tuple(dict.fromkeys(gaps)), + ) From 060203dcb27a623b0858324bd43f3a6a459fee72 Mon Sep 17 00:00:00 2001 From: cecilsd <127472276+cecilsd@users.noreply.github.com> Date: Thu, 19 Nov 2026 09:59:58 +0800 Subject: [PATCH 2/3] docs(runtime): clarify startup handoff for VMM integration Clarify that the startup report is a preparation snapshot and does not remeasure or validate subsequent storage commitment. Document full-reservation graph views, backed-block publication, physical VMM memory accounting, post-commit replay validation, and the current ordinary-storage requirement for PD/NIXL. --- docs/runtime.md | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/docs/runtime.md b/docs/runtime.md index fe08839a..fd8844d4 100644 --- a/docs/runtime.md +++ b/docs/runtime.md @@ -116,9 +116,23 @@ This preparation does not replace automatic KV sizing. The report describes the existing allocated KV pool, including FP8 scale storage when configured, and sizing still uses the current estimates and reserves. The graph and workspace allocations remain resident; the KV pool itself is not a transient workspace. -Future storage integration must preserve captured pointers, views and strides, -and validate replay after physical commitment before publishing readiness. Small explicit pools can reduce warmup coverage. Unseen exact graph -shapes retain bounded lazy capture. Longer contexts and mixed speculative/prompt +The report is a snapshot of preparation. Repeated `prepare_runtime()` calls return +that snapshot; they do not measure or validate later storage changes. With ordinary +KV tensors, `kv_bytes` is the usable payload including FP8 scales. Future VMM +integration must report physical mapping bytes separately, including alignment +overhead; PyTorch allocator counters do not include those mappings. + +Future storage integration must keep the full reserved tensor shape, pointers, +views and strides stable for captured graphs, while exposing only physically +backed blocks to the allocator. Capturing a view limited to the bootstrap blocks +can retain that smaller indexing bound after commitment. Never initialize the +unmapped part of a reservation with whole-tensor operations. Validate replay on +newly committed blocks before publishing readiness; this is a separate step from +the idempotent preparation call. PD/NIXL must retain ordinary storage until the +mapping and registration requirements are supported. + +Small explicit pools can reduce warmup coverage. Unseen exact graph shapes retain +bounded lazy capture. Longer contexts and mixed speculative/prompt overlap retain estimated transient allowances. Pipeline preparation reaches target verification before releasing its scratch state. The report is a lifecycle boundary for subsequent memory-planning integration, not a guarantee that every From 4a5b1989e58e8bcb01afcf31c6e54126f829ae82 Mon Sep 17 00:00:00 2001 From: cecilsd <127472276+cecilsd@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:20:14 +0800 Subject: [PATCH 3/3] fix(runtime): report startup coverage without combining independent limits Record completed recurrent decode batch/history pairs, including concurrent preparation, without adding startup workloads. Keep explicit estimated transient allowances for unmeasured combinations and random block speculation. Add regressions for capacity limits, staged/concurrent execution and the block/pipeline sampling distinction, and document the report contract. --- docs/runtime.md | 12 ++++- tests/test_runtime_startup.py | 90 +++++++++++++++++++++++++++++++++++ vllm_rlt/worker/warmup.py | 14 +++++- 3 files changed, 114 insertions(+), 2 deletions(-) diff --git a/docs/runtime.md b/docs/runtime.md index fd8844d4..6387d821 100644 --- a/docs/runtime.md +++ b/docs/runtime.md @@ -103,7 +103,17 @@ device high-water mark. The report is published only after successful cleanup. Single-request history covers the configured prefill limit and crosses a sliding window when committed KV permits. Decode batch buckets also cross a KV page. -Greedy and full-vocabulary top-p sampling use scratch request generators. +`decode_workloads` pairs the `batch_size` and `history_tokens` of each completed +recurrent decode traversal. History is the number of preceding tokens, excluding the +decode input itself. Independent maximum batch and history fields do not imply +that their combination was executed. Larger contexts and unreported batch/history +combinations still need an estimated transient allowance, even when the KV pool +could hold them. + +Greedy and full-vocabulary top-p sampling use scratch request generators. Block +speculation is warmed with greedy requests; random block speculation retains its +estimated transient allowance and is listed as a coverage gap. Pipeline speculation +supports greedy requests only. Async multi-stream preparation additionally submits an independent prompt core and coda before collection, followed by a decode core/coda pair, within the request/token/KV limits. Its diff --git a/tests/test_runtime_startup.py b/tests/test_runtime_startup.py index 19583dab..bf6f9d68 100644 --- a/tests/test_runtime_startup.py +++ b/tests/test_runtime_startup.py @@ -89,6 +89,89 @@ def test_small_kv_capacity_reports_reduced_coverage(): assert engine.cache_manager.num_used_blocks == 0 +@pytest.mark.parametrize("blocks,staged", [(64, False), (64, True), (8, False), (4, False)]) +def test_warmup_reports_completed_decode_batch_history_pairs(blocks, staged, monkeypatch): + from vllm_rlt.request import Operation + from vllm_rlt.worker.warmup import prepare_runner + + engine = make_engine(blocks=blocks, staged=staged) + runner = engine.model_runner + submit = runner.submit + observed = set() + + def observe(*args, batch): + ticket = submit(*args, batch=batch) + if batch.operation == Operation.RECURRENT and batch.emit_gate: + observed.add((len(batch.spans), batch.spans[0].token_start)) + return ticket + + monkeypatch.setattr(runner, "submit", observe) + report = prepare_runner(runner, engine.scheduler.config) + assert { + (workload["batch_size"], workload["history_tokens"]) + for workload in report["decode_workloads"] + } == observed + assert any("batch/history combinations" in gap for gap in report["coverage_gaps"]) + assert not engine.cache_manager._allocations and not engine.rng.generators + engine.close() + + +@pytest.mark.parametrize("mode", ["block", "pipeline"]) +@pytest.mark.parametrize( + "device,asynchronous", + [ + ("cpu", False), + pytest.param("cuda", False, marks=pytest.mark.gpu), + pytest.param("cuda", True, marks=pytest.mark.gpu), + ], +) +def test_warmup_reports_unmeasured_random_block_speculation( + mode, device, asynchronous, monkeypatch +): + from vllm_rlt import SpeculativeConfig + from vllm_rlt.worker.warmup import prepare_runner + + engine = StageEngine( + OuroForCausalLM(tiny_ouro_config(head_dim=64)).to(device), + attention_backend="triton" if device == "cuda" else "torch", + cache_config=CacheConfig(128, 2), + scheduler_config=SchedulerConfig( + max_num_seqs=3, max_num_batched_tokens=8, prefill_chunk_size=4 + ), + speculative_config=SpeculativeConfig(2, mode=mode), + execution_config=ExecutionConfig( + cuda_graphs=device == "cuda", async_scheduling=asynchronous + ), + _defer_runtime_preparation=True, + ) + speculation = engine.speculative_runner + submit = speculation.submit + temperatures = [] + + def observe(batch): + temperatures.extend(span.request.sampling_params.temperature for span in batch.requests) + return submit(batch) + + monkeypatch.setattr(speculation, "submit", observe) + report = prepare_runner(engine.model_runner, engine.scheduler.config, speculation) + assert temperatures and all(temperature == 0 for temperature in temperatures) + assert any("random block speculation" in gap for gap in report["coverage_gaps"]) == ( + mode == "block" + ) + assert not engine.cache_manager._allocations and not engine.rng.generators + + if mode == "block": + engine.add_request( + "sampled", + [1, 2, 3], + SamplingParams(max_tokens=6, ignore_eos=True, seed=42, temperature=0.8), + ) + while engine.has_unfinished_requests(): + engine.step() + assert any(temperature > 0 for temperature in temperatures) + engine.close() + + def test_warmup_populates_history_across_sliding_window(monkeypatch): from tests.test_dory import tiny_dory_config from vllm_rlt.models import DoryForCausalLM @@ -220,11 +303,14 @@ def test_async_startup_submits_independent_streams_before_collect(staged, monkey pending = [] pairs = [] decode_pairs = [] + decode_workloads = set() submit, collect = engine.model_runner.submit, Submission.collect def submitted(*args, batch): ids = {span.request.request_id for span in batch.spans} ticket = submit(*args, batch=batch) + if batch.operation == Operation.RECURRENT and batch.emit_gate: + decode_workloads.add((len(batch.spans), batch.spans[0].token_start)) if batch.operation == Operation.CODA: pairs.extend( (other, ticket) @@ -254,6 +340,10 @@ def collected(ticket): try: report = engine.prepare_runtime() assert pairs and decode_pairs + assert { + (workload["batch_size"], workload["history_tokens"]) + for workload in report["decode_workloads"] + } == decode_workloads assert report["concurrent_batches"] == (1, 2) assert "concurrent" in [phase["name"] for phase in report["phases"]] assert not engine.cache_manager._allocations and not engine.rng.generators diff --git a/vllm_rlt/worker/warmup.py b/vllm_rlt/worker/warmup.py index 783f0292..7e231de9 100644 --- a/vllm_rlt/worker/warmup.py +++ b/vllm_rlt/worker/warmup.py @@ -54,6 +54,7 @@ def prepare_runner(runner, scheduler, speculation=None, *, phase=None): queries, hits = cache.prefix_queries, cache.prefix_hits devices = [runner.device.index] if runner.device.type == "cuda" else [] gaps = [] + decode_workloads = set() extra = 2 + (2 * (speculation.config.num_speculative_tokens + 1) if speculation else 0) capacity = min( config.max_position_embeddings, cache.num_blocks // cache.storage_depths * cache.block_size @@ -194,6 +195,7 @@ def run_case(count, length, decode, *, concurrent=False, stochastic=False): ) execute(Operation.CODA, spans[:-1]) tickets[-1].collect() + decode_workloads.add((1, length)) return output = execute(Operation.CODA, spans) for request, token in zip(requests, output.token_ids): @@ -215,6 +217,7 @@ def run_case(count, length, decode, *, concurrent=False, stochastic=False): output = execute(Operation.CODA, [StateSpan(r, position, 1, 0) for r in requests]) for request, token in zip(requests, output.token_ids): request.generated_token_ids.append(token) + decode_workloads.add((count, position)) if speculation is not None: width = min( speculation.config.num_speculative_tokens + 1, @@ -347,7 +350,12 @@ def run_case(count, length, decode, *, concurrent=False, stochastic=False): buffers.imports.clear() if owners: gaps.append("unplanned exact graph shapes retain bounded lazy capture and graph reserve") - gaps.append("contexts beyond the reported history retain the estimated transient reserve") + gaps.append( + "contexts and batch/history combinations beyond the reported workloads " + "retain the estimated transient reserve" + ) + if speculation is not None and speculation.config.mode == "block": + gaps.append("random block speculation retains the estimated transient reserve") if runner.execution_config.async_scheduling and speculation is not None: gaps.append("mixed speculative/prompt overlap retains the estimated transient reserve") return dict( @@ -355,6 +363,10 @@ def run_case(count, length, decode, *, concurrent=False, stochastic=False): decode_batches=sizes, history_tokens=history, decode_history_tokens=batch_history, + decode_workloads=tuple( + dict(batch_size=count, history_tokens=position) + for count, position in sorted(decode_workloads) + ), concurrent_batches=concurrent_batches, graph_coverage=coverage, coverage_gaps=tuple(dict.fromkeys(gaps)),