diff --git a/docs/kv_storage.md b/docs/kv_storage.md new file mode 100644 index 00000000..f19b9e16 --- /dev/null +++ b/docs/kv_storage.md @@ -0,0 +1,79 @@ +# VMM-backed KV storage (RFC #192 stage 2) + +The KV pool can reserve its full address range once and map physical memory in +stages, so a CUDA graph captured against the K/V tensors stays valid while the +pool grows. This is the storage half of RFC #192: capacity is still decided by +`plan_cache`'s existing estimate, not by a warmup measurement. + +## Enabling it + +```bash +vllm-rlt serve --kv-vmm-bootstrap-blocks 64 +``` + +or in code: + +```python +CacheConfig(block_size=16, vmm_bootstrap_blocks=64) +``` + +At startup the engine: + +1. plans the block count as usual (`plan_cache`), +2. reserves one VMM range covering the K and V planes of that capacity, +3. maps only `vmm_bootstrap_blocks` blocks, +4. builds `KVCacheManager` against that pool, +5. maps the remaining blocks once the runners exist, then publishes the full + planned capacity. + +`stage.memory_plan` records `vmm`, `vmm_bootstrap_blocks`, `vmm_max_blocks`, +`vmm_committed_blocks` and `vmm_reserve_bytes`. + +Without the option nothing changes: the cache allocates its tensors with +`torch.empty` as before, `max_blocks == committed_blocks == published_blocks`, +and `publish_capacity` is unavailable. + +## Invariants + +- `published_blocks <= committed_blocks <= max_blocks` at all times. +- Publishing requires committed physical memory, so an allocation can never + address an unmapped block. +- The K/V tensors keep their `data_ptr`, shape and stride for the pool's whole + lifetime; growth only appends mappings. +- `GPUKVStorage.close()` is the only release path; dropping tensors does not + unmap anything. +- Only the committed prefix is addressable. Whole-tensor operations (`fill_`, + `clone`, a full `copy_`) fault while the tail is unmapped, so anything that + touches the K/V views must stay inside `committed_blocks`. + +## Boundaries + +- CUDA only, and only with automatic planning: `num_blocks` and + `kv_cache_memory_bytes` keep their fixed-capacity semantics and reject the + option. +- Not supported with a PD connector yet. NIXL registers the K/V planes once at + startup and peers cache `num_blocks`, so growth needs a re-registration + protocol that this stage does not implement. +- Growth happens once, during startup. There is no runtime expansion. + +## Verification + +```bash +python -m pytest tests/test_kv_storage.py -q # bookkeeping, CPU +CUDA_VISIBLE_DEVICES=0 python -m pytest tests/test_vmm_pool.py --run-gpu +OURO_CHECKPOINT=/path/to/Ouro-1.4B CUDA_VISIBLE_DEVICES=0 \ + python -m pytest tests/test_vmm_real_checkpoint.py --run-gpu -s +``` + +`tests/test_vmm_pool.py` covers pointer/shape/stride stability, data survival +across growth, new-block access, graph replay after growth, release accounting, +the publish/commit bound, and an engine-level run whose output matches the +non-VMM path on both the torch and Triton attention backends. + +`tests/test_vmm_real_checkpoint.py` runs the pinned Ouro snapshot and compares +the baseline `torch.empty` pool with a bootstrap-1 VMM pool: generated tokens, +KV bytes read back for the same positions in three layers, CUDA-graph capture, +prefix-cache hits, a clamped bootstrap, and repeated create/close. Driver +free-memory deltas are not a usable oracle on a device shared with other tenants, +so release is asserted through the pool's own counters and a 2 GiB mapping test. + diff --git a/tests/test_kv_storage.py b/tests/test_kv_storage.py new file mode 100644 index 00000000..2e702356 --- /dev/null +++ b/tests/test_kv_storage.py @@ -0,0 +1,220 @@ +"""Capacity publishing for VMM-backed KV storage (RFC #192 stage 2). + +These run on CPU against a stand-in pool: they cover the bookkeeping contract +(published <= committed <= max, growth only, no publishing before a commit). +The CUDA mapping itself is exercised in test_vmm_pool.py. +""" + +import pytest +import torch + +from vllm_rlt.core.kv_cache_manager import KVCacheManager + + +class FakeStorage: + """Stand-in for GPUKVStorage that keeps the K/V planes on the CPU.""" + + def __init__(self, *, max_blocks, num_layers, num_kv_heads, head_dim, block_size): + self.max_blocks = max_blocks + self.committed_blocks = 0 + self.dtype = torch.float32 + element = torch.empty((), dtype=self.dtype).element_size() + self.bytes_per_block = 2 * num_layers * block_size * num_kv_heads * head_dim * element + self.shape = (max_blocks, num_layers, block_size, num_kv_heads, head_dim) + self.keys_tensor = torch.zeros(self.shape, dtype=self.dtype) + self.values_tensor = torch.zeros(self.shape, dtype=self.dtype) + self.commits = [] + + def commit(self, blocks): + self.committed_blocks = max(self.committed_blocks, blocks) + self.commits.append(blocks) + return self.committed_blocks + + def keys(self, shape): + assert tuple(shape) == self.shape + return self.keys_tensor + + def values(self, shape): + assert tuple(shape) == self.shape + return self.values_tensor + + +def make_storage(max_blocks): + return FakeStorage( + max_blocks=max_blocks, num_layers=2, num_kv_heads=1, head_dim=4, block_size=2 + ) + + +def make_cache(storage=None, num_blocks=4, **kwargs): + parameters = dict( + num_layers=2, + num_kv_heads=1, + head_dim=4, + num_blocks=num_blocks, + block_size=2, + max_loops=1, + ) + parameters.update(kwargs) + return KVCacheManager(**parameters, storage=storage) + + +def test_storage_backed_cache_starts_at_bootstrap_capacity(): + storage = make_storage(10) + storage.commit(4) + cache = make_cache(storage, num_blocks=4) + assert cache.published_blocks == 4 + assert cache.committed_blocks == 4 + assert cache.max_blocks == 10 + assert cache.num_blocks == 4 + assert cache.num_free_blocks == 4 + # The planes span the whole reservation, not the bootstrap prefix. + assert cache.key_cache.shape == storage.shape + assert cache.value_cache.shape == storage.shape + + +def test_publish_capacity_extends_the_free_pool(): + storage = make_storage(10) + storage.commit(4) + cache = make_cache(storage, num_blocks=4) + assert cache.allocate("a", 8) # 4 pages: exactly the published blocks + assert cache.num_free_blocks == 0 + assert cache.allocate("b", 2) is False # no free blocks; admission must not raise + storage.commit(8) + assert cache.publish_capacity(8) == 8 + assert cache.num_blocks == 8 + assert cache.published_blocks == 8 + assert cache.committed_blocks == 8 + assert cache.num_free_blocks == 4 + assert cache.allocate("b", 2) + blocks = {page for table in cache.plane_block_tables("b") for page in table} + assert blocks <= set(range(8)) + assert blocks & set(range(4, 8)), "published blocks are the ones handed out" + + +def test_publish_never_exceeds_committed_memory(): + storage = make_storage(10) + storage.commit(4) + cache = make_cache(storage, num_blocks=4) + free_before, published_before = cache.num_free_blocks, cache.num_blocks + with pytest.raises(ValueError, match="committed"): + cache.publish_capacity(6) + assert cache.num_free_blocks == free_before + assert cache.num_blocks == published_before + assert cache.published_blocks == published_before + + +def test_publish_rejects_growth_past_the_reservation(): + storage = make_storage(10) + storage.commit(4) + cache = make_cache(storage, num_blocks=4) + storage.commit(10) + with pytest.raises(ValueError, match="reservation holds"): + cache.publish_capacity(11) + assert cache.num_blocks == 4 + + +def test_publish_is_monotonic(): + storage = make_storage(10) + storage.commit(8) + cache = make_cache(storage, num_blocks=4) + assert cache.publish_capacity(8) == 8 + with pytest.raises(ValueError, match="only grow"): + cache.publish_capacity(6) + assert cache.num_blocks == 8 + assert cache.publish_capacity(8) == 8 # idempotent + + +def test_publish_requires_storage(): + cache = make_cache(None) + with pytest.raises(RuntimeError, match="VMM-backed"): + cache.publish_capacity(8) + + +def test_plain_cache_keeps_fixed_capacity(): + cache = make_cache(None, num_blocks=6) + assert cache.storage is None + assert cache.max_blocks == cache.committed_blocks == cache.published_blocks == 6 + assert cache.num_blocks == 6 + + +def test_storage_must_be_committed_before_the_cache_is_built(): + storage = make_storage(10) + storage.commit(2) + with pytest.raises(ValueError, match="commit the pool"): + make_cache(storage, num_blocks=4) + + +def test_storage_must_cover_the_requested_blocks(): + storage = make_storage(2) + storage.commit(2) + with pytest.raises(ValueError, match="reserves"): + make_cache(storage, num_blocks=4) + + +def test_storage_layout_must_match_the_cache(): + storage = make_storage(10) + storage.bytes_per_block += 1 + storage.commit(4) + with pytest.raises(ValueError, match="bytes per block"): + make_cache(storage, num_blocks=4) + + +def test_published_blocks_are_writable_after_growth(): + storage = make_storage(10) + storage.commit(2) + cache = make_cache(storage, num_blocks=2) + assert cache.allocate("a", 4) # exactly the two published blocks + assert cache.allocate("b", 2) is False # nothing free, and admission must not raise + storage.commit(10) + cache.publish_capacity(10) + assert cache.allocate("b", 2) + blocks = cache.get_block_table("b", 0) + assert set(blocks) & set(range(2, 10)), "growth must hand out the newly published blocks" + key = torch.full((1, 1, 4), 3.0) + cache.write(0, ["b"], [0], [0], key, key + 1) + written = cache.key_cache[blocks[0], 0, 0] + assert torch.equal(written, key[0]) + assert torch.equal(cache.value_cache[blocks[0], 0, 0], key[0] + 1) + + +def test_watermark_tracks_published_capacity(): + storage = make_storage(20) + storage.commit(4) + cache = make_cache(storage, num_blocks=4, incremental_allocation=True, watermark_ratio=0.5) + assert cache.watermark_blocks == 2 + storage.commit(20) + cache.publish_capacity(20) + assert cache.watermark_blocks == 10 + + +def test_vmm_bootstrap_requires_automatic_planning(): + from vllm_rlt import CacheConfig + + with pytest.raises(ValueError, match="automatic KV planning"): + CacheConfig(num_blocks=8, vmm_bootstrap_blocks=2) + with pytest.raises(ValueError, match="automatic KV planning"): + CacheConfig(kv_cache_memory_bytes=1024, vmm_bootstrap_blocks=2) + with pytest.raises(ValueError, match="vmm_bootstrap_blocks"): + CacheConfig(vmm_bootstrap_blocks=0) + + +def test_vmm_storage_requires_cuda(): + from tests.helpers import tiny_ouro_model + from vllm_rlt import CacheConfig + from vllm_rlt.engine.stage_engine import StageEngine + + model = tiny_ouro_model() + with pytest.raises(ValueError, match="CUDA device"): + StageEngine(model, cache_config=CacheConfig(block_size=2, vmm_bootstrap_blocks=2)) + + +def test_runtime_args_expose_the_vmm_bootstrap_flag(): + import argparse + + from vllm_rlt.entrypoints.runtime_args import add_runtime_args + + parser = argparse.ArgumentParser() + add_runtime_args(parser) + assert parser.parse_args([]).kv_vmm_bootstrap_blocks is None + parsed = parser.parse_args(["--kv-vmm-bootstrap-blocks", "64"]) + assert parsed.kv_vmm_bootstrap_blocks == 64 diff --git a/tests/test_vmm_pool.py b/tests/test_vmm_pool.py new file mode 100644 index 00000000..c472ef57 --- /dev/null +++ b/tests/test_vmm_pool.py @@ -0,0 +1,341 @@ +"""CUDA VMM KV storage: stable address, growable backing, publishable blocks. + +Run with an explicitly selected device: + CUDA_VISIBLE_DEVICES=1 python -m pytest tests/test_vmm_pool.py --run-gpu + +Commits are padded to the driver allocation granularity, so a case that must +leave blocks uncommitted needs block bytes far above that granularity. +""" + +import gc + +import pytest +import torch + +from vllm_rlt.core.kv_cache_manager import KVCacheManager +from vllm_rlt.core.vmm_pool import GPUKVStorage + +pytestmark = pytest.mark.gpu + +MIB = 1 << 20 +# Small blocks keep the bookkeeping tests cheap. +SMALL = dict(num_layers=2, num_kv_heads=1, head_dim=4, block_size=2) +# ~16 MiB per block, so granularity padding cannot cover the whole reservation. +WIDE = dict(num_layers=2, num_kv_heads=8, head_dim=1024, block_size=128) +# 256 MiB of K+V per block: eight blocks map 2 GiB, enough signal to observe a +# release on a device shared with other tenants. +HEAVY = dict(num_layers=2, num_kv_heads=8, head_dim=2048, block_size=1024) + + +def build(*, max_blocks=64, dims=SMALL, committed=0, published=None, device=0): + """Return (storage, cache) with ``committed`` blocks mapped and ``published`` live.""" + element = torch.empty((), dtype=torch.float32).element_size() + storage = GPUKVStorage( + max_blocks=max_blocks, + bytes_per_block=( + 2 + * dims["num_layers"] + * dims["block_size"] + * dims["num_kv_heads"] + * dims["head_dim"] + * element + ), + device=f"cuda:{device}", + dtype=torch.float32, + ) + if committed: + storage.commit_blocks(committed) + published = published or storage.committed_blocks + if not published: + # No physical memory yet, so there is nothing a cache could publish. + return storage, None + cache = KVCacheManager( + num_layers=dims["num_layers"], + num_kv_heads=dims["num_kv_heads"], + head_dim=dims["head_dim"], + num_blocks=published or storage.committed_blocks, + block_size=dims["block_size"], + max_loops=1, + device=storage.device, + dtype=storage.dtype, + storage=storage, + ) + return storage, cache + + +def free_bytes(): + torch.cuda.synchronize() + return torch.cuda.mem_get_info()[0] + + +def test_reserve_costs_nothing_and_commit_costs_physical_memory(): + base = free_bytes() + storage, _ = build(max_blocks=8, dims=HEAVY) + try: + # Reserving only takes address space; nothing is mapped yet. + assert storage.committed_blocks == 0 + assert storage.committed_bytes == 0 + assert base - free_bytes() <= 8 * MIB, "reserving VA must not pin physical memory" + storage.commit_blocks(2) + assert storage.committed_blocks == 2 + # Expected cost derived from the KV layout, not from the pool's own + # counter: a plane sized as K+V instead of half would double this. + element = torch.empty((), dtype=torch.float32).element_size() + per_block = ( + 2 + * HEAVY["num_layers"] + * HEAVY["block_size"] + * HEAVY["num_kv_heads"] + * HEAVY["head_dim"] + * element + ) + plane = -(-(2 * (per_block // 2)) // storage.allocation_granularity) + expected = 2 * plane * storage.allocation_granularity + assert expected == 512 * MIB # a signal a shared device cannot mask + assert storage.committed_bytes == expected + assert abs((base - free_bytes()) - expected) <= 64 * MIB + finally: + storage.close() + torch.cuda.empty_cache() + + +def test_committed_bytes_follow_the_kv_layout(): + dims = WIDE + element = torch.empty((), dtype=torch.float32).element_size() + k_and_v_per_block = ( + 2 + * dims["num_layers"] + * dims["block_size"] + * dims["num_kv_heads"] + * dims["head_dim"] + * element + ) + storage, _ = build(max_blocks=8, dims=dims) + try: + # bytes_per_block counts K and V together, so each plane holds half. + assert storage.bytes_per_block == k_and_v_per_block + assert 2 * storage.plane_bytes_per_block == k_and_v_per_block + storage.commit_blocks(4) + plane = storage.plane_bytes_per_block * 4 + granularity = storage.allocation_granularity + mapped_plane = -(-plane // granularity) * granularity + assert storage.committed_bytes == 2 * mapped_plane + assert storage.reserve_bytes == 2 * storage.plane_bytes + assert storage.plane_bytes >= mapped_plane + finally: + storage.close() + + +def test_tensors_keep_address_shape_and_stride_across_commit(): + storage, cache = build(max_blocks=8, dims=WIDE, committed=2) + try: + before = [ + (t.data_ptr(), tuple(t.shape), t.stride()) for t in (cache.key_cache, cache.value_cache) + ] + storage.commit_blocks(8) + cache.publish_capacity(8) + after = [ + (t.data_ptr(), tuple(t.shape), t.stride()) for t in (cache.key_cache, cache.value_cache) + ] + assert before == after + # The two planes must not overlap. + keys_end = cache.key_cache.data_ptr() + cache.key_cache.numel() * 4 + assert keys_end <= cache.value_cache.data_ptr() + finally: + storage.close() + + +def test_existing_blocks_survive_growth_and_new_blocks_are_writable(): + # WIDE blocks are far bigger than the map granularity, so the commit really + # starts small and the growth below is a genuine second mapping. + storage, cache = build(max_blocks=8, dims=WIDE, committed=2) + try: + assert storage.committed_blocks == 2 + assert cache.allocate("a", WIDE["block_size"]) + shape = (1, WIDE["num_kv_heads"], WIDE["head_dim"]) + key = torch.full(shape, 5.0, device=storage.device) + cache.write(0, ["a"], [0], [0], key, key + 1) + torch.cuda.synchronize() + # Only the committed prefix is mapped, so snapshot that much: cloning the + # whole view would copy the unallocated tail and fault. + mapped = storage.committed_blocks + old = cache.key_cache[:mapped].clone() + + storage.commit_blocks(8) + cache.publish_capacity(8) + assert torch.equal(cache.key_cache[:mapped], old), "growth must not disturb mapped blocks" + + assert cache.allocate("b", WIDE["block_size"]) + blocks = cache.get_block_table("b", 0) + assert max(blocks) >= 2, "the new allocation must come from the newly published blocks" + fresh = torch.full(shape, 7.0, device=storage.device) + cache.write(0, ["b"], [0], [0], fresh, fresh + 1) + keys, values = cache.read(0, "b", 0, length=1) + assert torch.equal(keys[0], fresh[0]) + assert torch.equal(values[0], fresh[0] + 1) + finally: + storage.close() + + +def test_published_blocks_never_exceed_committed_memory(): + storage, _ = build(max_blocks=8, dims=WIDE, committed=2) + try: + committed = storage.committed_blocks + assert committed < 8, "granularity padding must leave blocks uncommitted here" + cache = KVCacheManager( + num_layers=WIDE["num_layers"], + num_kv_heads=WIDE["num_kv_heads"], + head_dim=WIDE["head_dim"], + num_blocks=committed, + block_size=WIDE["block_size"], + max_loops=1, + device=storage.device, + dtype=storage.dtype, + storage=storage, + ) + with pytest.raises(ValueError, match="committed"): + cache.publish_capacity(8) + assert cache.published_blocks == committed + assert cache.num_free_blocks == committed + storage.commit_blocks(8) + assert cache.publish_capacity(8) == 8 + assert cache.num_free_blocks == 8 + finally: + storage.close() + + +def test_graph_captured_before_growth_replays_on_the_kv_plane(): + storage, cache = build(max_blocks=8, dims=WIDE, committed=4) + try: + assert cache.allocate("a", WIDE["block_size"]) + block = cache.get_block_table("a", 0)[0] + window = cache.key_cache[block, 0, 0] + window.zero_() + + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + window.add_(1.0) # warm outside capture + torch.cuda.current_stream().wait_stream(stream) + + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + window.mul_(2.0).add_(3.0) + window.zero_() + graph.replay() + torch.cuda.synchronize() + expected = window.clone() + + storage.commit_blocks(8) # append physical memory after capture + cache.publish_capacity(8) + window.zero_() + graph.replay() + torch.cuda.synchronize() + assert torch.equal(window, expected) + finally: + storage.close() + + +def test_close_releases_the_reservation_and_is_idempotent(): + # A 2 GiB mapping gives a release signal far above neighbour noise on a shared + # device, and the window is measured around close() only. + storage, _ = build(max_blocks=8, dims=HEAVY) + storage.commit_blocks(8) + torch.cuda.synchronize() + committed = storage.committed_bytes + assert committed == 2 << 30 + held = free_bytes() + storage.close() + storage.close() # idempotent + torch.cuda.empty_cache() + assert storage.committed_bytes == 0 + assert storage.committed_blocks == 0 + assert free_bytes() - held >= committed - 64 * MIB, "close must return the mapping" + with pytest.raises(RuntimeError, match="closed"): + storage.commit_blocks(4) + + +def test_views_do_not_free_the_mapping(): + storage, cache = build(committed=4) + try: + address = cache.key_cache.data_ptr() + probe = torch.tensor([[1.0, 2.0, 3.0, 4.0]], device=storage.device) + cache.key_cache[0, 0, 0].copy_(probe) + torch.cuda.synchronize() + + del cache + gc.collect() + torch.cuda.empty_cache() + # A behavioural oracle: if the allocator had released the VA, the rebuilt + # view would fault or lose the data. Free-memory deltas are not usable + # here because other tests' cached blocks also return to the driver. + # Granularity padding may map the whole reservation; it must not shrink. + assert storage.committed_blocks >= 4 + rebuilt = KVCacheManager( + num_layers=SMALL["num_layers"], + num_kv_heads=SMALL["num_kv_heads"], + head_dim=SMALL["head_dim"], + num_blocks=storage.committed_blocks, + block_size=SMALL["block_size"], + max_loops=1, + device=storage.device, + dtype=storage.dtype, + storage=storage, + ) + assert rebuilt.key_cache.data_ptr() == address + assert torch.equal(rebuilt.key_cache[0, 0, 0], probe) + finally: + storage.close() + + +@pytest.mark.parametrize("backend", ["torch", "triton"]) +def test_engine_publishes_planned_capacity_after_bootstrap(backend): + from tests.helpers import LocalLLM, tiny_ouro_config + from vllm_rlt import CacheConfig, SamplingParams + from vllm_rlt.models import OuroForCausalLM + + torch.manual_seed(123) + base = OuroForCausalLM(tiny_ouro_config()).to(device="cuda", dtype=torch.bfloat16) + prompts = [[2, 3, 4, 5], [6, 7]] + params = [SamplingParams(max_tokens=4, ignore_eos=True, temperature=0.0) for _ in prompts] + + expected = LocalLLM(base, cache_config=CacheConfig(64, 2), attention_backend=backend).generate( + prompts, params + ) + + engine = LocalLLM( + base, + cache_config=CacheConfig(block_size=2, vmm_bootstrap_blocks=2), + attention_backend=backend, + ) + stage = engine.engine + try: + assert stage.kv_storage is not None + plan = stage.memory_plan + assert plan["vmm"] is True + assert plan["vmm_bootstrap_blocks"] == 2 + assert plan["vmm_max_blocks"] == stage.cache_manager.max_blocks + # Bootstrap is mapped first, then the planned capacity is published. + assert stage.cache_manager.published_blocks == plan["vmm_max_blocks"] + assert stage.cache_manager.committed_blocks >= stage.cache_manager.published_blocks + actual = engine.generate(prompts, params) + finally: + stage.close() + assert [(o.token_ids, o.exit_depths) for o in actual] == [ + (o.token_ids, o.exit_depths) for o in expected + ] + assert stage.kv_storage is None, "close must release the reservation" + + +def test_connector_is_rejected_with_vmm_storage(): + from tests.helpers import tiny_ouro_model + from vllm_rlt import CacheConfig + from vllm_rlt.engine.stage_engine import StageEngine + + model = tiny_ouro_model("cuda", torch.bfloat16) + with pytest.raises(ValueError, match="PD connector"): + StageEngine( + model, + cache_config=CacheConfig(block_size=2, vmm_bootstrap_blocks=2), + connector=object(), + ) diff --git a/tests/test_vmm_real_checkpoint.py b/tests/test_vmm_real_checkpoint.py new file mode 100644 index 00000000..398fc9ce --- /dev/null +++ b/tests/test_vmm_real_checkpoint.py @@ -0,0 +1,279 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Real-checkpoint parity for VMM-backed KV storage (RFC #192 stage 2). + +The storage tests use synthetic blocks; this one runs the pinned Ouro snapshot so +real layer/head shapes, real tokenizer ids and real generated tokens go through +the reservation. Set ``OURO_CHECKPOINT`` to the snapshot directory: + + OURO_CHECKPOINT=/data/rlt-perf/weights/Ouro-1.4B CUDA_VISIBLE_DEVICES=0 \ + python -m pytest tests/test_vmm_real_checkpoint.py -q --run-gpu + +Oracle provenance: the baseline is the ordinary ``torch.empty`` pool on the same +weights, prompts and sampling parameters. A VMM mapping that shifted, doubled or +truncated a plane changes either the generated tokens or the KV bytes read back +for the same positions. The two engines run sequentially because two automatic +plans cannot share one device budget. +""" + +import gc +import os + +import pytest +import torch + +from vllm_rlt import CacheConfig, ExecutionConfig, SamplingParams +from vllm_rlt.engine.stage_engine import StageEngine +from vllm_rlt.models import OuroForCausalLM, load_tokenizer + +CHECKPOINT = os.environ.get("OURO_CHECKPOINT") + +pytestmark = [ + pytest.mark.gpu, + pytest.mark.skipif( + not CHECKPOINT, reason="set OURO_CHECKPOINT to the pinned Ouro snapshot directory" + ), +] + +PROMPTS = ["The capital of France is", "In a distant future, humans have learned to"] +# Long enough to exceed two 16-token pages, so the prefix cache has something to +# publish. The text is real input, tokenized by the checkpoint's own tokenizer. +LONG_TEXT = ( + "The capital of France is Paris, a city on the Seine with the Eiffel Tower, " + "the Louvre, Notre-Dame and centuries of documented history." +) +MAX_TOKENS = 8 +# The France prompt tokenizes to exactly these ids on the pinned snapshot, which +# pins the real tokenizer as the source of the input. +FRANCE_IDS = [504, 3575, 282, 4649, 314] + + +@pytest.fixture(scope="module") +def model(): + return OuroForCausalLM.from_pretrained(CHECKPOINT, device="cuda:0", dtype=torch.bfloat16) + + +@pytest.fixture(scope="module") +def prompts(): + tokenizer = load_tokenizer(CHECKPOINT) + encoded = [tokenizer.encode(text) for text in PROMPTS] + assert encoded[0] == FRANCE_IDS, "the pinned snapshot must tokenize the prompt this way" + return encoded + + +@pytest.fixture(scope="module") +def long_prompt(): + ids = load_tokenizer(CHECKPOINT).encode(LONG_TEXT) + assert len(ids) >= 33, "prefix caching needs at least two full pages" + return ids + + +def free_bytes(): + torch.cuda.synchronize() + return torch.cuda.mem_get_info()[0] + + +def run( + model, + prompt_ids, + *, + cache_config, + execution_config=None, + layers=(0,), + backend="torch", + sequential=False, +): + """Drive one engine to completion; return outputs, plan, cache stats and KV bytes. + + ``sequential`` admits one request per wave, which is what the prefix cache + needs: a page is only publishable after the request that wrote it finishes. + """ + assert layers, "capture at least one layer so the KV bytes are compared" + stage = StageEngine( + model, + cache_config=cache_config, + attention_backend=backend, + execution_config=execution_config, + ) + waves = [[ids] for ids in prompt_ids] if sequential else [list(prompt_ids)] + tokens, captured, used = {}, {}, 0 + request_ids = [] + try: + for wave in waves: + for ids in wave: + rid = str(len(request_ids)) + request_ids.append(rid) + stage.add_request( + rid, + list(ids), + SamplingParams(max_tokens=MAX_TOKENS, temperature=0.0), + ) + while stage.has_unfinished_requests(): + for output in stage.step(): + if output.finished: + tokens[output.request_id] = ( + tuple(output.token_ids), + tuple(output.exit_depths), + ) + cache = stage.cache_manager + used = max(used, cache.num_used_blocks) + for rid in request_ids: + if not cache.has_allocation(rid): + continue + # depth 0 is the plane every loop writes, so it is comparable. + keys, _ = cache.read(layers[0], rid, 0) + if keys.shape[0] == 0: + continue + captured[rid] = [(layer, cache.read(layer, rid, 0)) for layer in layers] + plan = dict(stage.memory_plan) + stats = dict( + published=stage.cache_manager.published_blocks, + committed=stage.cache_manager.committed_blocks, + maximum=stage.cache_manager.max_blocks, + used=used, + prefix_hits=stage.cache_manager.prefix_hits, + prefix_queries=stage.cache_manager.prefix_queries, + ) + captures = 0 + for runner in (stage.model_runner, stage.speculative_runner): + for name in ("graphs", "coda_graphs"): + graphs = getattr(runner, name, None) + captures += int(getattr(graphs, "captures", 0) or 0) + finally: + stage.close() + return tokens, plan, stats, captured, captures + + +def assert_same_tokens_and_kv(baseline, vmm): + base_tokens, _, base_kv, _ = baseline[0], baseline[1], baseline[3], baseline[4] + vmm_tokens, vmm_kv = vmm[0], vmm[3] + assert base_tokens, "the baseline must have produced outputs" + assert vmm_tokens == base_tokens, "generated tokens must not depend on the pool" + assert base_kv, "the baseline must have captured KV before the request finished" + assert set(vmm_kv) == set(base_kv), "both runs must capture the same requests" + for rid, entries in base_kv.items(): + assert len(vmm_kv[rid]) == len(entries) + for (layer, (keys, values)), (other_layer, (other_keys, other_values)) in zip( + entries, vmm_kv[rid] + ): + assert layer == other_layer + assert keys.shape == other_keys.shape, f"layer {layer} prefix length differs" + assert torch.equal(keys, other_keys), f"layer {layer} K bytes differ" + assert torch.equal(values, other_values), f"layer {layer} V bytes differ" + + +def test_real_weights_growth_keeps_tokens_and_kv_bytes(model, prompts): + layers = (0, model.config.num_hidden_layers // 2, model.config.num_hidden_layers - 1) + baseline = run(model, prompts, cache_config=CacheConfig(), layers=layers) + gc.collect() + torch.cuda.empty_cache() + vmm = run( + model, + prompts, + cache_config=CacheConfig(vmm_bootstrap_blocks=1), + layers=layers, + ) + assert_same_tokens_and_kv(baseline, vmm) + + base_plan, base_stats = baseline[1], baseline[2] + vmm_plan, vmm_stats = vmm[1], vmm[2] + # The option really split the pool, and serving used grown memory. + assert vmm_plan["vmm"] is True + assert vmm_plan["vmm_bootstrap_blocks"] == 1 + assert vmm_plan["vmm_max_blocks"] > 1, "the device budget must allow a real growth" + assert vmm_stats["published"] == vmm_plan["vmm_max_blocks"] == vmm_stats["maximum"] + assert vmm_stats["committed"] >= vmm_stats["published"] + assert vmm_stats["used"] > 1, "requests must have used blocks the growth mapped" + # The baseline keeps its fixed-capacity bookkeeping. + assert "vmm" not in base_plan + assert base_stats["maximum"] == base_stats["published"] == base_stats["committed"] + + +def test_real_weights_growth_with_cuda_graphs(model, prompts): + # The torch backend has no graph capability; Triton is the graph path. + execution = ExecutionConfig(cuda_graphs=True) + baseline = run( + model, prompts, cache_config=CacheConfig(), execution_config=execution, backend="triton" + ) + gc.collect() + torch.cuda.empty_cache() + vmm = run( + model, + prompts, + cache_config=CacheConfig(vmm_bootstrap_blocks=1), + execution_config=execution, + backend="triton", + ) + assert_same_tokens_and_kv(baseline, vmm) + assert baseline[4] > 0, "the baseline must have captured a graph" + assert vmm[4] > 0, "the graph path must have captured at least one entry" + + +def test_real_weights_growth_with_prefix_caching(model, long_prompt): + # One page becomes publishable only after the request that wrote it finished, + # so the second identical request must be admitted after the first drains. + repeats = [long_prompt, long_prompt] + baseline = run( + model, + repeats, + cache_config=CacheConfig(enable_prefix_caching=True), + sequential=True, + ) + gc.collect() + torch.cuda.empty_cache() + vmm = run( + model, + repeats, + cache_config=CacheConfig(enable_prefix_caching=True, vmm_bootstrap_blocks=1), + sequential=True, + ) + assert_same_tokens_and_kv(baseline, vmm) + base_stats, vmm_stats = baseline[2], vmm[2] + assert base_stats["prefix_hits"] > 0, "the repeated prompt must hit published pages" + assert vmm_stats["prefix_hits"] == base_stats["prefix_hits"] + assert vmm_stats["prefix_queries"] == base_stats["prefix_queries"] + + +def test_real_weights_bootstrap_above_the_plan_clamps(model, prompts): + baseline = run(model, prompts, cache_config=CacheConfig()) + gc.collect() + torch.cuda.empty_cache() + vmm = run(model, prompts, cache_config=CacheConfig(vmm_bootstrap_blocks=10**7)) + assert_same_tokens_and_kv(baseline, vmm) + plan, stats = vmm[1], vmm[2] + assert plan["vmm_bootstrap_blocks"] == plan["vmm_max_blocks"], "clamped to the plan" + assert stats["published"] == stats["maximum"] + assert stats["committed"] >= stats["published"] + + +def test_real_weights_repeated_create_close_leaks_nothing(model, prompts): + # Driver free memory is not a usable oracle here: this device is shared with + # other tenants, and their allocations move between measurements. Assert on + # what this engine owns instead: every mapped byte returns, and no tensor + # outlives the cycle. + gc.collect() + torch.cuda.synchronize() + allocated_before = torch.cuda.memory_allocated() + for cycle in range(3): + stage = StageEngine( + model, + cache_config=CacheConfig(vmm_bootstrap_blocks=1), + attention_backend="torch", + ) + storage = stage.kv_storage + try: + assert storage is not None, "the pool must exist for this cycle" + stage.add_request("0", list(prompts[0]), SamplingParams(max_tokens=2, temperature=0.0)) + while stage.has_unfinished_requests(): + stage.step() + assert storage.committed_bytes > 0 + finally: + stage.close() + assert stage.kv_storage is None + assert storage.committed_bytes == 0, ( + f"cycle {cycle} left {storage.committed_bytes} mapped bytes" + ) + assert storage.committed_blocks == 0 + storage.close() # idempotent after the engine already closed it + gc.collect() + leaked = torch.cuda.memory_allocated() - allocated_before + assert leaked <= 64 * (1 << 20), f"cycle {cycle} left tensors alive: {leaked} bytes" diff --git a/vllm_rlt/config.py b/vllm_rlt/config.py index b5b58cb6..bcdea921 100644 --- a/vllm_rlt/config.py +++ b/vllm_rlt/config.py @@ -20,6 +20,12 @@ class CacheConfig: incremental_allocation: bool = False # Admission headroom for incremental allocation; otherwise effective watermark is zero. watermark_ratio: float = 0.0 + # Bootstrap block count for CUDA VMM-backed KV storage (RFC #192 stage 2). + # None keeps the ordinary torch allocation. When set, the pool reserves the + # planned capacity and maps only this many blocks up front, then maps the + # rest during startup. Automatic planning is required, so explicit + # num_blocks/kv_cache_memory_bytes keep their fixed-capacity semantics. + vmm_bootstrap_blocks: int | None = None def __post_init__(self): for name in ("enable_prefix_caching", "incremental_allocation"): @@ -45,6 +51,13 @@ def __post_init__(self): raise ValueError("specify num_blocks or kv_cache_memory_bytes, not both") if type(self.memory_reserve_bytes) is not int or self.memory_reserve_bytes < 0: raise ValueError("memory_reserve_bytes must be a nonnegative integer") + if self.vmm_bootstrap_blocks is not None: + _positive("vmm_bootstrap_blocks", self.vmm_bootstrap_blocks) + if self.num_blocks is not None or self.kv_cache_memory_bytes is not None: + raise ValueError( + "vmm_bootstrap_blocks requires automatic KV planning; " + "num_blocks/kv_cache_memory_bytes keep fixed capacity" + ) @dataclass(frozen=True) diff --git a/vllm_rlt/core/kv_cache_manager.py b/vllm_rlt/core/kv_cache_manager.py index 0d1e8641..b0cb7c1d 100644 --- a/vllm_rlt/core/kv_cache_manager.py +++ b/vllm_rlt/core/kv_cache_manager.py @@ -13,6 +13,7 @@ from copy import deepcopy from dataclasses import dataclass, field from numbers import Integral +from typing import Protocol import torch @@ -77,6 +78,23 @@ class KVSnapshot: values: torch.Tensor +class KVBlockStorage(Protocol): + """Block storage whose physical backing can grow while the VA stays fixed. + + ``GPUKVStorage`` implements this over CUDA VMM. The manager only reads the + capacity counters and the K/V planes; mapping stays the storage's job. + """ + + max_blocks: int + committed_blocks: int + bytes_per_block: int + dtype: torch.dtype + + def keys(self, shape) -> torch.Tensor: ... + + def values(self, shape) -> torch.Tensor: ... + + class KVCacheManager: """Own a fixed physical page pool shared by request/depth allocations. @@ -103,6 +121,7 @@ def __init__( recurrent_layers: Sequence[int] | None = None, *, attention: AttentionBackend | None = None, + storage: KVBlockStorage | None = None, ): for name, value in ( ("num_layers", num_layers), @@ -140,9 +159,19 @@ def __init__( ) self.attention_info = self.attention.info self.backend = self.attention_info["selected_backend"] - shape = (num_blocks, num_layers, block_size, num_kv_heads, head_dim) - self.key_cache = torch.empty(shape, device=self.device, dtype=dtype) - self.value_cache = torch.empty_like(self.key_cache) + self._storage = storage + self._watermark_ratio = watermark_ratio + if storage is None: + shape = (num_blocks, num_layers, block_size, num_kv_heads, head_dim) + self.key_cache = torch.empty(shape, device=self.device, dtype=dtype) + self.value_cache = torch.empty_like(self.key_cache) + self.max_blocks = num_blocks + self.committed_blocks = num_blocks + else: + self.key_cache, self.value_cache = self._attach_storage(storage, num_blocks) + # Requests are only ever given published blocks, and publishing requires + # committed physical memory, so published <= committed <= max holds. + self.published_blocks = num_blocks # Resolve an implicit CUDA index to the actual storage device once. self.device = self.key_cache.device self._free_blocks = list(reversed(range(num_blocks))) @@ -150,7 +179,7 @@ def __init__( self.enable_prefix_caching = enable_prefix_caching self.incremental_allocation = incremental_allocation # Full allocation already reserves each request's maximum KV capacity. - self.watermark_blocks = int(watermark_ratio * num_blocks) if incremental_allocation else 0 + self.watermark_blocks = self._watermark(num_blocks) self._refs = [0] * num_blocks self._prefixes = OrderedDict() self._pending_prefixes = [] @@ -183,6 +212,85 @@ def bytes_per_block(self) -> int: * self.key_cache.element_size() ) + @property + def storage(self) -> KVBlockStorage | None: + return self._storage + + def _watermark(self, blocks: int) -> int: + return int(self._watermark_ratio * blocks) if self.incremental_allocation else 0 + + def _attach_storage(self, storage: KVBlockStorage, num_blocks: int): + """Take the K/V planes from a pre-reserved pool instead of allocating them.""" + expected = ( + 2 + * self.num_layers + * self.block_size + * self.num_kv_heads + * self.head_dim + * torch.empty((), dtype=self.dtype).element_size() + ) + if storage.bytes_per_block != expected: + raise ValueError( + f"storage holds {storage.bytes_per_block} bytes per block, " + f"but this cache needs {expected}" + ) + if storage.dtype != self.dtype: + raise ValueError("storage dtype does not match the KV dtype") + if storage.max_blocks < num_blocks: + raise ValueError( + f"storage reserves {storage.max_blocks} blocks but {num_blocks} were requested" + ) + if storage.committed_blocks < num_blocks: + raise ValueError( + f"storage committed {storage.committed_blocks} blocks but {num_blocks} " + "were requested; commit the pool before building the cache" + ) + shape = ( + storage.max_blocks, + self.num_layers, + self.block_size, + self.num_kv_heads, + self.head_dim, + ) + keys, values = storage.keys(shape), storage.values(shape) + for name, tensor in (("keys", keys), ("values", values)): + if tensor.dtype != self.dtype or tuple(tensor.shape) != shape: + raise ValueError(f"storage {name} do not match the KV layout") + self.max_blocks = storage.max_blocks + self.committed_blocks = storage.committed_blocks + return keys, values + + def publish_capacity(self, blocks: int) -> int: + """Grow the allocatable pool after the storage mapped more blocks. + + The K/V tensors keep their address, shape and stride; only the free pool + and the published count grow. Uncommitted blocks are never published, so + no allocation can reference unmapped memory. + """ + if self._storage is None: + raise RuntimeError("capacity publishing requires VMM-backed storage") + if type(blocks) is not int or blocks < self.published_blocks: + raise ValueError("published capacity can only grow") + if blocks > self.max_blocks: + raise ValueError( + f"cannot publish {blocks} blocks; the reservation holds {self.max_blocks}" + ) + # Re-read the pool: a commit may have happened since the last publish. + self.committed_blocks = self._storage.committed_blocks + if blocks > self.committed_blocks: + raise ValueError( + f"cannot publish {blocks} blocks; the storage committed {self.committed_blocks}" + ) + if blocks == self.published_blocks: + return blocks + self._refs.extend([0] * (blocks - self.published_blocks)) + # _claim pops from the tail, so later blocks follow in ascending order. + self._free_blocks.extend(reversed(range(self.published_blocks, blocks))) + self.published_blocks = blocks + self.num_blocks = blocks + self.watermark_blocks = self._watermark(blocks) + return blocks + def required_blocks(self, max_tokens: int) -> int: if not isinstance(max_tokens, Integral) or isinstance(max_tokens, bool) or max_tokens <= 0: raise ValueError("max_tokens must be a positive integer") diff --git a/vllm_rlt/core/vmm_pool.py b/vllm_rlt/core/vmm_pool.py new file mode 100644 index 00000000..97039870 --- /dev/null +++ b/vllm_rlt/core/vmm_pool.py @@ -0,0 +1,309 @@ +"""CUDA VMM block pool with a stable virtual address and growable physical backing. + +RFC #192 stage 2 (incremental KV storage commitment). One pool owns the whole +reservation plus every physical mapping; the K and V tensors are views into it, +so there is exactly one lifecycle (reserve -> commit -> close) regardless of how +many tensor planes are exposed. + +The tensors always describe the full reservation. Only the committed prefix has +physical backing, so kernels may only address blocks below ``committed_blocks``. +Touching an unmapped block raises a sticky CUDA illegal-address error; the pool +therefore never publishes a block it has not mapped. +""" + +from __future__ import annotations + +import ctypes + +import torch + +_CU_MEM_ALLOCATION_TYPE_PINNED = 1 +_CU_MEM_LOCATION_TYPE_DEVICE = 1 +_CU_MEM_ACCESS_FLAGS_PROT_READWRITE = 3 +_CU_MEM_ALLOC_GRANULARITY_RECOMMENDED = 1 + +_TYPESTR = { + torch.float32: " None: + if code != 0: + message = ctypes.c_char_p() + self.lib.cuGetErrorString(code, ctypes.byref(message)) + text = message.value.decode() if message.value else "unknown" + raise RuntimeError(f"{name} failed with CUresult {code}: {text}") + + +_DRIVER: _Driver | None = None + + +def _driver() -> _Driver: + global _DRIVER + if _DRIVER is None: + _DRIVER = _Driver() + return _DRIVER + + +def _round_up(value: int, multiple: int) -> int: + return (value + multiple - 1) // multiple * multiple + + +class _ExternalBuffer: + """Carrier exposing a raw device pointer through __cuda_array_interface__.""" + + def __init__(self, ptr, shape, typestr): + self.__cuda_array_interface__ = { + "shape": tuple(shape), + "typestr": typestr, + "data": (int(ptr), False), + "version": 2, + "strides": None, + } + + +class GPUKVStorage: + """One VA reservation holding the K and V planes of a block pool. + + ``max_blocks`` fixes the logical capacity and therefore every tensor shape. + ``bytes_per_block`` is the K *and* V bytes of one block, matching + ``KVCacheManager.bytes_per_block``; each plane therefore holds half of it. + ``commit_blocks`` appends physical memory to both planes; ``close`` releases + everything. Nothing here knows about requests, block tables, or scheduling. + """ + + def __init__(self, *, max_blocks: int, bytes_per_block: int, device, dtype): + if type(max_blocks) is not int or max_blocks < 1: + raise ValueError("max_blocks must be a positive integer") + if type(bytes_per_block) is not int or bytes_per_block < 1: + raise ValueError("bytes_per_block must be a positive integer") + if bytes_per_block % 2: + raise ValueError("bytes_per_block must cover both K and V, so it must be even") + if dtype not in _TYPESTR: + raise ValueError("VMM KV storage supports float32, float16 and bfloat16") + self.dtype = dtype + self.max_blocks = max_blocks + self.bytes_per_block = bytes_per_block + self.device = torch.device(device) + if self.device.type != "cuda": + raise ValueError("VMM KV storage requires a CUDA device") + self.drv = _driver() + torch.cuda.set_device(self.device) + # Establish the primary context before cuMemAddressReserve attaches to it. + torch.empty(1, device=self.device) + # The driver only hands out physical memory the allocator has returned. + # A torch KV pool could reuse its cached segments, but cuMemCreate cannot, + # so return them before the first commit. Startup profiling leaves one + # such segment behind (the temporary KV probe). + torch.cuda.synchronize(self.device) + torch.cuda.empty_cache() + + self._prop = _AllocProp() + self._prop.type = _CU_MEM_ALLOCATION_TYPE_PINNED + self._prop.location.type = _CU_MEM_LOCATION_TYPE_DEVICE + self._prop.location.id = self.device.index or 0 + self._access = _AccessDesc() + self._access.location = self._prop.location + self._access.flags = _CU_MEM_ACCESS_FLAGS_PROT_READWRITE + + granularity = _SIZE() + self.drv.check( + "cuMemGetAllocationGranularity", + self.drv.lib.cuMemGetAllocationGranularity( + ctypes.byref(granularity), + ctypes.byref(self._prop), + _CU_MEM_ALLOC_GRANULARITY_RECOMMENDED, + ), + ) + self.allocation_granularity = int(granularity.value) + # half of bytes_per_block per plane: bytes_per_block already counts K and V. + self.plane_bytes_per_block = bytes_per_block // 2 + self.plane_bytes = _round_up( + max_blocks * self.plane_bytes_per_block, self.allocation_granularity + ) + self.reserve_bytes = 2 * self.plane_bytes + + base = _U64() + self.drv.check( + "cuMemAddressReserve", + self.drv.lib.cuMemAddressReserve( + ctypes.byref(base), self.reserve_bytes, self.allocation_granularity, 0, 0 + ), + ) + self.base = int(base.value) + self._mappings: list[tuple[int, int, int]] = [] + self._mapped_plane_bytes = 0 + self.committed_blocks = 0 + self._closed = False + + # -- capacity --------------------------------------------------------- + @property + def committed_bytes(self) -> int: + """Physical bytes mapped so far, K and V planes together.""" + return 2 * self._mapped_plane_bytes + + def commit_blocks(self, blocks: int) -> int: + """Map physical memory for ``blocks`` blocks in both planes. + + Returns the committed block count after the call, which can exceed + ``blocks`` because the last mapping is padded to the VMM granularity. + Commits only append; an existing mapping is never moved or replaced. + """ + self._check_open() + if type(blocks) is not int or blocks < 1: + raise ValueError("blocks must be a positive integer") + if blocks > self.max_blocks: + raise ValueError( + f"cannot commit {blocks} blocks; the reservation holds {self.max_blocks}" + ) + if blocks <= self.committed_blocks: + return self.committed_blocks + target = _round_up(blocks * self.plane_bytes_per_block, self.allocation_granularity) + if target > self.plane_bytes: + raise ValueError( + f"committing {blocks} blocks needs {target} bytes per plane, " + f"but the reservation holds {self.plane_bytes}" + ) + delta = target - self._mapped_plane_bytes + applied: list[tuple[int, int, int]] = [] + try: + for offset in (0, self.plane_bytes): + address = self.base + offset + self._mapped_plane_bytes + handle = _U64() + self.drv.check( + "cuMemCreate", + self.drv.lib.cuMemCreate( + ctypes.byref(handle), delta, ctypes.byref(self._prop), 0 + ), + ) + try: + self.drv.check( + "cuMemMap", self.drv.lib.cuMemMap(address, delta, 0, handle.value, 0) + ) + except Exception: + self.drv.lib.cuMemRelease(handle.value) + raise + try: + self.drv.check( + "cuMemSetAccess", + self.drv.lib.cuMemSetAccess(address, delta, ctypes.byref(self._access), 1), + ) + except Exception: # pragma: no cover - driver dependent + self.drv.lib.cuMemUnmap(address, delta) + self.drv.lib.cuMemRelease(handle.value) + raise + applied.append((address, delta, handle.value)) + except Exception: + for address, size, handle in reversed(applied): + self.drv.lib.cuMemUnmap(address, size) + self.drv.lib.cuMemRelease(handle) + raise + self._mappings.extend(applied) + self._mapped_plane_bytes = target + self.committed_blocks = min(target // self.plane_bytes_per_block, self.max_blocks) + return self.committed_blocks + + # -- tensors ---------------------------------------------------------- + def keys(self, shape) -> torch.Tensor: + return self._plane_tensor(0, shape) + + def values(self, shape) -> torch.Tensor: + return self._plane_tensor(self.plane_bytes, shape) + + def _plane_tensor(self, offset: int, shape) -> torch.Tensor: + self._check_open() + buffer = _ExternalBuffer(self.base + offset, shape, _TYPESTR[self.dtype]) + tensor = torch.as_tensor(buffer, device=self.device) + if tensor.dtype != self.dtype: + # Only bfloat16 reaches here: the interface cannot name it. + tensor = tensor.view(self.dtype) + # The mapping outlives every view; closing is the pool's decision alone. + tensor._vmm_owner = self + return tensor + + # -- lifecycle -------------------------------------------------------- + def close(self) -> None: + """Unmap, release handles, then free the reservation. Views become invalid.""" + if self._closed: + return + torch.cuda.synchronize(self.device) + for address, size, handle in reversed(self._mappings): + self.drv.check("cuMemUnmap", self.drv.lib.cuMemUnmap(address, size)) + self.drv.check("cuMemRelease", self.drv.lib.cuMemRelease(handle)) + self._mappings.clear() + self.drv.check( + "cuMemAddressFree", self.drv.lib.cuMemAddressFree(self.base, self.reserve_bytes) + ) + self._mapped_plane_bytes = 0 + self.committed_blocks = 0 + self._closed = True + + def _check_open(self) -> None: + if self._closed: + raise RuntimeError("the KV storage pool is closed") diff --git a/vllm_rlt/engine/stage_engine.py b/vllm_rlt/engine/stage_engine.py index e7c99067..6fe56947 100644 --- a/vllm_rlt/engine/stage_engine.py +++ b/vllm_rlt/engine/stage_engine.py @@ -14,6 +14,7 @@ from vllm_rlt.core.memory import plan_cache from vllm_rlt.core.scheduler import Scheduler from vllm_rlt.core.scheduling_policy import requires_staged_prefill +from vllm_rlt.core.vmm_pool import GPUKVStorage from vllm_rlt.engine.connector import StageConnector from vllm_rlt.engine.packed_prefill import packed_prefill_eligible from vllm_rlt.engine.preemption import PreemptionManager @@ -68,6 +69,14 @@ def __init__( raise ValueError(f"boundary-KV models require {required_layout} KV") if cache_config.enable_prefix_caching: raise ValueError("boundary-KV prefix caching is not yet supported") + if cache_config.vmm_bootstrap_blocks is not None: + if parameter.device.type != "cuda": + raise ValueError("vmm_bootstrap_blocks requires a CUDA device") + if connector is not None: + raise ValueError( + "vmm_bootstrap_blocks is not supported with a PD connector yet; " + "NIXL registers the KV planes once at startup" + ) self.exit_config = exit_config or ExitConfig() self.execution_config = resolve_cuda_graph_rows( execution_config or ExecutionConfig(), scheduler_config, speculative_config @@ -212,12 +221,25 @@ def __init__( speculative=speculative_config, staged_prefill=staged_prefill, ) + # RFC #192 stage 2: reserve the planned capacity once, map only the + # bootstrap blocks now, and map the remainder after the runners exist. + self.kv_storage = None + bootstrap_blocks = num_blocks + if cache_config.vmm_bootstrap_blocks is not None: + self.kv_storage = GPUKVStorage( + max_blocks=num_blocks, + bytes_per_block=self.memory_plan["bytes_per_block"], + device=parameter.device, + dtype=parameter.dtype, + ) + bootstrap_blocks = min(cache_config.vmm_bootstrap_blocks, num_blocks) + self.kv_storage.commit_blocks(bootstrap_blocks) self.cache_manager = KVCacheManager( num_layers=getattr(config, "num_kv_layers", config.num_hidden_layers), num_kv_heads=config.num_key_value_heads, head_dim=config.head_dim, max_loops=config.total_ut_steps, - num_blocks=num_blocks, + num_blocks=bootstrap_blocks, layout=cache_config.layout, block_size=cache_config.block_size, device=parameter.device, @@ -228,6 +250,7 @@ def __init__( incremental_allocation=cache_config.incremental_allocation, watermark_ratio=cache_config.watermark_ratio, recurrent_layers=getattr(model, "recurrent_kv_layers", None), + storage=self.kv_storage, ) self.scheduler = Scheduler( scheduler_config, @@ -296,6 +319,19 @@ def __init__( if scheduler_config.enable_preemption: self.scheduler.suspend_callback = self.preemption.suspend self.scheduler.resume_callback = self.preemption.resume + if self.kv_storage is not None: + # Every buffer the runners own now exists, so the remaining physical + # memory can be mapped into the same reservation and published. + self.kv_storage.commit_blocks(num_blocks) + self.cache_manager.publish_capacity(num_blocks) + self.memory_plan = dict( + self.memory_plan, + vmm=True, + vmm_bootstrap_blocks=bootstrap_blocks, + vmm_max_blocks=num_blocks, + vmm_committed_blocks=self.kv_storage.committed_blocks, + vmm_reserve_bytes=self.kv_storage.reserve_bytes, + ) @property def connector(self): @@ -439,6 +475,10 @@ def close(self): self.abort_request(rid) if self.async_speculative: self._discard_speculative() + # Device work is drained, so the reservation can be returned last. + if self.kv_storage is not None: + self.kv_storage.close() + self.kv_storage = None def _discard_speculative(self): """Release speculative owners after the caller has drained both streams.""" diff --git a/vllm_rlt/entrypoints/runtime_args.py b/vllm_rlt/entrypoints/runtime_args.py index 0995207f..cafbd003 100644 --- a/vllm_rlt/entrypoints/runtime_args.py +++ b/vllm_rlt/entrypoints/runtime_args.py @@ -22,6 +22,13 @@ def add_runtime_args(parser): parser.add_argument("--enable-prefix-caching", action="store_true") parser.add_argument("--incremental-kv", action="store_true") parser.add_argument("--kv-watermark", type=float, default=0.0) + parser.add_argument( + "--kv-vmm-bootstrap-blocks", + type=int, + default=None, + help="map only this many KV blocks at startup, then grow the pool to the planned " + "capacity (RFC #192 stage 2; CUDA only, automatic planning only)", + ) parser.add_argument("--scheduling-policy", choices=["fcfs", "priority"], default="fcfs") parser.add_argument("--enable-preemption", action="store_true") parser.add_argument( @@ -117,6 +124,7 @@ def runtime_configs(args): gpu_memory_utilization=args.gpu_memory_utilization, kv_cache_memory_bytes=args.kv_cache_memory_bytes, memory_reserve_bytes=args.memory_reserve_bytes, + vmm_bootstrap_blocks=args.kv_vmm_bootstrap_blocks, ), exit_config=ExitConfig( mode=args.exit_mode,