Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
90 changes: 90 additions & 0 deletions tests/test_dory.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

import json
import math
from collections import Counter

import pytest
import torch
Expand All @@ -14,6 +15,7 @@
from vllm_rlt.core.kv_cache_manager import KVCacheManager
from vllm_rlt.engine.llm_engine import LLMEngine
from vllm_rlt.kernels.paged_attention import torch_paged_attention
from vllm_rlt.layers import RotaryEmbedding
from vllm_rlt.models import AutoModelForCausalLM, DoryConfig, DoryForCausalLM

ARTIFACT_PAIRS = 36
Expand Down Expand Up @@ -385,6 +387,94 @@ def test_batched_positions_match_reference_for_per_loop_kv(model):
assert cache.num_used_blocks == 0


@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
@pytest.mark.parametrize("kv_mode", ["per_loop", "last_loop"])
def test_rotary_embeddings_are_reused_only_within_each_traversal(monkeypatch, dtype, kv_mode):
# Every stage contains repeated sliding and full attention layers.
layer_types = ("sliding_attention", "mlp", "full_attention", "mlp") * 6
config = tiny_dory_config(
num_hidden_layers=24,
n_input_layers=8,
n_recurrent_layers=8,
n_output_layers=8,
layer_types=layer_types,
recurrent_kv_cache_mode=kv_mode,
)
model = DoryForCausalLM(config).to(dtype=dtype).eval()
cache = make_cache(model)
assert cache.allocate("request", 2)
calls = Counter()
original = RotaryEmbedding.forward

def record(module, hidden, positions):
calls[module.rope_theta] += 1
return original(module, hidden, positions)

monkeypatch.setattr(RotaryEmbedding, "forward", record)
with torch.inference_mode():
for position in range(2):
batch = cache._prepare_batch(["request"], [0], [position])
calls.clear()
hidden = model.prelude_prepared(torch.tensor([4 + position]), batch, cache)
assert calls == {1e4: 1, 1e7: 1}
for depth in range(config.total_ut_steps):
core = cache._prepare_batch(["request"], [depth], [position])
calls.clear()
hidden, _ = model.recurrent_prepared(hidden, core, cache)
assert calls == {1e4: 1, 1e7: 1}
calls.clear()
model.coda_prepared(hidden, batch, cache)
assert calls == {1e4: 1, 1e7: 1}
cache.free("request")


@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
@pytest.mark.parametrize("kv_mode", ["per_loop", "last_loop"])
def test_reused_rotary_matches_per_layer_recomputation(monkeypatch, dtype, kv_mode):
config = tiny_dory_config(
num_hidden_layers=24,
n_input_layers=8,
n_recurrent_layers=8,
n_output_layers=8,
layer_types=("sliding_attention", "mlp", "full_attention", "mlp") * 6,
recurrent_kv_cache_mode=kv_mode,
)
model = DoryForCausalLM(config).to(dtype=dtype)
tokens = torch.tensor([4, 7, 9, 3, 12, 5])
with torch.inference_mode():
actual, cache = run_one_token_per_step(model, tokens)

def per_layer(backbone, hidden, batch, cache, layers):
for index in layers:
layer = backbone.layers[index]
embeddings = (
layer.mixer.rotary_emb(hidden, batch.position_ids)
if layer.is_attention
else None
)
hidden = layer(hidden, embeddings, batch, cache)
return hidden

monkeypatch.setattr(type(model.backbone), "forward", per_layer)
expected, reference_cache = run_one_token_per_step(model, tokens)
torch.testing.assert_close(actual["logits"], expected["logits"], atol=0, rtol=0)
for actual_loop, expected_loop in zip(actual["loops"], expected["loops"]):
torch.testing.assert_close(actual_loop, expected_loop, atol=0, rtol=0)
for layer in model.config.attention_layer_ids:
kv_layer = model.config.kv_layer_index(layer)
depths = (
range(model.config.total_ut_steps) if kv_layer in model.recurrent_kv_layers else (0,)
)
for depth in depths:
for actual_kv, expected_kv in zip(
cache.read(kv_layer, "request", depth),
reference_cache.read(kv_layer, "request", depth),
):
torch.testing.assert_close(actual_kv, expected_kv, atol=0, rtol=0)
cache.free("request")
reference_cache.free("request")


def test_recurrent_kv_is_laid_out_per_loop_plane(model):
tokens = torch.tensor([4, 7, 9, 3, 12, 5])
_, cache = run_one_token_per_step(model, tokens)
Expand Down
33 changes: 33 additions & 0 deletions tests/test_dory_gpu.py
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,39 @@ def test_dory_execution_modes_match_cpu_oracle(mode, max_num_seqs):
assert runner.prelude_graphs is None


@pytest.mark.parametrize("mode", EXECUTION_MODES)
def test_reused_rotary_embeddings_follow_positions_in_every_execution_mode(mode):
config = tiny_dory_config(
num_hidden_layers=24,
n_input_layers=8,
n_recurrent_layers=8,
n_output_layers=8,
layer_types=("sliding_attention", "mlp", "full_attention", "mlp") * 6,
)
torch.manual_seed(42)
reference = DoryForCausalLM(config).eval()
model = DoryForCausalLM(config).to(device=DEVICE, dtype=torch.float32).eval()
model.load_state_dict(reference.state_dict())
engine = make_engine(model, mode)
params = SamplingParams(max_tokens=6, min_loops=2, max_loops=2, ignore_eos=True)
for request_id, prompt in PROMPTS.items():
engine.add_request(request_id, prompt, params)
try:
finished = drain(engine)
for request_id, prompt in PROMPTS.items():
assert finished[request_id].token_ids == greedy_reference(reference, prompt, 6)
assert engine.cache_manager.num_used_blocks == 0
if "graph" in mode:
for graphs in (
engine.model_runner.prelude_graphs,
engine.model_runner.graphs,
engine.model_runner.coda_graphs,
):
assert graphs.replays > 0
finally:
engine.close()


@pytest.mark.parametrize("mode", EXECUTION_MODES)
@pytest.mark.parametrize("after", [0, 1, 3, 6, 9])
def test_dory_cancellation_releases_kv_in_every_execution_mode(mode, after):
Expand Down
29 changes: 19 additions & 10 deletions vllm_rlt/models/dory.py
Original file line number Diff line number Diff line change
Expand Up @@ -381,7 +381,7 @@ def __init__(self, config: DoryConfig, layer_idx: int) -> None:
def forward(
self,
hidden: torch.Tensor,
position_ids: torch.Tensor,
position_embeddings: tuple[torch.Tensor, torch.Tensor],
batch: "KVMetadata | DeviceKVMetadata",
cache: "KVCacheManager",
) -> torch.Tensor:
Expand All @@ -390,7 +390,7 @@ def forward(
k = self.k_proj(hidden).view(count, self.num_kv_heads, self.head_dim)
v = self.v_proj(hidden).view(count, self.num_kv_heads, self.head_dim)
gate = self.g_proj(hidden)
cos, sin = self.rotary_emb(hidden, position_ids)
cos, sin = position_embeddings
q, k = apply_rotary_pos_emb(q, k, cos, sin)
# nGPT Q/K normalization happens after RoPE: queries carry the square of
# the learned scale, keys only the unit-length normalization.
Expand Down Expand Up @@ -448,12 +448,12 @@ def __init__(self, config: DoryConfig, layer_idx: int) -> None:
def forward(
self,
hidden: torch.Tensor,
position_ids: torch.Tensor,
position_embeddings: tuple[torch.Tensor, torch.Tensor] | None,
batch: "KVMetadata | DeviceKVMetadata",
cache: "KVCacheManager",
) -> torch.Tensor:
if self.is_attention:
update = self.mixer(hidden, position_ids, batch, cache)
update = self.mixer(hidden, position_embeddings, batch, cache)
return slerp_alpha(hidden, update, self.attn_alpha, self.alpha_scale)
return slerp_alpha(hidden, self.mixer(hidden), self.mlp_alpha, self.alpha_scale)

Expand All @@ -475,6 +475,18 @@ def __init__(self, config: DoryConfig) -> None:
self.recurrent_layers = tuple(range(*recurrent_slice))
self.output_layers = tuple(range(*output_slice))

def forward(self, hidden, batch, cache, layers):
# Positions are shared within a traversal, but change between graph replays.
position_embeddings = {}
for index in layers:
layer = self.layers[index]
if layer.is_attention and layer.layer_type not in position_embeddings:
position_embeddings[layer.layer_type] = layer.mixer.rotary_emb(
hidden, batch.position_ids
)
hidden = layer(hidden, position_embeddings.get(layer.layer_type), batch, cache)
return hidden


class DoryForCausalLM(nn.Module):
"""Inference-only Dory with eager KV boundaries and a graphable recurrent core."""
Expand Down Expand Up @@ -547,8 +559,7 @@ def prelude_prepared(self, tokens, batch, cache, *, generators=None) -> torch.Te
if tokens.ndim != 1:
raise ValueError("prelude expects packed token_ids with shape [N]")
hidden = self.backbone.embeddings(tokens)
for index in self.backbone.input_layers:
hidden = self.backbone.layers[index](hidden, batch.position_ids, batch, cache)
hidden = self.backbone(hidden, batch, cache, self.backbone.input_layers)
# The retained injection input is half of the carried state, and loop zero
# starts from the same tensor, matching the reference ``state = input_h``.
return torch.cat((hidden, hidden), dim=-1)
Expand All @@ -572,8 +583,7 @@ def recurrent_prepared(self, hidden, batch, cache, *, compute_gate: bool = True)
raise ValueError("recurrent expects hidden with shape [N, hidden_size]")
state, injection = hidden.split(self.config.d_model, dim=-1)
current = justnorm(state + injection)
for index in self.backbone.recurrent_layers:
current = self.backbone.layers[index](current, batch.position_ids, batch, cache)
current = self.backbone(current, batch, cache, self.backbone.recurrent_layers)
state = justnorm(current)
# Dory has no exit gate; a constant gate keeps the scheduler on the
# configured depth instead of inventing an exit policy.
Expand All @@ -587,8 +597,7 @@ def recurrent_prepared(self, hidden, batch, cache, *, compute_gate: bool = True)
def coda_prepared(self, hidden, batch, cache, *, compute_logits: bool = True):
"""Run the attention-bearing output stage and the scaled LM head."""
state, _ = hidden.split(self.config.d_model, dim=-1)
for index in self.backbone.output_layers:
state = self.backbone.layers[index](state, batch.position_ids, batch, cache)
state = self.backbone(state, batch, cache, self.backbone.output_layers)
return self.logits(state) if compute_logits else None

def logits(self, state: torch.Tensor) -> torch.Tensor:
Expand Down
Loading