Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 22 additions & 9 deletions afd_plugin/compat/npu/feature_validation.py
Original file line number Diff line number Diff line change
Expand Up @@ -97,19 +97,25 @@ def _is_dsv4_target(vllm_config: VllmConfig) -> bool:
return any("DeepseekV4" in str(architecture) for architecture in architectures)


# ``connector_extra_config.quant_mode`` of the pre-quantized A2E transport. The
# transport lands with its own change; this guard keeps DBO rejected for it until
# the ubatch yield has been verified together with quantization.
_MXFP8_QUANT_MODE = 4


def _fail_if_unsupported_sync_attention_gate(
vllm_config: VllmConfig,
afd_config: AFDConfig,
extra_info: object,
*,
is_dsv4: bool,
) -> None:
"""Restrict the sync Attention-side gate prototype to what it supports today.
"""Restrict the sync Attention-side gate to what it supports today.

DBO and ubatching are rejected rather than merely unused: the sync shell does not
call the DBO yield, so enabling DBO here would interleave microbatches against a
transport that never yields. Not calling the yield is not the same as forbidding
the feature.
Native DBO runs through this shell: the forward resolves the ubatch stage before
the routing send, yields, and receives on the resolved stage, and the connector
keeps one AFD group per ubatch. The pre-quantized transport stays excluded until
its yield interaction is verified.
"""

parallel = vllm_config.parallel_config
Expand All @@ -122,10 +128,17 @@ def _fail_if_unsupported_sync_attention_gate(
f"TP={int(parallel.tensor_parallel_size)}",
)
if bool(parallel.enable_dbo) or bool(parallel.use_ubatching):
raise RuntimeError(
"Attention-side gate with the synchronous connector does not support "
"DBO or ubatching yet",
)
if int(parallel.num_ubatches) != 2:
raise RuntimeError(
"Attention-side gate with the synchronous connector requires "
f"exactly two microbatches; got {int(parallel.num_ubatches)}",
)
if int(getattr(extra_info, "quant_mode", 0)) == _MXFP8_QUANT_MODE:
raise RuntimeError(
"quant_mode=4 does not support DBO or ubatching yet: the ubatch "
"yield and the pre-quantized transport have not been verified "
"together",
)
if afd_config.connector != CAMP2P_CONNECTOR:
raise RuntimeError(
"Attention-side gate is only supported with "
Expand Down
18 changes: 14 additions & 4 deletions afd_plugin/model_executor/models/npu/deepseek_v4.py
Original file line number Diff line number Diff line change
Expand Up @@ -303,8 +303,9 @@ class AFDDeepseekV4SyncAttentionGateRemoteMoE(AFDDeepseekV4AttentionGateRemoteMo
send and the receive are separate calls;
- the shared experts run *between* those calls, which is the whole point of
moving them here: their work overlaps the FFN side's routed work;
- no DBO yield, because the first prototype keeps the sync path free of the
scheduling interaction the async shell needs.
- native DBO is supported: the stage is resolved before the routing send, the
forward yields between the send and the receive, and the receive reuses the
resolved stage so a peer ubatch cannot switch it mid-transfer.

Reusing ``RemoteFFNProxy._send_and_receive`` is not possible here: it issues the
send and the receive back to back, leaving no window for the shared experts.
Expand All @@ -327,10 +328,16 @@ def forward(
afd_metadata = get_afd_metadata_from_forward_context()
if afd_metadata is None:
raise RuntimeError("Sync Attention-side gate requires AFD forward metadata")
# Resolve the stage once and reuse it for the send and the receive. Reading
# it again after the yield would let the two diverge.
stage_idx = int(
getattr(get_forward_context(), "ubatch_idx", afd_metadata.stage_idx),
)
afd_metadata.stage_idx = stage_idx
context = AFDTransferContext(
metadata=AFDTransferMetadata.create_attention_metadata(
layer_idx=self.layer_idx,
stage_idx=afd_metadata.stage_idx,
stage_idx=stage_idx,
seq_len=int(hidden_states.shape[0]),
)
)
Expand All @@ -346,9 +353,12 @@ def forward(
if self.shared_experts is not None
else None
)
# Yield between send and receive: the FFN side's routed work is the window
# the peer ubatch runs in. The receive must use the stage resolved above.
maybe_apply_dbo_yield(hidden_states, role="attention")
output = afd_metadata.connector.recv_ffn_output(
ref_tensor=hidden_states,
ubatch_idx=afd_metadata.stage_idx,
ubatch_idx=stage_idx,
)
if shared_output is not None:
output = output + shared_output
Expand Down
4 changes: 2 additions & 2 deletions docs/design/module/connector_contracts.md
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ Unknown fields fail in the selected connector parser.
| --- | --- |
| `P2pNcclAFDConnector` | None; the mapping must be empty. |
| `DeepEPM2NAFDConnector` | `payload_format`, `expert_backend`, `capacity_tokens_per_rank`, `num_lanes`, `bucket_granularity`, `cpu_sync_min_tokens`, `num_sms`, `num_qps`, `expert_num_sms`, `cpu_timeout_s`, `gpu_timeout_s`, and a nested `network` mapping (`gin`, `hybrid`, `rdma_traffic_class`). All are optional; see the [DeepEP M2N guide](../../gpu/DEEPEP_M2N_CONNECTOR_USER_GUIDE.md#connector_extra_config) for defaults. |
| `CAMP2pAFDConnector` | `core_num`, optional `attn_core_num` / `ffn_core_num`, `compute_gate_on_attention`, `quant_mode`, and optional positive `hccl_buffer_size`. Core counts must be positive. Gate placement follows the common `afd.compute_gate_on_attention` field; the connector-local copy is kept for compatibility and does not select the mode. `true` requires DeepSeek-V4 at TP=1 without DBO or ubatching. Any nonzero quantization mode is rejected today. |
| `CAMP2pAFDConnector` | `core_num`, optional `attn_core_num` / `ffn_core_num`, `compute_gate_on_attention`, `quant_mode`, and optional positive `hccl_buffer_size`. Core counts must be positive. Gate placement follows the common `afd.compute_gate_on_attention` field; the connector-local copy is kept for compatibility and does not select the mode. `true` requires DeepSeek-V4 at TP=1; DBO/ubatching needs exactly two microbatches and Attention DP > 1 on the BF16 transport. Any nonzero quantization mode is rejected today. |
| `CAMAsyncAFDConnector` | `dynamicQuant`, `ffn_mega_moe`, `attn_ranks_per_dp`, and optional positive `hccl_buffer_size`. Attention microbatching uses native DBO configuration. |

The common `afd_process_group_timeout_s` defaults to 120 seconds and controls
Expand All @@ -122,7 +122,7 @@ model-routing selector. CUDA P2P supports both values; DeepEP M2N requires
backwards compatibility with older configurations and ignored for routing: the
synchronous NPU runtime selects Attention-side routing from the common field, so
deployments leave the connector-local copy unset. The supported synchronous
shape is `true` on both roles for DeepSeek-V4 at TP=1 with no DBO or ubatching.
shape is `true` on both roles for DeepSeek-V4 at TP=1; native DBO is supported on the BF16 transport with two microbatches and Attention DP > 1.
CAM async requires the common field to be `true`.

## Current connector modes
Expand Down
7 changes: 4 additions & 3 deletions docs/design/module/model_integration.md
Original file line number Diff line number Diff line change
Expand Up @@ -229,9 +229,10 @@ CUDA configurations and validation limits.
wraps the native Ascend model independently of the CUDA adapter. Attention
owns attention, mHC, normalization and finalization. Synchronous CAMP2P has two
shapes, selected by the common `compute_gate_on_attention` field: with `false`
the FFN role owns the native gate and MoE, and with `true` (DeepSeek-V4, TP=1, no
DBO or ubatching) the router and shared experts run on Attention and FFN executes
only the transported routed experts. The connector-local flag of the same name is
the FFN role owns the native gate and MoE, and with `true` (DeepSeek-V4, TP 1; DBO or
ubatching additionally requires Attention DP > 1 and exactly two microbatches on the
BF16 transport) the router and shared experts run on Attention and FFN executes only
the transported routed experts. The connector-local flag of the same name is
kept for compatibility and does not select the mode. Async CAM keeps the
Hash/top-k router and shared experts on Attention and sends only routed-expert
work; `dynamicQuant=1` is required.
Expand Down
14 changes: 10 additions & 4 deletions docs/npu/CAM_P2P_CONNECTOR_USER_GUIDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,10 @@ eager, `FULL`, or `FULL_DECODE_ONLY`. Plain MLA graph+DBO requires
`FULL_DECODE_ONLY`; `FULL` is rejected for that backend. SFA/DSA use the
generic graph path, whose accepted modes do not establish model qualification.
DBO requires Attention DP > 1, exactly two microbatches, and no speculative
decoding. The pair requires static EP, PP=PCP=DCP=1, and each
decoding. With the Attention-side gate enabled the sync connector also runs DBO:
the gate shell resolves the ubatch stage before the routing send, yields between
the send and the receive, and receives on the resolved stage. The pre-quantized
A2E transport is excluded from DBO until its yield interaction is verified. The pair requires static EP, PP=PCP=DCP=1, and each
role's ranks equal to DP x TP. Sequence parallelism follows the model
contract; synchronous DeepSeek-V2/V3 rejects effective model SP. Native MRV2
owns request slicing, capture, graph selection, replay and output merge.
Expand Down Expand Up @@ -134,7 +137,9 @@ Pass AFD configuration through vLLM's `--additional-config` option under the

Use the [common configuration reference](../configuration.md) for defaults
and aliases. CAMP2P requires `async_dp=false`; `compute_gate_on_attention` may be `true` on
both roles for DeepSeek-V4 at TP=1 without DBO or ubatching. For multi-machine placement, use
both roles for DeepSeek-V4 at TP=1. With that gate placement, native DBO needs
exactly two microbatches and Attention DP > 1 on the BF16 transport. For
multi-machine placement, use
the FFN machine's reachable IP as `host`.

### CAMP2P `connector_extra_config`
Expand Down Expand Up @@ -305,8 +310,9 @@ vllm serve /path/to/DeepSeek-V4 \
```

Drop `--enforce-eager` and add `--compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY", "cudagraph_capture_sizes": [8, 16, 32, 64, 128]}'` to run the same pair with ACL graphs.
The configuration is rejected when `compute_gate_on_attention=true` is combined with DBO,
ubatching, TP > 1, a connector other than `CAMP2pAFDConnector`, or a target other than DeepSeek-V4.
`compute_gate_on_attention=true` requires `CAMP2pAFDConnector`, DeepSeek-V4 and TP 1. With DBO
or ubatching it also requires Attention DP > 1 and exactly two microbatches on the BF16
transport; any other combination is rejected.

For model-specific launch and validation cases, see the
[E2E guide](../../tests/e2e/README.md).
Expand Down
2 changes: 1 addition & 1 deletion docs/support.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ qualify every model/topology combination on the current target.
| --- | --- | --- | --- | --- |
| CUDA V2 + `P2pNcclAFDConnector` | Eager or `FULL_DECODE_ONLY` native V2 CUDA Graph | Exactly two native microbatches; Attention DP1 or DP > 1 | Model-specific gate placement; PP/CP, elastic EP, EPLB, SP MoE, and compile SP rejected; role ranks equal DP x TP | Source-specific GPU validation is linked below; coverage is limited to the recorded model/topology combinations. DeepSeek-V2-Lite has known DBO shutdown failures. Qwen3.5/3.6 GDN DBO is unsupported. |
| CUDA V2 + `DeepEPM2NAFDConnector` | Eager or `FULL_DECODE_ONLY` native V2 CUDA Graph; SM90 or newer | Exactly two native microbatches, enabled on both roles; Attention DP1 or DP > 1, TP1 or TP2 | Qwen3 MoE (all layers MoE, no shared expert) or DeepSeek-V2; `compute_gate_on_attention=true`; FFN TP1 with EP over all FFN ranks, linear placement, `allgather_reducescatter` all2all and the Triton MoE weight layout; BF16 or block-128 FP8 payload; EPLB, PP/CP and SP MoE rejected | Historical vLLM 0.30 Qwen3-30B-A3B 2A2F GSM8K E2E for eager, graph, graph + DBO, and Attention TP2 eager/graph on H100 and H200, and for eager, graph and graph + DBO on GB200; two-node 2A2F eager, graph and graph + DBO with Attention and FFN on separate nodes. DeepSeek-V2-Lite: [E2E definitions](../tests/e2e/models/deepseek_v2_lite/test_deepseek_v2_lite.py). [GPU connector device tests](../tests/gpu/README.md) |
| Ascend V2 + `CAMP2pAFDConnector` | Eager, `FULL`, or `FULL_DECODE_ONLY` native V2 ACL Graph | Eager or graph native DBO with two ubatches and Attention DP > 1; plain MLA graph DBO requires `FULL_DECODE_ONLY` | `compute_gate_on_attention` may be `false`, or `true` for DeepSeek-V4 at TP=1 without DBO/ubatching; PP/CP, elastic EP and EPLB rejected; model-specific SP policy; role ranks equal DP x TP | Historical 0.30 V2-Lite 2A2F FULL 300-question comparison and 2A1F functional checks; current 0.32 Lite evidence is in the [validation record](https://github.com/vllm-project/afd-plugin/pull/507) |
| Ascend V2 + `CAMP2pAFDConnector` | Eager, `FULL`, or `FULL_DECODE_ONLY` native V2 ACL Graph | Eager or graph native DBO with two ubatches and Attention DP > 1; plain MLA graph DBO requires `FULL_DECODE_ONLY` | `compute_gate_on_attention` may be `false`, or `true` for DeepSeek-V4 at TP=1; gate-on-Attention DBO needs two ubatches and Attention DP > 1 on the BF16 transport; PP/CP, elastic EP and EPLB rejected; model-specific SP policy; role ranks equal DP x TP | Historical 0.30 V2-Lite 2A2F FULL 300-question comparison and 2A1F functional checks; current 0.32 Lite evidence is in the [validation record](https://github.com/vllm-project/afd-plugin/pull/507) |
| Ascend V2 + `CAMAsyncAFDConnector` | Eager Attention; layered FFN eager or FULL | Two native DBO microbatches on Attention, including DP1 and with FFN FULL; FFN stays outside the native DBO schedule | Experimental DeepSeek-V4 W4A8/W8A8 on A3/910C; `async_dp=true`, `compute_gate_on_attention=true`, and `dynamicQuant=1`; PP/CP unsupported; plugin-owned async CAM ops required | Historical DSV4 layered comparisons and limitations are in the [validation record](https://github.com/vllm-project/afd-plugin/pull/507); they predate removal of the legacy path |

The 300-question Ascend evidence in this table is historical. The 0.32
Expand Down
36 changes: 32 additions & 4 deletions tests/unit/compat/npu/test_sync_gate_feature_validation_cpu.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,17 +61,20 @@ def _afd_config(*, role: str = "attention"):
)


def _vllm_config(*, tp: int = 1, dbo: bool = False, ubatching: bool = False):
def _vllm_config(
*, tp: int = 1, dbo: bool = False, ubatching: bool = False, ubatches: int = 1
):
hf_config = SimpleNamespace(model_type=_DSV4)
return SimpleNamespace(
model_config=SimpleNamespace(
hf_config=hf_config,
hf_text_config=SimpleNamespace(model_type=_DSV4),
use_mla=False,
),
parallel_config=SimpleNamespace(
enable_dbo=dbo,
use_ubatching=ubatching,
num_ubatches=1,
num_ubatches=ubatches,
tensor_parallel_size=tp,
),
compilation_config=SimpleNamespace(cudagraph_mode=None),
Expand All @@ -91,12 +94,37 @@ def test_the_supported_shape_is_accepted_for_both_roles(extra_info, role):
assert extra_info.validated == 1


@pytest.mark.parametrize("role", ["attention", "ffn"])
def test_dbo_is_accepted_with_two_ubatches(extra_info, role):
"""Native DBO is part of the sync gate contract once it has two ubatches."""

feature_validation.fail_if_unsupported_npu_afd_features(
_vllm_config(dbo=True, ubatching=True, ubatches=2),
afd_config=_afd_config(role=role),
)

assert extra_info.validated == 1


def test_the_pre_quantized_transport_keeps_the_dbo_rejection(extra_info):
"""The MXFP8 transport stays out of DBO until its yield is verified."""

extra_info.quant_mode = 4
with pytest.raises(RuntimeError, match="does not support DBO or ubatching"):
feature_validation.fail_if_unsupported_npu_afd_features(
_vllm_config(dbo=True, ubatching=True, ubatches=2),
afd_config=_afd_config(role="attention"),
)


@pytest.mark.parametrize(
("kwargs", "match"),
[
({"dbo": True}, "DBO or ubatching"),
({"ubatching": True}, "DBO or ubatching"),
({"tp": 2}, "tensor-parallel size 1"),
(
{"dbo": True, "ubatching": True, "ubatches": 4},
"exactly two microbatches",
),
],
)
@pytest.mark.parametrize("role", ["attention", "ffn"])
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,21 @@ def recv_ffn_output(self, *, ref_tensor, ubatch_idx):
return ref_tensor * 0.5


@pytest.fixture(autouse=True)
def _stub_dbo_yield(monkeypatch):
"""Keep the protocol tests off the real DBO custom op.

The registered op dispatches through CUDA and rejects a CPU tensor before the
DBO check runs, so even a DBO-disabled run must not reach it.
"""

monkeypatch.setattr(
adapter,
"maybe_apply_dbo_yield",
lambda hidden_states, *, role: hidden_states,
)


def _fake_gate_topk(monkeypatch, events: list[tuple], topk_ids, topk_weights):
"""Replace the gate helper with one that records and returns fixed routing.

Expand Down Expand Up @@ -160,13 +175,17 @@ def forward(self, hidden_states):
return shell


def _install_metadata(monkeypatch, connector, events):
def _install_metadata(monkeypatch, connector, events, *, ubatch_idx=None):
afd_metadata = SimpleNamespace(connector=connector, stage_idx=7)
monkeypatch.setattr(
adapter,
"get_afd_metadata_from_forward_context",
lambda: afd_metadata,
)
forward_context = SimpleNamespace()
if ubatch_idx is not None:
forward_context.ubatch_idx = ubatch_idx
monkeypatch.setattr(adapter, "get_forward_context", lambda: forward_context)
return afd_metadata


Expand Down Expand Up @@ -236,3 +255,35 @@ def test_forward_refuses_a_context_without_afd_metadata(monkeypatch):

with pytest.raises(RuntimeError, match="AFD forward metadata"):
shell(torch.zeros(2, HIDDEN, dtype=torch.bfloat16))


def test_forward_resolves_the_stage_before_the_dbo_yield(monkeypatch):
"""The stage resolved before the yield is the one the receive uses."""

events: list[tuple] = []
connector = _RecordingConnector(events)
afd_metadata = _install_metadata(monkeypatch, connector, events, ubatch_idx=1)
forward_context = SimpleNamespace(ubatch_idx=1)
monkeypatch.setattr(adapter, "get_forward_context", lambda: forward_context)

def fake_yield(hidden_states, *, role):
events.append(("yield", role))
# The peer ubatch runs from here. Both stage sources move: the context the
# peer forward sees, and the shared metadata that already carries this
# forward's stage -- a receive that re-reads either must fail the test.
forward_context.ubatch_idx = 0
afd_metadata.stage_idx = 0
return hidden_states

monkeypatch.setattr(adapter, "maybe_apply_dbo_yield", fake_yield)
topk_ids = torch.zeros((2, TOP_K), dtype=torch.int32)
topk_weights = torch.ones((2, TOP_K), dtype=torch.float32)
_fake_gate_topk(monkeypatch, events, topk_ids, topk_weights)
shell = _shell(events=events, connector=connector, shared_experts=True)
hidden_states = torch.full((2, HIDDEN), 8.0, dtype=torch.bfloat16)

shell(hidden_states)

assert [event[0] for event in events] == ["gate", "send", "shared", "yield", "recv"]
assert events[1][2].metadata.stage_idx == 1
assert events[-1][1] == 1, "the receive followed the peer ubatch's stage"
Loading