diff --git a/afd_plugin/compat/npu/feature_validation.py b/afd_plugin/compat/npu/feature_validation.py index 22b7beea7..bcdf94c46 100644 --- a/afd_plugin/compat/npu/feature_validation.py +++ b/afd_plugin/compat/npu/feature_validation.py @@ -97,6 +97,12 @@ def _is_dsv4_target(vllm_config: VllmConfig) -> bool: return any("DeepseekV4" in str(architecture) for architecture in architectures) +# ``connector_extra_config.quant_mode`` of the pre-quantized A2E transport. The +# transport lands with its own change; this guard keeps DBO rejected for it until +# the ubatch yield has been verified together with quantization. +_MXFP8_QUANT_MODE = 4 + + def _fail_if_unsupported_sync_attention_gate( vllm_config: VllmConfig, afd_config: AFDConfig, @@ -104,12 +110,12 @@ def _fail_if_unsupported_sync_attention_gate( *, is_dsv4: bool, ) -> None: - """Restrict the sync Attention-side gate prototype to what it supports today. + """Restrict the sync Attention-side gate to what it supports today. - DBO and ubatching are rejected rather than merely unused: the sync shell does not - call the DBO yield, so enabling DBO here would interleave microbatches against a - transport that never yields. Not calling the yield is not the same as forbidding - the feature. + Native DBO runs through this shell: the forward resolves the ubatch stage before + the routing send, yields, and receives on the resolved stage, and the connector + keeps one AFD group per ubatch. The pre-quantized transport stays excluded until + its yield interaction is verified. """ parallel = vllm_config.parallel_config @@ -122,10 +128,17 @@ def _fail_if_unsupported_sync_attention_gate( f"TP={int(parallel.tensor_parallel_size)}", ) if bool(parallel.enable_dbo) or bool(parallel.use_ubatching): - raise RuntimeError( - "Attention-side gate with the synchronous connector does not support " - "DBO or ubatching yet", - ) + if int(parallel.num_ubatches) != 2: + raise RuntimeError( + "Attention-side gate with the synchronous connector requires " + f"exactly two microbatches; got {int(parallel.num_ubatches)}", + ) + if int(getattr(extra_info, "quant_mode", 0)) == _MXFP8_QUANT_MODE: + raise RuntimeError( + "quant_mode=4 does not support DBO or ubatching yet: the ubatch " + "yield and the pre-quantized transport have not been verified " + "together", + ) if afd_config.connector != CAMP2P_CONNECTOR: raise RuntimeError( "Attention-side gate is only supported with " diff --git a/afd_plugin/model_executor/models/npu/deepseek_v4.py b/afd_plugin/model_executor/models/npu/deepseek_v4.py index 51421102d..b82dd6da9 100644 --- a/afd_plugin/model_executor/models/npu/deepseek_v4.py +++ b/afd_plugin/model_executor/models/npu/deepseek_v4.py @@ -303,8 +303,9 @@ class AFDDeepseekV4SyncAttentionGateRemoteMoE(AFDDeepseekV4AttentionGateRemoteMo send and the receive are separate calls; - the shared experts run *between* those calls, which is the whole point of moving them here: their work overlaps the FFN side's routed work; - - no DBO yield, because the first prototype keeps the sync path free of the - scheduling interaction the async shell needs. + - native DBO is supported: the stage is resolved before the routing send, the + forward yields between the send and the receive, and the receive reuses the + resolved stage so a peer ubatch cannot switch it mid-transfer. Reusing ``RemoteFFNProxy._send_and_receive`` is not possible here: it issues the send and the receive back to back, leaving no window for the shared experts. @@ -327,10 +328,16 @@ def forward( afd_metadata = get_afd_metadata_from_forward_context() if afd_metadata is None: raise RuntimeError("Sync Attention-side gate requires AFD forward metadata") + # Resolve the stage once and reuse it for the send and the receive. Reading + # it again after the yield would let the two diverge. + stage_idx = int( + getattr(get_forward_context(), "ubatch_idx", afd_metadata.stage_idx), + ) + afd_metadata.stage_idx = stage_idx context = AFDTransferContext( metadata=AFDTransferMetadata.create_attention_metadata( layer_idx=self.layer_idx, - stage_idx=afd_metadata.stage_idx, + stage_idx=stage_idx, seq_len=int(hidden_states.shape[0]), ) ) @@ -346,9 +353,12 @@ def forward( if self.shared_experts is not None else None ) + # Yield between send and receive: the FFN side's routed work is the window + # the peer ubatch runs in. The receive must use the stage resolved above. + maybe_apply_dbo_yield(hidden_states, role="attention") output = afd_metadata.connector.recv_ffn_output( ref_tensor=hidden_states, - ubatch_idx=afd_metadata.stage_idx, + ubatch_idx=stage_idx, ) if shared_output is not None: output = output + shared_output diff --git a/docs/design/module/connector_contracts.md b/docs/design/module/connector_contracts.md index fd9cfff48..31841e3e7 100644 --- a/docs/design/module/connector_contracts.md +++ b/docs/design/module/connector_contracts.md @@ -107,7 +107,7 @@ Unknown fields fail in the selected connector parser. | --- | --- | | `P2pNcclAFDConnector` | None; the mapping must be empty. | | `DeepEPM2NAFDConnector` | `payload_format`, `expert_backend`, `capacity_tokens_per_rank`, `num_lanes`, `bucket_granularity`, `cpu_sync_min_tokens`, `num_sms`, `num_qps`, `expert_num_sms`, `cpu_timeout_s`, `gpu_timeout_s`, and a nested `network` mapping (`gin`, `hybrid`, `rdma_traffic_class`). All are optional; see the [DeepEP M2N guide](../../gpu/DEEPEP_M2N_CONNECTOR_USER_GUIDE.md#connector_extra_config) for defaults. | -| `CAMP2pAFDConnector` | `core_num`, optional `attn_core_num` / `ffn_core_num`, `compute_gate_on_attention`, `quant_mode`, and optional positive `hccl_buffer_size`. Core counts must be positive. Gate placement follows the common `afd.compute_gate_on_attention` field; the connector-local copy is kept for compatibility and does not select the mode. `true` requires DeepSeek-V4 at TP=1 without DBO or ubatching. Any nonzero quantization mode is rejected today. | +| `CAMP2pAFDConnector` | `core_num`, optional `attn_core_num` / `ffn_core_num`, `compute_gate_on_attention`, `quant_mode`, and optional positive `hccl_buffer_size`. Core counts must be positive. Gate placement follows the common `afd.compute_gate_on_attention` field; the connector-local copy is kept for compatibility and does not select the mode. `true` requires DeepSeek-V4 at TP=1; DBO/ubatching needs exactly two microbatches and Attention DP > 1 on the BF16 transport. Any nonzero quantization mode is rejected today. | | `CAMAsyncAFDConnector` | `dynamicQuant`, `ffn_mega_moe`, `attn_ranks_per_dp`, and optional positive `hccl_buffer_size`. Attention microbatching uses native DBO configuration. | The common `afd_process_group_timeout_s` defaults to 120 seconds and controls @@ -122,7 +122,7 @@ model-routing selector. CUDA P2P supports both values; DeepEP M2N requires backwards compatibility with older configurations and ignored for routing: the synchronous NPU runtime selects Attention-side routing from the common field, so deployments leave the connector-local copy unset. The supported synchronous -shape is `true` on both roles for DeepSeek-V4 at TP=1 with no DBO or ubatching. +shape is `true` on both roles for DeepSeek-V4 at TP=1; native DBO is supported on the BF16 transport with two microbatches and Attention DP > 1. CAM async requires the common field to be `true`. ## Current connector modes diff --git a/docs/design/module/model_integration.md b/docs/design/module/model_integration.md index 89ad1bce5..a1e30f6de 100644 --- a/docs/design/module/model_integration.md +++ b/docs/design/module/model_integration.md @@ -229,9 +229,10 @@ CUDA configurations and validation limits. wraps the native Ascend model independently of the CUDA adapter. Attention owns attention, mHC, normalization and finalization. Synchronous CAMP2P has two shapes, selected by the common `compute_gate_on_attention` field: with `false` -the FFN role owns the native gate and MoE, and with `true` (DeepSeek-V4, TP=1, no -DBO or ubatching) the router and shared experts run on Attention and FFN executes -only the transported routed experts. The connector-local flag of the same name is +the FFN role owns the native gate and MoE, and with `true` (DeepSeek-V4, TP 1; DBO or +ubatching additionally requires Attention DP > 1 and exactly two microbatches on the +BF16 transport) the router and shared experts run on Attention and FFN executes only +the transported routed experts. The connector-local flag of the same name is kept for compatibility and does not select the mode. Async CAM keeps the Hash/top-k router and shared experts on Attention and sends only routed-expert work; `dynamicQuant=1` is required. diff --git a/docs/npu/CAM_P2P_CONNECTOR_USER_GUIDE.md b/docs/npu/CAM_P2P_CONNECTOR_USER_GUIDE.md index 5fc067e5e..8b36f8896 100644 --- a/docs/npu/CAM_P2P_CONNECTOR_USER_GUIDE.md +++ b/docs/npu/CAM_P2P_CONNECTOR_USER_GUIDE.md @@ -60,7 +60,10 @@ eager, `FULL`, or `FULL_DECODE_ONLY`. Plain MLA graph+DBO requires `FULL_DECODE_ONLY`; `FULL` is rejected for that backend. SFA/DSA use the generic graph path, whose accepted modes do not establish model qualification. DBO requires Attention DP > 1, exactly two microbatches, and no speculative -decoding. The pair requires static EP, PP=PCP=DCP=1, and each +decoding. With the Attention-side gate enabled the sync connector also runs DBO: +the gate shell resolves the ubatch stage before the routing send, yields between +the send and the receive, and receives on the resolved stage. The pre-quantized +A2E transport is excluded from DBO until its yield interaction is verified. The pair requires static EP, PP=PCP=DCP=1, and each role's ranks equal to DP x TP. Sequence parallelism follows the model contract; synchronous DeepSeek-V2/V3 rejects effective model SP. Native MRV2 owns request slicing, capture, graph selection, replay and output merge. @@ -134,7 +137,9 @@ Pass AFD configuration through vLLM's `--additional-config` option under the Use the [common configuration reference](../configuration.md) for defaults and aliases. CAMP2P requires `async_dp=false`; `compute_gate_on_attention` may be `true` on -both roles for DeepSeek-V4 at TP=1 without DBO or ubatching. For multi-machine placement, use +both roles for DeepSeek-V4 at TP=1. With that gate placement, native DBO needs +exactly two microbatches and Attention DP > 1 on the BF16 transport. For +multi-machine placement, use the FFN machine's reachable IP as `host`. ### CAMP2P `connector_extra_config` @@ -305,8 +310,9 @@ vllm serve /path/to/DeepSeek-V4 \ ``` Drop `--enforce-eager` and add `--compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY", "cudagraph_capture_sizes": [8, 16, 32, 64, 128]}'` to run the same pair with ACL graphs. -The configuration is rejected when `compute_gate_on_attention=true` is combined with DBO, -ubatching, TP > 1, a connector other than `CAMP2pAFDConnector`, or a target other than DeepSeek-V4. +`compute_gate_on_attention=true` requires `CAMP2pAFDConnector`, DeepSeek-V4 and TP 1. With DBO +or ubatching it also requires Attention DP > 1 and exactly two microbatches on the BF16 +transport; any other combination is rejected. For model-specific launch and validation cases, see the [E2E guide](../../tests/e2e/README.md). diff --git a/docs/support.md b/docs/support.md index d45f44791..ecafd62b3 100644 --- a/docs/support.md +++ b/docs/support.md @@ -27,7 +27,7 @@ qualify every model/topology combination on the current target. | --- | --- | --- | --- | --- | | CUDA V2 + `P2pNcclAFDConnector` | Eager or `FULL_DECODE_ONLY` native V2 CUDA Graph | Exactly two native microbatches; Attention DP1 or DP > 1 | Model-specific gate placement; PP/CP, elastic EP, EPLB, SP MoE, and compile SP rejected; role ranks equal DP x TP | Source-specific GPU validation is linked below; coverage is limited to the recorded model/topology combinations. DeepSeek-V2-Lite has known DBO shutdown failures. Qwen3.5/3.6 GDN DBO is unsupported. | | CUDA V2 + `DeepEPM2NAFDConnector` | Eager or `FULL_DECODE_ONLY` native V2 CUDA Graph; SM90 or newer | Exactly two native microbatches, enabled on both roles; Attention DP1 or DP > 1, TP1 or TP2 | Qwen3 MoE (all layers MoE, no shared expert) or DeepSeek-V2; `compute_gate_on_attention=true`; FFN TP1 with EP over all FFN ranks, linear placement, `allgather_reducescatter` all2all and the Triton MoE weight layout; BF16 or block-128 FP8 payload; EPLB, PP/CP and SP MoE rejected | Historical vLLM 0.30 Qwen3-30B-A3B 2A2F GSM8K E2E for eager, graph, graph + DBO, and Attention TP2 eager/graph on H100 and H200, and for eager, graph and graph + DBO on GB200; two-node 2A2F eager, graph and graph + DBO with Attention and FFN on separate nodes. DeepSeek-V2-Lite: [E2E definitions](../tests/e2e/models/deepseek_v2_lite/test_deepseek_v2_lite.py). [GPU connector device tests](../tests/gpu/README.md) | -| Ascend V2 + `CAMP2pAFDConnector` | Eager, `FULL`, or `FULL_DECODE_ONLY` native V2 ACL Graph | Eager or graph native DBO with two ubatches and Attention DP > 1; plain MLA graph DBO requires `FULL_DECODE_ONLY` | `compute_gate_on_attention` may be `false`, or `true` for DeepSeek-V4 at TP=1 without DBO/ubatching; PP/CP, elastic EP and EPLB rejected; model-specific SP policy; role ranks equal DP x TP | Historical 0.30 V2-Lite 2A2F FULL 300-question comparison and 2A1F functional checks; current 0.32 Lite evidence is in the [validation record](https://github.com/vllm-project/afd-plugin/pull/507) | +| Ascend V2 + `CAMP2pAFDConnector` | Eager, `FULL`, or `FULL_DECODE_ONLY` native V2 ACL Graph | Eager or graph native DBO with two ubatches and Attention DP > 1; plain MLA graph DBO requires `FULL_DECODE_ONLY` | `compute_gate_on_attention` may be `false`, or `true` for DeepSeek-V4 at TP=1; gate-on-Attention DBO needs two ubatches and Attention DP > 1 on the BF16 transport; PP/CP, elastic EP and EPLB rejected; model-specific SP policy; role ranks equal DP x TP | Historical 0.30 V2-Lite 2A2F FULL 300-question comparison and 2A1F functional checks; current 0.32 Lite evidence is in the [validation record](https://github.com/vllm-project/afd-plugin/pull/507) | | Ascend V2 + `CAMAsyncAFDConnector` | Eager Attention; layered FFN eager or FULL | Two native DBO microbatches on Attention, including DP1 and with FFN FULL; FFN stays outside the native DBO schedule | Experimental DeepSeek-V4 W4A8/W8A8 on A3/910C; `async_dp=true`, `compute_gate_on_attention=true`, and `dynamicQuant=1`; PP/CP unsupported; plugin-owned async CAM ops required | Historical DSV4 layered comparisons and limitations are in the [validation record](https://github.com/vllm-project/afd-plugin/pull/507); they predate removal of the legacy path | The 300-question Ascend evidence in this table is historical. The 0.32 diff --git a/tests/unit/compat/npu/test_sync_gate_feature_validation_cpu.py b/tests/unit/compat/npu/test_sync_gate_feature_validation_cpu.py index ba89d2fd2..41670008b 100644 --- a/tests/unit/compat/npu/test_sync_gate_feature_validation_cpu.py +++ b/tests/unit/compat/npu/test_sync_gate_feature_validation_cpu.py @@ -61,17 +61,20 @@ def _afd_config(*, role: str = "attention"): ) -def _vllm_config(*, tp: int = 1, dbo: bool = False, ubatching: bool = False): +def _vllm_config( + *, tp: int = 1, dbo: bool = False, ubatching: bool = False, ubatches: int = 1 +): hf_config = SimpleNamespace(model_type=_DSV4) return SimpleNamespace( model_config=SimpleNamespace( hf_config=hf_config, hf_text_config=SimpleNamespace(model_type=_DSV4), + use_mla=False, ), parallel_config=SimpleNamespace( enable_dbo=dbo, use_ubatching=ubatching, - num_ubatches=1, + num_ubatches=ubatches, tensor_parallel_size=tp, ), compilation_config=SimpleNamespace(cudagraph_mode=None), @@ -91,12 +94,37 @@ def test_the_supported_shape_is_accepted_for_both_roles(extra_info, role): assert extra_info.validated == 1 +@pytest.mark.parametrize("role", ["attention", "ffn"]) +def test_dbo_is_accepted_with_two_ubatches(extra_info, role): + """Native DBO is part of the sync gate contract once it has two ubatches.""" + + feature_validation.fail_if_unsupported_npu_afd_features( + _vllm_config(dbo=True, ubatching=True, ubatches=2), + afd_config=_afd_config(role=role), + ) + + assert extra_info.validated == 1 + + +def test_the_pre_quantized_transport_keeps_the_dbo_rejection(extra_info): + """The MXFP8 transport stays out of DBO until its yield is verified.""" + + extra_info.quant_mode = 4 + with pytest.raises(RuntimeError, match="does not support DBO or ubatching"): + feature_validation.fail_if_unsupported_npu_afd_features( + _vllm_config(dbo=True, ubatching=True, ubatches=2), + afd_config=_afd_config(role="attention"), + ) + + @pytest.mark.parametrize( ("kwargs", "match"), [ - ({"dbo": True}, "DBO or ubatching"), - ({"ubatching": True}, "DBO or ubatching"), ({"tp": 2}, "tensor-parallel size 1"), + ( + {"dbo": True, "ubatching": True, "ubatches": 4}, + "exactly two microbatches", + ), ], ) @pytest.mark.parametrize("role", ["attention", "ffn"]) diff --git a/tests/unit/model_executor/models/test_sync_attention_gate_proxy.py b/tests/unit/model_executor/models/test_sync_attention_gate_proxy.py index e7aa6afb6..6f20dfbf6 100644 --- a/tests/unit/model_executor/models/test_sync_attention_gate_proxy.py +++ b/tests/unit/model_executor/models/test_sync_attention_gate_proxy.py @@ -97,6 +97,21 @@ def recv_ffn_output(self, *, ref_tensor, ubatch_idx): return ref_tensor * 0.5 +@pytest.fixture(autouse=True) +def _stub_dbo_yield(monkeypatch): + """Keep the protocol tests off the real DBO custom op. + + The registered op dispatches through CUDA and rejects a CPU tensor before the + DBO check runs, so even a DBO-disabled run must not reach it. + """ + + monkeypatch.setattr( + adapter, + "maybe_apply_dbo_yield", + lambda hidden_states, *, role: hidden_states, + ) + + def _fake_gate_topk(monkeypatch, events: list[tuple], topk_ids, topk_weights): """Replace the gate helper with one that records and returns fixed routing. @@ -160,13 +175,17 @@ def forward(self, hidden_states): return shell -def _install_metadata(monkeypatch, connector, events): +def _install_metadata(monkeypatch, connector, events, *, ubatch_idx=None): afd_metadata = SimpleNamespace(connector=connector, stage_idx=7) monkeypatch.setattr( adapter, "get_afd_metadata_from_forward_context", lambda: afd_metadata, ) + forward_context = SimpleNamespace() + if ubatch_idx is not None: + forward_context.ubatch_idx = ubatch_idx + monkeypatch.setattr(adapter, "get_forward_context", lambda: forward_context) return afd_metadata @@ -236,3 +255,35 @@ def test_forward_refuses_a_context_without_afd_metadata(monkeypatch): with pytest.raises(RuntimeError, match="AFD forward metadata"): shell(torch.zeros(2, HIDDEN, dtype=torch.bfloat16)) + + +def test_forward_resolves_the_stage_before_the_dbo_yield(monkeypatch): + """The stage resolved before the yield is the one the receive uses.""" + + events: list[tuple] = [] + connector = _RecordingConnector(events) + afd_metadata = _install_metadata(monkeypatch, connector, events, ubatch_idx=1) + forward_context = SimpleNamespace(ubatch_idx=1) + monkeypatch.setattr(adapter, "get_forward_context", lambda: forward_context) + + def fake_yield(hidden_states, *, role): + events.append(("yield", role)) + # The peer ubatch runs from here. Both stage sources move: the context the + # peer forward sees, and the shared metadata that already carries this + # forward's stage -- a receive that re-reads either must fail the test. + forward_context.ubatch_idx = 0 + afd_metadata.stage_idx = 0 + return hidden_states + + monkeypatch.setattr(adapter, "maybe_apply_dbo_yield", fake_yield) + topk_ids = torch.zeros((2, TOP_K), dtype=torch.int32) + topk_weights = torch.ones((2, TOP_K), dtype=torch.float32) + _fake_gate_topk(monkeypatch, events, topk_ids, topk_weights) + shell = _shell(events=events, connector=connector, shared_experts=True) + hidden_states = torch.full((2, HIDDEN), 8.0, dtype=torch.bfloat16) + + shell(hidden_states) + + assert [event[0] for event in events] == ["gate", "send", "shared", "yield", "recv"] + assert events[1][2].metadata.stage_idx == 1 + assert events[-1][1] == 1, "the receive followed the peer ubatch's stage"