From 2ea63b22b74feb6e5a748d09780fa075d5e644ed Mon Sep 17 00:00:00 2001 From: vensen Date: Thu, 3 Sep 2026 10:31:03 +0000 Subject: [PATCH 1/3] bench(rocm): add PR230 attention microprobe matrix Signed-off-by: vensen --- .github/workflows/ci.yml | 1 + .../benchmark_rocm_attention_ablation.py | 1122 +++++++++++++++++ .../test_rocm_attention_ablation_benchmark.py | 299 +++++ 3 files changed, 1422 insertions(+) create mode 100644 benchmarks/benchmark_rocm_attention_ablation.py create mode 100644 tests/test_rocm_attention_ablation_benchmark.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d3e5f6e2..5d8e9c9e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -89,6 +89,7 @@ jobs: tests/test_attention_cross_config_binding.py \ tests/test_attention_preprocess.py \ tests/test_attention_projection.py \ + tests/test_rocm_attention_ablation_benchmark.py \ tests/test_cp_attention.py \ tests/test_cp_attention_transformer_engine.py diff --git a/benchmarks/benchmark_rocm_attention_ablation.py b/benchmarks/benchmark_rocm_attention_ablation.py new file mode 100644 index 00000000..334ba55f --- /dev/null +++ b/benchmarks/benchmark_rocm_attention_ablation.py @@ -0,0 +1,1122 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""Execute the PR230 Attention row taxonomy as ROCm operator micro-probes. + +This does not claim PR230's frozen model/rollout replay: there is no checkpoint, +token stream, selected-token logprob, KL, or serving engine in this benchmark. +Most rows use the shared native HIP deterministic Attention reference core; A6 +and A7 use eager PyTorch-on-ROCm probes to isolate precision and merge order. +Every row records both implementations and the result scope explicitly. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import platform +import subprocess +import sys +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Callable, Mapping + +import torch +import torch.nn.functional as F + +from rl_engine.alignment.cross_config.attention_binding import ( + BindingErrorCode, + bind_attention_contracts, +) +from rl_engine.kernels.attention_contract import ( + STRICT_ATTENTION_REFERENCE_CORE_ID, + STRICT_ATTENTION_SCHEDULE_ID, + AttentionContract, + AttentionDType, + AttentionMode, + AttentionRole, + ReductionSpec, + ShardingSpec, + SplitKVSpec, + build_split_kv_runtime_plan_set, +) +from rl_engine.kernels.ops.pytorch.attention.debug_matrix import attention_debug_matrix + +QWEN3_Q_HEADS = 32 +QWEN3_KV_HEADS = 8 +QWEN3_HEAD_DIM = 128 +DEFAULT_SHAPES = ((1, 16), (1, 32), (1, 64), (1, 128), (2, 16), (2, 32), (2, 64), (2, 128)) +METRIC_NAMES = ("out", "lse", "dq", "dk", "dv") +ROCM_REFERENCE_BACKEND_ID = "rlkernel.rocm.deterministic_attention" +REPO_ROOT = Path(__file__).resolve().parents[1] +RESULT_SCHEMA = "rlkernel.rocm.attention_ablation_microprobe.v1" +RESULT_SCOPE = { + "kind": "operator_microprobe", + "pr230_row_taxonomy": True, + "frozen_rollout_replay": False, + "model_or_serving_execution": False, + "covered_metrics": ["out_max_abs", "lse_max_abs", "dq_max_abs", "dk_max_abs", "dv_max_abs"], + "excluded_metrics": [ + "train_rollout_logprob_abs_diff", + "mismatch_kl", + "mismatch_k3_kl", + ], +} +ROW_IMPLEMENTATIONS = { + "A0": (ROCM_REFERENCE_BACKEND_ID, ROCM_REFERENCE_BACKEND_ID), + "A1": ( + "rlkernel.rocm.deterministic_rope+native_attention", + "rlkernel.rocm.deterministic_rope+native_attention", + ), + "A2": ("torch.rocm.rms_norm+native_attention", ROCM_REFERENCE_BACKEND_ID), + "A3": (ROCM_REFERENCE_BACKEND_ID, ROCM_REFERENCE_BACKEND_ID), + "A5": (ROCM_REFERENCE_BACKEND_ID, "torch.rocm.index_select+native_attention"), + "A6": ( + "torch.rocm.explicit_fp32_serial_qk_accumulator", + "torch.rocm.explicit_bf16_serial_qk_accumulator", + ), + "A7": ( + "torch.rocm.fp32_chunk_merge_ascending", + "torch.rocm.fp32_chunk_merge_descending", + ), + "C0": (ROCM_REFERENCE_BACKEND_ID, "rlkernel.rocm.native_attention_per_tp2_head_shard"), + "C1": (ROCM_REFERENCE_BACKEND_ID, "rlkernel.rocm.native_attention_per_batch_row"), + "C2": ("rlkernel.rocm.native_attention_full_prefill", "rlkernel.rocm.native_attention_tail"), +} +ROW_REALIZATIONS = { + "A0": "Repeat the identical native HIP reference-core call.", + "A1": "Increment suffix RoPE positions while preserving Q/K/V tensors.", + "A2": "Apply or bypass unit-weight PyTorch RMSNorm before the native HIP core.", + "A3": "Toggle causal masking in the native HIP core.", + "A4": "Bind valid TP-rank-1 rollout and TP-rank-0 training contracts; do not run numerics.", + "A5": "Reverse four dense K/V tensor pages; this is not a paged-cache runtime.", + "A6": "Use identical FP32 products/order with explicit FP32 versus BF16 accumulator state.", + "A7": "Merge four dense chunks in opposite orders on one GPU; this is not a CP collective.", + "C0": "Compare full GQA with two contiguous TP=2 head shards on one GPU.", + "C1": "Compare a batch call with per-row calls to the same native HIP core.", + "C2": ( + "Compare a full-prefill tail with one trailing query over dense KV, without a serving " + "cache." + ), +} + + +@dataclass(frozen=True) +class Snapshot: + out: torch.Tensor + lse: torch.Tensor + dq: torch.Tensor + dk: torch.Tensor + dv: torch.Tensor + + +AttentionCall = Callable[ + [torch.Tensor, torch.Tensor, torch.Tensor], tuple[torch.Tensor, torch.Tensor] +] + + +def _seeded_tensor(shape: tuple[int, ...], *, device: torch.device, seed: int) -> torch.Tensor: + generator = torch.Generator(device=device).manual_seed(seed) + return torch.randn(shape, device=device, dtype=torch.bfloat16, generator=generator) + + +def _evaluate( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + dout: torch.Tensor, + call: AttentionCall, +) -> Snapshot: + qr = q.detach().clone().requires_grad_(True) + kr = k.detach().clone().requires_grad_(True) + vr = v.detach().clone().requires_grad_(True) + out, lse = call(qr, kr, vr) + if out.shape != dout.shape: + raise RuntimeError(f"upstream gradient shape {dout.shape} does not match {out.shape}") + dq, dk, dv = torch.autograd.grad( + out, + (qr, kr, vr), + grad_outputs=dout, + allow_unused=False, + ) + return Snapshot( + out.detach(), + lse.detach(), + dq.detach(), + dk.detach(), + dv.detach(), + ) + + +def _metric(left: torch.Tensor, right: torch.Tensor) -> dict[str, Any]: + if left.shape != right.shape: + raise ValueError(f"metric tensors differ in shape: {left.shape} != {right.shape}") + difference = (left.float() - right.float()).abs() + same_dtype = left.dtype == right.dtype + if same_dtype: + left_bytes = left.contiguous().view(torch.uint8).reshape(left.numel(), left.element_size()) + right_bytes = ( + right.contiguous().view(torch.uint8).reshape(right.numel(), right.element_size()) + ) + mismatch_count = int(torch.any(left_bytes != right_bytes, dim=1).sum().item()) + else: + mismatch_count = left.numel() + return { + "max_abs": 0.0 if difference.numel() == 0 else float(difference.max().item()), + "mismatch_count": mismatch_count, + "element_count": int(left.numel()), + "bitwise_equal": mismatch_count == 0, + "left_dtype": str(left.dtype), + "right_dtype": str(right.dtype), + "shape": list(left.shape), + } + + +def _compare(left: Snapshot, right: Snapshot) -> dict[str, dict[str, Any]]: + return {name: _metric(getattr(left, name), getattr(right, name)) for name in METRIC_NAMES} + + +def _native_call(operator: Any, *, causal: bool = True) -> AttentionCall: + def call(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor): + key_positions = torch.arange(k.size(2), device=k.device, dtype=torch.int64).repeat( + k.size(0), 1 + ) + result = operator.forward_with_lse( + q, + k, + v, + causal=causal, + scale=1.0 / math.sqrt(q.size(-1)), + query_position_ids=key_positions[:, -q.size(2) :], + key_position_ids=key_positions, + ) + return result.out, result.lse + + return call + + +def _rope_batch(operator: Any, tensor: torch.Tensor, positions: torch.Tensor) -> torch.Tensor: + return torch.cat( + [operator(tensor[index : index + 1], positions[index]) for index in range(tensor.size(0))] + ) + + +def _rope_call(attention: Any, rope: Any, positions: torch.Tensor) -> AttentionCall: + native = _native_call(attention) + + def call(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor): + return native(_rope_batch(rope, q, positions), _rope_batch(rope, k, positions), v) + + return call + + +def _qk_norm_call(attention: Any, weight: torch.Tensor, *, enabled: bool) -> AttentionCall: + native = _native_call(attention) + + def call(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor): + if enabled: + q = F.rms_norm(q, (q.size(-1),), weight, 1.0e-6) + k = F.rms_norm(k, (k.size(-1),), weight, 1.0e-6) + return native(q, k, v) + + return call + + +def _kv_page_call(attention: Any, permutation: torch.Tensor | None) -> AttentionCall: + native = _native_call(attention) + + def call(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor): + if permutation is not None: + k = k.index_select(2, permutation) + v = v.index_select(2, permutation) + return native(q, k, v) + + return call + + +def _dense_attention(*, accumulator_dtype: torch.dtype) -> AttentionCall: + if accumulator_dtype not in {torch.float32, torch.bfloat16}: + raise ValueError("accumulator_dtype must be torch.float32 or torch.bfloat16") + + def call(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor): + group = q.size(1) // k.size(1) + expanded_k = k.repeat_interleave(group, dim=1) + expanded_v = v.repeat_interleave(group, dim=1) + scores = torch.zeros( + (*q.shape[:-1], expanded_k.size(2)), + device=q.device, + dtype=accumulator_dtype, + ) + for index in range(q.size(-1)): + product = q[..., index].float().unsqueeze(-1) * expanded_k[ + ..., index + ].float().unsqueeze(-2) + scores = scores.float() + product + if accumulator_dtype is torch.bfloat16: + scores = scores.to(torch.bfloat16) + scores = scores.float() + scores = scores * (1.0 / math.sqrt(q.size(-1))) + q_index = torch.arange(q.size(2), device=q.device).unsqueeze(1) + k_index = torch.arange(k.size(2), device=q.device).unsqueeze(0) + scores = scores.masked_fill(k_index > q_index, float("-inf")) + lse = torch.logsumexp(scores, dim=-1) + out = torch.matmul(torch.softmax(scores, dim=-1), expanded_v.float()).to(q.dtype) + return out, lse + + return call + + +def _chunked_attention(order: str) -> AttentionCall: + if order not in {"ascending", "descending"}: + raise ValueError("chunk merge order must be ascending or descending") + + def call(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor): + group = q.size(1) // k.size(1) + expanded_k = k.repeat_interleave(group, dim=1) + expanded_v = v.repeat_interleave(group, dim=1).float() + sequence = k.size(2) + chunk_size = sequence // 4 + chunks: list[tuple[torch.Tensor, torch.Tensor, torch.Tensor]] = [] + q_positions = torch.arange(q.size(2), device=q.device).view(1, 1, -1, 1) + for start in range(0, sequence, chunk_size): + stop = min(start + chunk_size, sequence) + scores = torch.matmul(q.float(), expanded_k[:, :, start:stop].float().transpose(-1, -2)) + scores = scores * (1.0 / math.sqrt(q.size(-1))) + key_positions = torch.arange(start, stop, device=q.device).view(1, 1, 1, -1) + scores = scores.masked_fill(key_positions > q_positions, float("-inf")) + row_max = scores.max(dim=-1).values + valid = torch.isfinite(row_max) + safe_max = torch.where(valid, row_max, torch.zeros_like(row_max)) + weights = torch.exp(scores - safe_max.unsqueeze(-1)) + weights = torch.where(valid.unsqueeze(-1), weights, torch.zeros_like(weights)) + denominator = weights.sum(dim=-1) + numerator = torch.matmul(weights, expanded_v[:, :, start:stop]) + chunks.append((safe_max, denominator, numerator)) + + indices = range(len(chunks)) if order == "ascending" else reversed(range(len(chunks))) + merged_max: torch.Tensor | None = None + merged_denominator: torch.Tensor | None = None + merged_numerator: torch.Tensor | None = None + for index in indices: + row_max, denominator, numerator = chunks[index] + if merged_max is None: + merged_max, merged_denominator, merged_numerator = ( + row_max, + denominator, + numerator, + ) + continue + assert merged_denominator is not None and merged_numerator is not None + left_valid = merged_denominator > 0 + right_valid = denominator > 0 + new_max = torch.maximum(merged_max, row_max) + left_scale = torch.where( + left_valid, + torch.exp(merged_max - new_max), + torch.zeros_like(new_max), + ) + right_scale = torch.where( + right_valid, + torch.exp(row_max - new_max), + torch.zeros_like(new_max), + ) + merged_denominator = merged_denominator * left_scale + denominator * right_scale + merged_numerator = merged_numerator * left_scale.unsqueeze( + -1 + ) + numerator * right_scale.unsqueeze(-1) + merged_max = new_max + assert merged_max is not None + assert merged_denominator is not None and merged_numerator is not None + out = (merged_numerator / merged_denominator.unsqueeze(-1)).to(q.dtype) + lse = torch.log(merged_denominator) + merged_max + return out, lse + + return call + + +def _per_tp_partition_call(attention: Any) -> AttentionCall: + native = _native_call(attention) + + def call(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor): + tp_world_size = 2 + q_heads_per_rank = q.size(1) // tp_world_size + kv_heads_per_rank = k.size(1) // tp_world_size + outputs, lses = [], [] + for rank in range(tp_world_size): + result = native( + q[:, rank * q_heads_per_rank : (rank + 1) * q_heads_per_rank], + k[:, rank * kv_heads_per_rank : (rank + 1) * kv_heads_per_rank], + v[:, rank * kv_heads_per_rank : (rank + 1) * kv_heads_per_rank], + ) + outputs.append(result[0]) + lses.append(result[1]) + return torch.cat(outputs, dim=1), torch.cat(lses, dim=1) + + return call + + +def _per_batch_row_call(attention: Any) -> AttentionCall: + native = _native_call(attention) + + def call(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor): + rows = [native(q[i : i + 1], k[i : i + 1], v[i : i + 1]) for i in range(q.size(0))] + return torch.cat([row[0] for row in rows]), torch.cat([row[1] for row in rows]) + + return call + + +def _tail_call(attention: Any, *, full_query: bool) -> AttentionCall: + native = _native_call(attention) + + def call(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor): + if full_query: + out, lse = native(q, k, v) + return out[:, :, -1:], lse[:, :, -1:] + return native(q[:, :, -1:], k, v) + + return call + + +def _page_permutation(sequence: int, device: torch.device) -> torch.Tensor: + page_size = max(1, sequence // 4) + pages = [ + torch.arange(start, min(start + page_size, sequence), device=device) + for start in range(0, sequence, page_size) + ] + return torch.cat(list(reversed(pages))).to(torch.long) + + +def _topology_contract( + *, role: AttentionRole, batch: int, sequence: int, tp_rank: int +) -> AttentionContract: + tp_world_size = 2 + local_q_heads = QWEN3_Q_HEADS // tp_world_size + local_kv_heads = QWEN3_KV_HEADS // tp_world_size + return AttentionContract( + role=role, + mode=AttentionMode.PREFILL, + dtype=AttentionDType.BF16, + batch_size=batch, + query_sequence_length=sequence, + head_dim=QWEN3_HEAD_DIM, + causal=True, + causal_offsets=(0,) * batch, + sharding=ShardingSpec( + tp_rank=tp_rank, + tp_world_size=tp_world_size, + cp_rank=0, + cp_world_size=1, + global_q_heads=QWEN3_Q_HEADS, + global_kv_heads=QWEN3_KV_HEADS, + local_q_head_start=tp_rank * local_q_heads, + local_q_heads=local_q_heads, + local_kv_head_start=tp_rank * local_kv_heads, + local_kv_heads=local_kv_heads, + global_sequence_length=sequence, + local_sequence_length=sequence, + global_block_indices=(0,), + global_block_token_starts=(0,), + local_block_offsets=(0, sequence), + ), + reduction=ReductionSpec(), + split_kv=SplitKVSpec.disabled(), + ) + + +def _topology_gate(*, batch: int, sequence: int) -> dict[str, Any]: + """Exercise the production contract gate with different TP owners.""" + + rollout = _topology_contract( + role=AttentionRole.INFER, + batch=batch, + sequence=sequence, + tp_rank=1, + ) + training = _topology_contract( + role=AttentionRole.TRAIN, + batch=batch, + sequence=sequence, + tp_rank=0, + ) + plan_set = build_split_kv_runtime_plan_set( + (sequence,) * batch, + tp_world_size=2, + cp_world_size=1, + split_kv=SplitKVSpec.disabled(), + backend=ROCM_REFERENCE_BACKEND_ID, + ) + result = bind_attention_contracts( + rollout_contract=rollout, + training_contract=training, + rollout_identity={}, + training_identity={}, + rollout_backend_id=ROCM_REFERENCE_BACKEND_ID, + training_backend_id=ROCM_REFERENCE_BACKEND_ID, + rollout_split_kv_plan_set=plan_set, + training_split_kv_plan_set=plan_set, + require_full_identity=False, + ) + topology_issues = result.issues_by_code(BindingErrorCode.TOPOLOGY_MISMATCH) + unexpected_issues = tuple( + issue for issue in result.issues if issue.code is not BindingErrorCode.TOPOLOGY_MISMATCH + ) + if result.comparable or result.passed or not topology_issues or unexpected_issues: + raise RuntimeError("A4 did not isolate the topology comparability gate") + return result.to_dict() + + +def _case( + *, + batch: int, + sequence: int, + seed: int, + device: torch.device, + attention: Any, + rope: Any, +) -> list[dict[str, Any]]: + q = _seeded_tensor((batch, QWEN3_Q_HEADS, sequence, QWEN3_HEAD_DIM), device=device, seed=seed) + k = _seeded_tensor( + (batch, QWEN3_KV_HEADS, sequence, QWEN3_HEAD_DIM), device=device, seed=seed + 1 + ) + v = _seeded_tensor(k.shape, device=device, seed=seed + 2) + dout = _seeded_tensor(q.shape, device=device, seed=seed + 3) + native = _native_call(attention) + + positions = torch.arange(sequence, device=device, dtype=torch.int64).repeat(batch, 1) + changed_positions = positions.clone() + changed_positions[:, sequence // 2 :] += 1 + norm_weight = torch.ones(QWEN3_HEAD_DIM, device=device, dtype=torch.bfloat16) + permutation = _page_permutation(sequence, device) + + calls: dict[str, tuple[AttentionCall, AttentionCall, torch.Tensor]] = { + "A0": (native, native, dout), + "A1": ( + _rope_call(attention, rope, positions), + _rope_call(attention, rope, changed_positions), + dout, + ), + "A2": ( + _qk_norm_call(attention, norm_weight, enabled=True), + _qk_norm_call(attention, norm_weight, enabled=False), + dout, + ), + "A3": (native, _native_call(attention, causal=False), dout), + "A5": (native, _kv_page_call(attention, permutation), dout), + "A6": ( + _dense_attention(accumulator_dtype=torch.float32), + _dense_attention(accumulator_dtype=torch.bfloat16), + dout, + ), + "A7": (_chunked_attention("ascending"), _chunked_attention("descending"), dout), + "C0": (native, _per_tp_partition_call(attention), dout), + "C1": (native, _per_batch_row_call(attention), dout), + "C2": ( + _tail_call(attention, full_query=True), + _tail_call(attention, full_query=False), + dout[:, :, -1:].contiguous(), + ), + } + rows: list[dict[str, Any]] = [] + for matrix_row in attention_debug_matrix()["rows"]: + row_id = matrix_row["id"] + if row_id == "A4": + binding = _topology_gate(batch=batch, sequence=sequence) + rows.append( + { + "row_id": row_id, + "batch": batch, + "sequence": sequence, + "category": matrix_row["category"], + "probe": matrix_row["probe"], + "expected": matrix_row["expected"], + "comparable": binding["comparable"], + "passed": True, + "outcome": "rejected", + "realization": ROW_REALIZATIONS[row_id], + "gate_implementation": ( + "rl_engine.alignment.cross_config.bind_attention_contracts" + ), + "identity_errors": [issue["field"] for issue in binding["issues"]], + "binding_gate": binding, + "metrics": {name: None for name in METRIC_NAMES}, + } + ) + continue + baseline_call, candidate_call, row_dout = calls[row_id] + baseline = _evaluate(q, k, v, row_dout, baseline_call) + candidate = _evaluate(q, k, v, row_dout, candidate_call) + metrics = _compare(baseline, candidate) + mismatch_count = sum(metric["mismatch_count"] for metric in metrics.values()) + expected = matrix_row["expected"] + passed = ( + mismatch_count == 0 if expected in {"baseline", "exact_zero"} else mismatch_count > 0 + ) + baseline_implementation, candidate_implementation = ROW_IMPLEMENTATIONS[row_id] + rows.append( + { + "row_id": row_id, + "batch": batch, + "sequence": sequence, + "category": matrix_row["category"], + "probe": matrix_row["probe"], + "expected": expected, + "comparable": True, + "passed": passed, + "outcome": "matched" if mismatch_count == 0 else "drift_detected", + "realization": ROW_REALIZATIONS[row_id], + "implementations": { + "baseline": baseline_implementation, + "candidate": candidate_implementation, + }, + "metrics": metrics, + } + ) + return rows + + +def _environment(device: torch.device, attention: Any) -> dict[str, Any]: + properties = torch.cuda.get_device_properties(device) + return { + "python": platform.python_version(), + "pytorch": torch.__version__, + "hip_runtime": torch.version.hip, + "device_index": device.index, + "device_name": properties.name, + "architecture": getattr(properties, "gcnArchName", "unknown"), + "gpu_count": torch.cuda.device_count(), + "primary_backend_id": attention.backend_id, + "primary_core_id": attention.core_id, + "primary_schedule": attention.strict_schedule, + "primary_reference_only": attention.reference_only, + "primary_production_ready": attention.production_ready, + "execution_kind": "operator_only_rocm_reference", + } + + +def _git(*args: str) -> subprocess.CompletedProcess[bytes]: + return subprocess.run( + ["git", *args], + cwd=REPO_ROOT, + check=False, + capture_output=True, + ) + + +def _source_provenance() -> dict[str, Any]: + script_path = Path(__file__).resolve() + script_relative = script_path.relative_to(REPO_ROOT).as_posix() + revision_result = _git("rev-parse", "HEAD") + head_source = _git("show", f"HEAD:{script_relative}") + diff = _git("diff", "--binary", "HEAD") + if revision_result.returncode != 0 or diff.returncode != 0: + raise RuntimeError("unable to record RL-Kernel git provenance") + source = script_path.read_bytes() + tracked_diff = diff.stdout + return { + "revision": revision_result.stdout.decode().strip(), + "tracked_dirty": bool(tracked_diff), + "tracked_diff_sha256": (hashlib.sha256(tracked_diff).hexdigest() if tracked_diff else None), + "script_path": script_relative, + "script_sha256": hashlib.sha256(source).hexdigest(), + "script_matches_head": head_source.returncode == 0 and head_source.stdout == source, + } + + +def _aggregate(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + aggregate: list[dict[str, Any]] = [] + for matrix_row in attention_debug_matrix()["rows"]: + selected = [row for row in rows if row["row_id"] == matrix_row["id"]] + metrics: dict[str, Any] = {} + for name in METRIC_NAMES: + values = [row["metrics"][name] for row in selected if row["metrics"][name] is not None] + metrics[name] = ( + None + if not values + else { + "worst_max_abs": max(value["max_abs"] for value in values), + "total_mismatch_count": sum(value["mismatch_count"] for value in values), + "all_bitwise_equal": all(value["bitwise_equal"] for value in values), + } + ) + aggregate.append( + { + **matrix_row, + "case_count": len(selected), + "comparable": all(row["comparable"] for row in selected), + "passed": all(row["passed"] for row in selected), + "metrics": metrics, + } + ) + return aggregate + + +def _valid_digest(value: Any, *, lengths: tuple[int, ...]) -> bool: + return ( + isinstance(value, str) + and len(value) in lengths + and all(character in "0123456789abcdef" for character in value.lower()) + ) + + +def _validate_metric( + row_id: str, + name: str, + metric: Any, + *, + expected_dtype: str, + expected_shape: list[int], +) -> None: + if not isinstance(metric, Mapping) or set(metric) != { + "max_abs", + "mismatch_count", + "element_count", + "bitwise_equal", + "left_dtype", + "right_dtype", + "shape", + }: + raise ValueError(f"{row_id}.{name} has an invalid metric schema") + maximum = metric["max_abs"] + mismatches = metric["mismatch_count"] + elements = metric["element_count"] + bitwise_equal = metric["bitwise_equal"] + shape = metric["shape"] + if ( + isinstance(maximum, bool) + or not isinstance(maximum, (int, float)) + or not math.isfinite(maximum) + or maximum < 0 + ): + raise ValueError(f"{row_id}.{name}.max_abs must be finite and non-negative") + if ( + isinstance(mismatches, bool) + or not isinstance(mismatches, int) + or isinstance(elements, bool) + or not isinstance(elements, int) + or elements < 1 + or mismatches < 0 + or mismatches > elements + ): + raise ValueError(f"{row_id}.{name} has invalid element or mismatch counts") + if not isinstance(bitwise_equal, bool) or bitwise_equal != (mismatches == 0): + raise ValueError(f"{row_id}.{name} has inconsistent bitwise evidence") + if mismatches == 0 and maximum != 0: + raise ValueError(f"{row_id}.{name} has inconsistent numerical evidence") + if ( + metric["left_dtype"] != metric["right_dtype"] + or metric["left_dtype"] != expected_dtype + or shape != expected_shape + or elements != math.prod(expected_shape) + ): + raise ValueError(f"{row_id}.{name} has incompatible dtype or shape evidence") + + +def validate_payload(payload: Mapping[str, Any]) -> None: + if payload.get("schema_version") != RESULT_SCHEMA: + raise ValueError("unsupported ROCm Attention ablation result schema") + if payload.get("scope") != RESULT_SCOPE: + raise ValueError("result scope must identify an operator micro-probe, not a model replay") + manifest = attention_debug_matrix() + if payload.get("matrix_manifest") != manifest: + raise ValueError("result does not embed the exact PR230 matrix manifest") + + environment = payload.get("environment", {}) + if not isinstance(environment, Mapping) or ( + not environment.get("hip_runtime") + or environment.get("execution_kind") != "operator_only_rocm_reference" + or environment.get("primary_backend_id") != ROCM_REFERENCE_BACKEND_ID + or environment.get("primary_core_id") != STRICT_ATTENTION_REFERENCE_CORE_ID + or environment.get("primary_schedule") != STRICT_ATTENTION_SCHEDULE_ID + or environment.get("primary_reference_only") is not True + or environment.get("primary_production_ready") is not False + or "gfx942" not in environment.get("architecture", "") + ): + raise ValueError("result does not prove gfx942 ROCm reference execution") + for name in ("python", "pytorch", "hip_runtime", "device_name", "architecture"): + if not isinstance(environment.get(name), str) or not environment[name].strip(): + raise ValueError(f"environment.{name} must be a non-empty runtime readback") + hip_parts = environment["hip_runtime"].split(".") + if ( + len(hip_parts) < 2 + or not hip_parts[0].isdigit() + or not hip_parts[1].isdigit() + or "rocm" not in environment["pytorch"].lower() + or "amd" not in environment["device_name"].lower() + or "mi300x" not in environment["device_name"].lower() + ): + raise ValueError("environment does not identify an AMD MI300X ROCm runtime") + device_index = environment.get("device_index") + gpu_count = environment.get("gpu_count") + if ( + isinstance(device_index, bool) + or not isinstance(device_index, int) + or isinstance(gpu_count, bool) + or not isinstance(gpu_count, int) + or device_index < 0 + or gpu_count < 1 + or device_index >= gpu_count + ): + raise ValueError("environment GPU selection is invalid") + + command = payload.get("command") + if ( + not isinstance(command, list) + or len(command) < 2 + or any(not isinstance(argument, str) for argument in command) + or not command[1].endswith("benchmark_rocm_attention_ablation.py") + ): + raise ValueError("result does not record the benchmark command") + + provenance = payload.get("source_provenance", {}) + if not isinstance(provenance, Mapping) or ( + not _valid_digest(provenance.get("revision"), lengths=(40, 64)) + or provenance.get("tracked_dirty") is not False + or provenance.get("tracked_diff_sha256") is not None + or provenance.get("script_path") != "benchmarks/benchmark_rocm_attention_ablation.py" + or not _valid_digest(provenance.get("script_sha256"), lengths=(64,)) + or provenance.get("script_matches_head") is not True + ): + raise ValueError("result is not pinned to a clean committed runner") + + configuration = payload.get("configuration", {}) + raw_shapes = configuration.get("shapes") if isinstance(configuration, Mapping) else None + if not isinstance(raw_shapes, list) or any( + not isinstance(shape, (list, tuple)) or len(shape) != 2 for shape in raw_shapes + ): + raise ValueError("configuration.shapes must contain BxS pairs") + try: + shapes = tuple((shape[0], shape[1]) for shape in raw_shapes) + except (IndexError, TypeError): + raise ValueError("configuration.shapes must contain BxS pairs") from None + if ( + not shapes + or len(set(shapes)) != len(shapes) + or any( + isinstance(batch, bool) + or isinstance(sequence, bool) + or not isinstance(batch, int) + or not isinstance(sequence, int) + or batch < 1 + or sequence < 4 + or sequence % 4 + for batch, sequence in shapes + ) + ): + raise ValueError("configuration.shapes contains an invalid or duplicate shape") + if shapes != DEFAULT_SHAPES: + raise ValueError("publication results must cover the exact eight-shape ROCm sweep") + expected_configuration = { + "dtype": "bfloat16", + "q_heads": QWEN3_Q_HEADS, + "kv_heads": QWEN3_KV_HEADS, + "head_dim": QWEN3_HEAD_DIM, + } + if any(configuration.get(name) != value for name, value in expected_configuration.items()): + raise ValueError("result does not use the pinned Qwen3 BF16 Attention configuration") + seed = configuration.get("seed") + if isinstance(seed, bool) or not isinstance(seed, int) or seed < 0: + raise ValueError("configuration.seed must be a non-negative integer") + + expected_rows = [row["id"] for row in manifest["rows"]] + cases = payload.get("cases", []) + expected_case_keys = [ + (row_id, batch, sequence) for batch, sequence in shapes for row_id in expected_rows + ] + if ( + not isinstance(cases, list) + or [ + (case.get("row_id"), case.get("batch"), case.get("sequence")) + for case in cases + if isinstance(case, Mapping) + ] + != expected_case_keys + ): + raise ValueError("cases do not cover every PR230 row and configured shape exactly once") + + manifest_by_id = {row["id"]: row for row in manifest["rows"]} + for case in cases: + row_id = case["row_id"] + row_manifest = manifest_by_id[row_id] + if any(case.get(name) != row_manifest[name] for name in ("category", "probe", "expected")): + raise ValueError(f"{row_id} case metadata differs from the PR230 manifest") + if case.get("realization") != ROW_REALIZATIONS[row_id]: + raise ValueError(f"{row_id} does not disclose its operator-level realization") + if case.get("passed") is not True: + raise ValueError(f"{row_id} case did not satisfy its expected outcome") + metrics = case.get("metrics") + if not isinstance(metrics, Mapping) or set(metrics) != set(METRIC_NAMES): + raise ValueError(f"{row_id} has an invalid metric set") + if row_id == "A4": + binding = case.get("binding_gate", {}) + expected_binding = _topology_gate(batch=case["batch"], sequence=case["sequence"]) + issues = binding.get("issues", []) if isinstance(binding, Mapping) else [] + issue_evidence = [ + ( + issue.get("code"), + issue.get("tier"), + issue.get("field"), + issue.get("rollout"), + issue.get("training"), + ) + for issue in issues + if isinstance(issue, Mapping) + ] + expected_issue_evidence = [ + ("TOPOLOGY_MISMATCH", "identical", "sharding.tp_rank", 1, 0), + ( + "TOPOLOGY_MISMATCH", + "identical", + "sharding.local_q_head_start", + QWEN3_Q_HEADS // 2, + 0, + ), + ( + "TOPOLOGY_MISMATCH", + "identical", + "sharding.local_kv_head_start", + QWEN3_KV_HEADS // 2, + 0, + ), + ] + if ( + case.get("comparable") is not False + or case.get("outcome") != "rejected" + or case.get("gate_implementation") + != "rl_engine.alignment.cross_config.bind_attention_contracts" + or any(metrics[name] is not None for name in METRIC_NAMES) + or binding != expected_binding + or binding.get("comparable") is not False + or binding.get("passed") is not False + or binding.get("schema_version") != "cross_config.attention_binding.v3" + or not issues + or any(not isinstance(issue, Mapping) for issue in issues) + or issue_evidence != expected_issue_evidence + or case.get("identity_errors") != [issue.get("field") for issue in issues] + ): + raise ValueError("A4 lacks isolated topology-gate rejection evidence") + continue + + if case.get("comparable") is not True: + raise ValueError(f"{row_id} must contain a numerical comparison") + implementations = case.get("implementations", {}) + expected_implementations = ROW_IMPLEMENTATIONS[row_id] + if implementations != { + "baseline": expected_implementations[0], + "candidate": expected_implementations[1], + }: + raise ValueError(f"{row_id} implementation provenance is missing or incorrect") + batch, sequence = case["batch"], case["sequence"] + expected_shapes = { + "out": [batch, QWEN3_Q_HEADS, 1 if row_id == "C2" else sequence, QWEN3_HEAD_DIM], + "lse": [batch, QWEN3_Q_HEADS, 1 if row_id == "C2" else sequence], + "dq": [batch, QWEN3_Q_HEADS, sequence, QWEN3_HEAD_DIM], + "dk": [batch, QWEN3_KV_HEADS, sequence, QWEN3_HEAD_DIM], + "dv": [batch, QWEN3_KV_HEADS, sequence, QWEN3_HEAD_DIM], + } + for name in METRIC_NAMES: + _validate_metric( + row_id, + name, + metrics[name], + expected_dtype="torch.float32" if name == "lse" else "torch.bfloat16", + expected_shape=expected_shapes[name], + ) + mismatch_count = sum(metrics[name]["mismatch_count"] for name in METRIC_NAMES) + if row_manifest["expected"] in {"baseline", "exact_zero"}: + valid_outcome = mismatch_count == 0 and case.get("outcome") == "matched" + else: + valid_outcome = mismatch_count > 0 and case.get("outcome") == "drift_detected" + if not valid_outcome: + raise ValueError(f"{row_id} numerical evidence contradicts its expected outcome") + + aggregates = payload.get("matrix", []) + if not isinstance(aggregates, list) or any(not isinstance(row, Mapping) for row in aggregates): + raise ValueError("matrix summary must be a list of rows") + if [row.get("id") for row in aggregates] != expected_rows: + raise ValueError("result rows do not match the PR230 matrix") + reproduced = _aggregate(cases) + for row in reproduced: + if row["expected"] == "diagnostic" and any( + row["metrics"][name]["total_mismatch_count"] == 0 for name in METRIC_NAMES + ): + raise ValueError(f"{row['id']} must show a drift signature in all five metrics") + if aggregates != reproduced: + raise ValueError("matrix summary does not reproduce the per-shape case evidence") + failed = [row["id"] for row in aggregates if row.get("passed") is not True] + if failed: + raise ValueError("ROCm Attention ablation expectations failed: " + ", ".join(failed)) + + +def validate_repository_provenance(payload: Mapping[str, Any]) -> None: + """Bind recorded hashes to a real git object and the current runner.""" + + provenance = payload["source_provenance"] + revision = provenance["revision"] + script_path = provenance["script_path"] + committed_source = _git("show", f"{revision}:{script_path}") + current_source = (REPO_ROOT / script_path).read_bytes() + if ( + committed_source.returncode != 0 + or hashlib.sha256(committed_source.stdout).hexdigest() != provenance["script_sha256"] + or hashlib.sha256(current_source).hexdigest() != provenance["script_sha256"] + ): + raise ValueError("recorded runner hash is not backed by the RL-Kernel repository") + + +def _format_metric(value: Mapping[str, Any] | None) -> str: + if value is None: + return "—" + if value["total_mismatch_count"] == 0: + return "`0`" + return f"`{value['worst_max_abs']:.8g}`" + + +def _write_report(payload: Mapping[str, Any], path: Path) -> None: + environment = payload["environment"] + shapes = ", ".join( + f"B={batch}, S={sequence}" for batch, sequence in payload["configuration"]["shapes"] + ) + lines = [ + "# PR230 Attention taxonomy: ROCm operator micro-probes", + "", + "> This applies PR230's row taxonomy to deterministic operator micro-probes.", + "> It is not the frozen model/rollout replay from PR230: no checkpoint, token stream,", + "> selected-token logprob, KL, serving engine, or AITER production claim is included.", + "", + "## Environment", + "", + f"- GPU: {environment['device_name']} ({environment['architecture']})", + f"- PyTorch: {environment['pytorch']}; HIP: {environment['hip_runtime']}", + f"- RL-Kernel: `{payload['source_provenance']['revision']}`", + f"- Shapes: {shapes}", + "- Primary core: `rlkernel.rocm.deterministic_attention`", + "", + "## Matrix", + "", + "| Row | Factor | Comparable | Out | LSE | dQ | dK | dV | Result |", + "|---|---|:---:|---:|---:|---:|---:|---:|:---:|", + ] + for row in payload["matrix"]: + metrics = row["metrics"] + values = " | ".join(_format_metric(metrics[name]) for name in METRIC_NAMES) + result = "REJECTED" if row["id"] == "A4" and row["passed"] else "PASS" + if not row["passed"]: + result = "FAIL" + lines.append( + f"| {row['id']} | {row['label']} | {'yes' if row['comparable'] else 'no'} | " + f"{values} | **{result}** |" + ) + lines.extend(["", "## Probe realizations", ""]) + for row in attention_debug_matrix()["rows"]: + lines.append(f"- `{row['id']}` — {ROW_REALIZATIONS[row['id']]}") + lines.extend( + [ + "", + "A1-A3 and A5-A7 deliberately inject one mismatch and report the worst max-absolute", + "difference over the shape sweep. A4 is rejected by the repository's cross-config", + "binding gate before numerical comparison. A0 and C0-C2 must be bitwise zero for", + "Out/LSE/dQ/dK/dV.", + "", + "A6 and A7 are eager PyTorch-on-ROCm probes for accumulation and merge order; the", + "remaining numerical rows invoke the native deterministic HIP Attention core. This", + "is operator-only reference evidence, not full PR230 replay evidence.", + "", + "The complete per-shape mismatch counts and max-absolute values are in `results.json`.", + "", + "## Reproduce", + "", + "Run from the recorded clean commit and choose a new output directory:", + "", + "```bash", + "HIP_VISIBLE_DEVICES=0 CUDA_VISIBLE_DEVICES=0 python \\", + " benchmarks/benchmark_rocm_attention_ablation.py --device 0 \\", + " --output-dir /tmp/pr230_rocm_mi300x_ablation", + "```", + "", + ] + ) + path.write_text("\n".join(lines), encoding="utf-8") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--output-dir", + type=Path, + default=Path("benchmarks/results/pr230_rocm_mi300x_ablation"), + ) + parser.add_argument("--seed", type=int, default=230) + parser.add_argument("--device", type=int, default=0) + args = parser.parse_args() + + if args.output_dir.exists(): + raise SystemExit(f"refusing to overwrite existing output directory: {args.output_dir}") + source_provenance = _source_provenance() + if source_provenance["tracked_dirty"] or not source_provenance["script_matches_head"]: + raise SystemExit("refusing to publish evidence from an uncommitted runner or tracked tree") + if torch.version.hip is None or not torch.cuda.is_available(): + raise SystemExit("the ROCm Attention ablation matrix requires a ROCm GPU") + device = torch.device("cuda", args.device) + torch.cuda.set_device(device) + properties = torch.cuda.get_device_properties(device) + architecture = getattr(properties, "gcnArchName", "") + if "gfx942" not in architecture: + raise SystemExit(f"the checked-in matrix is pinned to gfx942, got {architecture!r}") + + from rl_engine.kernels.ops.cuda.attention.deterministic_attn import ( + RLKernelDeterministicAttentionCore, + ) + from rl_engine.kernels.ops.cuda.rotary_embedding.rope import RocmDeterministicRoPEOp + + attention = RLKernelDeterministicAttentionCore() + if attention.backend_id != ROCM_REFERENCE_BACKEND_ID: + raise SystemExit(f"unexpected reference backend on ROCm: {attention.backend_id}") + rope = RocmDeterministicRoPEOp() + rows: list[dict[str, Any]] = [] + for index, (batch, sequence) in enumerate(DEFAULT_SHAPES): + rows.extend( + _case( + batch=batch, + sequence=sequence, + seed=args.seed + index * 10, + device=device, + attention=attention, + rope=rope, + ) + ) + torch.cuda.synchronize(device) + + payload = { + "schema_version": RESULT_SCHEMA, + "scope": RESULT_SCOPE, + "created_at": datetime.now(timezone.utc).isoformat(), + "command": [sys.executable, *sys.argv], + "source_provenance": source_provenance, + "matrix_manifest": attention_debug_matrix(), + "environment": _environment(device, attention), + "configuration": { + "seed": args.seed, + "dtype": "bfloat16", + "q_heads": QWEN3_Q_HEADS, + "kv_heads": QWEN3_KV_HEADS, + "head_dim": QWEN3_HEAD_DIM, + "shapes": [list(shape) for shape in DEFAULT_SHAPES], + }, + "cases": rows, + "matrix": _aggregate(rows), + } + validate_payload(payload) + validate_repository_provenance(payload) + args.output_dir.mkdir(parents=True, exist_ok=False) + (args.output_dir / "results.json").write_text( + json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + _write_report(payload, args.output_dir / "report.md") + print(json.dumps({"output_dir": str(args.output_dir), "passed": True}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/test_rocm_attention_ablation_benchmark.py b/tests/test_rocm_attention_ablation_benchmark.py new file mode 100644 index 00000000..7bde5984 --- /dev/null +++ b/tests/test_rocm_attention_ablation_benchmark.py @@ -0,0 +1,299 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import copy +import importlib.util +import math +import sys +from pathlib import Path + +import pytest +import torch + +ROOT = Path(__file__).parents[1] +SCRIPT = ROOT / "benchmarks" / "benchmark_rocm_attention_ablation.py" +SPEC = importlib.util.spec_from_file_location("rocm_attention_ablation", SCRIPT) +assert SPEC is not None and SPEC.loader is not None +MODULE = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = MODULE +SPEC.loader.exec_module(MODULE) + + +def _metric(*, drift: bool, shape: list[int], dtype: str): + return { + "max_abs": 0.5 if drift else 0.0, + "mismatch_count": 1 if drift else 0, + "element_count": math.prod(shape), + "bitwise_equal": not drift, + "left_dtype": dtype, + "right_dtype": dtype, + "shape": shape, + } + + +def _valid_payload(): + shapes = [list(shape) for shape in MODULE.DEFAULT_SHAPES] + cases = [] + for batch, sequence in MODULE.DEFAULT_SHAPES: + for matrix_row in MODULE.attention_debug_matrix()["rows"]: + row_id = matrix_row["id"] + common = { + "row_id": row_id, + "batch": batch, + "sequence": sequence, + "category": matrix_row["category"], + "probe": matrix_row["probe"], + "expected": matrix_row["expected"], + "passed": True, + "realization": MODULE.ROW_REALIZATIONS[row_id], + } + if row_id == "A4": + binding = MODULE._topology_gate(batch=batch, sequence=sequence) + cases.append( + { + **common, + "comparable": False, + "outcome": "rejected", + "gate_implementation": ( + "rl_engine.alignment.cross_config.bind_attention_contracts" + ), + "identity_errors": [issue["field"] for issue in binding["issues"]], + "binding_gate": binding, + "metrics": {name: None for name in MODULE.METRIC_NAMES}, + } + ) + continue + drift = matrix_row["expected"] == "diagnostic" + implementations = MODULE.ROW_IMPLEMENTATIONS[row_id] + metric_shapes = { + "out": [ + batch, + MODULE.QWEN3_Q_HEADS, + 1 if row_id == "C2" else sequence, + MODULE.QWEN3_HEAD_DIM, + ], + "lse": [batch, MODULE.QWEN3_Q_HEADS, 1 if row_id == "C2" else sequence], + "dq": [batch, MODULE.QWEN3_Q_HEADS, sequence, MODULE.QWEN3_HEAD_DIM], + "dk": [batch, MODULE.QWEN3_KV_HEADS, sequence, MODULE.QWEN3_HEAD_DIM], + "dv": [batch, MODULE.QWEN3_KV_HEADS, sequence, MODULE.QWEN3_HEAD_DIM], + } + cases.append( + { + **common, + "comparable": True, + "outcome": "drift_detected" if drift else "matched", + "implementations": { + "baseline": implementations[0], + "candidate": implementations[1], + }, + "metrics": { + name: _metric( + drift=drift, + shape=metric_shapes[name], + dtype="torch.float32" if name == "lse" else "torch.bfloat16", + ) + for name in MODULE.METRIC_NAMES + }, + } + ) + return { + "schema_version": MODULE.RESULT_SCHEMA, + "scope": MODULE.RESULT_SCOPE, + "command": ["python", "benchmarks/benchmark_rocm_attention_ablation.py"], + "source_provenance": { + "revision": "a" * 40, + "tracked_dirty": False, + "tracked_diff_sha256": None, + "script_path": "benchmarks/benchmark_rocm_attention_ablation.py", + "script_sha256": "b" * 64, + "script_matches_head": True, + }, + "matrix_manifest": MODULE.attention_debug_matrix(), + "environment": { + "python": "3.10.0", + "pytorch": "2.12.0+rocm7.0", + "hip_runtime": "7.0", + "device_index": 0, + "device_name": "AMD Instinct MI300X", + "architecture": "gfx942:sramecc+:xnack-", + "gpu_count": 1, + "primary_backend_id": MODULE.ROCM_REFERENCE_BACKEND_ID, + "primary_core_id": MODULE.STRICT_ATTENTION_REFERENCE_CORE_ID, + "primary_schedule": MODULE.STRICT_ATTENTION_SCHEDULE_ID, + "primary_reference_only": True, + "primary_production_ready": False, + "execution_kind": "operator_only_rocm_reference", + }, + "configuration": { + "seed": 230, + "dtype": "bfloat16", + "q_heads": MODULE.QWEN3_Q_HEADS, + "kv_heads": MODULE.QWEN3_KV_HEADS, + "head_dim": MODULE.QWEN3_HEAD_DIM, + "shapes": shapes, + }, + "cases": cases, + "matrix": MODULE._aggregate(cases), + } + + +def test_metric_records_bitwise_and_numerical_drift(): + same = MODULE._metric(torch.tensor([1.0]), torch.tensor([1.0])) + drift = MODULE._metric(torch.tensor([1.0]), torch.tensor([1.5])) + + assert same == { + "max_abs": 0.0, + "mismatch_count": 0, + "element_count": 1, + "bitwise_equal": True, + "left_dtype": "torch.float32", + "right_dtype": "torch.float32", + "shape": [1], + } + assert drift["max_abs"] == 0.5 + assert drift["mismatch_count"] == 1 + assert drift["bitwise_equal"] is False + + +def test_metric_uses_raw_bits_and_requires_matching_dtype(): + signed_zero = MODULE._metric(torch.tensor([0.0]), torch.tensor([-0.0])) + mixed_dtype = MODULE._metric( + torch.tensor([1.0], dtype=torch.bfloat16), + torch.tensor([1.0], dtype=torch.float32), + ) + + assert signed_zero["max_abs"] == 0.0 + assert signed_zero["mismatch_count"] == 1 + assert signed_zero["bitwise_equal"] is False + assert mixed_dtype["mismatch_count"] == 1 + assert mixed_dtype["left_dtype"] != mixed_dtype["right_dtype"] + + +def test_chunk_merge_probe_has_attention_shapes_and_finite_values(): + generator = torch.Generator().manual_seed(230) + q = torch.randn(1, 4, 8, 128, dtype=torch.bfloat16, generator=generator) + k = torch.randn(1, 1, 8, 128, dtype=torch.bfloat16, generator=generator) + v = torch.randn(1, 1, 8, 128, dtype=torch.bfloat16, generator=generator) + dout = torch.randn(q.shape, dtype=torch.bfloat16, generator=generator) + dense = MODULE._evaluate( + q, + k, + v, + dout, + MODULE._dense_attention(accumulator_dtype=torch.float32), + ) + + for order in ("ascending", "descending"): + chunked = MODULE._evaluate(q, k, v, dout, MODULE._chunked_attention(order)) + assert chunked.out.shape == q.shape + assert chunked.lse.shape == q.shape[:-1] + assert all(torch.isfinite(getattr(chunked, name)).all() for name in MODULE.METRIC_NAMES) + maximums = { + name: MODULE._metric(getattr(dense, name), getattr(chunked, name))["max_abs"] + for name in MODULE.METRIC_NAMES + } + assert maximums["lse"] <= 1.0e-5 + assert all(maximums[name] <= 0.015625 for name in ("out", "dq", "dk", "dv")) + + +def test_explicit_accumulator_probe_changes_all_five_metrics(): + generator = torch.Generator().manual_seed(231) + q = torch.randn(1, 4, 4, 128, dtype=torch.bfloat16, generator=generator) + k = torch.randn(1, 1, 4, 128, dtype=torch.bfloat16, generator=generator) + v = torch.randn(1, 1, 4, 128, dtype=torch.bfloat16, generator=generator) + dout = torch.randn(q.shape, dtype=torch.bfloat16, generator=generator) + fp32 = MODULE._evaluate(q, k, v, dout, MODULE._dense_attention(accumulator_dtype=torch.float32)) + bf16 = MODULE._evaluate( + q, k, v, dout, MODULE._dense_attention(accumulator_dtype=torch.bfloat16) + ) + + assert all( + MODULE._metric(getattr(fp32, name), getattr(bf16, name))["mismatch_count"] > 0 + for name in MODULE.METRIC_NAMES + ) + + +def test_topology_probe_uses_binding_gate_and_isolates_ownership_mismatch(): + binding = MODULE._topology_gate(batch=2, sequence=32) + + assert binding["comparable"] is False + assert binding["passed"] is False + assert {issue["code"] for issue in binding["issues"]} == {"TOPOLOGY_MISMATCH"} + assert {issue["field"] for issue in binding["issues"]} == { + "sharding.tp_rank", + "sharding.local_q_head_start", + "sharding.local_kv_head_start", + } + + +def test_payload_validator_accepts_complete_pr230_rocm_evidence(): + payload = _valid_payload() + MODULE.validate_payload(payload) + + +def _invent_a4_issue(payload): + case = payload["cases"][4] + case["binding_gate"]["issues"] = [ + { + "code": "TOPOLOGY_MISMATCH", + "tier": "identical", + "field": "invented", + "rollout": 1, + "training": 0, + } + ] + case["identity_errors"] = ["invented"] + + +@pytest.mark.parametrize( + ("mutate", "message"), + [ + (lambda payload: payload.update(cases=[]), "cover every PR230 row"), + ( + lambda payload: payload["cases"][1]["metrics"]["out"].update(max_abs=float("nan")), + "finite and non-negative", + ), + ( + lambda payload: payload["cases"][4]["binding_gate"].update(comparable=True), + "topology-gate rejection", + ), + (_invent_a4_issue, "topology-gate rejection"), + ( + lambda payload: payload["cases"][0]["metrics"]["out"].update( + shape=[1], element_count=1 + ), + "incompatible dtype or shape", + ), + ( + lambda payload: payload["configuration"]["shapes"].pop(), + "exact eight-shape", + ), + ( + lambda payload: payload["configuration"].pop("seed"), + "configuration.seed", + ), + ( + lambda payload: payload["matrix"][0].update(case_count=0), + "does not reproduce", + ), + ( + lambda payload: payload["environment"].update(architecture="sm_90"), + "gfx942 ROCm", + ), + ( + lambda payload: payload["environment"].pop("device_name"), + "environment.device_name", + ), + ], +) +def test_payload_validator_rejects_incomplete_or_fabricated_evidence(mutate, message): + payload = copy.deepcopy(_valid_payload()) + mutate(payload) + with pytest.raises(ValueError, match=message): + MODULE.validate_payload(payload) + + +def test_repository_provenance_rejects_unbacked_hashes(): + with pytest.raises(ValueError, match="not backed"): + MODULE.validate_repository_provenance(_valid_payload()) From d0472ea8d95a9a0e77cea288746fe9b92c3435af Mon Sep 17 00:00:00 2001 From: vensen Date: Thu, 3 Sep 2026 10:43:31 +0000 Subject: [PATCH 2/3] bench(rocm): publish MI300X ablation results Signed-off-by: vensen --- .../pr230_rocm_mi300x_ablation/report.md | 64 + .../pr230_rocm_mi300x_ablation/results.json | 11234 ++++++++++++++++ .../test_rocm_attention_ablation_benchmark.py | 10 + 3 files changed, 11308 insertions(+) create mode 100644 benchmarks/results/pr230_rocm_mi300x_ablation/report.md create mode 100644 benchmarks/results/pr230_rocm_mi300x_ablation/results.json diff --git a/benchmarks/results/pr230_rocm_mi300x_ablation/report.md b/benchmarks/results/pr230_rocm_mi300x_ablation/report.md new file mode 100644 index 00000000..c76e4503 --- /dev/null +++ b/benchmarks/results/pr230_rocm_mi300x_ablation/report.md @@ -0,0 +1,64 @@ +# PR230 Attention taxonomy: ROCm operator micro-probes + +> This applies PR230's row taxonomy to deterministic operator micro-probes. +> It is not the frozen model/rollout replay from PR230: no checkpoint, token stream, +> selected-token logprob, KL, serving engine, or AITER production claim is included. + +## Environment + +- GPU: AMD Instinct MI300X VF (gfx942:sramecc+:xnack-) +- PyTorch: 2.12.0+rocm7.14.0a20260608; HIP: 7.14.60850 +- RL-Kernel: `2ea63b22b74feb6e5a748d09780fa075d5e644ed` +- Shapes: B=1, S=16, B=1, S=32, B=1, S=64, B=1, S=128, B=2, S=16, B=2, S=32, B=2, S=64, B=2, S=128 +- Primary core: `rlkernel.rocm.deterministic_attention` + +## Matrix + +| Row | Factor | Comparable | Out | LSE | dQ | dK | dV | Result | +|---|---|:---:|---:|---:|---:|---:|---:|:---:| +| A0 | Strict replay baseline | yes | `0` | `0` | `0` | `0` | `0` | **PASS** | +| A1 | Position / RoPE | yes | `0.7109375` | `0.39382553` | `1.4140625` | `2.234375` | `0.78320312` | **PASS** | +| A2 | Q/K preprocessing | yes | `0.58203125` | `1.0303755` | `1.015625` | `1.7578125` | `0.96875` | **PASS** | +| A3 | Mask / sequence boundary | yes | `4.21875` | `7.671814` | `4.1968994` | `6.15625` | `10.984375` | **PASS** | +| A4 | Topology / head ownership | no | — | — | — | — | — | **REJECTED** | +| A5 | KV-cache identity / layout | yes | `6` | `4.578392` | `4.2773438` | `6.6274414` | `13.233398` | **PASS** | +| A6 | Numerical policy | yes | `0.1171875` | `0.099507809` | `0.09375` | `0.09375` | `0.11328125` | **PASS** | +| A7 | Distributed schedule | yes | `0.001953125` | `9.5367432e-07` | `0.00390625` | `0.00390625` | `0.00390625` | **PASS** | +| C0 | Invariant control | yes | `0` | `0` | `0` | `0` | `0` | **PASS** | +| C1 | Invariant control | yes | `0` | `0` | `0` | `0` | `0` | **PASS** | +| C2 | Invariant control | yes | `0` | `0` | `0` | `0` | `0` | **PASS** | + +## Probe realizations + +- `A0` — Repeat the identical native HIP reference-core call. +- `A1` — Increment suffix RoPE positions while preserving Q/K/V tensors. +- `A2` — Apply or bypass unit-weight PyTorch RMSNorm before the native HIP core. +- `A3` — Toggle causal masking in the native HIP core. +- `A4` — Bind valid TP-rank-1 rollout and TP-rank-0 training contracts; do not run numerics. +- `A5` — Reverse four dense K/V tensor pages; this is not a paged-cache runtime. +- `A6` — Use identical FP32 products/order with explicit FP32 versus BF16 accumulator state. +- `A7` — Merge four dense chunks in opposite orders on one GPU; this is not a CP collective. +- `C0` — Compare full GQA with two contiguous TP=2 head shards on one GPU. +- `C1` — Compare a batch call with per-row calls to the same native HIP core. +- `C2` — Compare a full-prefill tail with one trailing query over dense KV, without a serving cache. + +A1-A3 and A5-A7 deliberately inject one mismatch and report the worst max-absolute +difference over the shape sweep. A4 is rejected by the repository's cross-config +binding gate before numerical comparison. A0 and C0-C2 must be bitwise zero for +Out/LSE/dQ/dK/dV. + +A6 and A7 are eager PyTorch-on-ROCm probes for accumulation and merge order; the +remaining numerical rows invoke the native deterministic HIP Attention core. This +is operator-only reference evidence, not full PR230 replay evidence. + +The complete per-shape mismatch counts and max-absolute values are in `results.json`. + +## Reproduce + +Run from the recorded clean commit and choose a new output directory: + +```bash +HIP_VISIBLE_DEVICES=0 CUDA_VISIBLE_DEVICES=0 python \ + benchmarks/benchmark_rocm_attention_ablation.py --device 0 \ + --output-dir /tmp/pr230_rocm_mi300x_ablation +``` diff --git a/benchmarks/results/pr230_rocm_mi300x_ablation/results.json b/benchmarks/results/pr230_rocm_mi300x_ablation/results.json new file mode 100644 index 00000000..6c08181f --- /dev/null +++ b/benchmarks/results/pr230_rocm_mi300x_ablation/results.json @@ -0,0 +1,11234 @@ +{ + "cases": [ + { + "batch": 1, + "category": "baseline", + "comparable": true, + "expected": "baseline", + "implementations": { + "baseline": "rlkernel.rocm.deterministic_attention", + "candidate": "rlkernel.rocm.deterministic_attention" + }, + "metrics": { + "dk": { + "bitwise_equal": true, + "element_count": 16384, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 16, + 128 + ] + }, + "dq": { + "bitwise_equal": true, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 16, + 128 + ] + }, + "dv": { + "bitwise_equal": true, + "element_count": 16384, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 16, + 128 + ] + }, + "lse": { + "bitwise_equal": true, + "element_count": 512, + "left_dtype": "torch.float32", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.float32", + "shape": [ + 1, + 32, + 16 + ] + }, + "out": { + "bitwise_equal": true, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 16, + 128 + ] + } + }, + "outcome": "matched", + "passed": true, + "probe": null, + "realization": "Repeat the identical native HIP reference-core call.", + "row_id": "A0", + "sequence": 16 + }, + { + "batch": 1, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "implementations": { + "baseline": "rlkernel.rocm.deterministic_rope+native_attention", + "candidate": "rlkernel.rocm.deterministic_rope+native_attention" + }, + "metrics": { + "dk": { + "bitwise_equal": false, + "element_count": 16384, + "left_dtype": "torch.bfloat16", + "max_abs": 1.6953125, + "mismatch_count": 15960, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 16, + 128 + ] + }, + "dq": { + "bitwise_equal": false, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 1.18359375, + "mismatch_count": 32219, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 16, + 128 + ] + }, + "dv": { + "bitwise_equal": false, + "element_count": 16384, + "left_dtype": "torch.bfloat16", + "max_abs": 0.513671875, + "mismatch_count": 15826, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 16, + 128 + ] + }, + "lse": { + "bitwise_equal": false, + "element_count": 512, + "left_dtype": "torch.float32", + "max_abs": 0.26604652404785156, + "mismatch_count": 256, + "right_dtype": "torch.float32", + "shape": [ + 1, + 32, + 16 + ] + }, + "out": { + "bitwise_equal": false, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 0.4765625, + "mismatch_count": 32138, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 16, + 128 + ] + } + }, + "outcome": "drift_detected", + "passed": true, + "probe": "position_ids", + "realization": "Increment suffix RoPE positions while preserving Q/K/V tensors.", + "row_id": "A1", + "sequence": 16 + }, + { + "batch": 1, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "implementations": { + "baseline": "torch.rocm.rms_norm+native_attention", + "candidate": "rlkernel.rocm.deterministic_attention" + }, + "metrics": { + "dk": { + "bitwise_equal": false, + "element_count": 16384, + "left_dtype": "torch.bfloat16", + "max_abs": 0.9453125, + "mismatch_count": 16189, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 16, + 128 + ] + }, + "dq": { + "bitwise_equal": false, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 0.546875, + "mismatch_count": 60603, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 16, + 128 + ] + }, + "dv": { + "bitwise_equal": false, + "element_count": 16384, + "left_dtype": "torch.bfloat16", + "max_abs": 0.5, + "mismatch_count": 15924, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 16, + 128 + ] + }, + "lse": { + "bitwise_equal": false, + "element_count": 512, + "left_dtype": "torch.float32", + "max_abs": 0.4451262950897217, + "mismatch_count": 512, + "right_dtype": "torch.float32", + "shape": [ + 1, + 32, + 16 + ] + }, + "out": { + "bitwise_equal": false, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 0.4296875, + "mismatch_count": 58032, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 16, + 128 + ] + } + }, + "outcome": "drift_detected", + "passed": true, + "probe": "qk_norm_disabled", + "realization": "Apply or bypass unit-weight PyTorch RMSNorm before the native HIP core.", + "row_id": "A2", + "sequence": 16 + }, + { + "batch": 1, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "implementations": { + "baseline": "rlkernel.rocm.deterministic_attention", + "candidate": "rlkernel.rocm.deterministic_attention" + }, + "metrics": { + "dk": { + "bitwise_equal": false, + "element_count": 16384, + "left_dtype": "torch.bfloat16", + "max_abs": 3.61328125, + "mismatch_count": 16354, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 16, + 128 + ] + }, + "dq": { + "bitwise_equal": false, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 3.5625, + "mismatch_count": 61247, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 16, + 128 + ] + }, + "dv": { + "bitwise_equal": false, + "element_count": 16384, + "left_dtype": "torch.bfloat16", + "max_abs": 7.330078125, + "mismatch_count": 16364, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 16, + 128 + ] + }, + "lse": { + "bitwise_equal": false, + "element_count": 512, + "left_dtype": "torch.float32", + "max_abs": 5.188642501831055, + "mismatch_count": 480, + "right_dtype": "torch.float32", + "shape": [ + 1, + 32, + 16 + ] + }, + "out": { + "bitwise_equal": false, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 3.4375, + "mismatch_count": 61228, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 16, + 128 + ] + } + }, + "outcome": "drift_detected", + "passed": true, + "probe": "causal_mask", + "realization": "Toggle causal masking in the native HIP core.", + "row_id": "A3", + "sequence": 16 + }, + { + "batch": 1, + "binding_gate": { + "binding_fingerprint": "36b3ae6b7b3315e4891e3fc59d23973f29af4697179fd58d6bc89e5b7643906a", + "comparable": false, + "identity_fingerprint": "c2a02818c11bd195873275c75f4d94ae20b0ae2f2e9daa7fcd5f51d9b409cb4c", + "issues": [ + { + "code": "TOPOLOGY_MISMATCH", + "field": "sharding.tp_rank", + "message": "sharding.tp_rank changes TP/CP ownership; the pair is not the same local attention problem", + "rollout": 1, + "tier": "identical", + "training": 0 + }, + { + "code": "TOPOLOGY_MISMATCH", + "field": "sharding.local_q_head_start", + "message": "sharding.local_q_head_start changes TP/CP ownership; the pair is not the same local attention problem", + "rollout": 16, + "tier": "identical", + "training": 0 + }, + { + "code": "TOPOLOGY_MISMATCH", + "field": "sharding.local_kv_head_start", + "message": "sharding.local_kv_head_start changes TP/CP ownership; the pair is not the same local attention problem", + "rollout": 4, + "tier": "identical", + "training": 0 + } + ], + "passed": false, + "provenance": { + "dtype": "bf16", + "lse_domain": "attention", + "rollout": { + "backend_id": "rlkernel.rocm.deterministic_attention", + "contract": { + "batch_size": 1, + "causal": true, + "causal_offsets": [ + 0 + ], + "dtype": "bf16", + "export_lse": true, + "head_dim": 128, + "kv_cache": null, + "lse_domain": "attention", + "mode": "prefill", + "projections": { + "o_proj": { + "acc_dtype": "fp32", + "backend_policy": "native_verified_then_common_deterministic", + "deterministic_backend": "rlkernel.cuda.det_gemm", + "input_dtype": "bf16", + "k_order": "ascending", + "output_dtype": "bf16", + "qkv_split_order": [], + "require_runtime_readback": true, + "sp_backward_collective": "all_gather", + "sp_forward_collective": "reduce_scatter", + "split_kv": "disabled", + "tp_backward_dgrad_collective": "none", + "tp_forward_collective": "all_reduce" + }, + "qkv": { + "acc_dtype": "fp32", + "backend_policy": "native_verified_then_common_deterministic", + "deterministic_backend": "rlkernel.cuda.det_gemm", + "input_dtype": "bf16", + "k_order": "ascending", + "output_dtype": "bf16", + "qkv_split_order": [ + "q", + "k", + "v" + ], + "require_runtime_readback": true, + "sp_backward_collective": "reduce_scatter", + "sp_forward_collective": "all_gather", + "split_kv": "disabled", + "tp_backward_dgrad_collective": "all_reduce", + "tp_forward_collective": "none" + } + }, + "query_sequence_length": 16, + "reduction": { + "acc_dtype": "fp32", + "downcast_at": "final_write", + "engine": "in_op_reference", + "merge": "online_softmax_lse", + "order": "global_block_index" + }, + "role": "infer", + "rope": null, + "semantic_operator": "standard_softmax_attention", + "sharding": { + "cp_rank": 0, + "cp_world_size": 1, + "global_block_indices": [ + 0 + ], + "global_block_token_starts": [ + 0 + ], + "global_kv_heads": 8, + "global_q_heads": 32, + "global_sequence_length": 16, + "local_block_offsets": [ + 0, + 16 + ], + "local_kv_head_start": 4, + "local_kv_heads": 4, + "local_q_head_start": 16, + "local_q_heads": 16, + "local_sequence_length": 16, + "packed_sequence_offsets": null, + "sp_rank": 0, + "sp_world_size": 1, + "tp_rank": 1, + "tp_world_size": 2 + }, + "split_kv": { + "fixed_split_size": null, + "mode": "disabled", + "strict_consistency": true + } + }, + "recorded": { + "backend_id": "rlkernel.rocm.deterministic_attention", + "mode": "prefill", + "reduction.engine": "in_op_reference" + } + }, + "split_kv_runtime": { + "rollout": { + "batch_size": 1, + "coverage": "complete_batch_tp_cp_owner_cartesian_product", + "cp_world_size": 1, + "entries": [ + { + "actual_split_boundaries": [ + [ + 0, + 16 + ] + ], + "actual_split_kv_count": 1, + "actual_split_kv_policy": "disabled", + "actual_split_kv_size": null, + "batch_index": 0, + "cp_rank": 0, + "expected_kv_range": [ + 0, + 16 + ], + "owner_cp_rank": 0, + "requested_split_kv_policy": "disabled", + "requested_split_kv_size": null, + "split_kv_accum_dtype": "fp32", + "split_kv_backend": "rlkernel.rocm.deterministic_attention", + "split_kv_downcast_at": "final_write", + "split_kv_fallback": false, + "split_kv_fallback_reason": null, + "split_kv_merge_order": "global_block_index", + "split_kv_plan_source": "contract_exact", + "tp_rank": 0 + }, + { + "actual_split_boundaries": [ + [ + 0, + 16 + ] + ], + "actual_split_kv_count": 1, + "actual_split_kv_policy": "disabled", + "actual_split_kv_size": null, + "batch_index": 0, + "cp_rank": 0, + "expected_kv_range": [ + 0, + 16 + ], + "owner_cp_rank": 0, + "requested_split_kv_policy": "disabled", + "requested_split_kv_size": null, + "split_kv_accum_dtype": "fp32", + "split_kv_backend": "rlkernel.rocm.deterministic_attention", + "split_kv_downcast_at": "final_write", + "split_kv_fallback": false, + "split_kv_fallback_reason": null, + "split_kv_merge_order": "global_block_index", + "split_kv_plan_source": "contract_exact", + "tp_rank": 1 + } + ], + "total_kv_tokens": [ + 16 + ], + "tp_world_size": 2 + }, + "training": { + "batch_size": 1, + "coverage": "complete_batch_tp_cp_owner_cartesian_product", + "cp_world_size": 1, + "entries": [ + { + "actual_split_boundaries": [ + [ + 0, + 16 + ] + ], + "actual_split_kv_count": 1, + "actual_split_kv_policy": "disabled", + "actual_split_kv_size": null, + "batch_index": 0, + "cp_rank": 0, + "expected_kv_range": [ + 0, + 16 + ], + "owner_cp_rank": 0, + "requested_split_kv_policy": "disabled", + "requested_split_kv_size": null, + "split_kv_accum_dtype": "fp32", + "split_kv_backend": "rlkernel.rocm.deterministic_attention", + "split_kv_downcast_at": "final_write", + "split_kv_fallback": false, + "split_kv_fallback_reason": null, + "split_kv_merge_order": "global_block_index", + "split_kv_plan_source": "contract_exact", + "tp_rank": 0 + }, + { + "actual_split_boundaries": [ + [ + 0, + 16 + ] + ], + "actual_split_kv_count": 1, + "actual_split_kv_policy": "disabled", + "actual_split_kv_size": null, + "batch_index": 0, + "cp_rank": 0, + "expected_kv_range": [ + 0, + 16 + ], + "owner_cp_rank": 0, + "requested_split_kv_policy": "disabled", + "requested_split_kv_size": null, + "split_kv_accum_dtype": "fp32", + "split_kv_backend": "rlkernel.rocm.deterministic_attention", + "split_kv_downcast_at": "final_write", + "split_kv_fallback": false, + "split_kv_fallback_reason": null, + "split_kv_merge_order": "global_block_index", + "split_kv_plan_source": "contract_exact", + "tp_rank": 1 + } + ], + "total_kv_tokens": [ + 16 + ], + "tp_world_size": 2 + } + }, + "training": { + "backend_id": "rlkernel.rocm.deterministic_attention", + "contract": { + "batch_size": 1, + "causal": true, + "causal_offsets": [ + 0 + ], + "dtype": "bf16", + "export_lse": true, + "head_dim": 128, + "kv_cache": null, + "lse_domain": "attention", + "mode": "prefill", + "projections": { + "o_proj": { + "acc_dtype": "fp32", + "backend_policy": "native_verified_then_common_deterministic", + "deterministic_backend": "rlkernel.cuda.det_gemm", + "input_dtype": "bf16", + "k_order": "ascending", + "output_dtype": "bf16", + "qkv_split_order": [], + "require_runtime_readback": true, + "sp_backward_collective": "all_gather", + "sp_forward_collective": "reduce_scatter", + "split_kv": "disabled", + "tp_backward_dgrad_collective": "none", + "tp_forward_collective": "all_reduce" + }, + "qkv": { + "acc_dtype": "fp32", + "backend_policy": "native_verified_then_common_deterministic", + "deterministic_backend": "rlkernel.cuda.det_gemm", + "input_dtype": "bf16", + "k_order": "ascending", + "output_dtype": "bf16", + "qkv_split_order": [ + "q", + "k", + "v" + ], + "require_runtime_readback": true, + "sp_backward_collective": "reduce_scatter", + "sp_forward_collective": "all_gather", + "split_kv": "disabled", + "tp_backward_dgrad_collective": "all_reduce", + "tp_forward_collective": "none" + } + }, + "query_sequence_length": 16, + "reduction": { + "acc_dtype": "fp32", + "downcast_at": "final_write", + "engine": "in_op_reference", + "merge": "online_softmax_lse", + "order": "global_block_index" + }, + "role": "train", + "rope": null, + "semantic_operator": "standard_softmax_attention", + "sharding": { + "cp_rank": 0, + "cp_world_size": 1, + "global_block_indices": [ + 0 + ], + "global_block_token_starts": [ + 0 + ], + "global_kv_heads": 8, + "global_q_heads": 32, + "global_sequence_length": 16, + "local_block_offsets": [ + 0, + 16 + ], + "local_kv_head_start": 0, + "local_kv_heads": 4, + "local_q_head_start": 0, + "local_q_heads": 16, + "local_sequence_length": 16, + "packed_sequence_offsets": null, + "sp_rank": 0, + "sp_world_size": 1, + "tp_rank": 0, + "tp_world_size": 2 + }, + "split_kv": { + "fixed_split_size": null, + "mode": "disabled", + "strict_consistency": true + } + }, + "recorded": { + "backend_id": "rlkernel.rocm.deterministic_attention", + "mode": "prefill", + "reduction.engine": "in_op_reference" + } + } + }, + "recorded_differences": {}, + "reduction_fingerprint": "4bfe65d607de1dc8d1b8c04041d48e9af4e81dc38614f0340d994616da273d6c", + "schema_version": "cross_config.attention_binding.v3" + }, + "category": "comparability_gate", + "comparable": false, + "expected": "rejected", + "gate_implementation": "rl_engine.alignment.cross_config.bind_attention_contracts", + "identity_errors": [ + "sharding.tp_rank", + "sharding.local_q_head_start", + "sharding.local_kv_head_start" + ], + "metrics": { + "dk": null, + "dq": null, + "dv": null, + "lse": null, + "out": null + }, + "outcome": "rejected", + "passed": true, + "probe": "tp_head_ownership", + "realization": "Bind valid TP-rank-1 rollout and TP-rank-0 training contracts; do not run numerics.", + "row_id": "A4", + "sequence": 16 + }, + { + "batch": 1, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "implementations": { + "baseline": "rlkernel.rocm.deterministic_attention", + "candidate": "torch.rocm.index_select+native_attention" + }, + "metrics": { + "dk": { + "bitwise_equal": false, + "element_count": 16384, + "left_dtype": "torch.bfloat16", + "max_abs": 5.06884765625, + "mismatch_count": 16365, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 16, + 128 + ] + }, + "dq": { + "bitwise_equal": false, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 3.7734375, + "mismatch_count": 57223, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 16, + 128 + ] + }, + "dv": { + "bitwise_equal": false, + "element_count": 16384, + "left_dtype": "torch.bfloat16", + "max_abs": 8.6474609375, + "mismatch_count": 16371, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 16, + 128 + ] + }, + "lse": { + "bitwise_equal": false, + "element_count": 512, + "left_dtype": "torch.float32", + "max_abs": 4.578392028808594, + "mismatch_count": 480, + "right_dtype": "torch.float32", + "shape": [ + 1, + 32, + 16 + ] + }, + "out": { + "bitwise_equal": false, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 4.40625, + "mismatch_count": 61329, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 16, + 128 + ] + } + }, + "outcome": "drift_detected", + "passed": true, + "probe": "kv_page_order", + "realization": "Reverse four dense K/V tensor pages; this is not a paged-cache runtime.", + "row_id": "A5", + "sequence": 16 + }, + { + "batch": 1, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "implementations": { + "baseline": "torch.rocm.explicit_fp32_serial_qk_accumulator", + "candidate": "torch.rocm.explicit_bf16_serial_qk_accumulator" + }, + "metrics": { + "dk": { + "bitwise_equal": false, + "element_count": 16384, + "left_dtype": "torch.bfloat16", + "max_abs": 0.05078125, + "mismatch_count": 13428, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 16, + 128 + ] + }, + "dq": { + "bitwise_equal": false, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0546875, + "mismatch_count": 47934, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 16, + 128 + ] + }, + "dv": { + "bitwise_equal": false, + "element_count": 16384, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0625, + "mismatch_count": 13654, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 16, + 128 + ] + }, + "lse": { + "bitwise_equal": false, + "element_count": 512, + "left_dtype": "torch.float32", + "max_abs": 0.05289459228515625, + "mismatch_count": 512, + "right_dtype": "torch.float32", + "shape": [ + 1, + 32, + 16 + ] + }, + "out": { + "bitwise_equal": false, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0546875, + "mismatch_count": 47319, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 16, + 128 + ] + } + }, + "outcome": "drift_detected", + "passed": true, + "probe": "accum_dtype", + "realization": "Use identical FP32 products/order with explicit FP32 versus BF16 accumulator state.", + "row_id": "A6", + "sequence": 16 + }, + { + "batch": 1, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "implementations": { + "baseline": "torch.rocm.fp32_chunk_merge_ascending", + "candidate": "torch.rocm.fp32_chunk_merge_descending" + }, + "metrics": { + "dk": { + "bitwise_equal": false, + "element_count": 16384, + "left_dtype": "torch.bfloat16", + "max_abs": 0.001953125, + "mismatch_count": 4, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 16, + 128 + ] + }, + "dq": { + "bitwise_equal": false, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 0.00390625, + "mismatch_count": 2, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 16, + 128 + ] + }, + "dv": { + "bitwise_equal": false, + "element_count": 16384, + "left_dtype": "torch.bfloat16", + "max_abs": 0.001953125, + "mismatch_count": 2, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 16, + 128 + ] + }, + "lse": { + "bitwise_equal": false, + "element_count": 512, + "left_dtype": "torch.float32", + "max_abs": 4.76837158203125e-07, + "mismatch_count": 28, + "right_dtype": "torch.float32", + "shape": [ + 1, + 32, + 16 + ] + }, + "out": { + "bitwise_equal": false, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 7.62939453125e-06, + "mismatch_count": 1, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 16, + 128 + ] + } + }, + "outcome": "drift_detected", + "passed": true, + "probe": "merge_order", + "realization": "Merge four dense chunks in opposite orders on one GPU; this is not a CP collective.", + "row_id": "A7", + "sequence": 16 + }, + { + "batch": 1, + "category": "invariant_control", + "comparable": true, + "expected": "exact_zero", + "implementations": { + "baseline": "rlkernel.rocm.deterministic_attention", + "candidate": "rlkernel.rocm.native_attention_per_tp2_head_shard" + }, + "metrics": { + "dk": { + "bitwise_equal": true, + "element_count": 16384, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 16, + 128 + ] + }, + "dq": { + "bitwise_equal": true, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 16, + 128 + ] + }, + "dv": { + "bitwise_equal": true, + "element_count": 16384, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 16, + 128 + ] + }, + "lse": { + "bitwise_equal": true, + "element_count": 512, + "left_dtype": "torch.float32", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.float32", + "shape": [ + 1, + 32, + 16 + ] + }, + "out": { + "bitwise_equal": true, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 16, + 128 + ] + } + }, + "outcome": "matched", + "passed": true, + "probe": "tp_partition_control", + "realization": "Compare full GQA with two contiguous TP=2 head shards on one GPU.", + "row_id": "C0", + "sequence": 16 + }, + { + "batch": 1, + "category": "invariant_control", + "comparable": true, + "expected": "exact_zero", + "implementations": { + "baseline": "rlkernel.rocm.deterministic_attention", + "candidate": "rlkernel.rocm.native_attention_per_batch_row" + }, + "metrics": { + "dk": { + "bitwise_equal": true, + "element_count": 16384, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 16, + 128 + ] + }, + "dq": { + "bitwise_equal": true, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 16, + 128 + ] + }, + "dv": { + "bitwise_equal": true, + "element_count": 16384, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 16, + 128 + ] + }, + "lse": { + "bitwise_equal": true, + "element_count": 512, + "left_dtype": "torch.float32", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.float32", + "shape": [ + 1, + 32, + 16 + ] + }, + "out": { + "bitwise_equal": true, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 16, + 128 + ] + } + }, + "outcome": "matched", + "passed": true, + "probe": "batch_composition_control", + "realization": "Compare a batch call with per-row calls to the same native HIP core.", + "row_id": "C1", + "sequence": 16 + }, + { + "batch": 1, + "category": "invariant_control", + "comparable": true, + "expected": "exact_zero", + "implementations": { + "baseline": "rlkernel.rocm.native_attention_full_prefill", + "candidate": "rlkernel.rocm.native_attention_tail" + }, + "metrics": { + "dk": { + "bitwise_equal": true, + "element_count": 16384, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 16, + 128 + ] + }, + "dq": { + "bitwise_equal": true, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 16, + 128 + ] + }, + "dv": { + "bitwise_equal": true, + "element_count": 16384, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 16, + 128 + ] + }, + "lse": { + "bitwise_equal": true, + "element_count": 32, + "left_dtype": "torch.float32", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.float32", + "shape": [ + 1, + 32, + 1 + ] + }, + "out": { + "bitwise_equal": true, + "element_count": 4096, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 1, + 128 + ] + } + }, + "outcome": "matched", + "passed": true, + "probe": "prefill_decode_tail_control", + "realization": "Compare a full-prefill tail with one trailing query over dense KV, without a serving cache.", + "row_id": "C2", + "sequence": 16 + }, + { + "batch": 1, + "category": "baseline", + "comparable": true, + "expected": "baseline", + "implementations": { + "baseline": "rlkernel.rocm.deterministic_attention", + "candidate": "rlkernel.rocm.deterministic_attention" + }, + "metrics": { + "dk": { + "bitwise_equal": true, + "element_count": 32768, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 32, + 128 + ] + }, + "dq": { + "bitwise_equal": true, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 32, + 128 + ] + }, + "dv": { + "bitwise_equal": true, + "element_count": 32768, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 32, + 128 + ] + }, + "lse": { + "bitwise_equal": true, + "element_count": 1024, + "left_dtype": "torch.float32", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.float32", + "shape": [ + 1, + 32, + 32 + ] + }, + "out": { + "bitwise_equal": true, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 32, + 128 + ] + } + }, + "outcome": "matched", + "passed": true, + "probe": null, + "realization": "Repeat the identical native HIP reference-core call.", + "row_id": "A0", + "sequence": 32 + }, + { + "batch": 1, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "implementations": { + "baseline": "rlkernel.rocm.deterministic_rope+native_attention", + "candidate": "rlkernel.rocm.deterministic_rope+native_attention" + }, + "metrics": { + "dk": { + "bitwise_equal": false, + "element_count": 32768, + "left_dtype": "torch.bfloat16", + "max_abs": 1.673828125, + "mismatch_count": 31886, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 32, + 128 + ] + }, + "dq": { + "bitwise_equal": false, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 1.0703125, + "mismatch_count": 64571, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 32, + 128 + ] + }, + "dv": { + "bitwise_equal": false, + "element_count": 32768, + "left_dtype": "torch.bfloat16", + "max_abs": 0.783203125, + "mismatch_count": 31476, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 32, + 128 + ] + }, + "lse": { + "bitwise_equal": false, + "element_count": 1024, + "left_dtype": "torch.float32", + "max_abs": 0.2042393684387207, + "mismatch_count": 512, + "right_dtype": "torch.float32", + "shape": [ + 1, + 32, + 32 + ] + }, + "out": { + "bitwise_equal": false, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.55078125, + "mismatch_count": 64372, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 32, + 128 + ] + } + }, + "outcome": "drift_detected", + "passed": true, + "probe": "position_ids", + "realization": "Increment suffix RoPE positions while preserving Q/K/V tensors.", + "row_id": "A1", + "sequence": 32 + }, + { + "batch": 1, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "implementations": { + "baseline": "torch.rocm.rms_norm+native_attention", + "candidate": "rlkernel.rocm.deterministic_attention" + }, + "metrics": { + "dk": { + "bitwise_equal": false, + "element_count": 32768, + "left_dtype": "torch.bfloat16", + "max_abs": 1.3720703125, + "mismatch_count": 32458, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 32, + 128 + ] + }, + "dq": { + "bitwise_equal": false, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.6953125, + "mismatch_count": 125431, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 32, + 128 + ] + }, + "dv": { + "bitwise_equal": false, + "element_count": 32768, + "left_dtype": "torch.bfloat16", + "max_abs": 0.6015625, + "mismatch_count": 32049, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 32, + 128 + ] + }, + "lse": { + "bitwise_equal": false, + "element_count": 1024, + "left_dtype": "torch.float32", + "max_abs": 0.41014528274536133, + "mismatch_count": 1024, + "right_dtype": "torch.float32", + "shape": [ + 1, + 32, + 32 + ] + }, + "out": { + "bitwise_equal": false, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.4609375, + "mismatch_count": 122609, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 32, + 128 + ] + } + }, + "outcome": "drift_detected", + "passed": true, + "probe": "qk_norm_disabled", + "realization": "Apply or bypass unit-weight PyTorch RMSNorm before the native HIP core.", + "row_id": "A2", + "sequence": 32 + }, + { + "batch": 1, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "implementations": { + "baseline": "rlkernel.rocm.deterministic_attention", + "candidate": "rlkernel.rocm.deterministic_attention" + }, + "metrics": { + "dk": { + "bitwise_equal": false, + "element_count": 32768, + "left_dtype": "torch.bfloat16", + "max_abs": 3.50390625, + "mismatch_count": 32724, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 32, + 128 + ] + }, + "dq": { + "bitwise_equal": false, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 2.78515625, + "mismatch_count": 126509, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 32, + 128 + ] + }, + "dv": { + "bitwise_equal": false, + "element_count": 32768, + "left_dtype": "torch.bfloat16", + "max_abs": 8.6015625, + "mismatch_count": 32713, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 32, + 128 + ] + }, + "lse": { + "bitwise_equal": false, + "element_count": 1024, + "left_dtype": "torch.float32", + "max_abs": 5.7191948890686035, + "mismatch_count": 992, + "right_dtype": "torch.float32", + "shape": [ + 1, + 32, + 32 + ] + }, + "out": { + "bitwise_equal": false, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 3.552734375, + "mismatch_count": 126458, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 32, + 128 + ] + } + }, + "outcome": "drift_detected", + "passed": true, + "probe": "causal_mask", + "realization": "Toggle causal masking in the native HIP core.", + "row_id": "A3", + "sequence": 32 + }, + { + "batch": 1, + "binding_gate": { + "binding_fingerprint": "c0353f09604293a8203cecf2acee5a48b38b055504e47de14ea0cdea6cd20342", + "comparable": false, + "identity_fingerprint": "c2a02818c11bd195873275c75f4d94ae20b0ae2f2e9daa7fcd5f51d9b409cb4c", + "issues": [ + { + "code": "TOPOLOGY_MISMATCH", + "field": "sharding.tp_rank", + "message": "sharding.tp_rank changes TP/CP ownership; the pair is not the same local attention problem", + "rollout": 1, + "tier": "identical", + "training": 0 + }, + { + "code": "TOPOLOGY_MISMATCH", + "field": "sharding.local_q_head_start", + "message": "sharding.local_q_head_start changes TP/CP ownership; the pair is not the same local attention problem", + "rollout": 16, + "tier": "identical", + "training": 0 + }, + { + "code": "TOPOLOGY_MISMATCH", + "field": "sharding.local_kv_head_start", + "message": "sharding.local_kv_head_start changes TP/CP ownership; the pair is not the same local attention problem", + "rollout": 4, + "tier": "identical", + "training": 0 + } + ], + "passed": false, + "provenance": { + "dtype": "bf16", + "lse_domain": "attention", + "rollout": { + "backend_id": "rlkernel.rocm.deterministic_attention", + "contract": { + "batch_size": 1, + "causal": true, + "causal_offsets": [ + 0 + ], + "dtype": "bf16", + "export_lse": true, + "head_dim": 128, + "kv_cache": null, + "lse_domain": "attention", + "mode": "prefill", + "projections": { + "o_proj": { + "acc_dtype": "fp32", + "backend_policy": "native_verified_then_common_deterministic", + "deterministic_backend": "rlkernel.cuda.det_gemm", + "input_dtype": "bf16", + "k_order": "ascending", + "output_dtype": "bf16", + "qkv_split_order": [], + "require_runtime_readback": true, + "sp_backward_collective": "all_gather", + "sp_forward_collective": "reduce_scatter", + "split_kv": "disabled", + "tp_backward_dgrad_collective": "none", + "tp_forward_collective": "all_reduce" + }, + "qkv": { + "acc_dtype": "fp32", + "backend_policy": "native_verified_then_common_deterministic", + "deterministic_backend": "rlkernel.cuda.det_gemm", + "input_dtype": "bf16", + "k_order": "ascending", + "output_dtype": "bf16", + "qkv_split_order": [ + "q", + "k", + "v" + ], + "require_runtime_readback": true, + "sp_backward_collective": "reduce_scatter", + "sp_forward_collective": "all_gather", + "split_kv": "disabled", + "tp_backward_dgrad_collective": "all_reduce", + "tp_forward_collective": "none" + } + }, + "query_sequence_length": 32, + "reduction": { + "acc_dtype": "fp32", + "downcast_at": "final_write", + "engine": "in_op_reference", + "merge": "online_softmax_lse", + "order": "global_block_index" + }, + "role": "infer", + "rope": null, + "semantic_operator": "standard_softmax_attention", + "sharding": { + "cp_rank": 0, + "cp_world_size": 1, + "global_block_indices": [ + 0 + ], + "global_block_token_starts": [ + 0 + ], + "global_kv_heads": 8, + "global_q_heads": 32, + "global_sequence_length": 32, + "local_block_offsets": [ + 0, + 32 + ], + "local_kv_head_start": 4, + "local_kv_heads": 4, + "local_q_head_start": 16, + "local_q_heads": 16, + "local_sequence_length": 32, + "packed_sequence_offsets": null, + "sp_rank": 0, + "sp_world_size": 1, + "tp_rank": 1, + "tp_world_size": 2 + }, + "split_kv": { + "fixed_split_size": null, + "mode": "disabled", + "strict_consistency": true + } + }, + "recorded": { + "backend_id": "rlkernel.rocm.deterministic_attention", + "mode": "prefill", + "reduction.engine": "in_op_reference" + } + }, + "split_kv_runtime": { + "rollout": { + "batch_size": 1, + "coverage": "complete_batch_tp_cp_owner_cartesian_product", + "cp_world_size": 1, + "entries": [ + { + "actual_split_boundaries": [ + [ + 0, + 32 + ] + ], + "actual_split_kv_count": 1, + "actual_split_kv_policy": "disabled", + "actual_split_kv_size": null, + "batch_index": 0, + "cp_rank": 0, + "expected_kv_range": [ + 0, + 32 + ], + "owner_cp_rank": 0, + "requested_split_kv_policy": "disabled", + "requested_split_kv_size": null, + "split_kv_accum_dtype": "fp32", + "split_kv_backend": "rlkernel.rocm.deterministic_attention", + "split_kv_downcast_at": "final_write", + "split_kv_fallback": false, + "split_kv_fallback_reason": null, + "split_kv_merge_order": "global_block_index", + "split_kv_plan_source": "contract_exact", + "tp_rank": 0 + }, + { + "actual_split_boundaries": [ + [ + 0, + 32 + ] + ], + "actual_split_kv_count": 1, + "actual_split_kv_policy": "disabled", + "actual_split_kv_size": null, + "batch_index": 0, + "cp_rank": 0, + "expected_kv_range": [ + 0, + 32 + ], + "owner_cp_rank": 0, + "requested_split_kv_policy": "disabled", + "requested_split_kv_size": null, + "split_kv_accum_dtype": "fp32", + "split_kv_backend": "rlkernel.rocm.deterministic_attention", + "split_kv_downcast_at": "final_write", + "split_kv_fallback": false, + "split_kv_fallback_reason": null, + "split_kv_merge_order": "global_block_index", + "split_kv_plan_source": "contract_exact", + "tp_rank": 1 + } + ], + "total_kv_tokens": [ + 32 + ], + "tp_world_size": 2 + }, + "training": { + "batch_size": 1, + "coverage": "complete_batch_tp_cp_owner_cartesian_product", + "cp_world_size": 1, + "entries": [ + { + "actual_split_boundaries": [ + [ + 0, + 32 + ] + ], + "actual_split_kv_count": 1, + "actual_split_kv_policy": "disabled", + "actual_split_kv_size": null, + "batch_index": 0, + "cp_rank": 0, + "expected_kv_range": [ + 0, + 32 + ], + "owner_cp_rank": 0, + "requested_split_kv_policy": "disabled", + "requested_split_kv_size": null, + "split_kv_accum_dtype": "fp32", + "split_kv_backend": "rlkernel.rocm.deterministic_attention", + "split_kv_downcast_at": "final_write", + "split_kv_fallback": false, + "split_kv_fallback_reason": null, + "split_kv_merge_order": "global_block_index", + "split_kv_plan_source": "contract_exact", + "tp_rank": 0 + }, + { + "actual_split_boundaries": [ + [ + 0, + 32 + ] + ], + "actual_split_kv_count": 1, + "actual_split_kv_policy": "disabled", + "actual_split_kv_size": null, + "batch_index": 0, + "cp_rank": 0, + "expected_kv_range": [ + 0, + 32 + ], + "owner_cp_rank": 0, + "requested_split_kv_policy": "disabled", + "requested_split_kv_size": null, + "split_kv_accum_dtype": "fp32", + "split_kv_backend": "rlkernel.rocm.deterministic_attention", + "split_kv_downcast_at": "final_write", + "split_kv_fallback": false, + "split_kv_fallback_reason": null, + "split_kv_merge_order": "global_block_index", + "split_kv_plan_source": "contract_exact", + "tp_rank": 1 + } + ], + "total_kv_tokens": [ + 32 + ], + "tp_world_size": 2 + } + }, + "training": { + "backend_id": "rlkernel.rocm.deterministic_attention", + "contract": { + "batch_size": 1, + "causal": true, + "causal_offsets": [ + 0 + ], + "dtype": "bf16", + "export_lse": true, + "head_dim": 128, + "kv_cache": null, + "lse_domain": "attention", + "mode": "prefill", + "projections": { + "o_proj": { + "acc_dtype": "fp32", + "backend_policy": "native_verified_then_common_deterministic", + "deterministic_backend": "rlkernel.cuda.det_gemm", + "input_dtype": "bf16", + "k_order": "ascending", + "output_dtype": "bf16", + "qkv_split_order": [], + "require_runtime_readback": true, + "sp_backward_collective": "all_gather", + "sp_forward_collective": "reduce_scatter", + "split_kv": "disabled", + "tp_backward_dgrad_collective": "none", + "tp_forward_collective": "all_reduce" + }, + "qkv": { + "acc_dtype": "fp32", + "backend_policy": "native_verified_then_common_deterministic", + "deterministic_backend": "rlkernel.cuda.det_gemm", + "input_dtype": "bf16", + "k_order": "ascending", + "output_dtype": "bf16", + "qkv_split_order": [ + "q", + "k", + "v" + ], + "require_runtime_readback": true, + "sp_backward_collective": "reduce_scatter", + "sp_forward_collective": "all_gather", + "split_kv": "disabled", + "tp_backward_dgrad_collective": "all_reduce", + "tp_forward_collective": "none" + } + }, + "query_sequence_length": 32, + "reduction": { + "acc_dtype": "fp32", + "downcast_at": "final_write", + "engine": "in_op_reference", + "merge": "online_softmax_lse", + "order": "global_block_index" + }, + "role": "train", + "rope": null, + "semantic_operator": "standard_softmax_attention", + "sharding": { + "cp_rank": 0, + "cp_world_size": 1, + "global_block_indices": [ + 0 + ], + "global_block_token_starts": [ + 0 + ], + "global_kv_heads": 8, + "global_q_heads": 32, + "global_sequence_length": 32, + "local_block_offsets": [ + 0, + 32 + ], + "local_kv_head_start": 0, + "local_kv_heads": 4, + "local_q_head_start": 0, + "local_q_heads": 16, + "local_sequence_length": 32, + "packed_sequence_offsets": null, + "sp_rank": 0, + "sp_world_size": 1, + "tp_rank": 0, + "tp_world_size": 2 + }, + "split_kv": { + "fixed_split_size": null, + "mode": "disabled", + "strict_consistency": true + } + }, + "recorded": { + "backend_id": "rlkernel.rocm.deterministic_attention", + "mode": "prefill", + "reduction.engine": "in_op_reference" + } + } + }, + "recorded_differences": {}, + "reduction_fingerprint": "4bfe65d607de1dc8d1b8c04041d48e9af4e81dc38614f0340d994616da273d6c", + "schema_version": "cross_config.attention_binding.v3" + }, + "category": "comparability_gate", + "comparable": false, + "expected": "rejected", + "gate_implementation": "rl_engine.alignment.cross_config.bind_attention_contracts", + "identity_errors": [ + "sharding.tp_rank", + "sharding.local_q_head_start", + "sharding.local_kv_head_start" + ], + "metrics": { + "dk": null, + "dq": null, + "dv": null, + "lse": null, + "out": null + }, + "outcome": "rejected", + "passed": true, + "probe": "tp_head_ownership", + "realization": "Bind valid TP-rank-1 rollout and TP-rank-0 training contracts; do not run numerics.", + "row_id": "A4", + "sequence": 32 + }, + { + "batch": 1, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "implementations": { + "baseline": "rlkernel.rocm.deterministic_attention", + "candidate": "torch.rocm.index_select+native_attention" + }, + "metrics": { + "dk": { + "bitwise_equal": false, + "element_count": 32768, + "left_dtype": "torch.bfloat16", + "max_abs": 5.302734375, + "mismatch_count": 32737, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 32, + 128 + ] + }, + "dq": { + "bitwise_equal": false, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 3.8125, + "mismatch_count": 122566, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 32, + 128 + ] + }, + "dv": { + "bitwise_equal": false, + "element_count": 32768, + "left_dtype": "torch.bfloat16", + "max_abs": 9.216796875, + "mismatch_count": 32725, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 32, + 128 + ] + }, + "lse": { + "bitwise_equal": false, + "element_count": 1024, + "left_dtype": "torch.float32", + "max_abs": 3.5017497539520264, + "mismatch_count": 992, + "right_dtype": "torch.float32", + "shape": [ + 1, + 32, + 32 + ] + }, + "out": { + "bitwise_equal": false, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 5.34375, + "mismatch_count": 126662, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 32, + 128 + ] + } + }, + "outcome": "drift_detected", + "passed": true, + "probe": "kv_page_order", + "realization": "Reverse four dense K/V tensor pages; this is not a paged-cache runtime.", + "row_id": "A5", + "sequence": 32 + }, + { + "batch": 1, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "implementations": { + "baseline": "torch.rocm.explicit_fp32_serial_qk_accumulator", + "candidate": "torch.rocm.explicit_bf16_serial_qk_accumulator" + }, + "metrics": { + "dk": { + "bitwise_equal": false, + "element_count": 32768, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0625, + "mismatch_count": 28012, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 32, + 128 + ] + }, + "dq": { + "bitwise_equal": false, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.05078125, + "mismatch_count": 102718, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 32, + 128 + ] + }, + "dv": { + "bitwise_equal": false, + "element_count": 32768, + "left_dtype": "torch.bfloat16", + "max_abs": 0.109375, + "mismatch_count": 28291, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 32, + 128 + ] + }, + "lse": { + "bitwise_equal": false, + "element_count": 1024, + "left_dtype": "torch.float32", + "max_abs": 0.058057308197021484, + "mismatch_count": 1024, + "right_dtype": "torch.float32", + "shape": [ + 1, + 32, + 32 + ] + }, + "out": { + "bitwise_equal": false, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.07421875, + "mismatch_count": 102924, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 32, + 128 + ] + } + }, + "outcome": "drift_detected", + "passed": true, + "probe": "accum_dtype", + "realization": "Use identical FP32 products/order with explicit FP32 versus BF16 accumulator state.", + "row_id": "A6", + "sequence": 32 + }, + { + "batch": 1, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "implementations": { + "baseline": "torch.rocm.fp32_chunk_merge_ascending", + "candidate": "torch.rocm.fp32_chunk_merge_descending" + }, + "metrics": { + "dk": { + "bitwise_equal": true, + "element_count": 32768, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 32, + 128 + ] + }, + "dq": { + "bitwise_equal": false, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.001953125, + "mismatch_count": 6, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 32, + 128 + ] + }, + "dv": { + "bitwise_equal": false, + "element_count": 32768, + "left_dtype": "torch.bfloat16", + "max_abs": 0.001953125, + "mismatch_count": 1, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 32, + 128 + ] + }, + "lse": { + "bitwise_equal": false, + "element_count": 1024, + "left_dtype": "torch.float32", + "max_abs": 4.76837158203125e-07, + "mismatch_count": 43, + "right_dtype": "torch.float32", + "shape": [ + 1, + 32, + 32 + ] + }, + "out": { + "bitwise_equal": false, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0009765625, + "mismatch_count": 3, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 32, + 128 + ] + } + }, + "outcome": "drift_detected", + "passed": true, + "probe": "merge_order", + "realization": "Merge four dense chunks in opposite orders on one GPU; this is not a CP collective.", + "row_id": "A7", + "sequence": 32 + }, + { + "batch": 1, + "category": "invariant_control", + "comparable": true, + "expected": "exact_zero", + "implementations": { + "baseline": "rlkernel.rocm.deterministic_attention", + "candidate": "rlkernel.rocm.native_attention_per_tp2_head_shard" + }, + "metrics": { + "dk": { + "bitwise_equal": true, + "element_count": 32768, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 32, + 128 + ] + }, + "dq": { + "bitwise_equal": true, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 32, + 128 + ] + }, + "dv": { + "bitwise_equal": true, + "element_count": 32768, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 32, + 128 + ] + }, + "lse": { + "bitwise_equal": true, + "element_count": 1024, + "left_dtype": "torch.float32", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.float32", + "shape": [ + 1, + 32, + 32 + ] + }, + "out": { + "bitwise_equal": true, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 32, + 128 + ] + } + }, + "outcome": "matched", + "passed": true, + "probe": "tp_partition_control", + "realization": "Compare full GQA with two contiguous TP=2 head shards on one GPU.", + "row_id": "C0", + "sequence": 32 + }, + { + "batch": 1, + "category": "invariant_control", + "comparable": true, + "expected": "exact_zero", + "implementations": { + "baseline": "rlkernel.rocm.deterministic_attention", + "candidate": "rlkernel.rocm.native_attention_per_batch_row" + }, + "metrics": { + "dk": { + "bitwise_equal": true, + "element_count": 32768, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 32, + 128 + ] + }, + "dq": { + "bitwise_equal": true, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 32, + 128 + ] + }, + "dv": { + "bitwise_equal": true, + "element_count": 32768, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 32, + 128 + ] + }, + "lse": { + "bitwise_equal": true, + "element_count": 1024, + "left_dtype": "torch.float32", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.float32", + "shape": [ + 1, + 32, + 32 + ] + }, + "out": { + "bitwise_equal": true, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 32, + 128 + ] + } + }, + "outcome": "matched", + "passed": true, + "probe": "batch_composition_control", + "realization": "Compare a batch call with per-row calls to the same native HIP core.", + "row_id": "C1", + "sequence": 32 + }, + { + "batch": 1, + "category": "invariant_control", + "comparable": true, + "expected": "exact_zero", + "implementations": { + "baseline": "rlkernel.rocm.native_attention_full_prefill", + "candidate": "rlkernel.rocm.native_attention_tail" + }, + "metrics": { + "dk": { + "bitwise_equal": true, + "element_count": 32768, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 32, + 128 + ] + }, + "dq": { + "bitwise_equal": true, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 32, + 128 + ] + }, + "dv": { + "bitwise_equal": true, + "element_count": 32768, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 32, + 128 + ] + }, + "lse": { + "bitwise_equal": true, + "element_count": 32, + "left_dtype": "torch.float32", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.float32", + "shape": [ + 1, + 32, + 1 + ] + }, + "out": { + "bitwise_equal": true, + "element_count": 4096, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 1, + 128 + ] + } + }, + "outcome": "matched", + "passed": true, + "probe": "prefill_decode_tail_control", + "realization": "Compare a full-prefill tail with one trailing query over dense KV, without a serving cache.", + "row_id": "C2", + "sequence": 32 + }, + { + "batch": 1, + "category": "baseline", + "comparable": true, + "expected": "baseline", + "implementations": { + "baseline": "rlkernel.rocm.deterministic_attention", + "candidate": "rlkernel.rocm.deterministic_attention" + }, + "metrics": { + "dk": { + "bitwise_equal": true, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 64, + 128 + ] + }, + "dq": { + "bitwise_equal": true, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 64, + 128 + ] + }, + "dv": { + "bitwise_equal": true, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 64, + 128 + ] + }, + "lse": { + "bitwise_equal": true, + "element_count": 2048, + "left_dtype": "torch.float32", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.float32", + "shape": [ + 1, + 32, + 64 + ] + }, + "out": { + "bitwise_equal": true, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 64, + 128 + ] + } + }, + "outcome": "matched", + "passed": true, + "probe": null, + "realization": "Repeat the identical native HIP reference-core call.", + "row_id": "A0", + "sequence": 64 + }, + { + "batch": 1, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "implementations": { + "baseline": "rlkernel.rocm.deterministic_rope+native_attention", + "candidate": "rlkernel.rocm.deterministic_rope+native_attention" + }, + "metrics": { + "dk": { + "bitwise_equal": false, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 1.470703125, + "mismatch_count": 63492, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 64, + 128 + ] + }, + "dq": { + "bitwise_equal": false, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 1.021484375, + "mismatch_count": 129300, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 64, + 128 + ] + }, + "dv": { + "bitwise_equal": false, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 0.50390625, + "mismatch_count": 62854, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 64, + 128 + ] + }, + "lse": { + "bitwise_equal": false, + "element_count": 2048, + "left_dtype": "torch.float32", + "max_abs": 0.16578149795532227, + "mismatch_count": 1024, + "right_dtype": "torch.float32", + "shape": [ + 1, + 32, + 64 + ] + }, + "out": { + "bitwise_equal": false, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 0.33056640625, + "mismatch_count": 129207, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 64, + 128 + ] + } + }, + "outcome": "drift_detected", + "passed": true, + "probe": "position_ids", + "realization": "Increment suffix RoPE positions while preserving Q/K/V tensors.", + "row_id": "A1", + "sequence": 64 + }, + { + "batch": 1, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "implementations": { + "baseline": "torch.rocm.rms_norm+native_attention", + "candidate": "rlkernel.rocm.deterministic_attention" + }, + "metrics": { + "dk": { + "bitwise_equal": false, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 1.4375, + "mismatch_count": 64945, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 64, + 128 + ] + }, + "dq": { + "bitwise_equal": false, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 0.92578125, + "mismatch_count": 255107, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 64, + 128 + ] + }, + "dv": { + "bitwise_equal": false, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 0.71875, + "mismatch_count": 64219, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 64, + 128 + ] + }, + "lse": { + "bitwise_equal": false, + "element_count": 2048, + "left_dtype": "torch.float32", + "max_abs": 0.41298937797546387, + "mismatch_count": 2048, + "right_dtype": "torch.float32", + "shape": [ + 1, + 32, + 64 + ] + }, + "out": { + "bitwise_equal": false, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 0.41015625, + "mismatch_count": 250177, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 64, + 128 + ] + } + }, + "outcome": "drift_detected", + "passed": true, + "probe": "qk_norm_disabled", + "realization": "Apply or bypass unit-weight PyTorch RMSNorm before the native HIP core.", + "row_id": "A2", + "sequence": 64 + }, + { + "batch": 1, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "implementations": { + "baseline": "rlkernel.rocm.deterministic_attention", + "candidate": "rlkernel.rocm.deterministic_attention" + }, + "metrics": { + "dk": { + "bitwise_equal": false, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 5.03515625, + "mismatch_count": 65433, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 64, + 128 + ] + }, + "dq": { + "bitwise_equal": false, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 3.56640625, + "mismatch_count": 256884, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 64, + 128 + ] + }, + "dv": { + "bitwise_equal": false, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 10.984375, + "mismatch_count": 65453, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 64, + 128 + ] + }, + "lse": { + "bitwise_equal": false, + "element_count": 2048, + "left_dtype": "torch.float32", + "max_abs": 6.717103481292725, + "mismatch_count": 2016, + "right_dtype": "torch.float32", + "shape": [ + 1, + 32, + 64 + ] + }, + "out": { + "bitwise_equal": false, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 3.27001953125, + "mismatch_count": 257021, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 64, + 128 + ] + } + }, + "outcome": "drift_detected", + "passed": true, + "probe": "causal_mask", + "realization": "Toggle causal masking in the native HIP core.", + "row_id": "A3", + "sequence": 64 + }, + { + "batch": 1, + "binding_gate": { + "binding_fingerprint": "291b5e9dd221cd3302bba8fe4c4ab4ae8ac6e45fce4f16ab0d2b5f77a16343ae", + "comparable": false, + "identity_fingerprint": "c2a02818c11bd195873275c75f4d94ae20b0ae2f2e9daa7fcd5f51d9b409cb4c", + "issues": [ + { + "code": "TOPOLOGY_MISMATCH", + "field": "sharding.tp_rank", + "message": "sharding.tp_rank changes TP/CP ownership; the pair is not the same local attention problem", + "rollout": 1, + "tier": "identical", + "training": 0 + }, + { + "code": "TOPOLOGY_MISMATCH", + "field": "sharding.local_q_head_start", + "message": "sharding.local_q_head_start changes TP/CP ownership; the pair is not the same local attention problem", + "rollout": 16, + "tier": "identical", + "training": 0 + }, + { + "code": "TOPOLOGY_MISMATCH", + "field": "sharding.local_kv_head_start", + "message": "sharding.local_kv_head_start changes TP/CP ownership; the pair is not the same local attention problem", + "rollout": 4, + "tier": "identical", + "training": 0 + } + ], + "passed": false, + "provenance": { + "dtype": "bf16", + "lse_domain": "attention", + "rollout": { + "backend_id": "rlkernel.rocm.deterministic_attention", + "contract": { + "batch_size": 1, + "causal": true, + "causal_offsets": [ + 0 + ], + "dtype": "bf16", + "export_lse": true, + "head_dim": 128, + "kv_cache": null, + "lse_domain": "attention", + "mode": "prefill", + "projections": { + "o_proj": { + "acc_dtype": "fp32", + "backend_policy": "native_verified_then_common_deterministic", + "deterministic_backend": "rlkernel.cuda.det_gemm", + "input_dtype": "bf16", + "k_order": "ascending", + "output_dtype": "bf16", + "qkv_split_order": [], + "require_runtime_readback": true, + "sp_backward_collective": "all_gather", + "sp_forward_collective": "reduce_scatter", + "split_kv": "disabled", + "tp_backward_dgrad_collective": "none", + "tp_forward_collective": "all_reduce" + }, + "qkv": { + "acc_dtype": "fp32", + "backend_policy": "native_verified_then_common_deterministic", + "deterministic_backend": "rlkernel.cuda.det_gemm", + "input_dtype": "bf16", + "k_order": "ascending", + "output_dtype": "bf16", + "qkv_split_order": [ + "q", + "k", + "v" + ], + "require_runtime_readback": true, + "sp_backward_collective": "reduce_scatter", + "sp_forward_collective": "all_gather", + "split_kv": "disabled", + "tp_backward_dgrad_collective": "all_reduce", + "tp_forward_collective": "none" + } + }, + "query_sequence_length": 64, + "reduction": { + "acc_dtype": "fp32", + "downcast_at": "final_write", + "engine": "in_op_reference", + "merge": "online_softmax_lse", + "order": "global_block_index" + }, + "role": "infer", + "rope": null, + "semantic_operator": "standard_softmax_attention", + "sharding": { + "cp_rank": 0, + "cp_world_size": 1, + "global_block_indices": [ + 0 + ], + "global_block_token_starts": [ + 0 + ], + "global_kv_heads": 8, + "global_q_heads": 32, + "global_sequence_length": 64, + "local_block_offsets": [ + 0, + 64 + ], + "local_kv_head_start": 4, + "local_kv_heads": 4, + "local_q_head_start": 16, + "local_q_heads": 16, + "local_sequence_length": 64, + "packed_sequence_offsets": null, + "sp_rank": 0, + "sp_world_size": 1, + "tp_rank": 1, + "tp_world_size": 2 + }, + "split_kv": { + "fixed_split_size": null, + "mode": "disabled", + "strict_consistency": true + } + }, + "recorded": { + "backend_id": "rlkernel.rocm.deterministic_attention", + "mode": "prefill", + "reduction.engine": "in_op_reference" + } + }, + "split_kv_runtime": { + "rollout": { + "batch_size": 1, + "coverage": "complete_batch_tp_cp_owner_cartesian_product", + "cp_world_size": 1, + "entries": [ + { + "actual_split_boundaries": [ + [ + 0, + 64 + ] + ], + "actual_split_kv_count": 1, + "actual_split_kv_policy": "disabled", + "actual_split_kv_size": null, + "batch_index": 0, + "cp_rank": 0, + "expected_kv_range": [ + 0, + 64 + ], + "owner_cp_rank": 0, + "requested_split_kv_policy": "disabled", + "requested_split_kv_size": null, + "split_kv_accum_dtype": "fp32", + "split_kv_backend": "rlkernel.rocm.deterministic_attention", + "split_kv_downcast_at": "final_write", + "split_kv_fallback": false, + "split_kv_fallback_reason": null, + "split_kv_merge_order": "global_block_index", + "split_kv_plan_source": "contract_exact", + "tp_rank": 0 + }, + { + "actual_split_boundaries": [ + [ + 0, + 64 + ] + ], + "actual_split_kv_count": 1, + "actual_split_kv_policy": "disabled", + "actual_split_kv_size": null, + "batch_index": 0, + "cp_rank": 0, + "expected_kv_range": [ + 0, + 64 + ], + "owner_cp_rank": 0, + "requested_split_kv_policy": "disabled", + "requested_split_kv_size": null, + "split_kv_accum_dtype": "fp32", + "split_kv_backend": "rlkernel.rocm.deterministic_attention", + "split_kv_downcast_at": "final_write", + "split_kv_fallback": false, + "split_kv_fallback_reason": null, + "split_kv_merge_order": "global_block_index", + "split_kv_plan_source": "contract_exact", + "tp_rank": 1 + } + ], + "total_kv_tokens": [ + 64 + ], + "tp_world_size": 2 + }, + "training": { + "batch_size": 1, + "coverage": "complete_batch_tp_cp_owner_cartesian_product", + "cp_world_size": 1, + "entries": [ + { + "actual_split_boundaries": [ + [ + 0, + 64 + ] + ], + "actual_split_kv_count": 1, + "actual_split_kv_policy": "disabled", + "actual_split_kv_size": null, + "batch_index": 0, + "cp_rank": 0, + "expected_kv_range": [ + 0, + 64 + ], + "owner_cp_rank": 0, + "requested_split_kv_policy": "disabled", + "requested_split_kv_size": null, + "split_kv_accum_dtype": "fp32", + "split_kv_backend": "rlkernel.rocm.deterministic_attention", + "split_kv_downcast_at": "final_write", + "split_kv_fallback": false, + "split_kv_fallback_reason": null, + "split_kv_merge_order": "global_block_index", + "split_kv_plan_source": "contract_exact", + "tp_rank": 0 + }, + { + "actual_split_boundaries": [ + [ + 0, + 64 + ] + ], + "actual_split_kv_count": 1, + "actual_split_kv_policy": "disabled", + "actual_split_kv_size": null, + "batch_index": 0, + "cp_rank": 0, + "expected_kv_range": [ + 0, + 64 + ], + "owner_cp_rank": 0, + "requested_split_kv_policy": "disabled", + "requested_split_kv_size": null, + "split_kv_accum_dtype": "fp32", + "split_kv_backend": "rlkernel.rocm.deterministic_attention", + "split_kv_downcast_at": "final_write", + "split_kv_fallback": false, + "split_kv_fallback_reason": null, + "split_kv_merge_order": "global_block_index", + "split_kv_plan_source": "contract_exact", + "tp_rank": 1 + } + ], + "total_kv_tokens": [ + 64 + ], + "tp_world_size": 2 + } + }, + "training": { + "backend_id": "rlkernel.rocm.deterministic_attention", + "contract": { + "batch_size": 1, + "causal": true, + "causal_offsets": [ + 0 + ], + "dtype": "bf16", + "export_lse": true, + "head_dim": 128, + "kv_cache": null, + "lse_domain": "attention", + "mode": "prefill", + "projections": { + "o_proj": { + "acc_dtype": "fp32", + "backend_policy": "native_verified_then_common_deterministic", + "deterministic_backend": "rlkernel.cuda.det_gemm", + "input_dtype": "bf16", + "k_order": "ascending", + "output_dtype": "bf16", + "qkv_split_order": [], + "require_runtime_readback": true, + "sp_backward_collective": "all_gather", + "sp_forward_collective": "reduce_scatter", + "split_kv": "disabled", + "tp_backward_dgrad_collective": "none", + "tp_forward_collective": "all_reduce" + }, + "qkv": { + "acc_dtype": "fp32", + "backend_policy": "native_verified_then_common_deterministic", + "deterministic_backend": "rlkernel.cuda.det_gemm", + "input_dtype": "bf16", + "k_order": "ascending", + "output_dtype": "bf16", + "qkv_split_order": [ + "q", + "k", + "v" + ], + "require_runtime_readback": true, + "sp_backward_collective": "reduce_scatter", + "sp_forward_collective": "all_gather", + "split_kv": "disabled", + "tp_backward_dgrad_collective": "all_reduce", + "tp_forward_collective": "none" + } + }, + "query_sequence_length": 64, + "reduction": { + "acc_dtype": "fp32", + "downcast_at": "final_write", + "engine": "in_op_reference", + "merge": "online_softmax_lse", + "order": "global_block_index" + }, + "role": "train", + "rope": null, + "semantic_operator": "standard_softmax_attention", + "sharding": { + "cp_rank": 0, + "cp_world_size": 1, + "global_block_indices": [ + 0 + ], + "global_block_token_starts": [ + 0 + ], + "global_kv_heads": 8, + "global_q_heads": 32, + "global_sequence_length": 64, + "local_block_offsets": [ + 0, + 64 + ], + "local_kv_head_start": 0, + "local_kv_heads": 4, + "local_q_head_start": 0, + "local_q_heads": 16, + "local_sequence_length": 64, + "packed_sequence_offsets": null, + "sp_rank": 0, + "sp_world_size": 1, + "tp_rank": 0, + "tp_world_size": 2 + }, + "split_kv": { + "fixed_split_size": null, + "mode": "disabled", + "strict_consistency": true + } + }, + "recorded": { + "backend_id": "rlkernel.rocm.deterministic_attention", + "mode": "prefill", + "reduction.engine": "in_op_reference" + } + } + }, + "recorded_differences": {}, + "reduction_fingerprint": "4bfe65d607de1dc8d1b8c04041d48e9af4e81dc38614f0340d994616da273d6c", + "schema_version": "cross_config.attention_binding.v3" + }, + "category": "comparability_gate", + "comparable": false, + "expected": "rejected", + "gate_implementation": "rl_engine.alignment.cross_config.bind_attention_contracts", + "identity_errors": [ + "sharding.tp_rank", + "sharding.local_q_head_start", + "sharding.local_kv_head_start" + ], + "metrics": { + "dk": null, + "dq": null, + "dv": null, + "lse": null, + "out": null + }, + "outcome": "rejected", + "passed": true, + "probe": "tp_head_ownership", + "realization": "Bind valid TP-rank-1 rollout and TP-rank-0 training contracts; do not run numerics.", + "row_id": "A4", + "sequence": 64 + }, + { + "batch": 1, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "implementations": { + "baseline": "rlkernel.rocm.deterministic_attention", + "candidate": "torch.rocm.index_select+native_attention" + }, + "metrics": { + "dk": { + "bitwise_equal": false, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 5.845947265625, + "mismatch_count": 65477, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 64, + 128 + ] + }, + "dq": { + "bitwise_equal": false, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 3.6943359375, + "mismatch_count": 253214, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 64, + 128 + ] + }, + "dv": { + "bitwise_equal": false, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 11.5634765625, + "mismatch_count": 65485, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 64, + 128 + ] + }, + "lse": { + "bitwise_equal": false, + "element_count": 2048, + "left_dtype": "torch.float32", + "max_abs": 3.160780906677246, + "mismatch_count": 2016, + "right_dtype": "torch.float32", + "shape": [ + 1, + 32, + 64 + ] + }, + "out": { + "bitwise_equal": false, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 6.0, + "mismatch_count": 257518, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 64, + 128 + ] + } + }, + "outcome": "drift_detected", + "passed": true, + "probe": "kv_page_order", + "realization": "Reverse four dense K/V tensor pages; this is not a paged-cache runtime.", + "row_id": "A5", + "sequence": 64 + }, + { + "batch": 1, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "implementations": { + "baseline": "torch.rocm.explicit_fp32_serial_qk_accumulator", + "candidate": "torch.rocm.explicit_bf16_serial_qk_accumulator" + }, + "metrics": { + "dk": { + "bitwise_equal": false, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 0.09375, + "mismatch_count": 57441, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 64, + 128 + ] + }, + "dq": { + "bitwise_equal": false, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0546875, + "mismatch_count": 218015, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 64, + 128 + ] + }, + "dv": { + "bitwise_equal": false, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 0.1015625, + "mismatch_count": 58082, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 64, + 128 + ] + }, + "lse": { + "bitwise_equal": false, + "element_count": 2048, + "left_dtype": "torch.float32", + "max_abs": 0.09950780868530273, + "mismatch_count": 2048, + "right_dtype": "torch.float32", + "shape": [ + 1, + 32, + 64 + ] + }, + "out": { + "bitwise_equal": false, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0859375, + "mismatch_count": 219875, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 64, + 128 + ] + } + }, + "outcome": "drift_detected", + "passed": true, + "probe": "accum_dtype", + "realization": "Use identical FP32 products/order with explicit FP32 versus BF16 accumulator state.", + "row_id": "A6", + "sequence": 64 + }, + { + "batch": 1, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "implementations": { + "baseline": "torch.rocm.fp32_chunk_merge_ascending", + "candidate": "torch.rocm.fp32_chunk_merge_descending" + }, + "metrics": { + "dk": { + "bitwise_equal": false, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 0.00048828125, + "mismatch_count": 2, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 64, + 128 + ] + }, + "dq": { + "bitwise_equal": false, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0009765625, + "mismatch_count": 10, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 64, + 128 + ] + }, + "dv": { + "bitwise_equal": true, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 64, + 128 + ] + }, + "lse": { + "bitwise_equal": false, + "element_count": 2048, + "left_dtype": "torch.float32", + "max_abs": 4.76837158203125e-07, + "mismatch_count": 62, + "right_dtype": "torch.float32", + "shape": [ + 1, + 32, + 64 + ] + }, + "out": { + "bitwise_equal": false, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0009765625, + "mismatch_count": 5, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 64, + 128 + ] + } + }, + "outcome": "drift_detected", + "passed": true, + "probe": "merge_order", + "realization": "Merge four dense chunks in opposite orders on one GPU; this is not a CP collective.", + "row_id": "A7", + "sequence": 64 + }, + { + "batch": 1, + "category": "invariant_control", + "comparable": true, + "expected": "exact_zero", + "implementations": { + "baseline": "rlkernel.rocm.deterministic_attention", + "candidate": "rlkernel.rocm.native_attention_per_tp2_head_shard" + }, + "metrics": { + "dk": { + "bitwise_equal": true, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 64, + 128 + ] + }, + "dq": { + "bitwise_equal": true, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 64, + 128 + ] + }, + "dv": { + "bitwise_equal": true, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 64, + 128 + ] + }, + "lse": { + "bitwise_equal": true, + "element_count": 2048, + "left_dtype": "torch.float32", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.float32", + "shape": [ + 1, + 32, + 64 + ] + }, + "out": { + "bitwise_equal": true, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 64, + 128 + ] + } + }, + "outcome": "matched", + "passed": true, + "probe": "tp_partition_control", + "realization": "Compare full GQA with two contiguous TP=2 head shards on one GPU.", + "row_id": "C0", + "sequence": 64 + }, + { + "batch": 1, + "category": "invariant_control", + "comparable": true, + "expected": "exact_zero", + "implementations": { + "baseline": "rlkernel.rocm.deterministic_attention", + "candidate": "rlkernel.rocm.native_attention_per_batch_row" + }, + "metrics": { + "dk": { + "bitwise_equal": true, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 64, + 128 + ] + }, + "dq": { + "bitwise_equal": true, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 64, + 128 + ] + }, + "dv": { + "bitwise_equal": true, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 64, + 128 + ] + }, + "lse": { + "bitwise_equal": true, + "element_count": 2048, + "left_dtype": "torch.float32", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.float32", + "shape": [ + 1, + 32, + 64 + ] + }, + "out": { + "bitwise_equal": true, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 64, + 128 + ] + } + }, + "outcome": "matched", + "passed": true, + "probe": "batch_composition_control", + "realization": "Compare a batch call with per-row calls to the same native HIP core.", + "row_id": "C1", + "sequence": 64 + }, + { + "batch": 1, + "category": "invariant_control", + "comparable": true, + "expected": "exact_zero", + "implementations": { + "baseline": "rlkernel.rocm.native_attention_full_prefill", + "candidate": "rlkernel.rocm.native_attention_tail" + }, + "metrics": { + "dk": { + "bitwise_equal": true, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 64, + 128 + ] + }, + "dq": { + "bitwise_equal": true, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 64, + 128 + ] + }, + "dv": { + "bitwise_equal": true, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 64, + 128 + ] + }, + "lse": { + "bitwise_equal": true, + "element_count": 32, + "left_dtype": "torch.float32", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.float32", + "shape": [ + 1, + 32, + 1 + ] + }, + "out": { + "bitwise_equal": true, + "element_count": 4096, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 1, + 128 + ] + } + }, + "outcome": "matched", + "passed": true, + "probe": "prefill_decode_tail_control", + "realization": "Compare a full-prefill tail with one trailing query over dense KV, without a serving cache.", + "row_id": "C2", + "sequence": 64 + }, + { + "batch": 1, + "category": "baseline", + "comparable": true, + "expected": "baseline", + "implementations": { + "baseline": "rlkernel.rocm.deterministic_attention", + "candidate": "rlkernel.rocm.deterministic_attention" + }, + "metrics": { + "dk": { + "bitwise_equal": true, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 128, + 128 + ] + }, + "dq": { + "bitwise_equal": true, + "element_count": 524288, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 128, + 128 + ] + }, + "dv": { + "bitwise_equal": true, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 128, + 128 + ] + }, + "lse": { + "bitwise_equal": true, + "element_count": 4096, + "left_dtype": "torch.float32", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.float32", + "shape": [ + 1, + 32, + 128 + ] + }, + "out": { + "bitwise_equal": true, + "element_count": 524288, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 128, + 128 + ] + } + }, + "outcome": "matched", + "passed": true, + "probe": null, + "realization": "Repeat the identical native HIP reference-core call.", + "row_id": "A0", + "sequence": 128 + }, + { + "batch": 1, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "implementations": { + "baseline": "rlkernel.rocm.deterministic_rope+native_attention", + "candidate": "rlkernel.rocm.deterministic_rope+native_attention" + }, + "metrics": { + "dk": { + "bitwise_equal": false, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 1.66015625, + "mismatch_count": 126148, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 128, + 128 + ] + }, + "dq": { + "bitwise_equal": false, + "element_count": 524288, + "left_dtype": "torch.bfloat16", + "max_abs": 0.8125, + "mismatch_count": 258977, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 128, + 128 + ] + }, + "dv": { + "bitwise_equal": false, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.4501953125, + "mismatch_count": 124473, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 128, + 128 + ] + }, + "lse": { + "bitwise_equal": false, + "element_count": 4096, + "left_dtype": "torch.float32", + "max_abs": 0.1391582489013672, + "mismatch_count": 2048, + "right_dtype": "torch.float32", + "shape": [ + 1, + 32, + 128 + ] + }, + "out": { + "bitwise_equal": false, + "element_count": 524288, + "left_dtype": "torch.bfloat16", + "max_abs": 0.41015625, + "mismatch_count": 258687, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 128, + 128 + ] + } + }, + "outcome": "drift_detected", + "passed": true, + "probe": "position_ids", + "realization": "Increment suffix RoPE positions while preserving Q/K/V tensors.", + "row_id": "A1", + "sequence": 128 + }, + { + "batch": 1, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "implementations": { + "baseline": "torch.rocm.rms_norm+native_attention", + "candidate": "rlkernel.rocm.deterministic_attention" + }, + "metrics": { + "dk": { + "bitwise_equal": false, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 1.453125, + "mismatch_count": 129998, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 128, + 128 + ] + }, + "dq": { + "bitwise_equal": false, + "element_count": 524288, + "left_dtype": "torch.bfloat16", + "max_abs": 0.88671875, + "mismatch_count": 514619, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 128, + 128 + ] + }, + "dv": { + "bitwise_equal": false, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.6484375, + "mismatch_count": 128904, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 128, + 128 + ] + }, + "lse": { + "bitwise_equal": false, + "element_count": 4096, + "left_dtype": "torch.float32", + "max_abs": 0.5336203575134277, + "mismatch_count": 4096, + "right_dtype": "torch.float32", + "shape": [ + 1, + 32, + 128 + ] + }, + "out": { + "bitwise_equal": false, + "element_count": 524288, + "left_dtype": "torch.bfloat16", + "max_abs": 0.38671875, + "mismatch_count": 507978, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 128, + 128 + ] + } + }, + "outcome": "drift_detected", + "passed": true, + "probe": "qk_norm_disabled", + "realization": "Apply or bypass unit-weight PyTorch RMSNorm before the native HIP core.", + "row_id": "A2", + "sequence": 128 + }, + { + "batch": 1, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "implementations": { + "baseline": "rlkernel.rocm.deterministic_attention", + "candidate": "rlkernel.rocm.deterministic_attention" + }, + "metrics": { + "dk": { + "bitwise_equal": false, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 6.15625, + "mismatch_count": 130889, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 128, + 128 + ] + }, + "dq": { + "bitwise_equal": false, + "element_count": 524288, + "left_dtype": "torch.bfloat16", + "max_abs": 3.3115234375, + "mismatch_count": 517813, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 128, + 128 + ] + }, + "dv": { + "bitwise_equal": false, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 9.1328125, + "mismatch_count": 130890, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 128, + 128 + ] + }, + "lse": { + "bitwise_equal": false, + "element_count": 4096, + "left_dtype": "torch.float32", + "max_abs": 7.67181396484375, + "mismatch_count": 4064, + "right_dtype": "torch.float32", + "shape": [ + 1, + 32, + 128 + ] + }, + "out": { + "bitwise_equal": false, + "element_count": 524288, + "left_dtype": "torch.bfloat16", + "max_abs": 3.380859375, + "mismatch_count": 518285, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 128, + 128 + ] + } + }, + "outcome": "drift_detected", + "passed": true, + "probe": "causal_mask", + "realization": "Toggle causal masking in the native HIP core.", + "row_id": "A3", + "sequence": 128 + }, + { + "batch": 1, + "binding_gate": { + "binding_fingerprint": "19a5c00a3fd47a426313eb74c50b43516adc1cba7c676476dd12ea0eca84d5a9", + "comparable": false, + "identity_fingerprint": "c2a02818c11bd195873275c75f4d94ae20b0ae2f2e9daa7fcd5f51d9b409cb4c", + "issues": [ + { + "code": "TOPOLOGY_MISMATCH", + "field": "sharding.tp_rank", + "message": "sharding.tp_rank changes TP/CP ownership; the pair is not the same local attention problem", + "rollout": 1, + "tier": "identical", + "training": 0 + }, + { + "code": "TOPOLOGY_MISMATCH", + "field": "sharding.local_q_head_start", + "message": "sharding.local_q_head_start changes TP/CP ownership; the pair is not the same local attention problem", + "rollout": 16, + "tier": "identical", + "training": 0 + }, + { + "code": "TOPOLOGY_MISMATCH", + "field": "sharding.local_kv_head_start", + "message": "sharding.local_kv_head_start changes TP/CP ownership; the pair is not the same local attention problem", + "rollout": 4, + "tier": "identical", + "training": 0 + } + ], + "passed": false, + "provenance": { + "dtype": "bf16", + "lse_domain": "attention", + "rollout": { + "backend_id": "rlkernel.rocm.deterministic_attention", + "contract": { + "batch_size": 1, + "causal": true, + "causal_offsets": [ + 0 + ], + "dtype": "bf16", + "export_lse": true, + "head_dim": 128, + "kv_cache": null, + "lse_domain": "attention", + "mode": "prefill", + "projections": { + "o_proj": { + "acc_dtype": "fp32", + "backend_policy": "native_verified_then_common_deterministic", + "deterministic_backend": "rlkernel.cuda.det_gemm", + "input_dtype": "bf16", + "k_order": "ascending", + "output_dtype": "bf16", + "qkv_split_order": [], + "require_runtime_readback": true, + "sp_backward_collective": "all_gather", + "sp_forward_collective": "reduce_scatter", + "split_kv": "disabled", + "tp_backward_dgrad_collective": "none", + "tp_forward_collective": "all_reduce" + }, + "qkv": { + "acc_dtype": "fp32", + "backend_policy": "native_verified_then_common_deterministic", + "deterministic_backend": "rlkernel.cuda.det_gemm", + "input_dtype": "bf16", + "k_order": "ascending", + "output_dtype": "bf16", + "qkv_split_order": [ + "q", + "k", + "v" + ], + "require_runtime_readback": true, + "sp_backward_collective": "reduce_scatter", + "sp_forward_collective": "all_gather", + "split_kv": "disabled", + "tp_backward_dgrad_collective": "all_reduce", + "tp_forward_collective": "none" + } + }, + "query_sequence_length": 128, + "reduction": { + "acc_dtype": "fp32", + "downcast_at": "final_write", + "engine": "in_op_reference", + "merge": "online_softmax_lse", + "order": "global_block_index" + }, + "role": "infer", + "rope": null, + "semantic_operator": "standard_softmax_attention", + "sharding": { + "cp_rank": 0, + "cp_world_size": 1, + "global_block_indices": [ + 0 + ], + "global_block_token_starts": [ + 0 + ], + "global_kv_heads": 8, + "global_q_heads": 32, + "global_sequence_length": 128, + "local_block_offsets": [ + 0, + 128 + ], + "local_kv_head_start": 4, + "local_kv_heads": 4, + "local_q_head_start": 16, + "local_q_heads": 16, + "local_sequence_length": 128, + "packed_sequence_offsets": null, + "sp_rank": 0, + "sp_world_size": 1, + "tp_rank": 1, + "tp_world_size": 2 + }, + "split_kv": { + "fixed_split_size": null, + "mode": "disabled", + "strict_consistency": true + } + }, + "recorded": { + "backend_id": "rlkernel.rocm.deterministic_attention", + "mode": "prefill", + "reduction.engine": "in_op_reference" + } + }, + "split_kv_runtime": { + "rollout": { + "batch_size": 1, + "coverage": "complete_batch_tp_cp_owner_cartesian_product", + "cp_world_size": 1, + "entries": [ + { + "actual_split_boundaries": [ + [ + 0, + 128 + ] + ], + "actual_split_kv_count": 1, + "actual_split_kv_policy": "disabled", + "actual_split_kv_size": null, + "batch_index": 0, + "cp_rank": 0, + "expected_kv_range": [ + 0, + 128 + ], + "owner_cp_rank": 0, + "requested_split_kv_policy": "disabled", + "requested_split_kv_size": null, + "split_kv_accum_dtype": "fp32", + "split_kv_backend": "rlkernel.rocm.deterministic_attention", + "split_kv_downcast_at": "final_write", + "split_kv_fallback": false, + "split_kv_fallback_reason": null, + "split_kv_merge_order": "global_block_index", + "split_kv_plan_source": "contract_exact", + "tp_rank": 0 + }, + { + "actual_split_boundaries": [ + [ + 0, + 128 + ] + ], + "actual_split_kv_count": 1, + "actual_split_kv_policy": "disabled", + "actual_split_kv_size": null, + "batch_index": 0, + "cp_rank": 0, + "expected_kv_range": [ + 0, + 128 + ], + "owner_cp_rank": 0, + "requested_split_kv_policy": "disabled", + "requested_split_kv_size": null, + "split_kv_accum_dtype": "fp32", + "split_kv_backend": "rlkernel.rocm.deterministic_attention", + "split_kv_downcast_at": "final_write", + "split_kv_fallback": false, + "split_kv_fallback_reason": null, + "split_kv_merge_order": "global_block_index", + "split_kv_plan_source": "contract_exact", + "tp_rank": 1 + } + ], + "total_kv_tokens": [ + 128 + ], + "tp_world_size": 2 + }, + "training": { + "batch_size": 1, + "coverage": "complete_batch_tp_cp_owner_cartesian_product", + "cp_world_size": 1, + "entries": [ + { + "actual_split_boundaries": [ + [ + 0, + 128 + ] + ], + "actual_split_kv_count": 1, + "actual_split_kv_policy": "disabled", + "actual_split_kv_size": null, + "batch_index": 0, + "cp_rank": 0, + "expected_kv_range": [ + 0, + 128 + ], + "owner_cp_rank": 0, + "requested_split_kv_policy": "disabled", + "requested_split_kv_size": null, + "split_kv_accum_dtype": "fp32", + "split_kv_backend": "rlkernel.rocm.deterministic_attention", + "split_kv_downcast_at": "final_write", + "split_kv_fallback": false, + "split_kv_fallback_reason": null, + "split_kv_merge_order": "global_block_index", + "split_kv_plan_source": "contract_exact", + "tp_rank": 0 + }, + { + "actual_split_boundaries": [ + [ + 0, + 128 + ] + ], + "actual_split_kv_count": 1, + "actual_split_kv_policy": "disabled", + "actual_split_kv_size": null, + "batch_index": 0, + "cp_rank": 0, + "expected_kv_range": [ + 0, + 128 + ], + "owner_cp_rank": 0, + "requested_split_kv_policy": "disabled", + "requested_split_kv_size": null, + "split_kv_accum_dtype": "fp32", + "split_kv_backend": "rlkernel.rocm.deterministic_attention", + "split_kv_downcast_at": "final_write", + "split_kv_fallback": false, + "split_kv_fallback_reason": null, + "split_kv_merge_order": "global_block_index", + "split_kv_plan_source": "contract_exact", + "tp_rank": 1 + } + ], + "total_kv_tokens": [ + 128 + ], + "tp_world_size": 2 + } + }, + "training": { + "backend_id": "rlkernel.rocm.deterministic_attention", + "contract": { + "batch_size": 1, + "causal": true, + "causal_offsets": [ + 0 + ], + "dtype": "bf16", + "export_lse": true, + "head_dim": 128, + "kv_cache": null, + "lse_domain": "attention", + "mode": "prefill", + "projections": { + "o_proj": { + "acc_dtype": "fp32", + "backend_policy": "native_verified_then_common_deterministic", + "deterministic_backend": "rlkernel.cuda.det_gemm", + "input_dtype": "bf16", + "k_order": "ascending", + "output_dtype": "bf16", + "qkv_split_order": [], + "require_runtime_readback": true, + "sp_backward_collective": "all_gather", + "sp_forward_collective": "reduce_scatter", + "split_kv": "disabled", + "tp_backward_dgrad_collective": "none", + "tp_forward_collective": "all_reduce" + }, + "qkv": { + "acc_dtype": "fp32", + "backend_policy": "native_verified_then_common_deterministic", + "deterministic_backend": "rlkernel.cuda.det_gemm", + "input_dtype": "bf16", + "k_order": "ascending", + "output_dtype": "bf16", + "qkv_split_order": [ + "q", + "k", + "v" + ], + "require_runtime_readback": true, + "sp_backward_collective": "reduce_scatter", + "sp_forward_collective": "all_gather", + "split_kv": "disabled", + "tp_backward_dgrad_collective": "all_reduce", + "tp_forward_collective": "none" + } + }, + "query_sequence_length": 128, + "reduction": { + "acc_dtype": "fp32", + "downcast_at": "final_write", + "engine": "in_op_reference", + "merge": "online_softmax_lse", + "order": "global_block_index" + }, + "role": "train", + "rope": null, + "semantic_operator": "standard_softmax_attention", + "sharding": { + "cp_rank": 0, + "cp_world_size": 1, + "global_block_indices": [ + 0 + ], + "global_block_token_starts": [ + 0 + ], + "global_kv_heads": 8, + "global_q_heads": 32, + "global_sequence_length": 128, + "local_block_offsets": [ + 0, + 128 + ], + "local_kv_head_start": 0, + "local_kv_heads": 4, + "local_q_head_start": 0, + "local_q_heads": 16, + "local_sequence_length": 128, + "packed_sequence_offsets": null, + "sp_rank": 0, + "sp_world_size": 1, + "tp_rank": 0, + "tp_world_size": 2 + }, + "split_kv": { + "fixed_split_size": null, + "mode": "disabled", + "strict_consistency": true + } + }, + "recorded": { + "backend_id": "rlkernel.rocm.deterministic_attention", + "mode": "prefill", + "reduction.engine": "in_op_reference" + } + } + }, + "recorded_differences": {}, + "reduction_fingerprint": "4bfe65d607de1dc8d1b8c04041d48e9af4e81dc38614f0340d994616da273d6c", + "schema_version": "cross_config.attention_binding.v3" + }, + "category": "comparability_gate", + "comparable": false, + "expected": "rejected", + "gate_implementation": "rl_engine.alignment.cross_config.bind_attention_contracts", + "identity_errors": [ + "sharding.tp_rank", + "sharding.local_q_head_start", + "sharding.local_kv_head_start" + ], + "metrics": { + "dk": null, + "dq": null, + "dv": null, + "lse": null, + "out": null + }, + "outcome": "rejected", + "passed": true, + "probe": "tp_head_ownership", + "realization": "Bind valid TP-rank-1 rollout and TP-rank-0 training contracts; do not run numerics.", + "row_id": "A4", + "sequence": 128 + }, + { + "batch": 1, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "implementations": { + "baseline": "rlkernel.rocm.deterministic_attention", + "candidate": "torch.rocm.index_select+native_attention" + }, + "metrics": { + "dk": { + "bitwise_equal": false, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 6.62744140625, + "mismatch_count": 130978, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 128, + 128 + ] + }, + "dq": { + "bitwise_equal": false, + "element_count": 524288, + "left_dtype": "torch.bfloat16", + "max_abs": 3.5849609375, + "mismatch_count": 514734, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 128, + 128 + ] + }, + "dv": { + "bitwise_equal": false, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 9.7392578125, + "mismatch_count": 130970, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 128, + 128 + ] + }, + "lse": { + "bitwise_equal": false, + "element_count": 4096, + "left_dtype": "torch.float32", + "max_abs": 3.14506196975708, + "mismatch_count": 4064, + "right_dtype": "torch.float32", + "shape": [ + 1, + 32, + 128 + ] + }, + "out": { + "bitwise_equal": false, + "element_count": 524288, + "left_dtype": "torch.bfloat16", + "max_abs": 4.75, + "mismatch_count": 519002, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 128, + 128 + ] + } + }, + "outcome": "drift_detected", + "passed": true, + "probe": "kv_page_order", + "realization": "Reverse four dense K/V tensor pages; this is not a paged-cache runtime.", + "row_id": "A5", + "sequence": 128 + }, + { + "batch": 1, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "implementations": { + "baseline": "torch.rocm.explicit_fp32_serial_qk_accumulator", + "candidate": "torch.rocm.explicit_bf16_serial_qk_accumulator" + }, + "metrics": { + "dk": { + "bitwise_equal": false, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0625, + "mismatch_count": 116599, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 128, + 128 + ] + }, + "dq": { + "bitwise_equal": false, + "element_count": 524288, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0546875, + "mismatch_count": 450629, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 128, + 128 + ] + }, + "dv": { + "bitwise_equal": false, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.09375, + "mismatch_count": 117447, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 128, + 128 + ] + }, + "lse": { + "bitwise_equal": false, + "element_count": 4096, + "left_dtype": "torch.float32", + "max_abs": 0.05309700965881348, + "mismatch_count": 4096, + "right_dtype": "torch.float32", + "shape": [ + 1, + 32, + 128 + ] + }, + "out": { + "bitwise_equal": false, + "element_count": 524288, + "left_dtype": "torch.bfloat16", + "max_abs": 0.09375, + "mismatch_count": 455338, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 128, + 128 + ] + } + }, + "outcome": "drift_detected", + "passed": true, + "probe": "accum_dtype", + "realization": "Use identical FP32 products/order with explicit FP32 versus BF16 accumulator state.", + "row_id": "A6", + "sequence": 128 + }, + { + "batch": 1, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "implementations": { + "baseline": "torch.rocm.fp32_chunk_merge_ascending", + "candidate": "torch.rocm.fp32_chunk_merge_descending" + }, + "metrics": { + "dk": { + "bitwise_equal": false, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.00390625, + "mismatch_count": 17, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 128, + 128 + ] + }, + "dq": { + "bitwise_equal": false, + "element_count": 524288, + "left_dtype": "torch.bfloat16", + "max_abs": 0.001953125, + "mismatch_count": 21, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 128, + 128 + ] + }, + "dv": { + "bitwise_equal": false, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.00390625, + "mismatch_count": 9, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 128, + 128 + ] + }, + "lse": { + "bitwise_equal": false, + "element_count": 4096, + "left_dtype": "torch.float32", + "max_abs": 9.5367431640625e-07, + "mismatch_count": 122, + "right_dtype": "torch.float32", + "shape": [ + 1, + 32, + 128 + ] + }, + "out": { + "bitwise_equal": false, + "element_count": 524288, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0009765625, + "mismatch_count": 12, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 128, + 128 + ] + } + }, + "outcome": "drift_detected", + "passed": true, + "probe": "merge_order", + "realization": "Merge four dense chunks in opposite orders on one GPU; this is not a CP collective.", + "row_id": "A7", + "sequence": 128 + }, + { + "batch": 1, + "category": "invariant_control", + "comparable": true, + "expected": "exact_zero", + "implementations": { + "baseline": "rlkernel.rocm.deterministic_attention", + "candidate": "rlkernel.rocm.native_attention_per_tp2_head_shard" + }, + "metrics": { + "dk": { + "bitwise_equal": true, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 128, + 128 + ] + }, + "dq": { + "bitwise_equal": true, + "element_count": 524288, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 128, + 128 + ] + }, + "dv": { + "bitwise_equal": true, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 128, + 128 + ] + }, + "lse": { + "bitwise_equal": true, + "element_count": 4096, + "left_dtype": "torch.float32", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.float32", + "shape": [ + 1, + 32, + 128 + ] + }, + "out": { + "bitwise_equal": true, + "element_count": 524288, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 128, + 128 + ] + } + }, + "outcome": "matched", + "passed": true, + "probe": "tp_partition_control", + "realization": "Compare full GQA with two contiguous TP=2 head shards on one GPU.", + "row_id": "C0", + "sequence": 128 + }, + { + "batch": 1, + "category": "invariant_control", + "comparable": true, + "expected": "exact_zero", + "implementations": { + "baseline": "rlkernel.rocm.deterministic_attention", + "candidate": "rlkernel.rocm.native_attention_per_batch_row" + }, + "metrics": { + "dk": { + "bitwise_equal": true, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 128, + 128 + ] + }, + "dq": { + "bitwise_equal": true, + "element_count": 524288, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 128, + 128 + ] + }, + "dv": { + "bitwise_equal": true, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 128, + 128 + ] + }, + "lse": { + "bitwise_equal": true, + "element_count": 4096, + "left_dtype": "torch.float32", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.float32", + "shape": [ + 1, + 32, + 128 + ] + }, + "out": { + "bitwise_equal": true, + "element_count": 524288, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 128, + 128 + ] + } + }, + "outcome": "matched", + "passed": true, + "probe": "batch_composition_control", + "realization": "Compare a batch call with per-row calls to the same native HIP core.", + "row_id": "C1", + "sequence": 128 + }, + { + "batch": 1, + "category": "invariant_control", + "comparable": true, + "expected": "exact_zero", + "implementations": { + "baseline": "rlkernel.rocm.native_attention_full_prefill", + "candidate": "rlkernel.rocm.native_attention_tail" + }, + "metrics": { + "dk": { + "bitwise_equal": true, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 128, + 128 + ] + }, + "dq": { + "bitwise_equal": true, + "element_count": 524288, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 128, + 128 + ] + }, + "dv": { + "bitwise_equal": true, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 8, + 128, + 128 + ] + }, + "lse": { + "bitwise_equal": true, + "element_count": 32, + "left_dtype": "torch.float32", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.float32", + "shape": [ + 1, + 32, + 1 + ] + }, + "out": { + "bitwise_equal": true, + "element_count": 4096, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 1, + 32, + 1, + 128 + ] + } + }, + "outcome": "matched", + "passed": true, + "probe": "prefill_decode_tail_control", + "realization": "Compare a full-prefill tail with one trailing query over dense KV, without a serving cache.", + "row_id": "C2", + "sequence": 128 + }, + { + "batch": 2, + "category": "baseline", + "comparable": true, + "expected": "baseline", + "implementations": { + "baseline": "rlkernel.rocm.deterministic_attention", + "candidate": "rlkernel.rocm.deterministic_attention" + }, + "metrics": { + "dk": { + "bitwise_equal": true, + "element_count": 32768, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 16, + 128 + ] + }, + "dq": { + "bitwise_equal": true, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 16, + 128 + ] + }, + "dv": { + "bitwise_equal": true, + "element_count": 32768, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 16, + 128 + ] + }, + "lse": { + "bitwise_equal": true, + "element_count": 1024, + "left_dtype": "torch.float32", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.float32", + "shape": [ + 2, + 32, + 16 + ] + }, + "out": { + "bitwise_equal": true, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 16, + 128 + ] + } + }, + "outcome": "matched", + "passed": true, + "probe": null, + "realization": "Repeat the identical native HIP reference-core call.", + "row_id": "A0", + "sequence": 16 + }, + { + "batch": 2, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "implementations": { + "baseline": "rlkernel.rocm.deterministic_rope+native_attention", + "candidate": "rlkernel.rocm.deterministic_rope+native_attention" + }, + "metrics": { + "dk": { + "bitwise_equal": false, + "element_count": 32768, + "left_dtype": "torch.bfloat16", + "max_abs": 1.78125, + "mismatch_count": 31944, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 16, + 128 + ] + }, + "dq": { + "bitwise_equal": false, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 1.3857421875, + "mismatch_count": 64515, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 16, + 128 + ] + }, + "dv": { + "bitwise_equal": false, + "element_count": 32768, + "left_dtype": "torch.bfloat16", + "max_abs": 0.64453125, + "mismatch_count": 31643, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 16, + 128 + ] + }, + "lse": { + "bitwise_equal": false, + "element_count": 1024, + "left_dtype": "torch.float32", + "max_abs": 0.3938255310058594, + "mismatch_count": 512, + "right_dtype": "torch.float32", + "shape": [ + 2, + 32, + 16 + ] + }, + "out": { + "bitwise_equal": false, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.671875, + "mismatch_count": 64375, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 16, + 128 + ] + } + }, + "outcome": "drift_detected", + "passed": true, + "probe": "position_ids", + "realization": "Increment suffix RoPE positions while preserving Q/K/V tensors.", + "row_id": "A1", + "sequence": 16 + }, + { + "batch": 2, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "implementations": { + "baseline": "torch.rocm.rms_norm+native_attention", + "candidate": "rlkernel.rocm.deterministic_attention" + }, + "metrics": { + "dk": { + "bitwise_equal": false, + "element_count": 32768, + "left_dtype": "torch.bfloat16", + "max_abs": 1.2265625, + "mismatch_count": 32375, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 16, + 128 + ] + }, + "dq": { + "bitwise_equal": false, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.65625, + "mismatch_count": 121092, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 16, + 128 + ] + }, + "dv": { + "bitwise_equal": false, + "element_count": 32768, + "left_dtype": "torch.bfloat16", + "max_abs": 0.455078125, + "mismatch_count": 31825, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 16, + 128 + ] + }, + "lse": { + "bitwise_equal": false, + "element_count": 1024, + "left_dtype": "torch.float32", + "max_abs": 0.4254317283630371, + "mismatch_count": 1024, + "right_dtype": "torch.float32", + "shape": [ + 2, + 32, + 16 + ] + }, + "out": { + "bitwise_equal": false, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.408203125, + "mismatch_count": 116343, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 16, + 128 + ] + } + }, + "outcome": "drift_detected", + "passed": true, + "probe": "qk_norm_disabled", + "realization": "Apply or bypass unit-weight PyTorch RMSNorm before the native HIP core.", + "row_id": "A2", + "sequence": 16 + }, + { + "batch": 2, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "implementations": { + "baseline": "rlkernel.rocm.deterministic_attention", + "candidate": "rlkernel.rocm.deterministic_attention" + }, + "metrics": { + "dk": { + "bitwise_equal": false, + "element_count": 32768, + "left_dtype": "torch.bfloat16", + "max_abs": 3.6875, + "mismatch_count": 32712, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 16, + 128 + ] + }, + "dq": { + "bitwise_equal": false, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 4.0546875, + "mismatch_count": 122436, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 16, + 128 + ] + }, + "dv": { + "bitwise_equal": false, + "element_count": 32768, + "left_dtype": "torch.bfloat16", + "max_abs": 7.96875, + "mismatch_count": 32708, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 16, + 128 + ] + }, + "lse": { + "bitwise_equal": false, + "element_count": 1024, + "left_dtype": "torch.float32", + "max_abs": 6.699570655822754, + "mismatch_count": 960, + "right_dtype": "torch.float32", + "shape": [ + 2, + 32, + 16 + ] + }, + "out": { + "bitwise_equal": false, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 4.21875, + "mismatch_count": 122371, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 16, + 128 + ] + } + }, + "outcome": "drift_detected", + "passed": true, + "probe": "causal_mask", + "realization": "Toggle causal masking in the native HIP core.", + "row_id": "A3", + "sequence": 16 + }, + { + "batch": 2, + "binding_gate": { + "binding_fingerprint": "aa9eef16d971dbbb005e00de631493bbbf265d71d4409a8517841df931c8ca4f", + "comparable": false, + "identity_fingerprint": "c2a02818c11bd195873275c75f4d94ae20b0ae2f2e9daa7fcd5f51d9b409cb4c", + "issues": [ + { + "code": "TOPOLOGY_MISMATCH", + "field": "sharding.tp_rank", + "message": "sharding.tp_rank changes TP/CP ownership; the pair is not the same local attention problem", + "rollout": 1, + "tier": "identical", + "training": 0 + }, + { + "code": "TOPOLOGY_MISMATCH", + "field": "sharding.local_q_head_start", + "message": "sharding.local_q_head_start changes TP/CP ownership; the pair is not the same local attention problem", + "rollout": 16, + "tier": "identical", + "training": 0 + }, + { + "code": "TOPOLOGY_MISMATCH", + "field": "sharding.local_kv_head_start", + "message": "sharding.local_kv_head_start changes TP/CP ownership; the pair is not the same local attention problem", + "rollout": 4, + "tier": "identical", + "training": 0 + } + ], + "passed": false, + "provenance": { + "dtype": "bf16", + "lse_domain": "attention", + "rollout": { + "backend_id": "rlkernel.rocm.deterministic_attention", + "contract": { + "batch_size": 2, + "causal": true, + "causal_offsets": [ + 0, + 0 + ], + "dtype": "bf16", + "export_lse": true, + "head_dim": 128, + "kv_cache": null, + "lse_domain": "attention", + "mode": "prefill", + "projections": { + "o_proj": { + "acc_dtype": "fp32", + "backend_policy": "native_verified_then_common_deterministic", + "deterministic_backend": "rlkernel.cuda.det_gemm", + "input_dtype": "bf16", + "k_order": "ascending", + "output_dtype": "bf16", + "qkv_split_order": [], + "require_runtime_readback": true, + "sp_backward_collective": "all_gather", + "sp_forward_collective": "reduce_scatter", + "split_kv": "disabled", + "tp_backward_dgrad_collective": "none", + "tp_forward_collective": "all_reduce" + }, + "qkv": { + "acc_dtype": "fp32", + "backend_policy": "native_verified_then_common_deterministic", + "deterministic_backend": "rlkernel.cuda.det_gemm", + "input_dtype": "bf16", + "k_order": "ascending", + "output_dtype": "bf16", + "qkv_split_order": [ + "q", + "k", + "v" + ], + "require_runtime_readback": true, + "sp_backward_collective": "reduce_scatter", + "sp_forward_collective": "all_gather", + "split_kv": "disabled", + "tp_backward_dgrad_collective": "all_reduce", + "tp_forward_collective": "none" + } + }, + "query_sequence_length": 16, + "reduction": { + "acc_dtype": "fp32", + "downcast_at": "final_write", + "engine": "in_op_reference", + "merge": "online_softmax_lse", + "order": "global_block_index" + }, + "role": "infer", + "rope": null, + "semantic_operator": "standard_softmax_attention", + "sharding": { + "cp_rank": 0, + "cp_world_size": 1, + "global_block_indices": [ + 0 + ], + "global_block_token_starts": [ + 0 + ], + "global_kv_heads": 8, + "global_q_heads": 32, + "global_sequence_length": 16, + "local_block_offsets": [ + 0, + 16 + ], + "local_kv_head_start": 4, + "local_kv_heads": 4, + "local_q_head_start": 16, + "local_q_heads": 16, + "local_sequence_length": 16, + "packed_sequence_offsets": null, + "sp_rank": 0, + "sp_world_size": 1, + "tp_rank": 1, + "tp_world_size": 2 + }, + "split_kv": { + "fixed_split_size": null, + "mode": "disabled", + "strict_consistency": true + } + }, + "recorded": { + "backend_id": "rlkernel.rocm.deterministic_attention", + "mode": "prefill", + "reduction.engine": "in_op_reference" + } + }, + "split_kv_runtime": { + "rollout": { + "batch_size": 2, + "coverage": "complete_batch_tp_cp_owner_cartesian_product", + "cp_world_size": 1, + "entries": [ + { + "actual_split_boundaries": [ + [ + 0, + 16 + ] + ], + "actual_split_kv_count": 1, + "actual_split_kv_policy": "disabled", + "actual_split_kv_size": null, + "batch_index": 0, + "cp_rank": 0, + "expected_kv_range": [ + 0, + 16 + ], + "owner_cp_rank": 0, + "requested_split_kv_policy": "disabled", + "requested_split_kv_size": null, + "split_kv_accum_dtype": "fp32", + "split_kv_backend": "rlkernel.rocm.deterministic_attention", + "split_kv_downcast_at": "final_write", + "split_kv_fallback": false, + "split_kv_fallback_reason": null, + "split_kv_merge_order": "global_block_index", + "split_kv_plan_source": "contract_exact", + "tp_rank": 0 + }, + { + "actual_split_boundaries": [ + [ + 0, + 16 + ] + ], + "actual_split_kv_count": 1, + "actual_split_kv_policy": "disabled", + "actual_split_kv_size": null, + "batch_index": 0, + "cp_rank": 0, + "expected_kv_range": [ + 0, + 16 + ], + "owner_cp_rank": 0, + "requested_split_kv_policy": "disabled", + "requested_split_kv_size": null, + "split_kv_accum_dtype": "fp32", + "split_kv_backend": "rlkernel.rocm.deterministic_attention", + "split_kv_downcast_at": "final_write", + "split_kv_fallback": false, + "split_kv_fallback_reason": null, + "split_kv_merge_order": "global_block_index", + "split_kv_plan_source": "contract_exact", + "tp_rank": 1 + }, + { + "actual_split_boundaries": [ + [ + 0, + 16 + ] + ], + "actual_split_kv_count": 1, + "actual_split_kv_policy": "disabled", + "actual_split_kv_size": null, + "batch_index": 1, + "cp_rank": 0, + "expected_kv_range": [ + 0, + 16 + ], + "owner_cp_rank": 0, + "requested_split_kv_policy": "disabled", + "requested_split_kv_size": null, + "split_kv_accum_dtype": "fp32", + "split_kv_backend": "rlkernel.rocm.deterministic_attention", + "split_kv_downcast_at": "final_write", + "split_kv_fallback": false, + "split_kv_fallback_reason": null, + "split_kv_merge_order": "global_block_index", + "split_kv_plan_source": "contract_exact", + "tp_rank": 0 + }, + { + "actual_split_boundaries": [ + [ + 0, + 16 + ] + ], + "actual_split_kv_count": 1, + "actual_split_kv_policy": "disabled", + "actual_split_kv_size": null, + "batch_index": 1, + "cp_rank": 0, + "expected_kv_range": [ + 0, + 16 + ], + "owner_cp_rank": 0, + "requested_split_kv_policy": "disabled", + "requested_split_kv_size": null, + "split_kv_accum_dtype": "fp32", + "split_kv_backend": "rlkernel.rocm.deterministic_attention", + "split_kv_downcast_at": "final_write", + "split_kv_fallback": false, + "split_kv_fallback_reason": null, + "split_kv_merge_order": "global_block_index", + "split_kv_plan_source": "contract_exact", + "tp_rank": 1 + } + ], + "total_kv_tokens": [ + 16, + 16 + ], + "tp_world_size": 2 + }, + "training": { + "batch_size": 2, + "coverage": "complete_batch_tp_cp_owner_cartesian_product", + "cp_world_size": 1, + "entries": [ + { + "actual_split_boundaries": [ + [ + 0, + 16 + ] + ], + "actual_split_kv_count": 1, + "actual_split_kv_policy": "disabled", + "actual_split_kv_size": null, + "batch_index": 0, + "cp_rank": 0, + "expected_kv_range": [ + 0, + 16 + ], + "owner_cp_rank": 0, + "requested_split_kv_policy": "disabled", + "requested_split_kv_size": null, + "split_kv_accum_dtype": "fp32", + "split_kv_backend": "rlkernel.rocm.deterministic_attention", + "split_kv_downcast_at": "final_write", + "split_kv_fallback": false, + "split_kv_fallback_reason": null, + "split_kv_merge_order": "global_block_index", + "split_kv_plan_source": "contract_exact", + "tp_rank": 0 + }, + { + "actual_split_boundaries": [ + [ + 0, + 16 + ] + ], + "actual_split_kv_count": 1, + "actual_split_kv_policy": "disabled", + "actual_split_kv_size": null, + "batch_index": 0, + "cp_rank": 0, + "expected_kv_range": [ + 0, + 16 + ], + "owner_cp_rank": 0, + "requested_split_kv_policy": "disabled", + "requested_split_kv_size": null, + "split_kv_accum_dtype": "fp32", + "split_kv_backend": "rlkernel.rocm.deterministic_attention", + "split_kv_downcast_at": "final_write", + "split_kv_fallback": false, + "split_kv_fallback_reason": null, + "split_kv_merge_order": "global_block_index", + "split_kv_plan_source": "contract_exact", + "tp_rank": 1 + }, + { + "actual_split_boundaries": [ + [ + 0, + 16 + ] + ], + "actual_split_kv_count": 1, + "actual_split_kv_policy": "disabled", + "actual_split_kv_size": null, + "batch_index": 1, + "cp_rank": 0, + "expected_kv_range": [ + 0, + 16 + ], + "owner_cp_rank": 0, + "requested_split_kv_policy": "disabled", + "requested_split_kv_size": null, + "split_kv_accum_dtype": "fp32", + "split_kv_backend": "rlkernel.rocm.deterministic_attention", + "split_kv_downcast_at": "final_write", + "split_kv_fallback": false, + "split_kv_fallback_reason": null, + "split_kv_merge_order": "global_block_index", + "split_kv_plan_source": "contract_exact", + "tp_rank": 0 + }, + { + "actual_split_boundaries": [ + [ + 0, + 16 + ] + ], + "actual_split_kv_count": 1, + "actual_split_kv_policy": "disabled", + "actual_split_kv_size": null, + "batch_index": 1, + "cp_rank": 0, + "expected_kv_range": [ + 0, + 16 + ], + "owner_cp_rank": 0, + "requested_split_kv_policy": "disabled", + "requested_split_kv_size": null, + "split_kv_accum_dtype": "fp32", + "split_kv_backend": "rlkernel.rocm.deterministic_attention", + "split_kv_downcast_at": "final_write", + "split_kv_fallback": false, + "split_kv_fallback_reason": null, + "split_kv_merge_order": "global_block_index", + "split_kv_plan_source": "contract_exact", + "tp_rank": 1 + } + ], + "total_kv_tokens": [ + 16, + 16 + ], + "tp_world_size": 2 + } + }, + "training": { + "backend_id": "rlkernel.rocm.deterministic_attention", + "contract": { + "batch_size": 2, + "causal": true, + "causal_offsets": [ + 0, + 0 + ], + "dtype": "bf16", + "export_lse": true, + "head_dim": 128, + "kv_cache": null, + "lse_domain": "attention", + "mode": "prefill", + "projections": { + "o_proj": { + "acc_dtype": "fp32", + "backend_policy": "native_verified_then_common_deterministic", + "deterministic_backend": "rlkernel.cuda.det_gemm", + "input_dtype": "bf16", + "k_order": "ascending", + "output_dtype": "bf16", + "qkv_split_order": [], + "require_runtime_readback": true, + "sp_backward_collective": "all_gather", + "sp_forward_collective": "reduce_scatter", + "split_kv": "disabled", + "tp_backward_dgrad_collective": "none", + "tp_forward_collective": "all_reduce" + }, + "qkv": { + "acc_dtype": "fp32", + "backend_policy": "native_verified_then_common_deterministic", + "deterministic_backend": "rlkernel.cuda.det_gemm", + "input_dtype": "bf16", + "k_order": "ascending", + "output_dtype": "bf16", + "qkv_split_order": [ + "q", + "k", + "v" + ], + "require_runtime_readback": true, + "sp_backward_collective": "reduce_scatter", + "sp_forward_collective": "all_gather", + "split_kv": "disabled", + "tp_backward_dgrad_collective": "all_reduce", + "tp_forward_collective": "none" + } + }, + "query_sequence_length": 16, + "reduction": { + "acc_dtype": "fp32", + "downcast_at": "final_write", + "engine": "in_op_reference", + "merge": "online_softmax_lse", + "order": "global_block_index" + }, + "role": "train", + "rope": null, + "semantic_operator": "standard_softmax_attention", + "sharding": { + "cp_rank": 0, + "cp_world_size": 1, + "global_block_indices": [ + 0 + ], + "global_block_token_starts": [ + 0 + ], + "global_kv_heads": 8, + "global_q_heads": 32, + "global_sequence_length": 16, + "local_block_offsets": [ + 0, + 16 + ], + "local_kv_head_start": 0, + "local_kv_heads": 4, + "local_q_head_start": 0, + "local_q_heads": 16, + "local_sequence_length": 16, + "packed_sequence_offsets": null, + "sp_rank": 0, + "sp_world_size": 1, + "tp_rank": 0, + "tp_world_size": 2 + }, + "split_kv": { + "fixed_split_size": null, + "mode": "disabled", + "strict_consistency": true + } + }, + "recorded": { + "backend_id": "rlkernel.rocm.deterministic_attention", + "mode": "prefill", + "reduction.engine": "in_op_reference" + } + } + }, + "recorded_differences": {}, + "reduction_fingerprint": "4bfe65d607de1dc8d1b8c04041d48e9af4e81dc38614f0340d994616da273d6c", + "schema_version": "cross_config.attention_binding.v3" + }, + "category": "comparability_gate", + "comparable": false, + "expected": "rejected", + "gate_implementation": "rl_engine.alignment.cross_config.bind_attention_contracts", + "identity_errors": [ + "sharding.tp_rank", + "sharding.local_q_head_start", + "sharding.local_kv_head_start" + ], + "metrics": { + "dk": null, + "dq": null, + "dv": null, + "lse": null, + "out": null + }, + "outcome": "rejected", + "passed": true, + "probe": "tp_head_ownership", + "realization": "Bind valid TP-rank-1 rollout and TP-rank-0 training contracts; do not run numerics.", + "row_id": "A4", + "sequence": 16 + }, + { + "batch": 2, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "implementations": { + "baseline": "rlkernel.rocm.deterministic_attention", + "candidate": "torch.rocm.index_select+native_attention" + }, + "metrics": { + "dk": { + "bitwise_equal": false, + "element_count": 32768, + "left_dtype": "torch.bfloat16", + "max_abs": 5.098358154296875, + "mismatch_count": 32726, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 16, + 128 + ] + }, + "dq": { + "bitwise_equal": false, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 3.9609375, + "mismatch_count": 114456, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 16, + 128 + ] + }, + "dv": { + "bitwise_equal": false, + "element_count": 32768, + "left_dtype": "torch.bfloat16", + "max_abs": 9.017578125, + "mismatch_count": 32739, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 16, + 128 + ] + }, + "lse": { + "bitwise_equal": false, + "element_count": 1024, + "left_dtype": "torch.float32", + "max_abs": 3.8384933471679688, + "mismatch_count": 960, + "right_dtype": "torch.float32", + "shape": [ + 2, + 32, + 16 + ] + }, + "out": { + "bitwise_equal": false, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 4.90625, + "mismatch_count": 122580, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 16, + 128 + ] + } + }, + "outcome": "drift_detected", + "passed": true, + "probe": "kv_page_order", + "realization": "Reverse four dense K/V tensor pages; this is not a paged-cache runtime.", + "row_id": "A5", + "sequence": 16 + }, + { + "batch": 2, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "implementations": { + "baseline": "torch.rocm.explicit_fp32_serial_qk_accumulator", + "candidate": "torch.rocm.explicit_bf16_serial_qk_accumulator" + }, + "metrics": { + "dk": { + "bitwise_equal": false, + "element_count": 32768, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0546875, + "mismatch_count": 27212, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 16, + 128 + ] + }, + "dq": { + "bitwise_equal": false, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0625, + "mismatch_count": 96463, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 16, + 128 + ] + }, + "dv": { + "bitwise_equal": false, + "element_count": 32768, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0703125, + "mismatch_count": 27390, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 16, + 128 + ] + }, + "lse": { + "bitwise_equal": false, + "element_count": 1024, + "left_dtype": "torch.float32", + "max_abs": 0.08946871757507324, + "mismatch_count": 1024, + "right_dtype": "torch.float32", + "shape": [ + 2, + 32, + 16 + ] + }, + "out": { + "bitwise_equal": false, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0703125, + "mismatch_count": 94303, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 16, + 128 + ] + } + }, + "outcome": "drift_detected", + "passed": true, + "probe": "accum_dtype", + "realization": "Use identical FP32 products/order with explicit FP32 versus BF16 accumulator state.", + "row_id": "A6", + "sequence": 16 + }, + { + "batch": 2, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "implementations": { + "baseline": "torch.rocm.fp32_chunk_merge_ascending", + "candidate": "torch.rocm.fp32_chunk_merge_descending" + }, + "metrics": { + "dk": { + "bitwise_equal": false, + "element_count": 32768, + "left_dtype": "torch.bfloat16", + "max_abs": 0.001953125, + "mismatch_count": 4, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 16, + 128 + ] + }, + "dq": { + "bitwise_equal": false, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.00048828125, + "mismatch_count": 3, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 16, + 128 + ] + }, + "dv": { + "bitwise_equal": true, + "element_count": 32768, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 16, + 128 + ] + }, + "lse": { + "bitwise_equal": false, + "element_count": 1024, + "left_dtype": "torch.float32", + "max_abs": 4.76837158203125e-07, + "mismatch_count": 71, + "right_dtype": "torch.float32", + "shape": [ + 2, + 32, + 16 + ] + }, + "out": { + "bitwise_equal": false, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 1.9073486328125e-06, + "mismatch_count": 2, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 16, + 128 + ] + } + }, + "outcome": "drift_detected", + "passed": true, + "probe": "merge_order", + "realization": "Merge four dense chunks in opposite orders on one GPU; this is not a CP collective.", + "row_id": "A7", + "sequence": 16 + }, + { + "batch": 2, + "category": "invariant_control", + "comparable": true, + "expected": "exact_zero", + "implementations": { + "baseline": "rlkernel.rocm.deterministic_attention", + "candidate": "rlkernel.rocm.native_attention_per_tp2_head_shard" + }, + "metrics": { + "dk": { + "bitwise_equal": true, + "element_count": 32768, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 16, + 128 + ] + }, + "dq": { + "bitwise_equal": true, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 16, + 128 + ] + }, + "dv": { + "bitwise_equal": true, + "element_count": 32768, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 16, + 128 + ] + }, + "lse": { + "bitwise_equal": true, + "element_count": 1024, + "left_dtype": "torch.float32", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.float32", + "shape": [ + 2, + 32, + 16 + ] + }, + "out": { + "bitwise_equal": true, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 16, + 128 + ] + } + }, + "outcome": "matched", + "passed": true, + "probe": "tp_partition_control", + "realization": "Compare full GQA with two contiguous TP=2 head shards on one GPU.", + "row_id": "C0", + "sequence": 16 + }, + { + "batch": 2, + "category": "invariant_control", + "comparable": true, + "expected": "exact_zero", + "implementations": { + "baseline": "rlkernel.rocm.deterministic_attention", + "candidate": "rlkernel.rocm.native_attention_per_batch_row" + }, + "metrics": { + "dk": { + "bitwise_equal": true, + "element_count": 32768, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 16, + 128 + ] + }, + "dq": { + "bitwise_equal": true, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 16, + 128 + ] + }, + "dv": { + "bitwise_equal": true, + "element_count": 32768, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 16, + 128 + ] + }, + "lse": { + "bitwise_equal": true, + "element_count": 1024, + "left_dtype": "torch.float32", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.float32", + "shape": [ + 2, + 32, + 16 + ] + }, + "out": { + "bitwise_equal": true, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 16, + 128 + ] + } + }, + "outcome": "matched", + "passed": true, + "probe": "batch_composition_control", + "realization": "Compare a batch call with per-row calls to the same native HIP core.", + "row_id": "C1", + "sequence": 16 + }, + { + "batch": 2, + "category": "invariant_control", + "comparable": true, + "expected": "exact_zero", + "implementations": { + "baseline": "rlkernel.rocm.native_attention_full_prefill", + "candidate": "rlkernel.rocm.native_attention_tail" + }, + "metrics": { + "dk": { + "bitwise_equal": true, + "element_count": 32768, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 16, + 128 + ] + }, + "dq": { + "bitwise_equal": true, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 16, + 128 + ] + }, + "dv": { + "bitwise_equal": true, + "element_count": 32768, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 16, + 128 + ] + }, + "lse": { + "bitwise_equal": true, + "element_count": 64, + "left_dtype": "torch.float32", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.float32", + "shape": [ + 2, + 32, + 1 + ] + }, + "out": { + "bitwise_equal": true, + "element_count": 8192, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 1, + 128 + ] + } + }, + "outcome": "matched", + "passed": true, + "probe": "prefill_decode_tail_control", + "realization": "Compare a full-prefill tail with one trailing query over dense KV, without a serving cache.", + "row_id": "C2", + "sequence": 16 + }, + { + "batch": 2, + "category": "baseline", + "comparable": true, + "expected": "baseline", + "implementations": { + "baseline": "rlkernel.rocm.deterministic_attention", + "candidate": "rlkernel.rocm.deterministic_attention" + }, + "metrics": { + "dk": { + "bitwise_equal": true, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 32, + 128 + ] + }, + "dq": { + "bitwise_equal": true, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 32, + 128 + ] + }, + "dv": { + "bitwise_equal": true, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 32, + 128 + ] + }, + "lse": { + "bitwise_equal": true, + "element_count": 2048, + "left_dtype": "torch.float32", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.float32", + "shape": [ + 2, + 32, + 32 + ] + }, + "out": { + "bitwise_equal": true, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 32, + 128 + ] + } + }, + "outcome": "matched", + "passed": true, + "probe": null, + "realization": "Repeat the identical native HIP reference-core call.", + "row_id": "A0", + "sequence": 32 + }, + { + "batch": 2, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "implementations": { + "baseline": "rlkernel.rocm.deterministic_rope+native_attention", + "candidate": "rlkernel.rocm.deterministic_rope+native_attention" + }, + "metrics": { + "dk": { + "bitwise_equal": false, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 2.234375, + "mismatch_count": 63791, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 32, + 128 + ] + }, + "dq": { + "bitwise_equal": false, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 1.4140625, + "mismatch_count": 129154, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 32, + 128 + ] + }, + "dv": { + "bitwise_equal": false, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 0.53125, + "mismatch_count": 63113, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 32, + 128 + ] + }, + "lse": { + "bitwise_equal": false, + "element_count": 2048, + "left_dtype": "torch.float32", + "max_abs": 0.3046402931213379, + "mismatch_count": 1024, + "right_dtype": "torch.float32", + "shape": [ + 2, + 32, + 32 + ] + }, + "out": { + "bitwise_equal": false, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 0.7109375, + "mismatch_count": 129019, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 32, + 128 + ] + } + }, + "outcome": "drift_detected", + "passed": true, + "probe": "position_ids", + "realization": "Increment suffix RoPE positions while preserving Q/K/V tensors.", + "row_id": "A1", + "sequence": 32 + }, + { + "batch": 2, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "implementations": { + "baseline": "torch.rocm.rms_norm+native_attention", + "candidate": "rlkernel.rocm.deterministic_attention" + }, + "metrics": { + "dk": { + "bitwise_equal": false, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 1.7578125, + "mismatch_count": 64890, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 32, + 128 + ] + }, + "dq": { + "bitwise_equal": false, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 0.7578125, + "mismatch_count": 250592, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 32, + 128 + ] + }, + "dv": { + "bitwise_equal": false, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 0.609375, + "mismatch_count": 64053, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 32, + 128 + ] + }, + "lse": { + "bitwise_equal": false, + "element_count": 2048, + "left_dtype": "torch.float32", + "max_abs": 0.3993661403656006, + "mismatch_count": 2048, + "right_dtype": "torch.float32", + "shape": [ + 2, + 32, + 32 + ] + }, + "out": { + "bitwise_equal": false, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 0.4453125, + "mismatch_count": 244117, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 32, + 128 + ] + } + }, + "outcome": "drift_detected", + "passed": true, + "probe": "qk_norm_disabled", + "realization": "Apply or bypass unit-weight PyTorch RMSNorm before the native HIP core.", + "row_id": "A2", + "sequence": 32 + }, + { + "batch": 2, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "implementations": { + "baseline": "rlkernel.rocm.deterministic_attention", + "candidate": "rlkernel.rocm.deterministic_attention" + }, + "metrics": { + "dk": { + "bitwise_equal": false, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 5.1640625, + "mismatch_count": 65446, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 32, + 128 + ] + }, + "dq": { + "bitwise_equal": false, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 4.1968994140625, + "mismatch_count": 252862, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 32, + 128 + ] + }, + "dv": { + "bitwise_equal": false, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 10.1328125, + "mismatch_count": 65455, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 32, + 128 + ] + }, + "lse": { + "bitwise_equal": false, + "element_count": 2048, + "left_dtype": "torch.float32", + "max_abs": 6.510496139526367, + "mismatch_count": 1984, + "right_dtype": "torch.float32", + "shape": [ + 2, + 32, + 32 + ] + }, + "out": { + "bitwise_equal": false, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 3.568359375, + "mismatch_count": 252854, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 32, + 128 + ] + } + }, + "outcome": "drift_detected", + "passed": true, + "probe": "causal_mask", + "realization": "Toggle causal masking in the native HIP core.", + "row_id": "A3", + "sequence": 32 + }, + { + "batch": 2, + "binding_gate": { + "binding_fingerprint": "5daa4663857ba9f0fd736b00ed6143ba2df4973fd0c9a24c390777b4c3719728", + "comparable": false, + "identity_fingerprint": "c2a02818c11bd195873275c75f4d94ae20b0ae2f2e9daa7fcd5f51d9b409cb4c", + "issues": [ + { + "code": "TOPOLOGY_MISMATCH", + "field": "sharding.tp_rank", + "message": "sharding.tp_rank changes TP/CP ownership; the pair is not the same local attention problem", + "rollout": 1, + "tier": "identical", + "training": 0 + }, + { + "code": "TOPOLOGY_MISMATCH", + "field": "sharding.local_q_head_start", + "message": "sharding.local_q_head_start changes TP/CP ownership; the pair is not the same local attention problem", + "rollout": 16, + "tier": "identical", + "training": 0 + }, + { + "code": "TOPOLOGY_MISMATCH", + "field": "sharding.local_kv_head_start", + "message": "sharding.local_kv_head_start changes TP/CP ownership; the pair is not the same local attention problem", + "rollout": 4, + "tier": "identical", + "training": 0 + } + ], + "passed": false, + "provenance": { + "dtype": "bf16", + "lse_domain": "attention", + "rollout": { + "backend_id": "rlkernel.rocm.deterministic_attention", + "contract": { + "batch_size": 2, + "causal": true, + "causal_offsets": [ + 0, + 0 + ], + "dtype": "bf16", + "export_lse": true, + "head_dim": 128, + "kv_cache": null, + "lse_domain": "attention", + "mode": "prefill", + "projections": { + "o_proj": { + "acc_dtype": "fp32", + "backend_policy": "native_verified_then_common_deterministic", + "deterministic_backend": "rlkernel.cuda.det_gemm", + "input_dtype": "bf16", + "k_order": "ascending", + "output_dtype": "bf16", + "qkv_split_order": [], + "require_runtime_readback": true, + "sp_backward_collective": "all_gather", + "sp_forward_collective": "reduce_scatter", + "split_kv": "disabled", + "tp_backward_dgrad_collective": "none", + "tp_forward_collective": "all_reduce" + }, + "qkv": { + "acc_dtype": "fp32", + "backend_policy": "native_verified_then_common_deterministic", + "deterministic_backend": "rlkernel.cuda.det_gemm", + "input_dtype": "bf16", + "k_order": "ascending", + "output_dtype": "bf16", + "qkv_split_order": [ + "q", + "k", + "v" + ], + "require_runtime_readback": true, + "sp_backward_collective": "reduce_scatter", + "sp_forward_collective": "all_gather", + "split_kv": "disabled", + "tp_backward_dgrad_collective": "all_reduce", + "tp_forward_collective": "none" + } + }, + "query_sequence_length": 32, + "reduction": { + "acc_dtype": "fp32", + "downcast_at": "final_write", + "engine": "in_op_reference", + "merge": "online_softmax_lse", + "order": "global_block_index" + }, + "role": "infer", + "rope": null, + "semantic_operator": "standard_softmax_attention", + "sharding": { + "cp_rank": 0, + "cp_world_size": 1, + "global_block_indices": [ + 0 + ], + "global_block_token_starts": [ + 0 + ], + "global_kv_heads": 8, + "global_q_heads": 32, + "global_sequence_length": 32, + "local_block_offsets": [ + 0, + 32 + ], + "local_kv_head_start": 4, + "local_kv_heads": 4, + "local_q_head_start": 16, + "local_q_heads": 16, + "local_sequence_length": 32, + "packed_sequence_offsets": null, + "sp_rank": 0, + "sp_world_size": 1, + "tp_rank": 1, + "tp_world_size": 2 + }, + "split_kv": { + "fixed_split_size": null, + "mode": "disabled", + "strict_consistency": true + } + }, + "recorded": { + "backend_id": "rlkernel.rocm.deterministic_attention", + "mode": "prefill", + "reduction.engine": "in_op_reference" + } + }, + "split_kv_runtime": { + "rollout": { + "batch_size": 2, + "coverage": "complete_batch_tp_cp_owner_cartesian_product", + "cp_world_size": 1, + "entries": [ + { + "actual_split_boundaries": [ + [ + 0, + 32 + ] + ], + "actual_split_kv_count": 1, + "actual_split_kv_policy": "disabled", + "actual_split_kv_size": null, + "batch_index": 0, + "cp_rank": 0, + "expected_kv_range": [ + 0, + 32 + ], + "owner_cp_rank": 0, + "requested_split_kv_policy": "disabled", + "requested_split_kv_size": null, + "split_kv_accum_dtype": "fp32", + "split_kv_backend": "rlkernel.rocm.deterministic_attention", + "split_kv_downcast_at": "final_write", + "split_kv_fallback": false, + "split_kv_fallback_reason": null, + "split_kv_merge_order": "global_block_index", + "split_kv_plan_source": "contract_exact", + "tp_rank": 0 + }, + { + "actual_split_boundaries": [ + [ + 0, + 32 + ] + ], + "actual_split_kv_count": 1, + "actual_split_kv_policy": "disabled", + "actual_split_kv_size": null, + "batch_index": 0, + "cp_rank": 0, + "expected_kv_range": [ + 0, + 32 + ], + "owner_cp_rank": 0, + "requested_split_kv_policy": "disabled", + "requested_split_kv_size": null, + "split_kv_accum_dtype": "fp32", + "split_kv_backend": "rlkernel.rocm.deterministic_attention", + "split_kv_downcast_at": "final_write", + "split_kv_fallback": false, + "split_kv_fallback_reason": null, + "split_kv_merge_order": "global_block_index", + "split_kv_plan_source": "contract_exact", + "tp_rank": 1 + }, + { + "actual_split_boundaries": [ + [ + 0, + 32 + ] + ], + "actual_split_kv_count": 1, + "actual_split_kv_policy": "disabled", + "actual_split_kv_size": null, + "batch_index": 1, + "cp_rank": 0, + "expected_kv_range": [ + 0, + 32 + ], + "owner_cp_rank": 0, + "requested_split_kv_policy": "disabled", + "requested_split_kv_size": null, + "split_kv_accum_dtype": "fp32", + "split_kv_backend": "rlkernel.rocm.deterministic_attention", + "split_kv_downcast_at": "final_write", + "split_kv_fallback": false, + "split_kv_fallback_reason": null, + "split_kv_merge_order": "global_block_index", + "split_kv_plan_source": "contract_exact", + "tp_rank": 0 + }, + { + "actual_split_boundaries": [ + [ + 0, + 32 + ] + ], + "actual_split_kv_count": 1, + "actual_split_kv_policy": "disabled", + "actual_split_kv_size": null, + "batch_index": 1, + "cp_rank": 0, + "expected_kv_range": [ + 0, + 32 + ], + "owner_cp_rank": 0, + "requested_split_kv_policy": "disabled", + "requested_split_kv_size": null, + "split_kv_accum_dtype": "fp32", + "split_kv_backend": "rlkernel.rocm.deterministic_attention", + "split_kv_downcast_at": "final_write", + "split_kv_fallback": false, + "split_kv_fallback_reason": null, + "split_kv_merge_order": "global_block_index", + "split_kv_plan_source": "contract_exact", + "tp_rank": 1 + } + ], + "total_kv_tokens": [ + 32, + 32 + ], + "tp_world_size": 2 + }, + "training": { + "batch_size": 2, + "coverage": "complete_batch_tp_cp_owner_cartesian_product", + "cp_world_size": 1, + "entries": [ + { + "actual_split_boundaries": [ + [ + 0, + 32 + ] + ], + "actual_split_kv_count": 1, + "actual_split_kv_policy": "disabled", + "actual_split_kv_size": null, + "batch_index": 0, + "cp_rank": 0, + "expected_kv_range": [ + 0, + 32 + ], + "owner_cp_rank": 0, + "requested_split_kv_policy": "disabled", + "requested_split_kv_size": null, + "split_kv_accum_dtype": "fp32", + "split_kv_backend": "rlkernel.rocm.deterministic_attention", + "split_kv_downcast_at": "final_write", + "split_kv_fallback": false, + "split_kv_fallback_reason": null, + "split_kv_merge_order": "global_block_index", + "split_kv_plan_source": "contract_exact", + "tp_rank": 0 + }, + { + "actual_split_boundaries": [ + [ + 0, + 32 + ] + ], + "actual_split_kv_count": 1, + "actual_split_kv_policy": "disabled", + "actual_split_kv_size": null, + "batch_index": 0, + "cp_rank": 0, + "expected_kv_range": [ + 0, + 32 + ], + "owner_cp_rank": 0, + "requested_split_kv_policy": "disabled", + "requested_split_kv_size": null, + "split_kv_accum_dtype": "fp32", + "split_kv_backend": "rlkernel.rocm.deterministic_attention", + "split_kv_downcast_at": "final_write", + "split_kv_fallback": false, + "split_kv_fallback_reason": null, + "split_kv_merge_order": "global_block_index", + "split_kv_plan_source": "contract_exact", + "tp_rank": 1 + }, + { + "actual_split_boundaries": [ + [ + 0, + 32 + ] + ], + "actual_split_kv_count": 1, + "actual_split_kv_policy": "disabled", + "actual_split_kv_size": null, + "batch_index": 1, + "cp_rank": 0, + "expected_kv_range": [ + 0, + 32 + ], + "owner_cp_rank": 0, + "requested_split_kv_policy": "disabled", + "requested_split_kv_size": null, + "split_kv_accum_dtype": "fp32", + "split_kv_backend": "rlkernel.rocm.deterministic_attention", + "split_kv_downcast_at": "final_write", + "split_kv_fallback": false, + "split_kv_fallback_reason": null, + "split_kv_merge_order": "global_block_index", + "split_kv_plan_source": "contract_exact", + "tp_rank": 0 + }, + { + "actual_split_boundaries": [ + [ + 0, + 32 + ] + ], + "actual_split_kv_count": 1, + "actual_split_kv_policy": "disabled", + "actual_split_kv_size": null, + "batch_index": 1, + "cp_rank": 0, + "expected_kv_range": [ + 0, + 32 + ], + "owner_cp_rank": 0, + "requested_split_kv_policy": "disabled", + "requested_split_kv_size": null, + "split_kv_accum_dtype": "fp32", + "split_kv_backend": "rlkernel.rocm.deterministic_attention", + "split_kv_downcast_at": "final_write", + "split_kv_fallback": false, + "split_kv_fallback_reason": null, + "split_kv_merge_order": "global_block_index", + "split_kv_plan_source": "contract_exact", + "tp_rank": 1 + } + ], + "total_kv_tokens": [ + 32, + 32 + ], + "tp_world_size": 2 + } + }, + "training": { + "backend_id": "rlkernel.rocm.deterministic_attention", + "contract": { + "batch_size": 2, + "causal": true, + "causal_offsets": [ + 0, + 0 + ], + "dtype": "bf16", + "export_lse": true, + "head_dim": 128, + "kv_cache": null, + "lse_domain": "attention", + "mode": "prefill", + "projections": { + "o_proj": { + "acc_dtype": "fp32", + "backend_policy": "native_verified_then_common_deterministic", + "deterministic_backend": "rlkernel.cuda.det_gemm", + "input_dtype": "bf16", + "k_order": "ascending", + "output_dtype": "bf16", + "qkv_split_order": [], + "require_runtime_readback": true, + "sp_backward_collective": "all_gather", + "sp_forward_collective": "reduce_scatter", + "split_kv": "disabled", + "tp_backward_dgrad_collective": "none", + "tp_forward_collective": "all_reduce" + }, + "qkv": { + "acc_dtype": "fp32", + "backend_policy": "native_verified_then_common_deterministic", + "deterministic_backend": "rlkernel.cuda.det_gemm", + "input_dtype": "bf16", + "k_order": "ascending", + "output_dtype": "bf16", + "qkv_split_order": [ + "q", + "k", + "v" + ], + "require_runtime_readback": true, + "sp_backward_collective": "reduce_scatter", + "sp_forward_collective": "all_gather", + "split_kv": "disabled", + "tp_backward_dgrad_collective": "all_reduce", + "tp_forward_collective": "none" + } + }, + "query_sequence_length": 32, + "reduction": { + "acc_dtype": "fp32", + "downcast_at": "final_write", + "engine": "in_op_reference", + "merge": "online_softmax_lse", + "order": "global_block_index" + }, + "role": "train", + "rope": null, + "semantic_operator": "standard_softmax_attention", + "sharding": { + "cp_rank": 0, + "cp_world_size": 1, + "global_block_indices": [ + 0 + ], + "global_block_token_starts": [ + 0 + ], + "global_kv_heads": 8, + "global_q_heads": 32, + "global_sequence_length": 32, + "local_block_offsets": [ + 0, + 32 + ], + "local_kv_head_start": 0, + "local_kv_heads": 4, + "local_q_head_start": 0, + "local_q_heads": 16, + "local_sequence_length": 32, + "packed_sequence_offsets": null, + "sp_rank": 0, + "sp_world_size": 1, + "tp_rank": 0, + "tp_world_size": 2 + }, + "split_kv": { + "fixed_split_size": null, + "mode": "disabled", + "strict_consistency": true + } + }, + "recorded": { + "backend_id": "rlkernel.rocm.deterministic_attention", + "mode": "prefill", + "reduction.engine": "in_op_reference" + } + } + }, + "recorded_differences": {}, + "reduction_fingerprint": "4bfe65d607de1dc8d1b8c04041d48e9af4e81dc38614f0340d994616da273d6c", + "schema_version": "cross_config.attention_binding.v3" + }, + "category": "comparability_gate", + "comparable": false, + "expected": "rejected", + "gate_implementation": "rl_engine.alignment.cross_config.bind_attention_contracts", + "identity_errors": [ + "sharding.tp_rank", + "sharding.local_q_head_start", + "sharding.local_kv_head_start" + ], + "metrics": { + "dk": null, + "dq": null, + "dv": null, + "lse": null, + "out": null + }, + "outcome": "rejected", + "passed": true, + "probe": "tp_head_ownership", + "realization": "Bind valid TP-rank-1 rollout and TP-rank-0 training contracts; do not run numerics.", + "row_id": "A4", + "sequence": 32 + }, + { + "batch": 2, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "implementations": { + "baseline": "rlkernel.rocm.deterministic_attention", + "candidate": "torch.rocm.index_select+native_attention" + }, + "metrics": { + "dk": { + "bitwise_equal": false, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 5.94140625, + "mismatch_count": 65470, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 32, + 128 + ] + }, + "dq": { + "bitwise_equal": false, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 4.27734375, + "mismatch_count": 245171, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 32, + 128 + ] + }, + "dv": { + "bitwise_equal": false, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 13.2333984375, + "mismatch_count": 65489, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 32, + 128 + ] + }, + "lse": { + "bitwise_equal": false, + "element_count": 2048, + "left_dtype": "torch.float32", + "max_abs": 3.504487991333008, + "mismatch_count": 1984, + "right_dtype": "torch.float32", + "shape": [ + 2, + 32, + 32 + ] + }, + "out": { + "bitwise_equal": false, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 5.21875, + "mismatch_count": 253373, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 32, + 128 + ] + } + }, + "outcome": "drift_detected", + "passed": true, + "probe": "kv_page_order", + "realization": "Reverse four dense K/V tensor pages; this is not a paged-cache runtime.", + "row_id": "A5", + "sequence": 32 + }, + { + "batch": 2, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "implementations": { + "baseline": "torch.rocm.explicit_fp32_serial_qk_accumulator", + "candidate": "torch.rocm.explicit_bf16_serial_qk_accumulator" + }, + "metrics": { + "dk": { + "bitwise_equal": false, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0625, + "mismatch_count": 56067, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 32, + 128 + ] + }, + "dq": { + "bitwise_equal": false, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0546875, + "mismatch_count": 206125, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 32, + 128 + ] + }, + "dv": { + "bitwise_equal": false, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 0.08984375, + "mismatch_count": 56748, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 32, + 128 + ] + }, + "lse": { + "bitwise_equal": false, + "element_count": 2048, + "left_dtype": "torch.float32", + "max_abs": 0.06517952680587769, + "mismatch_count": 2048, + "right_dtype": "torch.float32", + "shape": [ + 2, + 32, + 32 + ] + }, + "out": { + "bitwise_equal": false, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 0.078125, + "mismatch_count": 206686, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 32, + 128 + ] + } + }, + "outcome": "drift_detected", + "passed": true, + "probe": "accum_dtype", + "realization": "Use identical FP32 products/order with explicit FP32 versus BF16 accumulator state.", + "row_id": "A6", + "sequence": 32 + }, + { + "batch": 2, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "implementations": { + "baseline": "torch.rocm.fp32_chunk_merge_ascending", + "candidate": "torch.rocm.fp32_chunk_merge_descending" + }, + "metrics": { + "dk": { + "bitwise_equal": false, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 0.00390625, + "mismatch_count": 5, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 32, + 128 + ] + }, + "dq": { + "bitwise_equal": false, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0009765625, + "mismatch_count": 12, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 32, + 128 + ] + }, + "dv": { + "bitwise_equal": false, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 0.00390625, + "mismatch_count": 3, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 32, + 128 + ] + }, + "lse": { + "bitwise_equal": false, + "element_count": 2048, + "left_dtype": "torch.float32", + "max_abs": 4.76837158203125e-07, + "mismatch_count": 117, + "right_dtype": "torch.float32", + "shape": [ + 2, + 32, + 32 + ] + }, + "out": { + "bitwise_equal": false, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 0.00048828125, + "mismatch_count": 3, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 32, + 128 + ] + } + }, + "outcome": "drift_detected", + "passed": true, + "probe": "merge_order", + "realization": "Merge four dense chunks in opposite orders on one GPU; this is not a CP collective.", + "row_id": "A7", + "sequence": 32 + }, + { + "batch": 2, + "category": "invariant_control", + "comparable": true, + "expected": "exact_zero", + "implementations": { + "baseline": "rlkernel.rocm.deterministic_attention", + "candidate": "rlkernel.rocm.native_attention_per_tp2_head_shard" + }, + "metrics": { + "dk": { + "bitwise_equal": true, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 32, + 128 + ] + }, + "dq": { + "bitwise_equal": true, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 32, + 128 + ] + }, + "dv": { + "bitwise_equal": true, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 32, + 128 + ] + }, + "lse": { + "bitwise_equal": true, + "element_count": 2048, + "left_dtype": "torch.float32", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.float32", + "shape": [ + 2, + 32, + 32 + ] + }, + "out": { + "bitwise_equal": true, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 32, + 128 + ] + } + }, + "outcome": "matched", + "passed": true, + "probe": "tp_partition_control", + "realization": "Compare full GQA with two contiguous TP=2 head shards on one GPU.", + "row_id": "C0", + "sequence": 32 + }, + { + "batch": 2, + "category": "invariant_control", + "comparable": true, + "expected": "exact_zero", + "implementations": { + "baseline": "rlkernel.rocm.deterministic_attention", + "candidate": "rlkernel.rocm.native_attention_per_batch_row" + }, + "metrics": { + "dk": { + "bitwise_equal": true, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 32, + 128 + ] + }, + "dq": { + "bitwise_equal": true, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 32, + 128 + ] + }, + "dv": { + "bitwise_equal": true, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 32, + 128 + ] + }, + "lse": { + "bitwise_equal": true, + "element_count": 2048, + "left_dtype": "torch.float32", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.float32", + "shape": [ + 2, + 32, + 32 + ] + }, + "out": { + "bitwise_equal": true, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 32, + 128 + ] + } + }, + "outcome": "matched", + "passed": true, + "probe": "batch_composition_control", + "realization": "Compare a batch call with per-row calls to the same native HIP core.", + "row_id": "C1", + "sequence": 32 + }, + { + "batch": 2, + "category": "invariant_control", + "comparable": true, + "expected": "exact_zero", + "implementations": { + "baseline": "rlkernel.rocm.native_attention_full_prefill", + "candidate": "rlkernel.rocm.native_attention_tail" + }, + "metrics": { + "dk": { + "bitwise_equal": true, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 32, + 128 + ] + }, + "dq": { + "bitwise_equal": true, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 32, + 128 + ] + }, + "dv": { + "bitwise_equal": true, + "element_count": 65536, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 32, + 128 + ] + }, + "lse": { + "bitwise_equal": true, + "element_count": 64, + "left_dtype": "torch.float32", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.float32", + "shape": [ + 2, + 32, + 1 + ] + }, + "out": { + "bitwise_equal": true, + "element_count": 8192, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 1, + 128 + ] + } + }, + "outcome": "matched", + "passed": true, + "probe": "prefill_decode_tail_control", + "realization": "Compare a full-prefill tail with one trailing query over dense KV, without a serving cache.", + "row_id": "C2", + "sequence": 32 + }, + { + "batch": 2, + "category": "baseline", + "comparable": true, + "expected": "baseline", + "implementations": { + "baseline": "rlkernel.rocm.deterministic_attention", + "candidate": "rlkernel.rocm.deterministic_attention" + }, + "metrics": { + "dk": { + "bitwise_equal": true, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 64, + 128 + ] + }, + "dq": { + "bitwise_equal": true, + "element_count": 524288, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 64, + 128 + ] + }, + "dv": { + "bitwise_equal": true, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 64, + 128 + ] + }, + "lse": { + "bitwise_equal": true, + "element_count": 4096, + "left_dtype": "torch.float32", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.float32", + "shape": [ + 2, + 32, + 64 + ] + }, + "out": { + "bitwise_equal": true, + "element_count": 524288, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 64, + 128 + ] + } + }, + "outcome": "matched", + "passed": true, + "probe": null, + "realization": "Repeat the identical native HIP reference-core call.", + "row_id": "A0", + "sequence": 64 + }, + { + "batch": 2, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "implementations": { + "baseline": "rlkernel.rocm.deterministic_rope+native_attention", + "candidate": "rlkernel.rocm.deterministic_rope+native_attention" + }, + "metrics": { + "dk": { + "bitwise_equal": false, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 1.73046875, + "mismatch_count": 126801, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 64, + 128 + ] + }, + "dq": { + "bitwise_equal": false, + "element_count": 524288, + "left_dtype": "torch.bfloat16", + "max_abs": 1.0390625, + "mismatch_count": 258620, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 64, + 128 + ] + }, + "dv": { + "bitwise_equal": false, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.4765625, + "mismatch_count": 125563, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 64, + 128 + ] + }, + "lse": { + "bitwise_equal": false, + "element_count": 4096, + "left_dtype": "torch.float32", + "max_abs": 0.18473243713378906, + "mismatch_count": 2048, + "right_dtype": "torch.float32", + "shape": [ + 2, + 32, + 64 + ] + }, + "out": { + "bitwise_equal": false, + "element_count": 524288, + "left_dtype": "torch.bfloat16", + "max_abs": 0.40771484375, + "mismatch_count": 258446, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 64, + 128 + ] + } + }, + "outcome": "drift_detected", + "passed": true, + "probe": "position_ids", + "realization": "Increment suffix RoPE positions while preserving Q/K/V tensors.", + "row_id": "A1", + "sequence": 64 + }, + { + "batch": 2, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "implementations": { + "baseline": "torch.rocm.rms_norm+native_attention", + "candidate": "rlkernel.rocm.deterministic_attention" + }, + "metrics": { + "dk": { + "bitwise_equal": false, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 1.25, + "mismatch_count": 129907, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 64, + 128 + ] + }, + "dq": { + "bitwise_equal": false, + "element_count": 524288, + "left_dtype": "torch.bfloat16", + "max_abs": 1.015625, + "mismatch_count": 510233, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 64, + 128 + ] + }, + "dv": { + "bitwise_equal": false, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.96875, + "mismatch_count": 128678, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 64, + 128 + ] + }, + "lse": { + "bitwise_equal": false, + "element_count": 4096, + "left_dtype": "torch.float32", + "max_abs": 1.0303754806518555, + "mismatch_count": 4096, + "right_dtype": "torch.float32", + "shape": [ + 2, + 32, + 64 + ] + }, + "out": { + "bitwise_equal": false, + "element_count": 524288, + "left_dtype": "torch.bfloat16", + "max_abs": 0.490234375, + "mismatch_count": 500934, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 64, + 128 + ] + } + }, + "outcome": "drift_detected", + "passed": true, + "probe": "qk_norm_disabled", + "realization": "Apply or bypass unit-weight PyTorch RMSNorm before the native HIP core.", + "row_id": "A2", + "sequence": 64 + }, + { + "batch": 2, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "implementations": { + "baseline": "rlkernel.rocm.deterministic_attention", + "candidate": "rlkernel.rocm.deterministic_attention" + }, + "metrics": { + "dk": { + "bitwise_equal": false, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 4.3046875, + "mismatch_count": 130884, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 64, + 128 + ] + }, + "dq": { + "bitwise_equal": false, + "element_count": 524288, + "left_dtype": "torch.bfloat16", + "max_abs": 2.93408203125, + "mismatch_count": 513909, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 64, + 128 + ] + }, + "dv": { + "bitwise_equal": false, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 9.46484375, + "mismatch_count": 130910, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 64, + 128 + ] + }, + "lse": { + "bitwise_equal": false, + "element_count": 4096, + "left_dtype": "torch.float32", + "max_abs": 6.752752780914307, + "mismatch_count": 4032, + "right_dtype": "torch.float32", + "shape": [ + 2, + 32, + 64 + ] + }, + "out": { + "bitwise_equal": false, + "element_count": 524288, + "left_dtype": "torch.bfloat16", + "max_abs": 4.1802978515625, + "mismatch_count": 514012, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 64, + 128 + ] + } + }, + "outcome": "drift_detected", + "passed": true, + "probe": "causal_mask", + "realization": "Toggle causal masking in the native HIP core.", + "row_id": "A3", + "sequence": 64 + }, + { + "batch": 2, + "binding_gate": { + "binding_fingerprint": "e9c4483da82da5e16e5bdfbdbf4eeca2c280523521f62a3211bc316900df2ddb", + "comparable": false, + "identity_fingerprint": "c2a02818c11bd195873275c75f4d94ae20b0ae2f2e9daa7fcd5f51d9b409cb4c", + "issues": [ + { + "code": "TOPOLOGY_MISMATCH", + "field": "sharding.tp_rank", + "message": "sharding.tp_rank changes TP/CP ownership; the pair is not the same local attention problem", + "rollout": 1, + "tier": "identical", + "training": 0 + }, + { + "code": "TOPOLOGY_MISMATCH", + "field": "sharding.local_q_head_start", + "message": "sharding.local_q_head_start changes TP/CP ownership; the pair is not the same local attention problem", + "rollout": 16, + "tier": "identical", + "training": 0 + }, + { + "code": "TOPOLOGY_MISMATCH", + "field": "sharding.local_kv_head_start", + "message": "sharding.local_kv_head_start changes TP/CP ownership; the pair is not the same local attention problem", + "rollout": 4, + "tier": "identical", + "training": 0 + } + ], + "passed": false, + "provenance": { + "dtype": "bf16", + "lse_domain": "attention", + "rollout": { + "backend_id": "rlkernel.rocm.deterministic_attention", + "contract": { + "batch_size": 2, + "causal": true, + "causal_offsets": [ + 0, + 0 + ], + "dtype": "bf16", + "export_lse": true, + "head_dim": 128, + "kv_cache": null, + "lse_domain": "attention", + "mode": "prefill", + "projections": { + "o_proj": { + "acc_dtype": "fp32", + "backend_policy": "native_verified_then_common_deterministic", + "deterministic_backend": "rlkernel.cuda.det_gemm", + "input_dtype": "bf16", + "k_order": "ascending", + "output_dtype": "bf16", + "qkv_split_order": [], + "require_runtime_readback": true, + "sp_backward_collective": "all_gather", + "sp_forward_collective": "reduce_scatter", + "split_kv": "disabled", + "tp_backward_dgrad_collective": "none", + "tp_forward_collective": "all_reduce" + }, + "qkv": { + "acc_dtype": "fp32", + "backend_policy": "native_verified_then_common_deterministic", + "deterministic_backend": "rlkernel.cuda.det_gemm", + "input_dtype": "bf16", + "k_order": "ascending", + "output_dtype": "bf16", + "qkv_split_order": [ + "q", + "k", + "v" + ], + "require_runtime_readback": true, + "sp_backward_collective": "reduce_scatter", + "sp_forward_collective": "all_gather", + "split_kv": "disabled", + "tp_backward_dgrad_collective": "all_reduce", + "tp_forward_collective": "none" + } + }, + "query_sequence_length": 64, + "reduction": { + "acc_dtype": "fp32", + "downcast_at": "final_write", + "engine": "in_op_reference", + "merge": "online_softmax_lse", + "order": "global_block_index" + }, + "role": "infer", + "rope": null, + "semantic_operator": "standard_softmax_attention", + "sharding": { + "cp_rank": 0, + "cp_world_size": 1, + "global_block_indices": [ + 0 + ], + "global_block_token_starts": [ + 0 + ], + "global_kv_heads": 8, + "global_q_heads": 32, + "global_sequence_length": 64, + "local_block_offsets": [ + 0, + 64 + ], + "local_kv_head_start": 4, + "local_kv_heads": 4, + "local_q_head_start": 16, + "local_q_heads": 16, + "local_sequence_length": 64, + "packed_sequence_offsets": null, + "sp_rank": 0, + "sp_world_size": 1, + "tp_rank": 1, + "tp_world_size": 2 + }, + "split_kv": { + "fixed_split_size": null, + "mode": "disabled", + "strict_consistency": true + } + }, + "recorded": { + "backend_id": "rlkernel.rocm.deterministic_attention", + "mode": "prefill", + "reduction.engine": "in_op_reference" + } + }, + "split_kv_runtime": { + "rollout": { + "batch_size": 2, + "coverage": "complete_batch_tp_cp_owner_cartesian_product", + "cp_world_size": 1, + "entries": [ + { + "actual_split_boundaries": [ + [ + 0, + 64 + ] + ], + "actual_split_kv_count": 1, + "actual_split_kv_policy": "disabled", + "actual_split_kv_size": null, + "batch_index": 0, + "cp_rank": 0, + "expected_kv_range": [ + 0, + 64 + ], + "owner_cp_rank": 0, + "requested_split_kv_policy": "disabled", + "requested_split_kv_size": null, + "split_kv_accum_dtype": "fp32", + "split_kv_backend": "rlkernel.rocm.deterministic_attention", + "split_kv_downcast_at": "final_write", + "split_kv_fallback": false, + "split_kv_fallback_reason": null, + "split_kv_merge_order": "global_block_index", + "split_kv_plan_source": "contract_exact", + "tp_rank": 0 + }, + { + "actual_split_boundaries": [ + [ + 0, + 64 + ] + ], + "actual_split_kv_count": 1, + "actual_split_kv_policy": "disabled", + "actual_split_kv_size": null, + "batch_index": 0, + "cp_rank": 0, + "expected_kv_range": [ + 0, + 64 + ], + "owner_cp_rank": 0, + "requested_split_kv_policy": "disabled", + "requested_split_kv_size": null, + "split_kv_accum_dtype": "fp32", + "split_kv_backend": "rlkernel.rocm.deterministic_attention", + "split_kv_downcast_at": "final_write", + "split_kv_fallback": false, + "split_kv_fallback_reason": null, + "split_kv_merge_order": "global_block_index", + "split_kv_plan_source": "contract_exact", + "tp_rank": 1 + }, + { + "actual_split_boundaries": [ + [ + 0, + 64 + ] + ], + "actual_split_kv_count": 1, + "actual_split_kv_policy": "disabled", + "actual_split_kv_size": null, + "batch_index": 1, + "cp_rank": 0, + "expected_kv_range": [ + 0, + 64 + ], + "owner_cp_rank": 0, + "requested_split_kv_policy": "disabled", + "requested_split_kv_size": null, + "split_kv_accum_dtype": "fp32", + "split_kv_backend": "rlkernel.rocm.deterministic_attention", + "split_kv_downcast_at": "final_write", + "split_kv_fallback": false, + "split_kv_fallback_reason": null, + "split_kv_merge_order": "global_block_index", + "split_kv_plan_source": "contract_exact", + "tp_rank": 0 + }, + { + "actual_split_boundaries": [ + [ + 0, + 64 + ] + ], + "actual_split_kv_count": 1, + "actual_split_kv_policy": "disabled", + "actual_split_kv_size": null, + "batch_index": 1, + "cp_rank": 0, + "expected_kv_range": [ + 0, + 64 + ], + "owner_cp_rank": 0, + "requested_split_kv_policy": "disabled", + "requested_split_kv_size": null, + "split_kv_accum_dtype": "fp32", + "split_kv_backend": "rlkernel.rocm.deterministic_attention", + "split_kv_downcast_at": "final_write", + "split_kv_fallback": false, + "split_kv_fallback_reason": null, + "split_kv_merge_order": "global_block_index", + "split_kv_plan_source": "contract_exact", + "tp_rank": 1 + } + ], + "total_kv_tokens": [ + 64, + 64 + ], + "tp_world_size": 2 + }, + "training": { + "batch_size": 2, + "coverage": "complete_batch_tp_cp_owner_cartesian_product", + "cp_world_size": 1, + "entries": [ + { + "actual_split_boundaries": [ + [ + 0, + 64 + ] + ], + "actual_split_kv_count": 1, + "actual_split_kv_policy": "disabled", + "actual_split_kv_size": null, + "batch_index": 0, + "cp_rank": 0, + "expected_kv_range": [ + 0, + 64 + ], + "owner_cp_rank": 0, + "requested_split_kv_policy": "disabled", + "requested_split_kv_size": null, + "split_kv_accum_dtype": "fp32", + "split_kv_backend": "rlkernel.rocm.deterministic_attention", + "split_kv_downcast_at": "final_write", + "split_kv_fallback": false, + "split_kv_fallback_reason": null, + "split_kv_merge_order": "global_block_index", + "split_kv_plan_source": "contract_exact", + "tp_rank": 0 + }, + { + "actual_split_boundaries": [ + [ + 0, + 64 + ] + ], + "actual_split_kv_count": 1, + "actual_split_kv_policy": "disabled", + "actual_split_kv_size": null, + "batch_index": 0, + "cp_rank": 0, + "expected_kv_range": [ + 0, + 64 + ], + "owner_cp_rank": 0, + "requested_split_kv_policy": "disabled", + "requested_split_kv_size": null, + "split_kv_accum_dtype": "fp32", + "split_kv_backend": "rlkernel.rocm.deterministic_attention", + "split_kv_downcast_at": "final_write", + "split_kv_fallback": false, + "split_kv_fallback_reason": null, + "split_kv_merge_order": "global_block_index", + "split_kv_plan_source": "contract_exact", + "tp_rank": 1 + }, + { + "actual_split_boundaries": [ + [ + 0, + 64 + ] + ], + "actual_split_kv_count": 1, + "actual_split_kv_policy": "disabled", + "actual_split_kv_size": null, + "batch_index": 1, + "cp_rank": 0, + "expected_kv_range": [ + 0, + 64 + ], + "owner_cp_rank": 0, + "requested_split_kv_policy": "disabled", + "requested_split_kv_size": null, + "split_kv_accum_dtype": "fp32", + "split_kv_backend": "rlkernel.rocm.deterministic_attention", + "split_kv_downcast_at": "final_write", + "split_kv_fallback": false, + "split_kv_fallback_reason": null, + "split_kv_merge_order": "global_block_index", + "split_kv_plan_source": "contract_exact", + "tp_rank": 0 + }, + { + "actual_split_boundaries": [ + [ + 0, + 64 + ] + ], + "actual_split_kv_count": 1, + "actual_split_kv_policy": "disabled", + "actual_split_kv_size": null, + "batch_index": 1, + "cp_rank": 0, + "expected_kv_range": [ + 0, + 64 + ], + "owner_cp_rank": 0, + "requested_split_kv_policy": "disabled", + "requested_split_kv_size": null, + "split_kv_accum_dtype": "fp32", + "split_kv_backend": "rlkernel.rocm.deterministic_attention", + "split_kv_downcast_at": "final_write", + "split_kv_fallback": false, + "split_kv_fallback_reason": null, + "split_kv_merge_order": "global_block_index", + "split_kv_plan_source": "contract_exact", + "tp_rank": 1 + } + ], + "total_kv_tokens": [ + 64, + 64 + ], + "tp_world_size": 2 + } + }, + "training": { + "backend_id": "rlkernel.rocm.deterministic_attention", + "contract": { + "batch_size": 2, + "causal": true, + "causal_offsets": [ + 0, + 0 + ], + "dtype": "bf16", + "export_lse": true, + "head_dim": 128, + "kv_cache": null, + "lse_domain": "attention", + "mode": "prefill", + "projections": { + "o_proj": { + "acc_dtype": "fp32", + "backend_policy": "native_verified_then_common_deterministic", + "deterministic_backend": "rlkernel.cuda.det_gemm", + "input_dtype": "bf16", + "k_order": "ascending", + "output_dtype": "bf16", + "qkv_split_order": [], + "require_runtime_readback": true, + "sp_backward_collective": "all_gather", + "sp_forward_collective": "reduce_scatter", + "split_kv": "disabled", + "tp_backward_dgrad_collective": "none", + "tp_forward_collective": "all_reduce" + }, + "qkv": { + "acc_dtype": "fp32", + "backend_policy": "native_verified_then_common_deterministic", + "deterministic_backend": "rlkernel.cuda.det_gemm", + "input_dtype": "bf16", + "k_order": "ascending", + "output_dtype": "bf16", + "qkv_split_order": [ + "q", + "k", + "v" + ], + "require_runtime_readback": true, + "sp_backward_collective": "reduce_scatter", + "sp_forward_collective": "all_gather", + "split_kv": "disabled", + "tp_backward_dgrad_collective": "all_reduce", + "tp_forward_collective": "none" + } + }, + "query_sequence_length": 64, + "reduction": { + "acc_dtype": "fp32", + "downcast_at": "final_write", + "engine": "in_op_reference", + "merge": "online_softmax_lse", + "order": "global_block_index" + }, + "role": "train", + "rope": null, + "semantic_operator": "standard_softmax_attention", + "sharding": { + "cp_rank": 0, + "cp_world_size": 1, + "global_block_indices": [ + 0 + ], + "global_block_token_starts": [ + 0 + ], + "global_kv_heads": 8, + "global_q_heads": 32, + "global_sequence_length": 64, + "local_block_offsets": [ + 0, + 64 + ], + "local_kv_head_start": 0, + "local_kv_heads": 4, + "local_q_head_start": 0, + "local_q_heads": 16, + "local_sequence_length": 64, + "packed_sequence_offsets": null, + "sp_rank": 0, + "sp_world_size": 1, + "tp_rank": 0, + "tp_world_size": 2 + }, + "split_kv": { + "fixed_split_size": null, + "mode": "disabled", + "strict_consistency": true + } + }, + "recorded": { + "backend_id": "rlkernel.rocm.deterministic_attention", + "mode": "prefill", + "reduction.engine": "in_op_reference" + } + } + }, + "recorded_differences": {}, + "reduction_fingerprint": "4bfe65d607de1dc8d1b8c04041d48e9af4e81dc38614f0340d994616da273d6c", + "schema_version": "cross_config.attention_binding.v3" + }, + "category": "comparability_gate", + "comparable": false, + "expected": "rejected", + "gate_implementation": "rl_engine.alignment.cross_config.bind_attention_contracts", + "identity_errors": [ + "sharding.tp_rank", + "sharding.local_q_head_start", + "sharding.local_kv_head_start" + ], + "metrics": { + "dk": null, + "dq": null, + "dv": null, + "lse": null, + "out": null + }, + "outcome": "rejected", + "passed": true, + "probe": "tp_head_ownership", + "realization": "Bind valid TP-rank-1 rollout and TP-rank-0 training contracts; do not run numerics.", + "row_id": "A4", + "sequence": 64 + }, + { + "batch": 2, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "implementations": { + "baseline": "rlkernel.rocm.deterministic_attention", + "candidate": "torch.rocm.index_select+native_attention" + }, + "metrics": { + "dk": { + "bitwise_equal": false, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 5.31640625, + "mismatch_count": 130971, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 64, + 128 + ] + }, + "dq": { + "bitwise_equal": false, + "element_count": 524288, + "left_dtype": "torch.bfloat16", + "max_abs": 3.9140625, + "mismatch_count": 506631, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 64, + 128 + ] + }, + "dv": { + "bitwise_equal": false, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 10.416015625, + "mismatch_count": 130967, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 64, + 128 + ] + }, + "lse": { + "bitwise_equal": false, + "element_count": 4096, + "left_dtype": "torch.float32", + "max_abs": 3.361884593963623, + "mismatch_count": 4032, + "right_dtype": "torch.float32", + "shape": [ + 2, + 32, + 64 + ] + }, + "out": { + "bitwise_equal": false, + "element_count": 524288, + "left_dtype": "torch.bfloat16", + "max_abs": 5.015625, + "mismatch_count": 514873, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 64, + 128 + ] + } + }, + "outcome": "drift_detected", + "passed": true, + "probe": "kv_page_order", + "realization": "Reverse four dense K/V tensor pages; this is not a paged-cache runtime.", + "row_id": "A5", + "sequence": 64 + }, + { + "batch": 2, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "implementations": { + "baseline": "torch.rocm.explicit_fp32_serial_qk_accumulator", + "candidate": "torch.rocm.explicit_bf16_serial_qk_accumulator" + }, + "metrics": { + "dk": { + "bitwise_equal": false, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.09375, + "mismatch_count": 114541, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 64, + 128 + ] + }, + "dq": { + "bitwise_equal": false, + "element_count": 524288, + "left_dtype": "torch.bfloat16", + "max_abs": 0.09375, + "mismatch_count": 433929, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 64, + 128 + ] + }, + "dv": { + "bitwise_equal": false, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.11328125, + "mismatch_count": 115947, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 64, + 128 + ] + }, + "lse": { + "bitwise_equal": false, + "element_count": 4096, + "left_dtype": "torch.float32", + "max_abs": 0.08368206024169922, + "mismatch_count": 4096, + "right_dtype": "torch.float32", + "shape": [ + 2, + 32, + 64 + ] + }, + "out": { + "bitwise_equal": false, + "element_count": 524288, + "left_dtype": "torch.bfloat16", + "max_abs": 0.1171875, + "mismatch_count": 437879, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 64, + 128 + ] + } + }, + "outcome": "drift_detected", + "passed": true, + "probe": "accum_dtype", + "realization": "Use identical FP32 products/order with explicit FP32 versus BF16 accumulator state.", + "row_id": "A6", + "sequence": 64 + }, + { + "batch": 2, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "implementations": { + "baseline": "torch.rocm.fp32_chunk_merge_ascending", + "candidate": "torch.rocm.fp32_chunk_merge_descending" + }, + "metrics": { + "dk": { + "bitwise_equal": false, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.001953125, + "mismatch_count": 11, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 64, + 128 + ] + }, + "dq": { + "bitwise_equal": false, + "element_count": 524288, + "left_dtype": "torch.bfloat16", + "max_abs": 0.00390625, + "mismatch_count": 19, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 64, + 128 + ] + }, + "dv": { + "bitwise_equal": false, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0009765625, + "mismatch_count": 5, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 64, + 128 + ] + }, + "lse": { + "bitwise_equal": false, + "element_count": 4096, + "left_dtype": "torch.float32", + "max_abs": 4.76837158203125e-07, + "mismatch_count": 138, + "right_dtype": "torch.float32", + "shape": [ + 2, + 32, + 64 + ] + }, + "out": { + "bitwise_equal": false, + "element_count": 524288, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0009765625, + "mismatch_count": 14, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 64, + 128 + ] + } + }, + "outcome": "drift_detected", + "passed": true, + "probe": "merge_order", + "realization": "Merge four dense chunks in opposite orders on one GPU; this is not a CP collective.", + "row_id": "A7", + "sequence": 64 + }, + { + "batch": 2, + "category": "invariant_control", + "comparable": true, + "expected": "exact_zero", + "implementations": { + "baseline": "rlkernel.rocm.deterministic_attention", + "candidate": "rlkernel.rocm.native_attention_per_tp2_head_shard" + }, + "metrics": { + "dk": { + "bitwise_equal": true, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 64, + 128 + ] + }, + "dq": { + "bitwise_equal": true, + "element_count": 524288, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 64, + 128 + ] + }, + "dv": { + "bitwise_equal": true, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 64, + 128 + ] + }, + "lse": { + "bitwise_equal": true, + "element_count": 4096, + "left_dtype": "torch.float32", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.float32", + "shape": [ + 2, + 32, + 64 + ] + }, + "out": { + "bitwise_equal": true, + "element_count": 524288, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 64, + 128 + ] + } + }, + "outcome": "matched", + "passed": true, + "probe": "tp_partition_control", + "realization": "Compare full GQA with two contiguous TP=2 head shards on one GPU.", + "row_id": "C0", + "sequence": 64 + }, + { + "batch": 2, + "category": "invariant_control", + "comparable": true, + "expected": "exact_zero", + "implementations": { + "baseline": "rlkernel.rocm.deterministic_attention", + "candidate": "rlkernel.rocm.native_attention_per_batch_row" + }, + "metrics": { + "dk": { + "bitwise_equal": true, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 64, + 128 + ] + }, + "dq": { + "bitwise_equal": true, + "element_count": 524288, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 64, + 128 + ] + }, + "dv": { + "bitwise_equal": true, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 64, + 128 + ] + }, + "lse": { + "bitwise_equal": true, + "element_count": 4096, + "left_dtype": "torch.float32", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.float32", + "shape": [ + 2, + 32, + 64 + ] + }, + "out": { + "bitwise_equal": true, + "element_count": 524288, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 64, + 128 + ] + } + }, + "outcome": "matched", + "passed": true, + "probe": "batch_composition_control", + "realization": "Compare a batch call with per-row calls to the same native HIP core.", + "row_id": "C1", + "sequence": 64 + }, + { + "batch": 2, + "category": "invariant_control", + "comparable": true, + "expected": "exact_zero", + "implementations": { + "baseline": "rlkernel.rocm.native_attention_full_prefill", + "candidate": "rlkernel.rocm.native_attention_tail" + }, + "metrics": { + "dk": { + "bitwise_equal": true, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 64, + 128 + ] + }, + "dq": { + "bitwise_equal": true, + "element_count": 524288, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 64, + 128 + ] + }, + "dv": { + "bitwise_equal": true, + "element_count": 131072, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 64, + 128 + ] + }, + "lse": { + "bitwise_equal": true, + "element_count": 64, + "left_dtype": "torch.float32", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.float32", + "shape": [ + 2, + 32, + 1 + ] + }, + "out": { + "bitwise_equal": true, + "element_count": 8192, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 1, + 128 + ] + } + }, + "outcome": "matched", + "passed": true, + "probe": "prefill_decode_tail_control", + "realization": "Compare a full-prefill tail with one trailing query over dense KV, without a serving cache.", + "row_id": "C2", + "sequence": 64 + }, + { + "batch": 2, + "category": "baseline", + "comparable": true, + "expected": "baseline", + "implementations": { + "baseline": "rlkernel.rocm.deterministic_attention", + "candidate": "rlkernel.rocm.deterministic_attention" + }, + "metrics": { + "dk": { + "bitwise_equal": true, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 128, + 128 + ] + }, + "dq": { + "bitwise_equal": true, + "element_count": 1048576, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 128, + 128 + ] + }, + "dv": { + "bitwise_equal": true, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 128, + 128 + ] + }, + "lse": { + "bitwise_equal": true, + "element_count": 8192, + "left_dtype": "torch.float32", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.float32", + "shape": [ + 2, + 32, + 128 + ] + }, + "out": { + "bitwise_equal": true, + "element_count": 1048576, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 128, + 128 + ] + } + }, + "outcome": "matched", + "passed": true, + "probe": null, + "realization": "Repeat the identical native HIP reference-core call.", + "row_id": "A0", + "sequence": 128 + }, + { + "batch": 2, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "implementations": { + "baseline": "rlkernel.rocm.deterministic_rope+native_attention", + "candidate": "rlkernel.rocm.deterministic_rope+native_attention" + }, + "metrics": { + "dk": { + "bitwise_equal": false, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 1.580078125, + "mismatch_count": 252194, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 128, + 128 + ] + }, + "dq": { + "bitwise_equal": false, + "element_count": 1048576, + "left_dtype": "torch.bfloat16", + "max_abs": 1.04150390625, + "mismatch_count": 517844, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 128, + 128 + ] + }, + "dv": { + "bitwise_equal": false, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 0.3916015625, + "mismatch_count": 248733, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 128, + 128 + ] + }, + "lse": { + "bitwise_equal": false, + "element_count": 8192, + "left_dtype": "torch.float32", + "max_abs": 0.1368560791015625, + "mismatch_count": 4096, + "right_dtype": "torch.float32", + "shape": [ + 2, + 32, + 128 + ] + }, + "out": { + "bitwise_equal": false, + "element_count": 1048576, + "left_dtype": "torch.bfloat16", + "max_abs": 0.416015625, + "mismatch_count": 517567, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 128, + 128 + ] + } + }, + "outcome": "drift_detected", + "passed": true, + "probe": "position_ids", + "realization": "Increment suffix RoPE positions while preserving Q/K/V tensors.", + "row_id": "A1", + "sequence": 128 + }, + { + "batch": 2, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "implementations": { + "baseline": "torch.rocm.rms_norm+native_attention", + "candidate": "rlkernel.rocm.deterministic_attention" + }, + "metrics": { + "dk": { + "bitwise_equal": false, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 1.4921875, + "mismatch_count": 259978, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 128, + 128 + ] + }, + "dq": { + "bitwise_equal": false, + "element_count": 1048576, + "left_dtype": "torch.bfloat16", + "max_abs": 0.7890625, + "mismatch_count": 1030092, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 128, + 128 + ] + }, + "dv": { + "bitwise_equal": false, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 0.890625, + "mismatch_count": 257810, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 128, + 128 + ] + }, + "lse": { + "bitwise_equal": false, + "element_count": 8192, + "left_dtype": "torch.float32", + "max_abs": 0.3836212158203125, + "mismatch_count": 8192, + "right_dtype": "torch.float32", + "shape": [ + 2, + 32, + 128 + ] + }, + "out": { + "bitwise_equal": false, + "element_count": 1048576, + "left_dtype": "torch.bfloat16", + "max_abs": 0.58203125, + "mismatch_count": 1017881, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 128, + 128 + ] + } + }, + "outcome": "drift_detected", + "passed": true, + "probe": "qk_norm_disabled", + "realization": "Apply or bypass unit-weight PyTorch RMSNorm before the native HIP core.", + "row_id": "A2", + "sequence": 128 + }, + { + "batch": 2, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "implementations": { + "baseline": "rlkernel.rocm.deterministic_attention", + "candidate": "rlkernel.rocm.deterministic_attention" + }, + "metrics": { + "dk": { + "bitwise_equal": false, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 5.357421875, + "mismatch_count": 261780, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 128, + 128 + ] + }, + "dq": { + "bitwise_equal": false, + "element_count": 1048576, + "left_dtype": "torch.bfloat16", + "max_abs": 3.1474609375, + "mismatch_count": 1035801, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 128, + 128 + ] + }, + "dv": { + "bitwise_equal": false, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 9.1572265625, + "mismatch_count": 261805, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 128, + 128 + ] + }, + "lse": { + "bitwise_equal": false, + "element_count": 8192, + "left_dtype": "torch.float32", + "max_abs": 7.316828727722168, + "mismatch_count": 8128, + "right_dtype": "torch.float32", + "shape": [ + 2, + 32, + 128 + ] + }, + "out": { + "bitwise_equal": false, + "element_count": 1048576, + "left_dtype": "torch.bfloat16", + "max_abs": 3.4052734375, + "mismatch_count": 1036536, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 128, + 128 + ] + } + }, + "outcome": "drift_detected", + "passed": true, + "probe": "causal_mask", + "realization": "Toggle causal masking in the native HIP core.", + "row_id": "A3", + "sequence": 128 + }, + { + "batch": 2, + "binding_gate": { + "binding_fingerprint": "03ccee373f91bcf85226a86a89472e7f94b881fc788c51ffbe3d8625753af201", + "comparable": false, + "identity_fingerprint": "c2a02818c11bd195873275c75f4d94ae20b0ae2f2e9daa7fcd5f51d9b409cb4c", + "issues": [ + { + "code": "TOPOLOGY_MISMATCH", + "field": "sharding.tp_rank", + "message": "sharding.tp_rank changes TP/CP ownership; the pair is not the same local attention problem", + "rollout": 1, + "tier": "identical", + "training": 0 + }, + { + "code": "TOPOLOGY_MISMATCH", + "field": "sharding.local_q_head_start", + "message": "sharding.local_q_head_start changes TP/CP ownership; the pair is not the same local attention problem", + "rollout": 16, + "tier": "identical", + "training": 0 + }, + { + "code": "TOPOLOGY_MISMATCH", + "field": "sharding.local_kv_head_start", + "message": "sharding.local_kv_head_start changes TP/CP ownership; the pair is not the same local attention problem", + "rollout": 4, + "tier": "identical", + "training": 0 + } + ], + "passed": false, + "provenance": { + "dtype": "bf16", + "lse_domain": "attention", + "rollout": { + "backend_id": "rlkernel.rocm.deterministic_attention", + "contract": { + "batch_size": 2, + "causal": true, + "causal_offsets": [ + 0, + 0 + ], + "dtype": "bf16", + "export_lse": true, + "head_dim": 128, + "kv_cache": null, + "lse_domain": "attention", + "mode": "prefill", + "projections": { + "o_proj": { + "acc_dtype": "fp32", + "backend_policy": "native_verified_then_common_deterministic", + "deterministic_backend": "rlkernel.cuda.det_gemm", + "input_dtype": "bf16", + "k_order": "ascending", + "output_dtype": "bf16", + "qkv_split_order": [], + "require_runtime_readback": true, + "sp_backward_collective": "all_gather", + "sp_forward_collective": "reduce_scatter", + "split_kv": "disabled", + "tp_backward_dgrad_collective": "none", + "tp_forward_collective": "all_reduce" + }, + "qkv": { + "acc_dtype": "fp32", + "backend_policy": "native_verified_then_common_deterministic", + "deterministic_backend": "rlkernel.cuda.det_gemm", + "input_dtype": "bf16", + "k_order": "ascending", + "output_dtype": "bf16", + "qkv_split_order": [ + "q", + "k", + "v" + ], + "require_runtime_readback": true, + "sp_backward_collective": "reduce_scatter", + "sp_forward_collective": "all_gather", + "split_kv": "disabled", + "tp_backward_dgrad_collective": "all_reduce", + "tp_forward_collective": "none" + } + }, + "query_sequence_length": 128, + "reduction": { + "acc_dtype": "fp32", + "downcast_at": "final_write", + "engine": "in_op_reference", + "merge": "online_softmax_lse", + "order": "global_block_index" + }, + "role": "infer", + "rope": null, + "semantic_operator": "standard_softmax_attention", + "sharding": { + "cp_rank": 0, + "cp_world_size": 1, + "global_block_indices": [ + 0 + ], + "global_block_token_starts": [ + 0 + ], + "global_kv_heads": 8, + "global_q_heads": 32, + "global_sequence_length": 128, + "local_block_offsets": [ + 0, + 128 + ], + "local_kv_head_start": 4, + "local_kv_heads": 4, + "local_q_head_start": 16, + "local_q_heads": 16, + "local_sequence_length": 128, + "packed_sequence_offsets": null, + "sp_rank": 0, + "sp_world_size": 1, + "tp_rank": 1, + "tp_world_size": 2 + }, + "split_kv": { + "fixed_split_size": null, + "mode": "disabled", + "strict_consistency": true + } + }, + "recorded": { + "backend_id": "rlkernel.rocm.deterministic_attention", + "mode": "prefill", + "reduction.engine": "in_op_reference" + } + }, + "split_kv_runtime": { + "rollout": { + "batch_size": 2, + "coverage": "complete_batch_tp_cp_owner_cartesian_product", + "cp_world_size": 1, + "entries": [ + { + "actual_split_boundaries": [ + [ + 0, + 128 + ] + ], + "actual_split_kv_count": 1, + "actual_split_kv_policy": "disabled", + "actual_split_kv_size": null, + "batch_index": 0, + "cp_rank": 0, + "expected_kv_range": [ + 0, + 128 + ], + "owner_cp_rank": 0, + "requested_split_kv_policy": "disabled", + "requested_split_kv_size": null, + "split_kv_accum_dtype": "fp32", + "split_kv_backend": "rlkernel.rocm.deterministic_attention", + "split_kv_downcast_at": "final_write", + "split_kv_fallback": false, + "split_kv_fallback_reason": null, + "split_kv_merge_order": "global_block_index", + "split_kv_plan_source": "contract_exact", + "tp_rank": 0 + }, + { + "actual_split_boundaries": [ + [ + 0, + 128 + ] + ], + "actual_split_kv_count": 1, + "actual_split_kv_policy": "disabled", + "actual_split_kv_size": null, + "batch_index": 0, + "cp_rank": 0, + "expected_kv_range": [ + 0, + 128 + ], + "owner_cp_rank": 0, + "requested_split_kv_policy": "disabled", + "requested_split_kv_size": null, + "split_kv_accum_dtype": "fp32", + "split_kv_backend": "rlkernel.rocm.deterministic_attention", + "split_kv_downcast_at": "final_write", + "split_kv_fallback": false, + "split_kv_fallback_reason": null, + "split_kv_merge_order": "global_block_index", + "split_kv_plan_source": "contract_exact", + "tp_rank": 1 + }, + { + "actual_split_boundaries": [ + [ + 0, + 128 + ] + ], + "actual_split_kv_count": 1, + "actual_split_kv_policy": "disabled", + "actual_split_kv_size": null, + "batch_index": 1, + "cp_rank": 0, + "expected_kv_range": [ + 0, + 128 + ], + "owner_cp_rank": 0, + "requested_split_kv_policy": "disabled", + "requested_split_kv_size": null, + "split_kv_accum_dtype": "fp32", + "split_kv_backend": "rlkernel.rocm.deterministic_attention", + "split_kv_downcast_at": "final_write", + "split_kv_fallback": false, + "split_kv_fallback_reason": null, + "split_kv_merge_order": "global_block_index", + "split_kv_plan_source": "contract_exact", + "tp_rank": 0 + }, + { + "actual_split_boundaries": [ + [ + 0, + 128 + ] + ], + "actual_split_kv_count": 1, + "actual_split_kv_policy": "disabled", + "actual_split_kv_size": null, + "batch_index": 1, + "cp_rank": 0, + "expected_kv_range": [ + 0, + 128 + ], + "owner_cp_rank": 0, + "requested_split_kv_policy": "disabled", + "requested_split_kv_size": null, + "split_kv_accum_dtype": "fp32", + "split_kv_backend": "rlkernel.rocm.deterministic_attention", + "split_kv_downcast_at": "final_write", + "split_kv_fallback": false, + "split_kv_fallback_reason": null, + "split_kv_merge_order": "global_block_index", + "split_kv_plan_source": "contract_exact", + "tp_rank": 1 + } + ], + "total_kv_tokens": [ + 128, + 128 + ], + "tp_world_size": 2 + }, + "training": { + "batch_size": 2, + "coverage": "complete_batch_tp_cp_owner_cartesian_product", + "cp_world_size": 1, + "entries": [ + { + "actual_split_boundaries": [ + [ + 0, + 128 + ] + ], + "actual_split_kv_count": 1, + "actual_split_kv_policy": "disabled", + "actual_split_kv_size": null, + "batch_index": 0, + "cp_rank": 0, + "expected_kv_range": [ + 0, + 128 + ], + "owner_cp_rank": 0, + "requested_split_kv_policy": "disabled", + "requested_split_kv_size": null, + "split_kv_accum_dtype": "fp32", + "split_kv_backend": "rlkernel.rocm.deterministic_attention", + "split_kv_downcast_at": "final_write", + "split_kv_fallback": false, + "split_kv_fallback_reason": null, + "split_kv_merge_order": "global_block_index", + "split_kv_plan_source": "contract_exact", + "tp_rank": 0 + }, + { + "actual_split_boundaries": [ + [ + 0, + 128 + ] + ], + "actual_split_kv_count": 1, + "actual_split_kv_policy": "disabled", + "actual_split_kv_size": null, + "batch_index": 0, + "cp_rank": 0, + "expected_kv_range": [ + 0, + 128 + ], + "owner_cp_rank": 0, + "requested_split_kv_policy": "disabled", + "requested_split_kv_size": null, + "split_kv_accum_dtype": "fp32", + "split_kv_backend": "rlkernel.rocm.deterministic_attention", + "split_kv_downcast_at": "final_write", + "split_kv_fallback": false, + "split_kv_fallback_reason": null, + "split_kv_merge_order": "global_block_index", + "split_kv_plan_source": "contract_exact", + "tp_rank": 1 + }, + { + "actual_split_boundaries": [ + [ + 0, + 128 + ] + ], + "actual_split_kv_count": 1, + "actual_split_kv_policy": "disabled", + "actual_split_kv_size": null, + "batch_index": 1, + "cp_rank": 0, + "expected_kv_range": [ + 0, + 128 + ], + "owner_cp_rank": 0, + "requested_split_kv_policy": "disabled", + "requested_split_kv_size": null, + "split_kv_accum_dtype": "fp32", + "split_kv_backend": "rlkernel.rocm.deterministic_attention", + "split_kv_downcast_at": "final_write", + "split_kv_fallback": false, + "split_kv_fallback_reason": null, + "split_kv_merge_order": "global_block_index", + "split_kv_plan_source": "contract_exact", + "tp_rank": 0 + }, + { + "actual_split_boundaries": [ + [ + 0, + 128 + ] + ], + "actual_split_kv_count": 1, + "actual_split_kv_policy": "disabled", + "actual_split_kv_size": null, + "batch_index": 1, + "cp_rank": 0, + "expected_kv_range": [ + 0, + 128 + ], + "owner_cp_rank": 0, + "requested_split_kv_policy": "disabled", + "requested_split_kv_size": null, + "split_kv_accum_dtype": "fp32", + "split_kv_backend": "rlkernel.rocm.deterministic_attention", + "split_kv_downcast_at": "final_write", + "split_kv_fallback": false, + "split_kv_fallback_reason": null, + "split_kv_merge_order": "global_block_index", + "split_kv_plan_source": "contract_exact", + "tp_rank": 1 + } + ], + "total_kv_tokens": [ + 128, + 128 + ], + "tp_world_size": 2 + } + }, + "training": { + "backend_id": "rlkernel.rocm.deterministic_attention", + "contract": { + "batch_size": 2, + "causal": true, + "causal_offsets": [ + 0, + 0 + ], + "dtype": "bf16", + "export_lse": true, + "head_dim": 128, + "kv_cache": null, + "lse_domain": "attention", + "mode": "prefill", + "projections": { + "o_proj": { + "acc_dtype": "fp32", + "backend_policy": "native_verified_then_common_deterministic", + "deterministic_backend": "rlkernel.cuda.det_gemm", + "input_dtype": "bf16", + "k_order": "ascending", + "output_dtype": "bf16", + "qkv_split_order": [], + "require_runtime_readback": true, + "sp_backward_collective": "all_gather", + "sp_forward_collective": "reduce_scatter", + "split_kv": "disabled", + "tp_backward_dgrad_collective": "none", + "tp_forward_collective": "all_reduce" + }, + "qkv": { + "acc_dtype": "fp32", + "backend_policy": "native_verified_then_common_deterministic", + "deterministic_backend": "rlkernel.cuda.det_gemm", + "input_dtype": "bf16", + "k_order": "ascending", + "output_dtype": "bf16", + "qkv_split_order": [ + "q", + "k", + "v" + ], + "require_runtime_readback": true, + "sp_backward_collective": "reduce_scatter", + "sp_forward_collective": "all_gather", + "split_kv": "disabled", + "tp_backward_dgrad_collective": "all_reduce", + "tp_forward_collective": "none" + } + }, + "query_sequence_length": 128, + "reduction": { + "acc_dtype": "fp32", + "downcast_at": "final_write", + "engine": "in_op_reference", + "merge": "online_softmax_lse", + "order": "global_block_index" + }, + "role": "train", + "rope": null, + "semantic_operator": "standard_softmax_attention", + "sharding": { + "cp_rank": 0, + "cp_world_size": 1, + "global_block_indices": [ + 0 + ], + "global_block_token_starts": [ + 0 + ], + "global_kv_heads": 8, + "global_q_heads": 32, + "global_sequence_length": 128, + "local_block_offsets": [ + 0, + 128 + ], + "local_kv_head_start": 0, + "local_kv_heads": 4, + "local_q_head_start": 0, + "local_q_heads": 16, + "local_sequence_length": 128, + "packed_sequence_offsets": null, + "sp_rank": 0, + "sp_world_size": 1, + "tp_rank": 0, + "tp_world_size": 2 + }, + "split_kv": { + "fixed_split_size": null, + "mode": "disabled", + "strict_consistency": true + } + }, + "recorded": { + "backend_id": "rlkernel.rocm.deterministic_attention", + "mode": "prefill", + "reduction.engine": "in_op_reference" + } + } + }, + "recorded_differences": {}, + "reduction_fingerprint": "4bfe65d607de1dc8d1b8c04041d48e9af4e81dc38614f0340d994616da273d6c", + "schema_version": "cross_config.attention_binding.v3" + }, + "category": "comparability_gate", + "comparable": false, + "expected": "rejected", + "gate_implementation": "rl_engine.alignment.cross_config.bind_attention_contracts", + "identity_errors": [ + "sharding.tp_rank", + "sharding.local_q_head_start", + "sharding.local_kv_head_start" + ], + "metrics": { + "dk": null, + "dq": null, + "dv": null, + "lse": null, + "out": null + }, + "outcome": "rejected", + "passed": true, + "probe": "tp_head_ownership", + "realization": "Bind valid TP-rank-1 rollout and TP-rank-0 training contracts; do not run numerics.", + "row_id": "A4", + "sequence": 128 + }, + { + "batch": 2, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "implementations": { + "baseline": "rlkernel.rocm.deterministic_attention", + "candidate": "torch.rocm.index_select+native_attention" + }, + "metrics": { + "dk": { + "bitwise_equal": false, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 5.84375, + "mismatch_count": 261932, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 128, + 128 + ] + }, + "dq": { + "bitwise_equal": false, + "element_count": 1048576, + "left_dtype": "torch.bfloat16", + "max_abs": 4.1328125, + "mismatch_count": 1029699, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 128, + 128 + ] + }, + "dv": { + "bitwise_equal": false, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 9.7470703125, + "mismatch_count": 261935, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 128, + 128 + ] + }, + "lse": { + "bitwise_equal": false, + "element_count": 8192, + "left_dtype": "torch.float32", + "max_abs": 3.6807479858398438, + "mismatch_count": 8128, + "right_dtype": "torch.float32", + "shape": [ + 2, + 32, + 128 + ] + }, + "out": { + "bitwise_equal": false, + "element_count": 1048576, + "left_dtype": "torch.bfloat16", + "max_abs": 5.421875, + "mismatch_count": 1038063, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 128, + 128 + ] + } + }, + "outcome": "drift_detected", + "passed": true, + "probe": "kv_page_order", + "realization": "Reverse four dense K/V tensor pages; this is not a paged-cache runtime.", + "row_id": "A5", + "sequence": 128 + }, + { + "batch": 2, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "implementations": { + "baseline": "torch.rocm.explicit_fp32_serial_qk_accumulator", + "candidate": "torch.rocm.explicit_bf16_serial_qk_accumulator" + }, + "metrics": { + "dk": { + "bitwise_equal": false, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 0.078125, + "mismatch_count": 232744, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 128, + 128 + ] + }, + "dq": { + "bitwise_equal": false, + "element_count": 1048576, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0703125, + "mismatch_count": 900263, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 128, + 128 + ] + }, + "dv": { + "bitwise_equal": false, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 0.09375, + "mismatch_count": 235097, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 128, + 128 + ] + }, + "lse": { + "bitwise_equal": false, + "element_count": 8192, + "left_dtype": "torch.float32", + "max_abs": 0.08346295356750488, + "mismatch_count": 8190, + "right_dtype": "torch.float32", + "shape": [ + 2, + 32, + 128 + ] + }, + "out": { + "bitwise_equal": false, + "element_count": 1048576, + "left_dtype": "torch.bfloat16", + "max_abs": 0.078125, + "mismatch_count": 909350, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 128, + 128 + ] + } + }, + "outcome": "drift_detected", + "passed": true, + "probe": "accum_dtype", + "realization": "Use identical FP32 products/order with explicit FP32 versus BF16 accumulator state.", + "row_id": "A6", + "sequence": 128 + }, + { + "batch": 2, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "implementations": { + "baseline": "torch.rocm.fp32_chunk_merge_ascending", + "candidate": "torch.rocm.fp32_chunk_merge_descending" + }, + "metrics": { + "dk": { + "bitwise_equal": false, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0009765625, + "mismatch_count": 17, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 128, + 128 + ] + }, + "dq": { + "bitwise_equal": false, + "element_count": 1048576, + "left_dtype": "torch.bfloat16", + "max_abs": 0.00390625, + "mismatch_count": 43, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 128, + 128 + ] + }, + "dv": { + "bitwise_equal": false, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 0.001953125, + "mismatch_count": 10, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 128, + 128 + ] + }, + "lse": { + "bitwise_equal": false, + "element_count": 8192, + "left_dtype": "torch.float32", + "max_abs": 9.5367431640625e-07, + "mismatch_count": 279, + "right_dtype": "torch.float32", + "shape": [ + 2, + 32, + 128 + ] + }, + "out": { + "bitwise_equal": false, + "element_count": 1048576, + "left_dtype": "torch.bfloat16", + "max_abs": 0.001953125, + "mismatch_count": 26, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 128, + 128 + ] + } + }, + "outcome": "drift_detected", + "passed": true, + "probe": "merge_order", + "realization": "Merge four dense chunks in opposite orders on one GPU; this is not a CP collective.", + "row_id": "A7", + "sequence": 128 + }, + { + "batch": 2, + "category": "invariant_control", + "comparable": true, + "expected": "exact_zero", + "implementations": { + "baseline": "rlkernel.rocm.deterministic_attention", + "candidate": "rlkernel.rocm.native_attention_per_tp2_head_shard" + }, + "metrics": { + "dk": { + "bitwise_equal": true, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 128, + 128 + ] + }, + "dq": { + "bitwise_equal": true, + "element_count": 1048576, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 128, + 128 + ] + }, + "dv": { + "bitwise_equal": true, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 128, + 128 + ] + }, + "lse": { + "bitwise_equal": true, + "element_count": 8192, + "left_dtype": "torch.float32", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.float32", + "shape": [ + 2, + 32, + 128 + ] + }, + "out": { + "bitwise_equal": true, + "element_count": 1048576, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 128, + 128 + ] + } + }, + "outcome": "matched", + "passed": true, + "probe": "tp_partition_control", + "realization": "Compare full GQA with two contiguous TP=2 head shards on one GPU.", + "row_id": "C0", + "sequence": 128 + }, + { + "batch": 2, + "category": "invariant_control", + "comparable": true, + "expected": "exact_zero", + "implementations": { + "baseline": "rlkernel.rocm.deterministic_attention", + "candidate": "rlkernel.rocm.native_attention_per_batch_row" + }, + "metrics": { + "dk": { + "bitwise_equal": true, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 128, + 128 + ] + }, + "dq": { + "bitwise_equal": true, + "element_count": 1048576, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 128, + 128 + ] + }, + "dv": { + "bitwise_equal": true, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 128, + 128 + ] + }, + "lse": { + "bitwise_equal": true, + "element_count": 8192, + "left_dtype": "torch.float32", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.float32", + "shape": [ + 2, + 32, + 128 + ] + }, + "out": { + "bitwise_equal": true, + "element_count": 1048576, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 128, + 128 + ] + } + }, + "outcome": "matched", + "passed": true, + "probe": "batch_composition_control", + "realization": "Compare a batch call with per-row calls to the same native HIP core.", + "row_id": "C1", + "sequence": 128 + }, + { + "batch": 2, + "category": "invariant_control", + "comparable": true, + "expected": "exact_zero", + "implementations": { + "baseline": "rlkernel.rocm.native_attention_full_prefill", + "candidate": "rlkernel.rocm.native_attention_tail" + }, + "metrics": { + "dk": { + "bitwise_equal": true, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 128, + 128 + ] + }, + "dq": { + "bitwise_equal": true, + "element_count": 1048576, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 128, + 128 + ] + }, + "dv": { + "bitwise_equal": true, + "element_count": 262144, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 8, + 128, + 128 + ] + }, + "lse": { + "bitwise_equal": true, + "element_count": 64, + "left_dtype": "torch.float32", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.float32", + "shape": [ + 2, + 32, + 1 + ] + }, + "out": { + "bitwise_equal": true, + "element_count": 8192, + "left_dtype": "torch.bfloat16", + "max_abs": 0.0, + "mismatch_count": 0, + "right_dtype": "torch.bfloat16", + "shape": [ + 2, + 32, + 1, + 128 + ] + } + }, + "outcome": "matched", + "passed": true, + "probe": "prefill_decode_tail_control", + "realization": "Compare a full-prefill tail with one trailing query over dense KV, without a serving cache.", + "row_id": "C2", + "sequence": 128 + } + ], + "command": [ + "/opt/venv/bin/python", + "benchmarks/benchmark_rocm_attention_ablation.py", + "--device", + "0", + "--output-dir", + "benchmarks/results/pr230_rocm_mi300x_ablation" + ], + "configuration": { + "dtype": "bfloat16", + "head_dim": 128, + "kv_heads": 8, + "q_heads": 32, + "seed": 230, + "shapes": [ + [ + 1, + 16 + ], + [ + 1, + 32 + ], + [ + 1, + 64 + ], + [ + 1, + 128 + ], + [ + 2, + 16 + ], + [ + 2, + 32 + ], + [ + 2, + 64 + ], + [ + 2, + 128 + ] + ] + }, + "created_at": "2026-09-03T10:42:33.755824+00:00", + "environment": { + "architecture": "gfx942:sramecc+:xnack-", + "device_index": 0, + "device_name": "AMD Instinct MI300X VF", + "execution_kind": "operator_only_rocm_reference", + "gpu_count": 1, + "hip_runtime": "7.14.60850", + "primary_backend_id": "rlkernel.rocm.deterministic_attention", + "primary_core_id": "rlkernel.attention.deterministic_core.v1", + "primary_production_ready": false, + "primary_reference_only": true, + "primary_schedule": "single_batch_single_query_global_kv_blocks", + "python": "3.12.3", + "pytorch": "2.12.0+rocm7.14.0a20260608" + }, + "matrix": [ + { + "case_count": 8, + "category": "baseline", + "comparable": true, + "expected": "baseline", + "id": "A0", + "label": "Strict replay baseline", + "metrics": { + "dk": { + "all_bitwise_equal": true, + "total_mismatch_count": 0, + "worst_max_abs": 0.0 + }, + "dq": { + "all_bitwise_equal": true, + "total_mismatch_count": 0, + "worst_max_abs": 0.0 + }, + "dv": { + "all_bitwise_equal": true, + "total_mismatch_count": 0, + "worst_max_abs": 0.0 + }, + "lse": { + "all_bitwise_equal": true, + "total_mismatch_count": 0, + "worst_max_abs": 0.0 + }, + "out": { + "all_bitwise_equal": true, + "total_mismatch_count": 0, + "worst_max_abs": 0.0 + } + }, + "passed": true, + "probe": null, + "root_cause_axis": null + }, + { + "case_count": 8, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "id": "A1", + "label": "Position / RoPE", + "metrics": { + "dk": { + "all_bitwise_equal": false, + "total_mismatch_count": 712216, + "worst_max_abs": 2.234375 + }, + "dq": { + "all_bitwise_equal": false, + "total_mismatch_count": 1455200, + "worst_max_abs": 1.4140625 + }, + "dv": { + "all_bitwise_equal": false, + "total_mismatch_count": 703681, + "worst_max_abs": 0.783203125 + }, + "lse": { + "all_bitwise_equal": false, + "total_mismatch_count": 11520, + "worst_max_abs": 0.3938255310058594 + }, + "out": { + "all_bitwise_equal": false, + "total_mismatch_count": 1453811, + "worst_max_abs": 0.7109375 + } + }, + "passed": true, + "probe": "position_ids", + "root_cause_axis": "position_rope" + }, + { + "case_count": 8, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "id": "A2", + "label": "Q/K preprocessing", + "metrics": { + "dk": { + "all_bitwise_equal": false, + "total_mismatch_count": 730740, + "worst_max_abs": 1.7578125 + }, + "dq": { + "all_bitwise_equal": false, + "total_mismatch_count": 2867769, + "worst_max_abs": 1.015625 + }, + "dv": { + "all_bitwise_equal": false, + "total_mismatch_count": 723462, + "worst_max_abs": 0.96875 + }, + "lse": { + "all_bitwise_equal": false, + "total_mismatch_count": 23040, + "worst_max_abs": 1.0303754806518555 + }, + "out": { + "all_bitwise_equal": false, + "total_mismatch_count": 2818071, + "worst_max_abs": 0.58203125 + } + }, + "passed": true, + "probe": "qk_norm_disabled", + "root_cause_axis": "qk_preprocessing" + }, + { + "case_count": 8, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "id": "A3", + "label": "Mask / sequence boundary", + "metrics": { + "dk": { + "all_bitwise_equal": false, + "total_mismatch_count": 736222, + "worst_max_abs": 6.15625 + }, + "dq": { + "all_bitwise_equal": false, + "total_mismatch_count": 2887461, + "worst_max_abs": 4.1968994140625 + }, + "dv": { + "all_bitwise_equal": false, + "total_mismatch_count": 736298, + "worst_max_abs": 10.984375 + }, + "lse": { + "all_bitwise_equal": false, + "total_mismatch_count": 22656, + "worst_max_abs": 7.67181396484375 + }, + "out": { + "all_bitwise_equal": false, + "total_mismatch_count": 2888765, + "worst_max_abs": 4.21875 + } + }, + "passed": true, + "probe": "causal_mask", + "root_cause_axis": "mask_sequence_boundary" + }, + { + "case_count": 8, + "category": "comparability_gate", + "comparable": false, + "expected": "rejected", + "id": "A4", + "label": "Topology / head ownership", + "metrics": { + "dk": null, + "dq": null, + "dv": null, + "lse": null, + "out": null + }, + "passed": true, + "probe": "tp_head_ownership", + "root_cause_axis": "topology_head_ownership" + }, + { + "case_count": 8, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "id": "A5", + "label": "KV-cache identity / layout", + "metrics": { + "dk": { + "all_bitwise_equal": false, + "total_mismatch_count": 736656, + "worst_max_abs": 6.62744140625 + }, + "dq": { + "all_bitwise_equal": false, + "total_mismatch_count": 2843694, + "worst_max_abs": 4.27734375 + }, + "dv": { + "all_bitwise_equal": false, + "total_mismatch_count": 736681, + "worst_max_abs": 13.2333984375 + }, + "lse": { + "all_bitwise_equal": false, + "total_mismatch_count": 22656, + "worst_max_abs": 4.578392028808594 + }, + "out": { + "all_bitwise_equal": false, + "total_mismatch_count": 2893400, + "worst_max_abs": 6.0 + } + }, + "passed": true, + "probe": "kv_page_order", + "root_cause_axis": "kv_cache_identity_layout" + }, + { + "case_count": 8, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "id": "A6", + "label": "Numerical policy", + "metrics": { + "dk": { + "all_bitwise_equal": false, + "total_mismatch_count": 646044, + "worst_max_abs": 0.09375 + }, + "dq": { + "all_bitwise_equal": false, + "total_mismatch_count": 2456076, + "worst_max_abs": 0.09375 + }, + "dv": { + "all_bitwise_equal": false, + "total_mismatch_count": 652656, + "worst_max_abs": 0.11328125 + }, + "lse": { + "all_bitwise_equal": false, + "total_mismatch_count": 23038, + "worst_max_abs": 0.09950780868530273 + }, + "out": { + "all_bitwise_equal": false, + "total_mismatch_count": 2473674, + "worst_max_abs": 0.1171875 + } + }, + "passed": true, + "probe": "accum_dtype", + "root_cause_axis": "numerical_policy" + }, + { + "case_count": 8, + "category": "root_cause", + "comparable": true, + "expected": "diagnostic", + "id": "A7", + "label": "Distributed schedule", + "metrics": { + "dk": { + "all_bitwise_equal": false, + "total_mismatch_count": 60, + "worst_max_abs": 0.00390625 + }, + "dq": { + "all_bitwise_equal": false, + "total_mismatch_count": 116, + "worst_max_abs": 0.00390625 + }, + "dv": { + "all_bitwise_equal": false, + "total_mismatch_count": 30, + "worst_max_abs": 0.00390625 + }, + "lse": { + "all_bitwise_equal": false, + "total_mismatch_count": 860, + "worst_max_abs": 9.5367431640625e-07 + }, + "out": { + "all_bitwise_equal": false, + "total_mismatch_count": 66, + "worst_max_abs": 0.001953125 + } + }, + "passed": true, + "probe": "merge_order", + "root_cause_axis": "distributed_schedule" + }, + { + "case_count": 8, + "category": "invariant_control", + "comparable": true, + "expected": "exact_zero", + "id": "C0", + "label": "Invariant control", + "metrics": { + "dk": { + "all_bitwise_equal": true, + "total_mismatch_count": 0, + "worst_max_abs": 0.0 + }, + "dq": { + "all_bitwise_equal": true, + "total_mismatch_count": 0, + "worst_max_abs": 0.0 + }, + "dv": { + "all_bitwise_equal": true, + "total_mismatch_count": 0, + "worst_max_abs": 0.0 + }, + "lse": { + "all_bitwise_equal": true, + "total_mismatch_count": 0, + "worst_max_abs": 0.0 + }, + "out": { + "all_bitwise_equal": true, + "total_mismatch_count": 0, + "worst_max_abs": 0.0 + } + }, + "passed": true, + "probe": "tp_partition_control", + "root_cause_axis": null + }, + { + "case_count": 8, + "category": "invariant_control", + "comparable": true, + "expected": "exact_zero", + "id": "C1", + "label": "Invariant control", + "metrics": { + "dk": { + "all_bitwise_equal": true, + "total_mismatch_count": 0, + "worst_max_abs": 0.0 + }, + "dq": { + "all_bitwise_equal": true, + "total_mismatch_count": 0, + "worst_max_abs": 0.0 + }, + "dv": { + "all_bitwise_equal": true, + "total_mismatch_count": 0, + "worst_max_abs": 0.0 + }, + "lse": { + "all_bitwise_equal": true, + "total_mismatch_count": 0, + "worst_max_abs": 0.0 + }, + "out": { + "all_bitwise_equal": true, + "total_mismatch_count": 0, + "worst_max_abs": 0.0 + } + }, + "passed": true, + "probe": "batch_composition_control", + "root_cause_axis": null + }, + { + "case_count": 8, + "category": "invariant_control", + "comparable": true, + "expected": "exact_zero", + "id": "C2", + "label": "Invariant control", + "metrics": { + "dk": { + "all_bitwise_equal": true, + "total_mismatch_count": 0, + "worst_max_abs": 0.0 + }, + "dq": { + "all_bitwise_equal": true, + "total_mismatch_count": 0, + "worst_max_abs": 0.0 + }, + "dv": { + "all_bitwise_equal": true, + "total_mismatch_count": 0, + "worst_max_abs": 0.0 + }, + "lse": { + "all_bitwise_equal": true, + "total_mismatch_count": 0, + "worst_max_abs": 0.0 + }, + "out": { + "all_bitwise_equal": true, + "total_mismatch_count": 0, + "worst_max_abs": 0.0 + } + }, + "passed": true, + "probe": "prefill_decode_tail_control", + "root_cause_axis": null + } + ], + "matrix_manifest": { + "baseline_row": "A0", + "cartesian_product": false, + "method": "fixed_replay_one_at_a_time", + "metrics": [ + "train_rollout_logprob_abs_diff", + "mismatch_kl", + "mismatch_k3_kl", + "out_max_abs", + "lse_max_abs", + "dq_max_abs", + "dk_max_abs", + "dv_max_abs" + ], + "replay_identity": "same checkpoint, token IDs, selected-token IDs, masks, positions, cache metadata, and pre-update model state", + "row_baseline": "each diagnostic row is compared with its own phase-local A0 baseline", + "rows": [ + { + "category": "baseline", + "expected": "baseline", + "id": "A0", + "label": "Strict replay baseline", + "probe": null, + "root_cause_axis": null + }, + { + "category": "root_cause", + "expected": "diagnostic", + "id": "A1", + "label": "Position / RoPE", + "probe": "position_ids", + "root_cause_axis": "position_rope" + }, + { + "category": "root_cause", + "expected": "diagnostic", + "id": "A2", + "label": "Q/K preprocessing", + "probe": "qk_norm_disabled", + "root_cause_axis": "qk_preprocessing" + }, + { + "category": "root_cause", + "expected": "diagnostic", + "id": "A3", + "label": "Mask / sequence boundary", + "probe": "causal_mask", + "root_cause_axis": "mask_sequence_boundary" + }, + { + "category": "comparability_gate", + "expected": "rejected", + "id": "A4", + "label": "Topology / head ownership", + "probe": "tp_head_ownership", + "root_cause_axis": "topology_head_ownership" + }, + { + "category": "root_cause", + "expected": "diagnostic", + "id": "A5", + "label": "KV-cache identity / layout", + "probe": "kv_page_order", + "root_cause_axis": "kv_cache_identity_layout" + }, + { + "category": "root_cause", + "expected": "diagnostic", + "id": "A6", + "label": "Numerical policy", + "probe": "accum_dtype", + "root_cause_axis": "numerical_policy" + }, + { + "category": "root_cause", + "expected": "diagnostic", + "id": "A7", + "label": "Distributed schedule", + "probe": "merge_order", + "root_cause_axis": "distributed_schedule" + }, + { + "category": "invariant_control", + "expected": "exact_zero", + "id": "C0", + "label": "Invariant control", + "probe": "tp_partition_control", + "root_cause_axis": null + }, + { + "category": "invariant_control", + "expected": "exact_zero", + "id": "C1", + "label": "Invariant control", + "probe": "batch_composition_control", + "root_cause_axis": null + }, + { + "category": "invariant_control", + "expected": "exact_zero", + "id": "C2", + "label": "Invariant control", + "probe": "prefill_decode_tail_control", + "root_cause_axis": null + } + ], + "schema_version": "rlkernel.attention.debug_matrix.v1", + "topology_gate": [ + "checkpoint/model/token identity", + "TP head ownership", + "CP sequence ownership", + "actual Split-KV plan" + ] + }, + "schema_version": "rlkernel.rocm.attention_ablation_microprobe.v1", + "scope": { + "covered_metrics": [ + "out_max_abs", + "lse_max_abs", + "dq_max_abs", + "dk_max_abs", + "dv_max_abs" + ], + "excluded_metrics": [ + "train_rollout_logprob_abs_diff", + "mismatch_kl", + "mismatch_k3_kl" + ], + "frozen_rollout_replay": false, + "kind": "operator_microprobe", + "model_or_serving_execution": false, + "pr230_row_taxonomy": true + }, + "source_provenance": { + "revision": "2ea63b22b74feb6e5a748d09780fa075d5e644ed", + "script_matches_head": true, + "script_path": "benchmarks/benchmark_rocm_attention_ablation.py", + "script_sha256": "0139a15ee6bae14e9ee8ca7cc32146956b10d70d201d37086a0a45315f5816ac", + "tracked_diff_sha256": null, + "tracked_dirty": false + } +} diff --git a/tests/test_rocm_attention_ablation_benchmark.py b/tests/test_rocm_attention_ablation_benchmark.py index 7bde5984..974fe9ac 100644 --- a/tests/test_rocm_attention_ablation_benchmark.py +++ b/tests/test_rocm_attention_ablation_benchmark.py @@ -4,6 +4,7 @@ import copy import importlib.util +import json import math import sys from pathlib import Path @@ -13,6 +14,7 @@ ROOT = Path(__file__).parents[1] SCRIPT = ROOT / "benchmarks" / "benchmark_rocm_attention_ablation.py" +CHECKED_IN_RESULT = ROOT / "benchmarks" / "results" / "pr230_rocm_mi300x_ablation" / "results.json" SPEC = importlib.util.spec_from_file_location("rocm_attention_ablation", SCRIPT) assert SPEC is not None and SPEC.loader is not None MODULE = importlib.util.module_from_spec(SPEC) @@ -297,3 +299,11 @@ def test_payload_validator_rejects_incomplete_or_fabricated_evidence(mutate, mes def test_repository_provenance_rejects_unbacked_hashes(): with pytest.raises(ValueError, match="not backed"): MODULE.validate_repository_provenance(_valid_payload()) + + +def test_checked_in_mi300x_matrix_is_complete_and_source_backed(): + payload = json.loads(CHECKED_IN_RESULT.read_text(encoding="utf-8")) + + MODULE.validate_payload(payload) + MODULE.validate_repository_provenance(payload) + assert all(row["passed"] for row in payload["matrix"]) From 9bd46b9c1b6d76d992df401aa4b22c74a17a41a9 Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 3 Sep 2026 07:23:01 -0500 Subject: [PATCH 3/3] feat(rocm): add end-to-end attention ablation matrix Signed-off-by: Codex --- .github/workflows/ci.yml | 3 +- .../benchmark_rocm_attention_ablation.py | 1122 -- .../pr230_rocm_mi300x_ablation/report.md | 64 - .../pr230_rocm_mi300x_ablation/results.json | 11234 ---------------- .../vime_qwen3_8b_rocm_ablation/README.md | 93 + .../rocm_python_entrypoint.sh | 30 + examples/vime_qwen3_8b_rocm_ablation/run.py | 81 + rl_engine/integrations/__init__.py | 12 + rl_engine/integrations/framework_operators.py | 115 +- rl_engine/integrations/rocm_ablation.py | 588 + rl_engine/integrations/runtime.py | 4 +- rl_engine/integrations/vllm_runtime.py | 83 +- .../kernels/ops/pytorch/attention/ablation.py | 107 +- rl_engine/kernels/registry.py | 11 +- tests/test_attention_ablation.py | 39 + tests/test_framework_runtime_adapters.py | 264 +- .../test_rocm_attention_ablation_benchmark.py | 309 - tests/test_rocm_e2e_ablation.py | 303 + 18 files changed, 1651 insertions(+), 12811 deletions(-) delete mode 100644 benchmarks/benchmark_rocm_attention_ablation.py delete mode 100644 benchmarks/results/pr230_rocm_mi300x_ablation/report.md delete mode 100644 benchmarks/results/pr230_rocm_mi300x_ablation/results.json create mode 100644 examples/vime_qwen3_8b_rocm_ablation/README.md create mode 100755 examples/vime_qwen3_8b_rocm_ablation/rocm_python_entrypoint.sh create mode 100644 examples/vime_qwen3_8b_rocm_ablation/run.py create mode 100644 rl_engine/integrations/rocm_ablation.py delete mode 100644 tests/test_rocm_attention_ablation_benchmark.py create mode 100644 tests/test_rocm_e2e_ablation.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5d8e9c9e..711b8df4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -89,7 +89,8 @@ jobs: tests/test_attention_cross_config_binding.py \ tests/test_attention_preprocess.py \ tests/test_attention_projection.py \ - tests/test_rocm_attention_ablation_benchmark.py \ + tests/test_framework_runtime_adapters.py \ + tests/test_rocm_e2e_ablation.py \ tests/test_cp_attention.py \ tests/test_cp_attention_transformer_engine.py diff --git a/benchmarks/benchmark_rocm_attention_ablation.py b/benchmarks/benchmark_rocm_attention_ablation.py deleted file mode 100644 index 334ba55f..00000000 --- a/benchmarks/benchmark_rocm_attention_ablation.py +++ /dev/null @@ -1,1122 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# Copyright (c) 2026 RL-Kernel Contributors - -"""Execute the PR230 Attention row taxonomy as ROCm operator micro-probes. - -This does not claim PR230's frozen model/rollout replay: there is no checkpoint, -token stream, selected-token logprob, KL, or serving engine in this benchmark. -Most rows use the shared native HIP deterministic Attention reference core; A6 -and A7 use eager PyTorch-on-ROCm probes to isolate precision and merge order. -Every row records both implementations and the result scope explicitly. -""" - -from __future__ import annotations - -import argparse -import hashlib -import json -import math -import platform -import subprocess -import sys -from dataclasses import dataclass -from datetime import datetime, timezone -from pathlib import Path -from typing import Any, Callable, Mapping - -import torch -import torch.nn.functional as F - -from rl_engine.alignment.cross_config.attention_binding import ( - BindingErrorCode, - bind_attention_contracts, -) -from rl_engine.kernels.attention_contract import ( - STRICT_ATTENTION_REFERENCE_CORE_ID, - STRICT_ATTENTION_SCHEDULE_ID, - AttentionContract, - AttentionDType, - AttentionMode, - AttentionRole, - ReductionSpec, - ShardingSpec, - SplitKVSpec, - build_split_kv_runtime_plan_set, -) -from rl_engine.kernels.ops.pytorch.attention.debug_matrix import attention_debug_matrix - -QWEN3_Q_HEADS = 32 -QWEN3_KV_HEADS = 8 -QWEN3_HEAD_DIM = 128 -DEFAULT_SHAPES = ((1, 16), (1, 32), (1, 64), (1, 128), (2, 16), (2, 32), (2, 64), (2, 128)) -METRIC_NAMES = ("out", "lse", "dq", "dk", "dv") -ROCM_REFERENCE_BACKEND_ID = "rlkernel.rocm.deterministic_attention" -REPO_ROOT = Path(__file__).resolve().parents[1] -RESULT_SCHEMA = "rlkernel.rocm.attention_ablation_microprobe.v1" -RESULT_SCOPE = { - "kind": "operator_microprobe", - "pr230_row_taxonomy": True, - "frozen_rollout_replay": False, - "model_or_serving_execution": False, - "covered_metrics": ["out_max_abs", "lse_max_abs", "dq_max_abs", "dk_max_abs", "dv_max_abs"], - "excluded_metrics": [ - "train_rollout_logprob_abs_diff", - "mismatch_kl", - "mismatch_k3_kl", - ], -} -ROW_IMPLEMENTATIONS = { - "A0": (ROCM_REFERENCE_BACKEND_ID, ROCM_REFERENCE_BACKEND_ID), - "A1": ( - "rlkernel.rocm.deterministic_rope+native_attention", - "rlkernel.rocm.deterministic_rope+native_attention", - ), - "A2": ("torch.rocm.rms_norm+native_attention", ROCM_REFERENCE_BACKEND_ID), - "A3": (ROCM_REFERENCE_BACKEND_ID, ROCM_REFERENCE_BACKEND_ID), - "A5": (ROCM_REFERENCE_BACKEND_ID, "torch.rocm.index_select+native_attention"), - "A6": ( - "torch.rocm.explicit_fp32_serial_qk_accumulator", - "torch.rocm.explicit_bf16_serial_qk_accumulator", - ), - "A7": ( - "torch.rocm.fp32_chunk_merge_ascending", - "torch.rocm.fp32_chunk_merge_descending", - ), - "C0": (ROCM_REFERENCE_BACKEND_ID, "rlkernel.rocm.native_attention_per_tp2_head_shard"), - "C1": (ROCM_REFERENCE_BACKEND_ID, "rlkernel.rocm.native_attention_per_batch_row"), - "C2": ("rlkernel.rocm.native_attention_full_prefill", "rlkernel.rocm.native_attention_tail"), -} -ROW_REALIZATIONS = { - "A0": "Repeat the identical native HIP reference-core call.", - "A1": "Increment suffix RoPE positions while preserving Q/K/V tensors.", - "A2": "Apply or bypass unit-weight PyTorch RMSNorm before the native HIP core.", - "A3": "Toggle causal masking in the native HIP core.", - "A4": "Bind valid TP-rank-1 rollout and TP-rank-0 training contracts; do not run numerics.", - "A5": "Reverse four dense K/V tensor pages; this is not a paged-cache runtime.", - "A6": "Use identical FP32 products/order with explicit FP32 versus BF16 accumulator state.", - "A7": "Merge four dense chunks in opposite orders on one GPU; this is not a CP collective.", - "C0": "Compare full GQA with two contiguous TP=2 head shards on one GPU.", - "C1": "Compare a batch call with per-row calls to the same native HIP core.", - "C2": ( - "Compare a full-prefill tail with one trailing query over dense KV, without a serving " - "cache." - ), -} - - -@dataclass(frozen=True) -class Snapshot: - out: torch.Tensor - lse: torch.Tensor - dq: torch.Tensor - dk: torch.Tensor - dv: torch.Tensor - - -AttentionCall = Callable[ - [torch.Tensor, torch.Tensor, torch.Tensor], tuple[torch.Tensor, torch.Tensor] -] - - -def _seeded_tensor(shape: tuple[int, ...], *, device: torch.device, seed: int) -> torch.Tensor: - generator = torch.Generator(device=device).manual_seed(seed) - return torch.randn(shape, device=device, dtype=torch.bfloat16, generator=generator) - - -def _evaluate( - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - dout: torch.Tensor, - call: AttentionCall, -) -> Snapshot: - qr = q.detach().clone().requires_grad_(True) - kr = k.detach().clone().requires_grad_(True) - vr = v.detach().clone().requires_grad_(True) - out, lse = call(qr, kr, vr) - if out.shape != dout.shape: - raise RuntimeError(f"upstream gradient shape {dout.shape} does not match {out.shape}") - dq, dk, dv = torch.autograd.grad( - out, - (qr, kr, vr), - grad_outputs=dout, - allow_unused=False, - ) - return Snapshot( - out.detach(), - lse.detach(), - dq.detach(), - dk.detach(), - dv.detach(), - ) - - -def _metric(left: torch.Tensor, right: torch.Tensor) -> dict[str, Any]: - if left.shape != right.shape: - raise ValueError(f"metric tensors differ in shape: {left.shape} != {right.shape}") - difference = (left.float() - right.float()).abs() - same_dtype = left.dtype == right.dtype - if same_dtype: - left_bytes = left.contiguous().view(torch.uint8).reshape(left.numel(), left.element_size()) - right_bytes = ( - right.contiguous().view(torch.uint8).reshape(right.numel(), right.element_size()) - ) - mismatch_count = int(torch.any(left_bytes != right_bytes, dim=1).sum().item()) - else: - mismatch_count = left.numel() - return { - "max_abs": 0.0 if difference.numel() == 0 else float(difference.max().item()), - "mismatch_count": mismatch_count, - "element_count": int(left.numel()), - "bitwise_equal": mismatch_count == 0, - "left_dtype": str(left.dtype), - "right_dtype": str(right.dtype), - "shape": list(left.shape), - } - - -def _compare(left: Snapshot, right: Snapshot) -> dict[str, dict[str, Any]]: - return {name: _metric(getattr(left, name), getattr(right, name)) for name in METRIC_NAMES} - - -def _native_call(operator: Any, *, causal: bool = True) -> AttentionCall: - def call(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor): - key_positions = torch.arange(k.size(2), device=k.device, dtype=torch.int64).repeat( - k.size(0), 1 - ) - result = operator.forward_with_lse( - q, - k, - v, - causal=causal, - scale=1.0 / math.sqrt(q.size(-1)), - query_position_ids=key_positions[:, -q.size(2) :], - key_position_ids=key_positions, - ) - return result.out, result.lse - - return call - - -def _rope_batch(operator: Any, tensor: torch.Tensor, positions: torch.Tensor) -> torch.Tensor: - return torch.cat( - [operator(tensor[index : index + 1], positions[index]) for index in range(tensor.size(0))] - ) - - -def _rope_call(attention: Any, rope: Any, positions: torch.Tensor) -> AttentionCall: - native = _native_call(attention) - - def call(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor): - return native(_rope_batch(rope, q, positions), _rope_batch(rope, k, positions), v) - - return call - - -def _qk_norm_call(attention: Any, weight: torch.Tensor, *, enabled: bool) -> AttentionCall: - native = _native_call(attention) - - def call(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor): - if enabled: - q = F.rms_norm(q, (q.size(-1),), weight, 1.0e-6) - k = F.rms_norm(k, (k.size(-1),), weight, 1.0e-6) - return native(q, k, v) - - return call - - -def _kv_page_call(attention: Any, permutation: torch.Tensor | None) -> AttentionCall: - native = _native_call(attention) - - def call(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor): - if permutation is not None: - k = k.index_select(2, permutation) - v = v.index_select(2, permutation) - return native(q, k, v) - - return call - - -def _dense_attention(*, accumulator_dtype: torch.dtype) -> AttentionCall: - if accumulator_dtype not in {torch.float32, torch.bfloat16}: - raise ValueError("accumulator_dtype must be torch.float32 or torch.bfloat16") - - def call(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor): - group = q.size(1) // k.size(1) - expanded_k = k.repeat_interleave(group, dim=1) - expanded_v = v.repeat_interleave(group, dim=1) - scores = torch.zeros( - (*q.shape[:-1], expanded_k.size(2)), - device=q.device, - dtype=accumulator_dtype, - ) - for index in range(q.size(-1)): - product = q[..., index].float().unsqueeze(-1) * expanded_k[ - ..., index - ].float().unsqueeze(-2) - scores = scores.float() + product - if accumulator_dtype is torch.bfloat16: - scores = scores.to(torch.bfloat16) - scores = scores.float() - scores = scores * (1.0 / math.sqrt(q.size(-1))) - q_index = torch.arange(q.size(2), device=q.device).unsqueeze(1) - k_index = torch.arange(k.size(2), device=q.device).unsqueeze(0) - scores = scores.masked_fill(k_index > q_index, float("-inf")) - lse = torch.logsumexp(scores, dim=-1) - out = torch.matmul(torch.softmax(scores, dim=-1), expanded_v.float()).to(q.dtype) - return out, lse - - return call - - -def _chunked_attention(order: str) -> AttentionCall: - if order not in {"ascending", "descending"}: - raise ValueError("chunk merge order must be ascending or descending") - - def call(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor): - group = q.size(1) // k.size(1) - expanded_k = k.repeat_interleave(group, dim=1) - expanded_v = v.repeat_interleave(group, dim=1).float() - sequence = k.size(2) - chunk_size = sequence // 4 - chunks: list[tuple[torch.Tensor, torch.Tensor, torch.Tensor]] = [] - q_positions = torch.arange(q.size(2), device=q.device).view(1, 1, -1, 1) - for start in range(0, sequence, chunk_size): - stop = min(start + chunk_size, sequence) - scores = torch.matmul(q.float(), expanded_k[:, :, start:stop].float().transpose(-1, -2)) - scores = scores * (1.0 / math.sqrt(q.size(-1))) - key_positions = torch.arange(start, stop, device=q.device).view(1, 1, 1, -1) - scores = scores.masked_fill(key_positions > q_positions, float("-inf")) - row_max = scores.max(dim=-1).values - valid = torch.isfinite(row_max) - safe_max = torch.where(valid, row_max, torch.zeros_like(row_max)) - weights = torch.exp(scores - safe_max.unsqueeze(-1)) - weights = torch.where(valid.unsqueeze(-1), weights, torch.zeros_like(weights)) - denominator = weights.sum(dim=-1) - numerator = torch.matmul(weights, expanded_v[:, :, start:stop]) - chunks.append((safe_max, denominator, numerator)) - - indices = range(len(chunks)) if order == "ascending" else reversed(range(len(chunks))) - merged_max: torch.Tensor | None = None - merged_denominator: torch.Tensor | None = None - merged_numerator: torch.Tensor | None = None - for index in indices: - row_max, denominator, numerator = chunks[index] - if merged_max is None: - merged_max, merged_denominator, merged_numerator = ( - row_max, - denominator, - numerator, - ) - continue - assert merged_denominator is not None and merged_numerator is not None - left_valid = merged_denominator > 0 - right_valid = denominator > 0 - new_max = torch.maximum(merged_max, row_max) - left_scale = torch.where( - left_valid, - torch.exp(merged_max - new_max), - torch.zeros_like(new_max), - ) - right_scale = torch.where( - right_valid, - torch.exp(row_max - new_max), - torch.zeros_like(new_max), - ) - merged_denominator = merged_denominator * left_scale + denominator * right_scale - merged_numerator = merged_numerator * left_scale.unsqueeze( - -1 - ) + numerator * right_scale.unsqueeze(-1) - merged_max = new_max - assert merged_max is not None - assert merged_denominator is not None and merged_numerator is not None - out = (merged_numerator / merged_denominator.unsqueeze(-1)).to(q.dtype) - lse = torch.log(merged_denominator) + merged_max - return out, lse - - return call - - -def _per_tp_partition_call(attention: Any) -> AttentionCall: - native = _native_call(attention) - - def call(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor): - tp_world_size = 2 - q_heads_per_rank = q.size(1) // tp_world_size - kv_heads_per_rank = k.size(1) // tp_world_size - outputs, lses = [], [] - for rank in range(tp_world_size): - result = native( - q[:, rank * q_heads_per_rank : (rank + 1) * q_heads_per_rank], - k[:, rank * kv_heads_per_rank : (rank + 1) * kv_heads_per_rank], - v[:, rank * kv_heads_per_rank : (rank + 1) * kv_heads_per_rank], - ) - outputs.append(result[0]) - lses.append(result[1]) - return torch.cat(outputs, dim=1), torch.cat(lses, dim=1) - - return call - - -def _per_batch_row_call(attention: Any) -> AttentionCall: - native = _native_call(attention) - - def call(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor): - rows = [native(q[i : i + 1], k[i : i + 1], v[i : i + 1]) for i in range(q.size(0))] - return torch.cat([row[0] for row in rows]), torch.cat([row[1] for row in rows]) - - return call - - -def _tail_call(attention: Any, *, full_query: bool) -> AttentionCall: - native = _native_call(attention) - - def call(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor): - if full_query: - out, lse = native(q, k, v) - return out[:, :, -1:], lse[:, :, -1:] - return native(q[:, :, -1:], k, v) - - return call - - -def _page_permutation(sequence: int, device: torch.device) -> torch.Tensor: - page_size = max(1, sequence // 4) - pages = [ - torch.arange(start, min(start + page_size, sequence), device=device) - for start in range(0, sequence, page_size) - ] - return torch.cat(list(reversed(pages))).to(torch.long) - - -def _topology_contract( - *, role: AttentionRole, batch: int, sequence: int, tp_rank: int -) -> AttentionContract: - tp_world_size = 2 - local_q_heads = QWEN3_Q_HEADS // tp_world_size - local_kv_heads = QWEN3_KV_HEADS // tp_world_size - return AttentionContract( - role=role, - mode=AttentionMode.PREFILL, - dtype=AttentionDType.BF16, - batch_size=batch, - query_sequence_length=sequence, - head_dim=QWEN3_HEAD_DIM, - causal=True, - causal_offsets=(0,) * batch, - sharding=ShardingSpec( - tp_rank=tp_rank, - tp_world_size=tp_world_size, - cp_rank=0, - cp_world_size=1, - global_q_heads=QWEN3_Q_HEADS, - global_kv_heads=QWEN3_KV_HEADS, - local_q_head_start=tp_rank * local_q_heads, - local_q_heads=local_q_heads, - local_kv_head_start=tp_rank * local_kv_heads, - local_kv_heads=local_kv_heads, - global_sequence_length=sequence, - local_sequence_length=sequence, - global_block_indices=(0,), - global_block_token_starts=(0,), - local_block_offsets=(0, sequence), - ), - reduction=ReductionSpec(), - split_kv=SplitKVSpec.disabled(), - ) - - -def _topology_gate(*, batch: int, sequence: int) -> dict[str, Any]: - """Exercise the production contract gate with different TP owners.""" - - rollout = _topology_contract( - role=AttentionRole.INFER, - batch=batch, - sequence=sequence, - tp_rank=1, - ) - training = _topology_contract( - role=AttentionRole.TRAIN, - batch=batch, - sequence=sequence, - tp_rank=0, - ) - plan_set = build_split_kv_runtime_plan_set( - (sequence,) * batch, - tp_world_size=2, - cp_world_size=1, - split_kv=SplitKVSpec.disabled(), - backend=ROCM_REFERENCE_BACKEND_ID, - ) - result = bind_attention_contracts( - rollout_contract=rollout, - training_contract=training, - rollout_identity={}, - training_identity={}, - rollout_backend_id=ROCM_REFERENCE_BACKEND_ID, - training_backend_id=ROCM_REFERENCE_BACKEND_ID, - rollout_split_kv_plan_set=plan_set, - training_split_kv_plan_set=plan_set, - require_full_identity=False, - ) - topology_issues = result.issues_by_code(BindingErrorCode.TOPOLOGY_MISMATCH) - unexpected_issues = tuple( - issue for issue in result.issues if issue.code is not BindingErrorCode.TOPOLOGY_MISMATCH - ) - if result.comparable or result.passed or not topology_issues or unexpected_issues: - raise RuntimeError("A4 did not isolate the topology comparability gate") - return result.to_dict() - - -def _case( - *, - batch: int, - sequence: int, - seed: int, - device: torch.device, - attention: Any, - rope: Any, -) -> list[dict[str, Any]]: - q = _seeded_tensor((batch, QWEN3_Q_HEADS, sequence, QWEN3_HEAD_DIM), device=device, seed=seed) - k = _seeded_tensor( - (batch, QWEN3_KV_HEADS, sequence, QWEN3_HEAD_DIM), device=device, seed=seed + 1 - ) - v = _seeded_tensor(k.shape, device=device, seed=seed + 2) - dout = _seeded_tensor(q.shape, device=device, seed=seed + 3) - native = _native_call(attention) - - positions = torch.arange(sequence, device=device, dtype=torch.int64).repeat(batch, 1) - changed_positions = positions.clone() - changed_positions[:, sequence // 2 :] += 1 - norm_weight = torch.ones(QWEN3_HEAD_DIM, device=device, dtype=torch.bfloat16) - permutation = _page_permutation(sequence, device) - - calls: dict[str, tuple[AttentionCall, AttentionCall, torch.Tensor]] = { - "A0": (native, native, dout), - "A1": ( - _rope_call(attention, rope, positions), - _rope_call(attention, rope, changed_positions), - dout, - ), - "A2": ( - _qk_norm_call(attention, norm_weight, enabled=True), - _qk_norm_call(attention, norm_weight, enabled=False), - dout, - ), - "A3": (native, _native_call(attention, causal=False), dout), - "A5": (native, _kv_page_call(attention, permutation), dout), - "A6": ( - _dense_attention(accumulator_dtype=torch.float32), - _dense_attention(accumulator_dtype=torch.bfloat16), - dout, - ), - "A7": (_chunked_attention("ascending"), _chunked_attention("descending"), dout), - "C0": (native, _per_tp_partition_call(attention), dout), - "C1": (native, _per_batch_row_call(attention), dout), - "C2": ( - _tail_call(attention, full_query=True), - _tail_call(attention, full_query=False), - dout[:, :, -1:].contiguous(), - ), - } - rows: list[dict[str, Any]] = [] - for matrix_row in attention_debug_matrix()["rows"]: - row_id = matrix_row["id"] - if row_id == "A4": - binding = _topology_gate(batch=batch, sequence=sequence) - rows.append( - { - "row_id": row_id, - "batch": batch, - "sequence": sequence, - "category": matrix_row["category"], - "probe": matrix_row["probe"], - "expected": matrix_row["expected"], - "comparable": binding["comparable"], - "passed": True, - "outcome": "rejected", - "realization": ROW_REALIZATIONS[row_id], - "gate_implementation": ( - "rl_engine.alignment.cross_config.bind_attention_contracts" - ), - "identity_errors": [issue["field"] for issue in binding["issues"]], - "binding_gate": binding, - "metrics": {name: None for name in METRIC_NAMES}, - } - ) - continue - baseline_call, candidate_call, row_dout = calls[row_id] - baseline = _evaluate(q, k, v, row_dout, baseline_call) - candidate = _evaluate(q, k, v, row_dout, candidate_call) - metrics = _compare(baseline, candidate) - mismatch_count = sum(metric["mismatch_count"] for metric in metrics.values()) - expected = matrix_row["expected"] - passed = ( - mismatch_count == 0 if expected in {"baseline", "exact_zero"} else mismatch_count > 0 - ) - baseline_implementation, candidate_implementation = ROW_IMPLEMENTATIONS[row_id] - rows.append( - { - "row_id": row_id, - "batch": batch, - "sequence": sequence, - "category": matrix_row["category"], - "probe": matrix_row["probe"], - "expected": expected, - "comparable": True, - "passed": passed, - "outcome": "matched" if mismatch_count == 0 else "drift_detected", - "realization": ROW_REALIZATIONS[row_id], - "implementations": { - "baseline": baseline_implementation, - "candidate": candidate_implementation, - }, - "metrics": metrics, - } - ) - return rows - - -def _environment(device: torch.device, attention: Any) -> dict[str, Any]: - properties = torch.cuda.get_device_properties(device) - return { - "python": platform.python_version(), - "pytorch": torch.__version__, - "hip_runtime": torch.version.hip, - "device_index": device.index, - "device_name": properties.name, - "architecture": getattr(properties, "gcnArchName", "unknown"), - "gpu_count": torch.cuda.device_count(), - "primary_backend_id": attention.backend_id, - "primary_core_id": attention.core_id, - "primary_schedule": attention.strict_schedule, - "primary_reference_only": attention.reference_only, - "primary_production_ready": attention.production_ready, - "execution_kind": "operator_only_rocm_reference", - } - - -def _git(*args: str) -> subprocess.CompletedProcess[bytes]: - return subprocess.run( - ["git", *args], - cwd=REPO_ROOT, - check=False, - capture_output=True, - ) - - -def _source_provenance() -> dict[str, Any]: - script_path = Path(__file__).resolve() - script_relative = script_path.relative_to(REPO_ROOT).as_posix() - revision_result = _git("rev-parse", "HEAD") - head_source = _git("show", f"HEAD:{script_relative}") - diff = _git("diff", "--binary", "HEAD") - if revision_result.returncode != 0 or diff.returncode != 0: - raise RuntimeError("unable to record RL-Kernel git provenance") - source = script_path.read_bytes() - tracked_diff = diff.stdout - return { - "revision": revision_result.stdout.decode().strip(), - "tracked_dirty": bool(tracked_diff), - "tracked_diff_sha256": (hashlib.sha256(tracked_diff).hexdigest() if tracked_diff else None), - "script_path": script_relative, - "script_sha256": hashlib.sha256(source).hexdigest(), - "script_matches_head": head_source.returncode == 0 and head_source.stdout == source, - } - - -def _aggregate(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: - aggregate: list[dict[str, Any]] = [] - for matrix_row in attention_debug_matrix()["rows"]: - selected = [row for row in rows if row["row_id"] == matrix_row["id"]] - metrics: dict[str, Any] = {} - for name in METRIC_NAMES: - values = [row["metrics"][name] for row in selected if row["metrics"][name] is not None] - metrics[name] = ( - None - if not values - else { - "worst_max_abs": max(value["max_abs"] for value in values), - "total_mismatch_count": sum(value["mismatch_count"] for value in values), - "all_bitwise_equal": all(value["bitwise_equal"] for value in values), - } - ) - aggregate.append( - { - **matrix_row, - "case_count": len(selected), - "comparable": all(row["comparable"] for row in selected), - "passed": all(row["passed"] for row in selected), - "metrics": metrics, - } - ) - return aggregate - - -def _valid_digest(value: Any, *, lengths: tuple[int, ...]) -> bool: - return ( - isinstance(value, str) - and len(value) in lengths - and all(character in "0123456789abcdef" for character in value.lower()) - ) - - -def _validate_metric( - row_id: str, - name: str, - metric: Any, - *, - expected_dtype: str, - expected_shape: list[int], -) -> None: - if not isinstance(metric, Mapping) or set(metric) != { - "max_abs", - "mismatch_count", - "element_count", - "bitwise_equal", - "left_dtype", - "right_dtype", - "shape", - }: - raise ValueError(f"{row_id}.{name} has an invalid metric schema") - maximum = metric["max_abs"] - mismatches = metric["mismatch_count"] - elements = metric["element_count"] - bitwise_equal = metric["bitwise_equal"] - shape = metric["shape"] - if ( - isinstance(maximum, bool) - or not isinstance(maximum, (int, float)) - or not math.isfinite(maximum) - or maximum < 0 - ): - raise ValueError(f"{row_id}.{name}.max_abs must be finite and non-negative") - if ( - isinstance(mismatches, bool) - or not isinstance(mismatches, int) - or isinstance(elements, bool) - or not isinstance(elements, int) - or elements < 1 - or mismatches < 0 - or mismatches > elements - ): - raise ValueError(f"{row_id}.{name} has invalid element or mismatch counts") - if not isinstance(bitwise_equal, bool) or bitwise_equal != (mismatches == 0): - raise ValueError(f"{row_id}.{name} has inconsistent bitwise evidence") - if mismatches == 0 and maximum != 0: - raise ValueError(f"{row_id}.{name} has inconsistent numerical evidence") - if ( - metric["left_dtype"] != metric["right_dtype"] - or metric["left_dtype"] != expected_dtype - or shape != expected_shape - or elements != math.prod(expected_shape) - ): - raise ValueError(f"{row_id}.{name} has incompatible dtype or shape evidence") - - -def validate_payload(payload: Mapping[str, Any]) -> None: - if payload.get("schema_version") != RESULT_SCHEMA: - raise ValueError("unsupported ROCm Attention ablation result schema") - if payload.get("scope") != RESULT_SCOPE: - raise ValueError("result scope must identify an operator micro-probe, not a model replay") - manifest = attention_debug_matrix() - if payload.get("matrix_manifest") != manifest: - raise ValueError("result does not embed the exact PR230 matrix manifest") - - environment = payload.get("environment", {}) - if not isinstance(environment, Mapping) or ( - not environment.get("hip_runtime") - or environment.get("execution_kind") != "operator_only_rocm_reference" - or environment.get("primary_backend_id") != ROCM_REFERENCE_BACKEND_ID - or environment.get("primary_core_id") != STRICT_ATTENTION_REFERENCE_CORE_ID - or environment.get("primary_schedule") != STRICT_ATTENTION_SCHEDULE_ID - or environment.get("primary_reference_only") is not True - or environment.get("primary_production_ready") is not False - or "gfx942" not in environment.get("architecture", "") - ): - raise ValueError("result does not prove gfx942 ROCm reference execution") - for name in ("python", "pytorch", "hip_runtime", "device_name", "architecture"): - if not isinstance(environment.get(name), str) or not environment[name].strip(): - raise ValueError(f"environment.{name} must be a non-empty runtime readback") - hip_parts = environment["hip_runtime"].split(".") - if ( - len(hip_parts) < 2 - or not hip_parts[0].isdigit() - or not hip_parts[1].isdigit() - or "rocm" not in environment["pytorch"].lower() - or "amd" not in environment["device_name"].lower() - or "mi300x" not in environment["device_name"].lower() - ): - raise ValueError("environment does not identify an AMD MI300X ROCm runtime") - device_index = environment.get("device_index") - gpu_count = environment.get("gpu_count") - if ( - isinstance(device_index, bool) - or not isinstance(device_index, int) - or isinstance(gpu_count, bool) - or not isinstance(gpu_count, int) - or device_index < 0 - or gpu_count < 1 - or device_index >= gpu_count - ): - raise ValueError("environment GPU selection is invalid") - - command = payload.get("command") - if ( - not isinstance(command, list) - or len(command) < 2 - or any(not isinstance(argument, str) for argument in command) - or not command[1].endswith("benchmark_rocm_attention_ablation.py") - ): - raise ValueError("result does not record the benchmark command") - - provenance = payload.get("source_provenance", {}) - if not isinstance(provenance, Mapping) or ( - not _valid_digest(provenance.get("revision"), lengths=(40, 64)) - or provenance.get("tracked_dirty") is not False - or provenance.get("tracked_diff_sha256") is not None - or provenance.get("script_path") != "benchmarks/benchmark_rocm_attention_ablation.py" - or not _valid_digest(provenance.get("script_sha256"), lengths=(64,)) - or provenance.get("script_matches_head") is not True - ): - raise ValueError("result is not pinned to a clean committed runner") - - configuration = payload.get("configuration", {}) - raw_shapes = configuration.get("shapes") if isinstance(configuration, Mapping) else None - if not isinstance(raw_shapes, list) or any( - not isinstance(shape, (list, tuple)) or len(shape) != 2 for shape in raw_shapes - ): - raise ValueError("configuration.shapes must contain BxS pairs") - try: - shapes = tuple((shape[0], shape[1]) for shape in raw_shapes) - except (IndexError, TypeError): - raise ValueError("configuration.shapes must contain BxS pairs") from None - if ( - not shapes - or len(set(shapes)) != len(shapes) - or any( - isinstance(batch, bool) - or isinstance(sequence, bool) - or not isinstance(batch, int) - or not isinstance(sequence, int) - or batch < 1 - or sequence < 4 - or sequence % 4 - for batch, sequence in shapes - ) - ): - raise ValueError("configuration.shapes contains an invalid or duplicate shape") - if shapes != DEFAULT_SHAPES: - raise ValueError("publication results must cover the exact eight-shape ROCm sweep") - expected_configuration = { - "dtype": "bfloat16", - "q_heads": QWEN3_Q_HEADS, - "kv_heads": QWEN3_KV_HEADS, - "head_dim": QWEN3_HEAD_DIM, - } - if any(configuration.get(name) != value for name, value in expected_configuration.items()): - raise ValueError("result does not use the pinned Qwen3 BF16 Attention configuration") - seed = configuration.get("seed") - if isinstance(seed, bool) or not isinstance(seed, int) or seed < 0: - raise ValueError("configuration.seed must be a non-negative integer") - - expected_rows = [row["id"] for row in manifest["rows"]] - cases = payload.get("cases", []) - expected_case_keys = [ - (row_id, batch, sequence) for batch, sequence in shapes for row_id in expected_rows - ] - if ( - not isinstance(cases, list) - or [ - (case.get("row_id"), case.get("batch"), case.get("sequence")) - for case in cases - if isinstance(case, Mapping) - ] - != expected_case_keys - ): - raise ValueError("cases do not cover every PR230 row and configured shape exactly once") - - manifest_by_id = {row["id"]: row for row in manifest["rows"]} - for case in cases: - row_id = case["row_id"] - row_manifest = manifest_by_id[row_id] - if any(case.get(name) != row_manifest[name] for name in ("category", "probe", "expected")): - raise ValueError(f"{row_id} case metadata differs from the PR230 manifest") - if case.get("realization") != ROW_REALIZATIONS[row_id]: - raise ValueError(f"{row_id} does not disclose its operator-level realization") - if case.get("passed") is not True: - raise ValueError(f"{row_id} case did not satisfy its expected outcome") - metrics = case.get("metrics") - if not isinstance(metrics, Mapping) or set(metrics) != set(METRIC_NAMES): - raise ValueError(f"{row_id} has an invalid metric set") - if row_id == "A4": - binding = case.get("binding_gate", {}) - expected_binding = _topology_gate(batch=case["batch"], sequence=case["sequence"]) - issues = binding.get("issues", []) if isinstance(binding, Mapping) else [] - issue_evidence = [ - ( - issue.get("code"), - issue.get("tier"), - issue.get("field"), - issue.get("rollout"), - issue.get("training"), - ) - for issue in issues - if isinstance(issue, Mapping) - ] - expected_issue_evidence = [ - ("TOPOLOGY_MISMATCH", "identical", "sharding.tp_rank", 1, 0), - ( - "TOPOLOGY_MISMATCH", - "identical", - "sharding.local_q_head_start", - QWEN3_Q_HEADS // 2, - 0, - ), - ( - "TOPOLOGY_MISMATCH", - "identical", - "sharding.local_kv_head_start", - QWEN3_KV_HEADS // 2, - 0, - ), - ] - if ( - case.get("comparable") is not False - or case.get("outcome") != "rejected" - or case.get("gate_implementation") - != "rl_engine.alignment.cross_config.bind_attention_contracts" - or any(metrics[name] is not None for name in METRIC_NAMES) - or binding != expected_binding - or binding.get("comparable") is not False - or binding.get("passed") is not False - or binding.get("schema_version") != "cross_config.attention_binding.v3" - or not issues - or any(not isinstance(issue, Mapping) for issue in issues) - or issue_evidence != expected_issue_evidence - or case.get("identity_errors") != [issue.get("field") for issue in issues] - ): - raise ValueError("A4 lacks isolated topology-gate rejection evidence") - continue - - if case.get("comparable") is not True: - raise ValueError(f"{row_id} must contain a numerical comparison") - implementations = case.get("implementations", {}) - expected_implementations = ROW_IMPLEMENTATIONS[row_id] - if implementations != { - "baseline": expected_implementations[0], - "candidate": expected_implementations[1], - }: - raise ValueError(f"{row_id} implementation provenance is missing or incorrect") - batch, sequence = case["batch"], case["sequence"] - expected_shapes = { - "out": [batch, QWEN3_Q_HEADS, 1 if row_id == "C2" else sequence, QWEN3_HEAD_DIM], - "lse": [batch, QWEN3_Q_HEADS, 1 if row_id == "C2" else sequence], - "dq": [batch, QWEN3_Q_HEADS, sequence, QWEN3_HEAD_DIM], - "dk": [batch, QWEN3_KV_HEADS, sequence, QWEN3_HEAD_DIM], - "dv": [batch, QWEN3_KV_HEADS, sequence, QWEN3_HEAD_DIM], - } - for name in METRIC_NAMES: - _validate_metric( - row_id, - name, - metrics[name], - expected_dtype="torch.float32" if name == "lse" else "torch.bfloat16", - expected_shape=expected_shapes[name], - ) - mismatch_count = sum(metrics[name]["mismatch_count"] for name in METRIC_NAMES) - if row_manifest["expected"] in {"baseline", "exact_zero"}: - valid_outcome = mismatch_count == 0 and case.get("outcome") == "matched" - else: - valid_outcome = mismatch_count > 0 and case.get("outcome") == "drift_detected" - if not valid_outcome: - raise ValueError(f"{row_id} numerical evidence contradicts its expected outcome") - - aggregates = payload.get("matrix", []) - if not isinstance(aggregates, list) or any(not isinstance(row, Mapping) for row in aggregates): - raise ValueError("matrix summary must be a list of rows") - if [row.get("id") for row in aggregates] != expected_rows: - raise ValueError("result rows do not match the PR230 matrix") - reproduced = _aggregate(cases) - for row in reproduced: - if row["expected"] == "diagnostic" and any( - row["metrics"][name]["total_mismatch_count"] == 0 for name in METRIC_NAMES - ): - raise ValueError(f"{row['id']} must show a drift signature in all five metrics") - if aggregates != reproduced: - raise ValueError("matrix summary does not reproduce the per-shape case evidence") - failed = [row["id"] for row in aggregates if row.get("passed") is not True] - if failed: - raise ValueError("ROCm Attention ablation expectations failed: " + ", ".join(failed)) - - -def validate_repository_provenance(payload: Mapping[str, Any]) -> None: - """Bind recorded hashes to a real git object and the current runner.""" - - provenance = payload["source_provenance"] - revision = provenance["revision"] - script_path = provenance["script_path"] - committed_source = _git("show", f"{revision}:{script_path}") - current_source = (REPO_ROOT / script_path).read_bytes() - if ( - committed_source.returncode != 0 - or hashlib.sha256(committed_source.stdout).hexdigest() != provenance["script_sha256"] - or hashlib.sha256(current_source).hexdigest() != provenance["script_sha256"] - ): - raise ValueError("recorded runner hash is not backed by the RL-Kernel repository") - - -def _format_metric(value: Mapping[str, Any] | None) -> str: - if value is None: - return "—" - if value["total_mismatch_count"] == 0: - return "`0`" - return f"`{value['worst_max_abs']:.8g}`" - - -def _write_report(payload: Mapping[str, Any], path: Path) -> None: - environment = payload["environment"] - shapes = ", ".join( - f"B={batch}, S={sequence}" for batch, sequence in payload["configuration"]["shapes"] - ) - lines = [ - "# PR230 Attention taxonomy: ROCm operator micro-probes", - "", - "> This applies PR230's row taxonomy to deterministic operator micro-probes.", - "> It is not the frozen model/rollout replay from PR230: no checkpoint, token stream,", - "> selected-token logprob, KL, serving engine, or AITER production claim is included.", - "", - "## Environment", - "", - f"- GPU: {environment['device_name']} ({environment['architecture']})", - f"- PyTorch: {environment['pytorch']}; HIP: {environment['hip_runtime']}", - f"- RL-Kernel: `{payload['source_provenance']['revision']}`", - f"- Shapes: {shapes}", - "- Primary core: `rlkernel.rocm.deterministic_attention`", - "", - "## Matrix", - "", - "| Row | Factor | Comparable | Out | LSE | dQ | dK | dV | Result |", - "|---|---|:---:|---:|---:|---:|---:|---:|:---:|", - ] - for row in payload["matrix"]: - metrics = row["metrics"] - values = " | ".join(_format_metric(metrics[name]) for name in METRIC_NAMES) - result = "REJECTED" if row["id"] == "A4" and row["passed"] else "PASS" - if not row["passed"]: - result = "FAIL" - lines.append( - f"| {row['id']} | {row['label']} | {'yes' if row['comparable'] else 'no'} | " - f"{values} | **{result}** |" - ) - lines.extend(["", "## Probe realizations", ""]) - for row in attention_debug_matrix()["rows"]: - lines.append(f"- `{row['id']}` — {ROW_REALIZATIONS[row['id']]}") - lines.extend( - [ - "", - "A1-A3 and A5-A7 deliberately inject one mismatch and report the worst max-absolute", - "difference over the shape sweep. A4 is rejected by the repository's cross-config", - "binding gate before numerical comparison. A0 and C0-C2 must be bitwise zero for", - "Out/LSE/dQ/dK/dV.", - "", - "A6 and A7 are eager PyTorch-on-ROCm probes for accumulation and merge order; the", - "remaining numerical rows invoke the native deterministic HIP Attention core. This", - "is operator-only reference evidence, not full PR230 replay evidence.", - "", - "The complete per-shape mismatch counts and max-absolute values are in `results.json`.", - "", - "## Reproduce", - "", - "Run from the recorded clean commit and choose a new output directory:", - "", - "```bash", - "HIP_VISIBLE_DEVICES=0 CUDA_VISIBLE_DEVICES=0 python \\", - " benchmarks/benchmark_rocm_attention_ablation.py --device 0 \\", - " --output-dir /tmp/pr230_rocm_mi300x_ablation", - "```", - "", - ] - ) - path.write_text("\n".join(lines), encoding="utf-8") - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--output-dir", - type=Path, - default=Path("benchmarks/results/pr230_rocm_mi300x_ablation"), - ) - parser.add_argument("--seed", type=int, default=230) - parser.add_argument("--device", type=int, default=0) - args = parser.parse_args() - - if args.output_dir.exists(): - raise SystemExit(f"refusing to overwrite existing output directory: {args.output_dir}") - source_provenance = _source_provenance() - if source_provenance["tracked_dirty"] or not source_provenance["script_matches_head"]: - raise SystemExit("refusing to publish evidence from an uncommitted runner or tracked tree") - if torch.version.hip is None or not torch.cuda.is_available(): - raise SystemExit("the ROCm Attention ablation matrix requires a ROCm GPU") - device = torch.device("cuda", args.device) - torch.cuda.set_device(device) - properties = torch.cuda.get_device_properties(device) - architecture = getattr(properties, "gcnArchName", "") - if "gfx942" not in architecture: - raise SystemExit(f"the checked-in matrix is pinned to gfx942, got {architecture!r}") - - from rl_engine.kernels.ops.cuda.attention.deterministic_attn import ( - RLKernelDeterministicAttentionCore, - ) - from rl_engine.kernels.ops.cuda.rotary_embedding.rope import RocmDeterministicRoPEOp - - attention = RLKernelDeterministicAttentionCore() - if attention.backend_id != ROCM_REFERENCE_BACKEND_ID: - raise SystemExit(f"unexpected reference backend on ROCm: {attention.backend_id}") - rope = RocmDeterministicRoPEOp() - rows: list[dict[str, Any]] = [] - for index, (batch, sequence) in enumerate(DEFAULT_SHAPES): - rows.extend( - _case( - batch=batch, - sequence=sequence, - seed=args.seed + index * 10, - device=device, - attention=attention, - rope=rope, - ) - ) - torch.cuda.synchronize(device) - - payload = { - "schema_version": RESULT_SCHEMA, - "scope": RESULT_SCOPE, - "created_at": datetime.now(timezone.utc).isoformat(), - "command": [sys.executable, *sys.argv], - "source_provenance": source_provenance, - "matrix_manifest": attention_debug_matrix(), - "environment": _environment(device, attention), - "configuration": { - "seed": args.seed, - "dtype": "bfloat16", - "q_heads": QWEN3_Q_HEADS, - "kv_heads": QWEN3_KV_HEADS, - "head_dim": QWEN3_HEAD_DIM, - "shapes": [list(shape) for shape in DEFAULT_SHAPES], - }, - "cases": rows, - "matrix": _aggregate(rows), - } - validate_payload(payload) - validate_repository_provenance(payload) - args.output_dir.mkdir(parents=True, exist_ok=False) - (args.output_dir / "results.json").write_text( - json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8" - ) - _write_report(payload, args.output_dir / "report.md") - print(json.dumps({"output_dir": str(args.output_dir), "passed": True}, indent=2)) - - -if __name__ == "__main__": - main() diff --git a/benchmarks/results/pr230_rocm_mi300x_ablation/report.md b/benchmarks/results/pr230_rocm_mi300x_ablation/report.md deleted file mode 100644 index c76e4503..00000000 --- a/benchmarks/results/pr230_rocm_mi300x_ablation/report.md +++ /dev/null @@ -1,64 +0,0 @@ -# PR230 Attention taxonomy: ROCm operator micro-probes - -> This applies PR230's row taxonomy to deterministic operator micro-probes. -> It is not the frozen model/rollout replay from PR230: no checkpoint, token stream, -> selected-token logprob, KL, serving engine, or AITER production claim is included. - -## Environment - -- GPU: AMD Instinct MI300X VF (gfx942:sramecc+:xnack-) -- PyTorch: 2.12.0+rocm7.14.0a20260608; HIP: 7.14.60850 -- RL-Kernel: `2ea63b22b74feb6e5a748d09780fa075d5e644ed` -- Shapes: B=1, S=16, B=1, S=32, B=1, S=64, B=1, S=128, B=2, S=16, B=2, S=32, B=2, S=64, B=2, S=128 -- Primary core: `rlkernel.rocm.deterministic_attention` - -## Matrix - -| Row | Factor | Comparable | Out | LSE | dQ | dK | dV | Result | -|---|---|:---:|---:|---:|---:|---:|---:|:---:| -| A0 | Strict replay baseline | yes | `0` | `0` | `0` | `0` | `0` | **PASS** | -| A1 | Position / RoPE | yes | `0.7109375` | `0.39382553` | `1.4140625` | `2.234375` | `0.78320312` | **PASS** | -| A2 | Q/K preprocessing | yes | `0.58203125` | `1.0303755` | `1.015625` | `1.7578125` | `0.96875` | **PASS** | -| A3 | Mask / sequence boundary | yes | `4.21875` | `7.671814` | `4.1968994` | `6.15625` | `10.984375` | **PASS** | -| A4 | Topology / head ownership | no | — | — | — | — | — | **REJECTED** | -| A5 | KV-cache identity / layout | yes | `6` | `4.578392` | `4.2773438` | `6.6274414` | `13.233398` | **PASS** | -| A6 | Numerical policy | yes | `0.1171875` | `0.099507809` | `0.09375` | `0.09375` | `0.11328125` | **PASS** | -| A7 | Distributed schedule | yes | `0.001953125` | `9.5367432e-07` | `0.00390625` | `0.00390625` | `0.00390625` | **PASS** | -| C0 | Invariant control | yes | `0` | `0` | `0` | `0` | `0` | **PASS** | -| C1 | Invariant control | yes | `0` | `0` | `0` | `0` | `0` | **PASS** | -| C2 | Invariant control | yes | `0` | `0` | `0` | `0` | `0` | **PASS** | - -## Probe realizations - -- `A0` — Repeat the identical native HIP reference-core call. -- `A1` — Increment suffix RoPE positions while preserving Q/K/V tensors. -- `A2` — Apply or bypass unit-weight PyTorch RMSNorm before the native HIP core. -- `A3` — Toggle causal masking in the native HIP core. -- `A4` — Bind valid TP-rank-1 rollout and TP-rank-0 training contracts; do not run numerics. -- `A5` — Reverse four dense K/V tensor pages; this is not a paged-cache runtime. -- `A6` — Use identical FP32 products/order with explicit FP32 versus BF16 accumulator state. -- `A7` — Merge four dense chunks in opposite orders on one GPU; this is not a CP collective. -- `C0` — Compare full GQA with two contiguous TP=2 head shards on one GPU. -- `C1` — Compare a batch call with per-row calls to the same native HIP core. -- `C2` — Compare a full-prefill tail with one trailing query over dense KV, without a serving cache. - -A1-A3 and A5-A7 deliberately inject one mismatch and report the worst max-absolute -difference over the shape sweep. A4 is rejected by the repository's cross-config -binding gate before numerical comparison. A0 and C0-C2 must be bitwise zero for -Out/LSE/dQ/dK/dV. - -A6 and A7 are eager PyTorch-on-ROCm probes for accumulation and merge order; the -remaining numerical rows invoke the native deterministic HIP Attention core. This -is operator-only reference evidence, not full PR230 replay evidence. - -The complete per-shape mismatch counts and max-absolute values are in `results.json`. - -## Reproduce - -Run from the recorded clean commit and choose a new output directory: - -```bash -HIP_VISIBLE_DEVICES=0 CUDA_VISIBLE_DEVICES=0 python \ - benchmarks/benchmark_rocm_attention_ablation.py --device 0 \ - --output-dir /tmp/pr230_rocm_mi300x_ablation -``` diff --git a/benchmarks/results/pr230_rocm_mi300x_ablation/results.json b/benchmarks/results/pr230_rocm_mi300x_ablation/results.json deleted file mode 100644 index 6c08181f..00000000 --- a/benchmarks/results/pr230_rocm_mi300x_ablation/results.json +++ /dev/null @@ -1,11234 +0,0 @@ -{ - "cases": [ - { - "batch": 1, - "category": "baseline", - "comparable": true, - "expected": "baseline", - "implementations": { - "baseline": "rlkernel.rocm.deterministic_attention", - "candidate": "rlkernel.rocm.deterministic_attention" - }, - "metrics": { - "dk": { - "bitwise_equal": true, - "element_count": 16384, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 16, - 128 - ] - }, - "dq": { - "bitwise_equal": true, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 16, - 128 - ] - }, - "dv": { - "bitwise_equal": true, - "element_count": 16384, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 16, - 128 - ] - }, - "lse": { - "bitwise_equal": true, - "element_count": 512, - "left_dtype": "torch.float32", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.float32", - "shape": [ - 1, - 32, - 16 - ] - }, - "out": { - "bitwise_equal": true, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 16, - 128 - ] - } - }, - "outcome": "matched", - "passed": true, - "probe": null, - "realization": "Repeat the identical native HIP reference-core call.", - "row_id": "A0", - "sequence": 16 - }, - { - "batch": 1, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "implementations": { - "baseline": "rlkernel.rocm.deterministic_rope+native_attention", - "candidate": "rlkernel.rocm.deterministic_rope+native_attention" - }, - "metrics": { - "dk": { - "bitwise_equal": false, - "element_count": 16384, - "left_dtype": "torch.bfloat16", - "max_abs": 1.6953125, - "mismatch_count": 15960, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 16, - 128 - ] - }, - "dq": { - "bitwise_equal": false, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 1.18359375, - "mismatch_count": 32219, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 16, - 128 - ] - }, - "dv": { - "bitwise_equal": false, - "element_count": 16384, - "left_dtype": "torch.bfloat16", - "max_abs": 0.513671875, - "mismatch_count": 15826, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 16, - 128 - ] - }, - "lse": { - "bitwise_equal": false, - "element_count": 512, - "left_dtype": "torch.float32", - "max_abs": 0.26604652404785156, - "mismatch_count": 256, - "right_dtype": "torch.float32", - "shape": [ - 1, - 32, - 16 - ] - }, - "out": { - "bitwise_equal": false, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 0.4765625, - "mismatch_count": 32138, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 16, - 128 - ] - } - }, - "outcome": "drift_detected", - "passed": true, - "probe": "position_ids", - "realization": "Increment suffix RoPE positions while preserving Q/K/V tensors.", - "row_id": "A1", - "sequence": 16 - }, - { - "batch": 1, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "implementations": { - "baseline": "torch.rocm.rms_norm+native_attention", - "candidate": "rlkernel.rocm.deterministic_attention" - }, - "metrics": { - "dk": { - "bitwise_equal": false, - "element_count": 16384, - "left_dtype": "torch.bfloat16", - "max_abs": 0.9453125, - "mismatch_count": 16189, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 16, - 128 - ] - }, - "dq": { - "bitwise_equal": false, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 0.546875, - "mismatch_count": 60603, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 16, - 128 - ] - }, - "dv": { - "bitwise_equal": false, - "element_count": 16384, - "left_dtype": "torch.bfloat16", - "max_abs": 0.5, - "mismatch_count": 15924, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 16, - 128 - ] - }, - "lse": { - "bitwise_equal": false, - "element_count": 512, - "left_dtype": "torch.float32", - "max_abs": 0.4451262950897217, - "mismatch_count": 512, - "right_dtype": "torch.float32", - "shape": [ - 1, - 32, - 16 - ] - }, - "out": { - "bitwise_equal": false, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 0.4296875, - "mismatch_count": 58032, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 16, - 128 - ] - } - }, - "outcome": "drift_detected", - "passed": true, - "probe": "qk_norm_disabled", - "realization": "Apply or bypass unit-weight PyTorch RMSNorm before the native HIP core.", - "row_id": "A2", - "sequence": 16 - }, - { - "batch": 1, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "implementations": { - "baseline": "rlkernel.rocm.deterministic_attention", - "candidate": "rlkernel.rocm.deterministic_attention" - }, - "metrics": { - "dk": { - "bitwise_equal": false, - "element_count": 16384, - "left_dtype": "torch.bfloat16", - "max_abs": 3.61328125, - "mismatch_count": 16354, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 16, - 128 - ] - }, - "dq": { - "bitwise_equal": false, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 3.5625, - "mismatch_count": 61247, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 16, - 128 - ] - }, - "dv": { - "bitwise_equal": false, - "element_count": 16384, - "left_dtype": "torch.bfloat16", - "max_abs": 7.330078125, - "mismatch_count": 16364, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 16, - 128 - ] - }, - "lse": { - "bitwise_equal": false, - "element_count": 512, - "left_dtype": "torch.float32", - "max_abs": 5.188642501831055, - "mismatch_count": 480, - "right_dtype": "torch.float32", - "shape": [ - 1, - 32, - 16 - ] - }, - "out": { - "bitwise_equal": false, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 3.4375, - "mismatch_count": 61228, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 16, - 128 - ] - } - }, - "outcome": "drift_detected", - "passed": true, - "probe": "causal_mask", - "realization": "Toggle causal masking in the native HIP core.", - "row_id": "A3", - "sequence": 16 - }, - { - "batch": 1, - "binding_gate": { - "binding_fingerprint": "36b3ae6b7b3315e4891e3fc59d23973f29af4697179fd58d6bc89e5b7643906a", - "comparable": false, - "identity_fingerprint": "c2a02818c11bd195873275c75f4d94ae20b0ae2f2e9daa7fcd5f51d9b409cb4c", - "issues": [ - { - "code": "TOPOLOGY_MISMATCH", - "field": "sharding.tp_rank", - "message": "sharding.tp_rank changes TP/CP ownership; the pair is not the same local attention problem", - "rollout": 1, - "tier": "identical", - "training": 0 - }, - { - "code": "TOPOLOGY_MISMATCH", - "field": "sharding.local_q_head_start", - "message": "sharding.local_q_head_start changes TP/CP ownership; the pair is not the same local attention problem", - "rollout": 16, - "tier": "identical", - "training": 0 - }, - { - "code": "TOPOLOGY_MISMATCH", - "field": "sharding.local_kv_head_start", - "message": "sharding.local_kv_head_start changes TP/CP ownership; the pair is not the same local attention problem", - "rollout": 4, - "tier": "identical", - "training": 0 - } - ], - "passed": false, - "provenance": { - "dtype": "bf16", - "lse_domain": "attention", - "rollout": { - "backend_id": "rlkernel.rocm.deterministic_attention", - "contract": { - "batch_size": 1, - "causal": true, - "causal_offsets": [ - 0 - ], - "dtype": "bf16", - "export_lse": true, - "head_dim": 128, - "kv_cache": null, - "lse_domain": "attention", - "mode": "prefill", - "projections": { - "o_proj": { - "acc_dtype": "fp32", - "backend_policy": "native_verified_then_common_deterministic", - "deterministic_backend": "rlkernel.cuda.det_gemm", - "input_dtype": "bf16", - "k_order": "ascending", - "output_dtype": "bf16", - "qkv_split_order": [], - "require_runtime_readback": true, - "sp_backward_collective": "all_gather", - "sp_forward_collective": "reduce_scatter", - "split_kv": "disabled", - "tp_backward_dgrad_collective": "none", - "tp_forward_collective": "all_reduce" - }, - "qkv": { - "acc_dtype": "fp32", - "backend_policy": "native_verified_then_common_deterministic", - "deterministic_backend": "rlkernel.cuda.det_gemm", - "input_dtype": "bf16", - "k_order": "ascending", - "output_dtype": "bf16", - "qkv_split_order": [ - "q", - "k", - "v" - ], - "require_runtime_readback": true, - "sp_backward_collective": "reduce_scatter", - "sp_forward_collective": "all_gather", - "split_kv": "disabled", - "tp_backward_dgrad_collective": "all_reduce", - "tp_forward_collective": "none" - } - }, - "query_sequence_length": 16, - "reduction": { - "acc_dtype": "fp32", - "downcast_at": "final_write", - "engine": "in_op_reference", - "merge": "online_softmax_lse", - "order": "global_block_index" - }, - "role": "infer", - "rope": null, - "semantic_operator": "standard_softmax_attention", - "sharding": { - "cp_rank": 0, - "cp_world_size": 1, - "global_block_indices": [ - 0 - ], - "global_block_token_starts": [ - 0 - ], - "global_kv_heads": 8, - "global_q_heads": 32, - "global_sequence_length": 16, - "local_block_offsets": [ - 0, - 16 - ], - "local_kv_head_start": 4, - "local_kv_heads": 4, - "local_q_head_start": 16, - "local_q_heads": 16, - "local_sequence_length": 16, - "packed_sequence_offsets": null, - "sp_rank": 0, - "sp_world_size": 1, - "tp_rank": 1, - "tp_world_size": 2 - }, - "split_kv": { - "fixed_split_size": null, - "mode": "disabled", - "strict_consistency": true - } - }, - "recorded": { - "backend_id": "rlkernel.rocm.deterministic_attention", - "mode": "prefill", - "reduction.engine": "in_op_reference" - } - }, - "split_kv_runtime": { - "rollout": { - "batch_size": 1, - "coverage": "complete_batch_tp_cp_owner_cartesian_product", - "cp_world_size": 1, - "entries": [ - { - "actual_split_boundaries": [ - [ - 0, - 16 - ] - ], - "actual_split_kv_count": 1, - "actual_split_kv_policy": "disabled", - "actual_split_kv_size": null, - "batch_index": 0, - "cp_rank": 0, - "expected_kv_range": [ - 0, - 16 - ], - "owner_cp_rank": 0, - "requested_split_kv_policy": "disabled", - "requested_split_kv_size": null, - "split_kv_accum_dtype": "fp32", - "split_kv_backend": "rlkernel.rocm.deterministic_attention", - "split_kv_downcast_at": "final_write", - "split_kv_fallback": false, - "split_kv_fallback_reason": null, - "split_kv_merge_order": "global_block_index", - "split_kv_plan_source": "contract_exact", - "tp_rank": 0 - }, - { - "actual_split_boundaries": [ - [ - 0, - 16 - ] - ], - "actual_split_kv_count": 1, - "actual_split_kv_policy": "disabled", - "actual_split_kv_size": null, - "batch_index": 0, - "cp_rank": 0, - "expected_kv_range": [ - 0, - 16 - ], - "owner_cp_rank": 0, - "requested_split_kv_policy": "disabled", - "requested_split_kv_size": null, - "split_kv_accum_dtype": "fp32", - "split_kv_backend": "rlkernel.rocm.deterministic_attention", - "split_kv_downcast_at": "final_write", - "split_kv_fallback": false, - "split_kv_fallback_reason": null, - "split_kv_merge_order": "global_block_index", - "split_kv_plan_source": "contract_exact", - "tp_rank": 1 - } - ], - "total_kv_tokens": [ - 16 - ], - "tp_world_size": 2 - }, - "training": { - "batch_size": 1, - "coverage": "complete_batch_tp_cp_owner_cartesian_product", - "cp_world_size": 1, - "entries": [ - { - "actual_split_boundaries": [ - [ - 0, - 16 - ] - ], - "actual_split_kv_count": 1, - "actual_split_kv_policy": "disabled", - "actual_split_kv_size": null, - "batch_index": 0, - "cp_rank": 0, - "expected_kv_range": [ - 0, - 16 - ], - "owner_cp_rank": 0, - "requested_split_kv_policy": "disabled", - "requested_split_kv_size": null, - "split_kv_accum_dtype": "fp32", - "split_kv_backend": "rlkernel.rocm.deterministic_attention", - "split_kv_downcast_at": "final_write", - "split_kv_fallback": false, - "split_kv_fallback_reason": null, - "split_kv_merge_order": "global_block_index", - "split_kv_plan_source": "contract_exact", - "tp_rank": 0 - }, - { - "actual_split_boundaries": [ - [ - 0, - 16 - ] - ], - "actual_split_kv_count": 1, - "actual_split_kv_policy": "disabled", - "actual_split_kv_size": null, - "batch_index": 0, - "cp_rank": 0, - "expected_kv_range": [ - 0, - 16 - ], - "owner_cp_rank": 0, - "requested_split_kv_policy": "disabled", - "requested_split_kv_size": null, - "split_kv_accum_dtype": "fp32", - "split_kv_backend": "rlkernel.rocm.deterministic_attention", - "split_kv_downcast_at": "final_write", - "split_kv_fallback": false, - "split_kv_fallback_reason": null, - "split_kv_merge_order": "global_block_index", - "split_kv_plan_source": "contract_exact", - "tp_rank": 1 - } - ], - "total_kv_tokens": [ - 16 - ], - "tp_world_size": 2 - } - }, - "training": { - "backend_id": "rlkernel.rocm.deterministic_attention", - "contract": { - "batch_size": 1, - "causal": true, - "causal_offsets": [ - 0 - ], - "dtype": "bf16", - "export_lse": true, - "head_dim": 128, - "kv_cache": null, - "lse_domain": "attention", - "mode": "prefill", - "projections": { - "o_proj": { - "acc_dtype": "fp32", - "backend_policy": "native_verified_then_common_deterministic", - "deterministic_backend": "rlkernel.cuda.det_gemm", - "input_dtype": "bf16", - "k_order": "ascending", - "output_dtype": "bf16", - "qkv_split_order": [], - "require_runtime_readback": true, - "sp_backward_collective": "all_gather", - "sp_forward_collective": "reduce_scatter", - "split_kv": "disabled", - "tp_backward_dgrad_collective": "none", - "tp_forward_collective": "all_reduce" - }, - "qkv": { - "acc_dtype": "fp32", - "backend_policy": "native_verified_then_common_deterministic", - "deterministic_backend": "rlkernel.cuda.det_gemm", - "input_dtype": "bf16", - "k_order": "ascending", - "output_dtype": "bf16", - "qkv_split_order": [ - "q", - "k", - "v" - ], - "require_runtime_readback": true, - "sp_backward_collective": "reduce_scatter", - "sp_forward_collective": "all_gather", - "split_kv": "disabled", - "tp_backward_dgrad_collective": "all_reduce", - "tp_forward_collective": "none" - } - }, - "query_sequence_length": 16, - "reduction": { - "acc_dtype": "fp32", - "downcast_at": "final_write", - "engine": "in_op_reference", - "merge": "online_softmax_lse", - "order": "global_block_index" - }, - "role": "train", - "rope": null, - "semantic_operator": "standard_softmax_attention", - "sharding": { - "cp_rank": 0, - "cp_world_size": 1, - "global_block_indices": [ - 0 - ], - "global_block_token_starts": [ - 0 - ], - "global_kv_heads": 8, - "global_q_heads": 32, - "global_sequence_length": 16, - "local_block_offsets": [ - 0, - 16 - ], - "local_kv_head_start": 0, - "local_kv_heads": 4, - "local_q_head_start": 0, - "local_q_heads": 16, - "local_sequence_length": 16, - "packed_sequence_offsets": null, - "sp_rank": 0, - "sp_world_size": 1, - "tp_rank": 0, - "tp_world_size": 2 - }, - "split_kv": { - "fixed_split_size": null, - "mode": "disabled", - "strict_consistency": true - } - }, - "recorded": { - "backend_id": "rlkernel.rocm.deterministic_attention", - "mode": "prefill", - "reduction.engine": "in_op_reference" - } - } - }, - "recorded_differences": {}, - "reduction_fingerprint": "4bfe65d607de1dc8d1b8c04041d48e9af4e81dc38614f0340d994616da273d6c", - "schema_version": "cross_config.attention_binding.v3" - }, - "category": "comparability_gate", - "comparable": false, - "expected": "rejected", - "gate_implementation": "rl_engine.alignment.cross_config.bind_attention_contracts", - "identity_errors": [ - "sharding.tp_rank", - "sharding.local_q_head_start", - "sharding.local_kv_head_start" - ], - "metrics": { - "dk": null, - "dq": null, - "dv": null, - "lse": null, - "out": null - }, - "outcome": "rejected", - "passed": true, - "probe": "tp_head_ownership", - "realization": "Bind valid TP-rank-1 rollout and TP-rank-0 training contracts; do not run numerics.", - "row_id": "A4", - "sequence": 16 - }, - { - "batch": 1, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "implementations": { - "baseline": "rlkernel.rocm.deterministic_attention", - "candidate": "torch.rocm.index_select+native_attention" - }, - "metrics": { - "dk": { - "bitwise_equal": false, - "element_count": 16384, - "left_dtype": "torch.bfloat16", - "max_abs": 5.06884765625, - "mismatch_count": 16365, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 16, - 128 - ] - }, - "dq": { - "bitwise_equal": false, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 3.7734375, - "mismatch_count": 57223, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 16, - 128 - ] - }, - "dv": { - "bitwise_equal": false, - "element_count": 16384, - "left_dtype": "torch.bfloat16", - "max_abs": 8.6474609375, - "mismatch_count": 16371, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 16, - 128 - ] - }, - "lse": { - "bitwise_equal": false, - "element_count": 512, - "left_dtype": "torch.float32", - "max_abs": 4.578392028808594, - "mismatch_count": 480, - "right_dtype": "torch.float32", - "shape": [ - 1, - 32, - 16 - ] - }, - "out": { - "bitwise_equal": false, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 4.40625, - "mismatch_count": 61329, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 16, - 128 - ] - } - }, - "outcome": "drift_detected", - "passed": true, - "probe": "kv_page_order", - "realization": "Reverse four dense K/V tensor pages; this is not a paged-cache runtime.", - "row_id": "A5", - "sequence": 16 - }, - { - "batch": 1, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "implementations": { - "baseline": "torch.rocm.explicit_fp32_serial_qk_accumulator", - "candidate": "torch.rocm.explicit_bf16_serial_qk_accumulator" - }, - "metrics": { - "dk": { - "bitwise_equal": false, - "element_count": 16384, - "left_dtype": "torch.bfloat16", - "max_abs": 0.05078125, - "mismatch_count": 13428, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 16, - 128 - ] - }, - "dq": { - "bitwise_equal": false, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0546875, - "mismatch_count": 47934, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 16, - 128 - ] - }, - "dv": { - "bitwise_equal": false, - "element_count": 16384, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0625, - "mismatch_count": 13654, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 16, - 128 - ] - }, - "lse": { - "bitwise_equal": false, - "element_count": 512, - "left_dtype": "torch.float32", - "max_abs": 0.05289459228515625, - "mismatch_count": 512, - "right_dtype": "torch.float32", - "shape": [ - 1, - 32, - 16 - ] - }, - "out": { - "bitwise_equal": false, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0546875, - "mismatch_count": 47319, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 16, - 128 - ] - } - }, - "outcome": "drift_detected", - "passed": true, - "probe": "accum_dtype", - "realization": "Use identical FP32 products/order with explicit FP32 versus BF16 accumulator state.", - "row_id": "A6", - "sequence": 16 - }, - { - "batch": 1, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "implementations": { - "baseline": "torch.rocm.fp32_chunk_merge_ascending", - "candidate": "torch.rocm.fp32_chunk_merge_descending" - }, - "metrics": { - "dk": { - "bitwise_equal": false, - "element_count": 16384, - "left_dtype": "torch.bfloat16", - "max_abs": 0.001953125, - "mismatch_count": 4, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 16, - 128 - ] - }, - "dq": { - "bitwise_equal": false, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 0.00390625, - "mismatch_count": 2, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 16, - 128 - ] - }, - "dv": { - "bitwise_equal": false, - "element_count": 16384, - "left_dtype": "torch.bfloat16", - "max_abs": 0.001953125, - "mismatch_count": 2, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 16, - 128 - ] - }, - "lse": { - "bitwise_equal": false, - "element_count": 512, - "left_dtype": "torch.float32", - "max_abs": 4.76837158203125e-07, - "mismatch_count": 28, - "right_dtype": "torch.float32", - "shape": [ - 1, - 32, - 16 - ] - }, - "out": { - "bitwise_equal": false, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 7.62939453125e-06, - "mismatch_count": 1, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 16, - 128 - ] - } - }, - "outcome": "drift_detected", - "passed": true, - "probe": "merge_order", - "realization": "Merge four dense chunks in opposite orders on one GPU; this is not a CP collective.", - "row_id": "A7", - "sequence": 16 - }, - { - "batch": 1, - "category": "invariant_control", - "comparable": true, - "expected": "exact_zero", - "implementations": { - "baseline": "rlkernel.rocm.deterministic_attention", - "candidate": "rlkernel.rocm.native_attention_per_tp2_head_shard" - }, - "metrics": { - "dk": { - "bitwise_equal": true, - "element_count": 16384, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 16, - 128 - ] - }, - "dq": { - "bitwise_equal": true, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 16, - 128 - ] - }, - "dv": { - "bitwise_equal": true, - "element_count": 16384, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 16, - 128 - ] - }, - "lse": { - "bitwise_equal": true, - "element_count": 512, - "left_dtype": "torch.float32", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.float32", - "shape": [ - 1, - 32, - 16 - ] - }, - "out": { - "bitwise_equal": true, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 16, - 128 - ] - } - }, - "outcome": "matched", - "passed": true, - "probe": "tp_partition_control", - "realization": "Compare full GQA with two contiguous TP=2 head shards on one GPU.", - "row_id": "C0", - "sequence": 16 - }, - { - "batch": 1, - "category": "invariant_control", - "comparable": true, - "expected": "exact_zero", - "implementations": { - "baseline": "rlkernel.rocm.deterministic_attention", - "candidate": "rlkernel.rocm.native_attention_per_batch_row" - }, - "metrics": { - "dk": { - "bitwise_equal": true, - "element_count": 16384, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 16, - 128 - ] - }, - "dq": { - "bitwise_equal": true, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 16, - 128 - ] - }, - "dv": { - "bitwise_equal": true, - "element_count": 16384, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 16, - 128 - ] - }, - "lse": { - "bitwise_equal": true, - "element_count": 512, - "left_dtype": "torch.float32", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.float32", - "shape": [ - 1, - 32, - 16 - ] - }, - "out": { - "bitwise_equal": true, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 16, - 128 - ] - } - }, - "outcome": "matched", - "passed": true, - "probe": "batch_composition_control", - "realization": "Compare a batch call with per-row calls to the same native HIP core.", - "row_id": "C1", - "sequence": 16 - }, - { - "batch": 1, - "category": "invariant_control", - "comparable": true, - "expected": "exact_zero", - "implementations": { - "baseline": "rlkernel.rocm.native_attention_full_prefill", - "candidate": "rlkernel.rocm.native_attention_tail" - }, - "metrics": { - "dk": { - "bitwise_equal": true, - "element_count": 16384, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 16, - 128 - ] - }, - "dq": { - "bitwise_equal": true, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 16, - 128 - ] - }, - "dv": { - "bitwise_equal": true, - "element_count": 16384, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 16, - 128 - ] - }, - "lse": { - "bitwise_equal": true, - "element_count": 32, - "left_dtype": "torch.float32", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.float32", - "shape": [ - 1, - 32, - 1 - ] - }, - "out": { - "bitwise_equal": true, - "element_count": 4096, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 1, - 128 - ] - } - }, - "outcome": "matched", - "passed": true, - "probe": "prefill_decode_tail_control", - "realization": "Compare a full-prefill tail with one trailing query over dense KV, without a serving cache.", - "row_id": "C2", - "sequence": 16 - }, - { - "batch": 1, - "category": "baseline", - "comparable": true, - "expected": "baseline", - "implementations": { - "baseline": "rlkernel.rocm.deterministic_attention", - "candidate": "rlkernel.rocm.deterministic_attention" - }, - "metrics": { - "dk": { - "bitwise_equal": true, - "element_count": 32768, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 32, - 128 - ] - }, - "dq": { - "bitwise_equal": true, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 32, - 128 - ] - }, - "dv": { - "bitwise_equal": true, - "element_count": 32768, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 32, - 128 - ] - }, - "lse": { - "bitwise_equal": true, - "element_count": 1024, - "left_dtype": "torch.float32", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.float32", - "shape": [ - 1, - 32, - 32 - ] - }, - "out": { - "bitwise_equal": true, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 32, - 128 - ] - } - }, - "outcome": "matched", - "passed": true, - "probe": null, - "realization": "Repeat the identical native HIP reference-core call.", - "row_id": "A0", - "sequence": 32 - }, - { - "batch": 1, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "implementations": { - "baseline": "rlkernel.rocm.deterministic_rope+native_attention", - "candidate": "rlkernel.rocm.deterministic_rope+native_attention" - }, - "metrics": { - "dk": { - "bitwise_equal": false, - "element_count": 32768, - "left_dtype": "torch.bfloat16", - "max_abs": 1.673828125, - "mismatch_count": 31886, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 32, - 128 - ] - }, - "dq": { - "bitwise_equal": false, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 1.0703125, - "mismatch_count": 64571, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 32, - 128 - ] - }, - "dv": { - "bitwise_equal": false, - "element_count": 32768, - "left_dtype": "torch.bfloat16", - "max_abs": 0.783203125, - "mismatch_count": 31476, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 32, - 128 - ] - }, - "lse": { - "bitwise_equal": false, - "element_count": 1024, - "left_dtype": "torch.float32", - "max_abs": 0.2042393684387207, - "mismatch_count": 512, - "right_dtype": "torch.float32", - "shape": [ - 1, - 32, - 32 - ] - }, - "out": { - "bitwise_equal": false, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.55078125, - "mismatch_count": 64372, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 32, - 128 - ] - } - }, - "outcome": "drift_detected", - "passed": true, - "probe": "position_ids", - "realization": "Increment suffix RoPE positions while preserving Q/K/V tensors.", - "row_id": "A1", - "sequence": 32 - }, - { - "batch": 1, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "implementations": { - "baseline": "torch.rocm.rms_norm+native_attention", - "candidate": "rlkernel.rocm.deterministic_attention" - }, - "metrics": { - "dk": { - "bitwise_equal": false, - "element_count": 32768, - "left_dtype": "torch.bfloat16", - "max_abs": 1.3720703125, - "mismatch_count": 32458, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 32, - 128 - ] - }, - "dq": { - "bitwise_equal": false, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.6953125, - "mismatch_count": 125431, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 32, - 128 - ] - }, - "dv": { - "bitwise_equal": false, - "element_count": 32768, - "left_dtype": "torch.bfloat16", - "max_abs": 0.6015625, - "mismatch_count": 32049, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 32, - 128 - ] - }, - "lse": { - "bitwise_equal": false, - "element_count": 1024, - "left_dtype": "torch.float32", - "max_abs": 0.41014528274536133, - "mismatch_count": 1024, - "right_dtype": "torch.float32", - "shape": [ - 1, - 32, - 32 - ] - }, - "out": { - "bitwise_equal": false, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.4609375, - "mismatch_count": 122609, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 32, - 128 - ] - } - }, - "outcome": "drift_detected", - "passed": true, - "probe": "qk_norm_disabled", - "realization": "Apply or bypass unit-weight PyTorch RMSNorm before the native HIP core.", - "row_id": "A2", - "sequence": 32 - }, - { - "batch": 1, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "implementations": { - "baseline": "rlkernel.rocm.deterministic_attention", - "candidate": "rlkernel.rocm.deterministic_attention" - }, - "metrics": { - "dk": { - "bitwise_equal": false, - "element_count": 32768, - "left_dtype": "torch.bfloat16", - "max_abs": 3.50390625, - "mismatch_count": 32724, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 32, - 128 - ] - }, - "dq": { - "bitwise_equal": false, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 2.78515625, - "mismatch_count": 126509, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 32, - 128 - ] - }, - "dv": { - "bitwise_equal": false, - "element_count": 32768, - "left_dtype": "torch.bfloat16", - "max_abs": 8.6015625, - "mismatch_count": 32713, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 32, - 128 - ] - }, - "lse": { - "bitwise_equal": false, - "element_count": 1024, - "left_dtype": "torch.float32", - "max_abs": 5.7191948890686035, - "mismatch_count": 992, - "right_dtype": "torch.float32", - "shape": [ - 1, - 32, - 32 - ] - }, - "out": { - "bitwise_equal": false, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 3.552734375, - "mismatch_count": 126458, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 32, - 128 - ] - } - }, - "outcome": "drift_detected", - "passed": true, - "probe": "causal_mask", - "realization": "Toggle causal masking in the native HIP core.", - "row_id": "A3", - "sequence": 32 - }, - { - "batch": 1, - "binding_gate": { - "binding_fingerprint": "c0353f09604293a8203cecf2acee5a48b38b055504e47de14ea0cdea6cd20342", - "comparable": false, - "identity_fingerprint": "c2a02818c11bd195873275c75f4d94ae20b0ae2f2e9daa7fcd5f51d9b409cb4c", - "issues": [ - { - "code": "TOPOLOGY_MISMATCH", - "field": "sharding.tp_rank", - "message": "sharding.tp_rank changes TP/CP ownership; the pair is not the same local attention problem", - "rollout": 1, - "tier": "identical", - "training": 0 - }, - { - "code": "TOPOLOGY_MISMATCH", - "field": "sharding.local_q_head_start", - "message": "sharding.local_q_head_start changes TP/CP ownership; the pair is not the same local attention problem", - "rollout": 16, - "tier": "identical", - "training": 0 - }, - { - "code": "TOPOLOGY_MISMATCH", - "field": "sharding.local_kv_head_start", - "message": "sharding.local_kv_head_start changes TP/CP ownership; the pair is not the same local attention problem", - "rollout": 4, - "tier": "identical", - "training": 0 - } - ], - "passed": false, - "provenance": { - "dtype": "bf16", - "lse_domain": "attention", - "rollout": { - "backend_id": "rlkernel.rocm.deterministic_attention", - "contract": { - "batch_size": 1, - "causal": true, - "causal_offsets": [ - 0 - ], - "dtype": "bf16", - "export_lse": true, - "head_dim": 128, - "kv_cache": null, - "lse_domain": "attention", - "mode": "prefill", - "projections": { - "o_proj": { - "acc_dtype": "fp32", - "backend_policy": "native_verified_then_common_deterministic", - "deterministic_backend": "rlkernel.cuda.det_gemm", - "input_dtype": "bf16", - "k_order": "ascending", - "output_dtype": "bf16", - "qkv_split_order": [], - "require_runtime_readback": true, - "sp_backward_collective": "all_gather", - "sp_forward_collective": "reduce_scatter", - "split_kv": "disabled", - "tp_backward_dgrad_collective": "none", - "tp_forward_collective": "all_reduce" - }, - "qkv": { - "acc_dtype": "fp32", - "backend_policy": "native_verified_then_common_deterministic", - "deterministic_backend": "rlkernel.cuda.det_gemm", - "input_dtype": "bf16", - "k_order": "ascending", - "output_dtype": "bf16", - "qkv_split_order": [ - "q", - "k", - "v" - ], - "require_runtime_readback": true, - "sp_backward_collective": "reduce_scatter", - "sp_forward_collective": "all_gather", - "split_kv": "disabled", - "tp_backward_dgrad_collective": "all_reduce", - "tp_forward_collective": "none" - } - }, - "query_sequence_length": 32, - "reduction": { - "acc_dtype": "fp32", - "downcast_at": "final_write", - "engine": "in_op_reference", - "merge": "online_softmax_lse", - "order": "global_block_index" - }, - "role": "infer", - "rope": null, - "semantic_operator": "standard_softmax_attention", - "sharding": { - "cp_rank": 0, - "cp_world_size": 1, - "global_block_indices": [ - 0 - ], - "global_block_token_starts": [ - 0 - ], - "global_kv_heads": 8, - "global_q_heads": 32, - "global_sequence_length": 32, - "local_block_offsets": [ - 0, - 32 - ], - "local_kv_head_start": 4, - "local_kv_heads": 4, - "local_q_head_start": 16, - "local_q_heads": 16, - "local_sequence_length": 32, - "packed_sequence_offsets": null, - "sp_rank": 0, - "sp_world_size": 1, - "tp_rank": 1, - "tp_world_size": 2 - }, - "split_kv": { - "fixed_split_size": null, - "mode": "disabled", - "strict_consistency": true - } - }, - "recorded": { - "backend_id": "rlkernel.rocm.deterministic_attention", - "mode": "prefill", - "reduction.engine": "in_op_reference" - } - }, - "split_kv_runtime": { - "rollout": { - "batch_size": 1, - "coverage": "complete_batch_tp_cp_owner_cartesian_product", - "cp_world_size": 1, - "entries": [ - { - "actual_split_boundaries": [ - [ - 0, - 32 - ] - ], - "actual_split_kv_count": 1, - "actual_split_kv_policy": "disabled", - "actual_split_kv_size": null, - "batch_index": 0, - "cp_rank": 0, - "expected_kv_range": [ - 0, - 32 - ], - "owner_cp_rank": 0, - "requested_split_kv_policy": "disabled", - "requested_split_kv_size": null, - "split_kv_accum_dtype": "fp32", - "split_kv_backend": "rlkernel.rocm.deterministic_attention", - "split_kv_downcast_at": "final_write", - "split_kv_fallback": false, - "split_kv_fallback_reason": null, - "split_kv_merge_order": "global_block_index", - "split_kv_plan_source": "contract_exact", - "tp_rank": 0 - }, - { - "actual_split_boundaries": [ - [ - 0, - 32 - ] - ], - "actual_split_kv_count": 1, - "actual_split_kv_policy": "disabled", - "actual_split_kv_size": null, - "batch_index": 0, - "cp_rank": 0, - "expected_kv_range": [ - 0, - 32 - ], - "owner_cp_rank": 0, - "requested_split_kv_policy": "disabled", - "requested_split_kv_size": null, - "split_kv_accum_dtype": "fp32", - "split_kv_backend": "rlkernel.rocm.deterministic_attention", - "split_kv_downcast_at": "final_write", - "split_kv_fallback": false, - "split_kv_fallback_reason": null, - "split_kv_merge_order": "global_block_index", - "split_kv_plan_source": "contract_exact", - "tp_rank": 1 - } - ], - "total_kv_tokens": [ - 32 - ], - "tp_world_size": 2 - }, - "training": { - "batch_size": 1, - "coverage": "complete_batch_tp_cp_owner_cartesian_product", - "cp_world_size": 1, - "entries": [ - { - "actual_split_boundaries": [ - [ - 0, - 32 - ] - ], - "actual_split_kv_count": 1, - "actual_split_kv_policy": "disabled", - "actual_split_kv_size": null, - "batch_index": 0, - "cp_rank": 0, - "expected_kv_range": [ - 0, - 32 - ], - "owner_cp_rank": 0, - "requested_split_kv_policy": "disabled", - "requested_split_kv_size": null, - "split_kv_accum_dtype": "fp32", - "split_kv_backend": "rlkernel.rocm.deterministic_attention", - "split_kv_downcast_at": "final_write", - "split_kv_fallback": false, - "split_kv_fallback_reason": null, - "split_kv_merge_order": "global_block_index", - "split_kv_plan_source": "contract_exact", - "tp_rank": 0 - }, - { - "actual_split_boundaries": [ - [ - 0, - 32 - ] - ], - "actual_split_kv_count": 1, - "actual_split_kv_policy": "disabled", - "actual_split_kv_size": null, - "batch_index": 0, - "cp_rank": 0, - "expected_kv_range": [ - 0, - 32 - ], - "owner_cp_rank": 0, - "requested_split_kv_policy": "disabled", - "requested_split_kv_size": null, - "split_kv_accum_dtype": "fp32", - "split_kv_backend": "rlkernel.rocm.deterministic_attention", - "split_kv_downcast_at": "final_write", - "split_kv_fallback": false, - "split_kv_fallback_reason": null, - "split_kv_merge_order": "global_block_index", - "split_kv_plan_source": "contract_exact", - "tp_rank": 1 - } - ], - "total_kv_tokens": [ - 32 - ], - "tp_world_size": 2 - } - }, - "training": { - "backend_id": "rlkernel.rocm.deterministic_attention", - "contract": { - "batch_size": 1, - "causal": true, - "causal_offsets": [ - 0 - ], - "dtype": "bf16", - "export_lse": true, - "head_dim": 128, - "kv_cache": null, - "lse_domain": "attention", - "mode": "prefill", - "projections": { - "o_proj": { - "acc_dtype": "fp32", - "backend_policy": "native_verified_then_common_deterministic", - "deterministic_backend": "rlkernel.cuda.det_gemm", - "input_dtype": "bf16", - "k_order": "ascending", - "output_dtype": "bf16", - "qkv_split_order": [], - "require_runtime_readback": true, - "sp_backward_collective": "all_gather", - "sp_forward_collective": "reduce_scatter", - "split_kv": "disabled", - "tp_backward_dgrad_collective": "none", - "tp_forward_collective": "all_reduce" - }, - "qkv": { - "acc_dtype": "fp32", - "backend_policy": "native_verified_then_common_deterministic", - "deterministic_backend": "rlkernel.cuda.det_gemm", - "input_dtype": "bf16", - "k_order": "ascending", - "output_dtype": "bf16", - "qkv_split_order": [ - "q", - "k", - "v" - ], - "require_runtime_readback": true, - "sp_backward_collective": "reduce_scatter", - "sp_forward_collective": "all_gather", - "split_kv": "disabled", - "tp_backward_dgrad_collective": "all_reduce", - "tp_forward_collective": "none" - } - }, - "query_sequence_length": 32, - "reduction": { - "acc_dtype": "fp32", - "downcast_at": "final_write", - "engine": "in_op_reference", - "merge": "online_softmax_lse", - "order": "global_block_index" - }, - "role": "train", - "rope": null, - "semantic_operator": "standard_softmax_attention", - "sharding": { - "cp_rank": 0, - "cp_world_size": 1, - "global_block_indices": [ - 0 - ], - "global_block_token_starts": [ - 0 - ], - "global_kv_heads": 8, - "global_q_heads": 32, - "global_sequence_length": 32, - "local_block_offsets": [ - 0, - 32 - ], - "local_kv_head_start": 0, - "local_kv_heads": 4, - "local_q_head_start": 0, - "local_q_heads": 16, - "local_sequence_length": 32, - "packed_sequence_offsets": null, - "sp_rank": 0, - "sp_world_size": 1, - "tp_rank": 0, - "tp_world_size": 2 - }, - "split_kv": { - "fixed_split_size": null, - "mode": "disabled", - "strict_consistency": true - } - }, - "recorded": { - "backend_id": "rlkernel.rocm.deterministic_attention", - "mode": "prefill", - "reduction.engine": "in_op_reference" - } - } - }, - "recorded_differences": {}, - "reduction_fingerprint": "4bfe65d607de1dc8d1b8c04041d48e9af4e81dc38614f0340d994616da273d6c", - "schema_version": "cross_config.attention_binding.v3" - }, - "category": "comparability_gate", - "comparable": false, - "expected": "rejected", - "gate_implementation": "rl_engine.alignment.cross_config.bind_attention_contracts", - "identity_errors": [ - "sharding.tp_rank", - "sharding.local_q_head_start", - "sharding.local_kv_head_start" - ], - "metrics": { - "dk": null, - "dq": null, - "dv": null, - "lse": null, - "out": null - }, - "outcome": "rejected", - "passed": true, - "probe": "tp_head_ownership", - "realization": "Bind valid TP-rank-1 rollout and TP-rank-0 training contracts; do not run numerics.", - "row_id": "A4", - "sequence": 32 - }, - { - "batch": 1, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "implementations": { - "baseline": "rlkernel.rocm.deterministic_attention", - "candidate": "torch.rocm.index_select+native_attention" - }, - "metrics": { - "dk": { - "bitwise_equal": false, - "element_count": 32768, - "left_dtype": "torch.bfloat16", - "max_abs": 5.302734375, - "mismatch_count": 32737, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 32, - 128 - ] - }, - "dq": { - "bitwise_equal": false, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 3.8125, - "mismatch_count": 122566, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 32, - 128 - ] - }, - "dv": { - "bitwise_equal": false, - "element_count": 32768, - "left_dtype": "torch.bfloat16", - "max_abs": 9.216796875, - "mismatch_count": 32725, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 32, - 128 - ] - }, - "lse": { - "bitwise_equal": false, - "element_count": 1024, - "left_dtype": "torch.float32", - "max_abs": 3.5017497539520264, - "mismatch_count": 992, - "right_dtype": "torch.float32", - "shape": [ - 1, - 32, - 32 - ] - }, - "out": { - "bitwise_equal": false, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 5.34375, - "mismatch_count": 126662, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 32, - 128 - ] - } - }, - "outcome": "drift_detected", - "passed": true, - "probe": "kv_page_order", - "realization": "Reverse four dense K/V tensor pages; this is not a paged-cache runtime.", - "row_id": "A5", - "sequence": 32 - }, - { - "batch": 1, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "implementations": { - "baseline": "torch.rocm.explicit_fp32_serial_qk_accumulator", - "candidate": "torch.rocm.explicit_bf16_serial_qk_accumulator" - }, - "metrics": { - "dk": { - "bitwise_equal": false, - "element_count": 32768, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0625, - "mismatch_count": 28012, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 32, - 128 - ] - }, - "dq": { - "bitwise_equal": false, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.05078125, - "mismatch_count": 102718, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 32, - 128 - ] - }, - "dv": { - "bitwise_equal": false, - "element_count": 32768, - "left_dtype": "torch.bfloat16", - "max_abs": 0.109375, - "mismatch_count": 28291, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 32, - 128 - ] - }, - "lse": { - "bitwise_equal": false, - "element_count": 1024, - "left_dtype": "torch.float32", - "max_abs": 0.058057308197021484, - "mismatch_count": 1024, - "right_dtype": "torch.float32", - "shape": [ - 1, - 32, - 32 - ] - }, - "out": { - "bitwise_equal": false, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.07421875, - "mismatch_count": 102924, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 32, - 128 - ] - } - }, - "outcome": "drift_detected", - "passed": true, - "probe": "accum_dtype", - "realization": "Use identical FP32 products/order with explicit FP32 versus BF16 accumulator state.", - "row_id": "A6", - "sequence": 32 - }, - { - "batch": 1, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "implementations": { - "baseline": "torch.rocm.fp32_chunk_merge_ascending", - "candidate": "torch.rocm.fp32_chunk_merge_descending" - }, - "metrics": { - "dk": { - "bitwise_equal": true, - "element_count": 32768, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 32, - 128 - ] - }, - "dq": { - "bitwise_equal": false, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.001953125, - "mismatch_count": 6, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 32, - 128 - ] - }, - "dv": { - "bitwise_equal": false, - "element_count": 32768, - "left_dtype": "torch.bfloat16", - "max_abs": 0.001953125, - "mismatch_count": 1, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 32, - 128 - ] - }, - "lse": { - "bitwise_equal": false, - "element_count": 1024, - "left_dtype": "torch.float32", - "max_abs": 4.76837158203125e-07, - "mismatch_count": 43, - "right_dtype": "torch.float32", - "shape": [ - 1, - 32, - 32 - ] - }, - "out": { - "bitwise_equal": false, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0009765625, - "mismatch_count": 3, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 32, - 128 - ] - } - }, - "outcome": "drift_detected", - "passed": true, - "probe": "merge_order", - "realization": "Merge four dense chunks in opposite orders on one GPU; this is not a CP collective.", - "row_id": "A7", - "sequence": 32 - }, - { - "batch": 1, - "category": "invariant_control", - "comparable": true, - "expected": "exact_zero", - "implementations": { - "baseline": "rlkernel.rocm.deterministic_attention", - "candidate": "rlkernel.rocm.native_attention_per_tp2_head_shard" - }, - "metrics": { - "dk": { - "bitwise_equal": true, - "element_count": 32768, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 32, - 128 - ] - }, - "dq": { - "bitwise_equal": true, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 32, - 128 - ] - }, - "dv": { - "bitwise_equal": true, - "element_count": 32768, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 32, - 128 - ] - }, - "lse": { - "bitwise_equal": true, - "element_count": 1024, - "left_dtype": "torch.float32", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.float32", - "shape": [ - 1, - 32, - 32 - ] - }, - "out": { - "bitwise_equal": true, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 32, - 128 - ] - } - }, - "outcome": "matched", - "passed": true, - "probe": "tp_partition_control", - "realization": "Compare full GQA with two contiguous TP=2 head shards on one GPU.", - "row_id": "C0", - "sequence": 32 - }, - { - "batch": 1, - "category": "invariant_control", - "comparable": true, - "expected": "exact_zero", - "implementations": { - "baseline": "rlkernel.rocm.deterministic_attention", - "candidate": "rlkernel.rocm.native_attention_per_batch_row" - }, - "metrics": { - "dk": { - "bitwise_equal": true, - "element_count": 32768, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 32, - 128 - ] - }, - "dq": { - "bitwise_equal": true, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 32, - 128 - ] - }, - "dv": { - "bitwise_equal": true, - "element_count": 32768, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 32, - 128 - ] - }, - "lse": { - "bitwise_equal": true, - "element_count": 1024, - "left_dtype": "torch.float32", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.float32", - "shape": [ - 1, - 32, - 32 - ] - }, - "out": { - "bitwise_equal": true, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 32, - 128 - ] - } - }, - "outcome": "matched", - "passed": true, - "probe": "batch_composition_control", - "realization": "Compare a batch call with per-row calls to the same native HIP core.", - "row_id": "C1", - "sequence": 32 - }, - { - "batch": 1, - "category": "invariant_control", - "comparable": true, - "expected": "exact_zero", - "implementations": { - "baseline": "rlkernel.rocm.native_attention_full_prefill", - "candidate": "rlkernel.rocm.native_attention_tail" - }, - "metrics": { - "dk": { - "bitwise_equal": true, - "element_count": 32768, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 32, - 128 - ] - }, - "dq": { - "bitwise_equal": true, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 32, - 128 - ] - }, - "dv": { - "bitwise_equal": true, - "element_count": 32768, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 32, - 128 - ] - }, - "lse": { - "bitwise_equal": true, - "element_count": 32, - "left_dtype": "torch.float32", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.float32", - "shape": [ - 1, - 32, - 1 - ] - }, - "out": { - "bitwise_equal": true, - "element_count": 4096, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 1, - 128 - ] - } - }, - "outcome": "matched", - "passed": true, - "probe": "prefill_decode_tail_control", - "realization": "Compare a full-prefill tail with one trailing query over dense KV, without a serving cache.", - "row_id": "C2", - "sequence": 32 - }, - { - "batch": 1, - "category": "baseline", - "comparable": true, - "expected": "baseline", - "implementations": { - "baseline": "rlkernel.rocm.deterministic_attention", - "candidate": "rlkernel.rocm.deterministic_attention" - }, - "metrics": { - "dk": { - "bitwise_equal": true, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 64, - 128 - ] - }, - "dq": { - "bitwise_equal": true, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 64, - 128 - ] - }, - "dv": { - "bitwise_equal": true, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 64, - 128 - ] - }, - "lse": { - "bitwise_equal": true, - "element_count": 2048, - "left_dtype": "torch.float32", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.float32", - "shape": [ - 1, - 32, - 64 - ] - }, - "out": { - "bitwise_equal": true, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 64, - 128 - ] - } - }, - "outcome": "matched", - "passed": true, - "probe": null, - "realization": "Repeat the identical native HIP reference-core call.", - "row_id": "A0", - "sequence": 64 - }, - { - "batch": 1, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "implementations": { - "baseline": "rlkernel.rocm.deterministic_rope+native_attention", - "candidate": "rlkernel.rocm.deterministic_rope+native_attention" - }, - "metrics": { - "dk": { - "bitwise_equal": false, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 1.470703125, - "mismatch_count": 63492, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 64, - 128 - ] - }, - "dq": { - "bitwise_equal": false, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 1.021484375, - "mismatch_count": 129300, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 64, - 128 - ] - }, - "dv": { - "bitwise_equal": false, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 0.50390625, - "mismatch_count": 62854, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 64, - 128 - ] - }, - "lse": { - "bitwise_equal": false, - "element_count": 2048, - "left_dtype": "torch.float32", - "max_abs": 0.16578149795532227, - "mismatch_count": 1024, - "right_dtype": "torch.float32", - "shape": [ - 1, - 32, - 64 - ] - }, - "out": { - "bitwise_equal": false, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 0.33056640625, - "mismatch_count": 129207, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 64, - 128 - ] - } - }, - "outcome": "drift_detected", - "passed": true, - "probe": "position_ids", - "realization": "Increment suffix RoPE positions while preserving Q/K/V tensors.", - "row_id": "A1", - "sequence": 64 - }, - { - "batch": 1, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "implementations": { - "baseline": "torch.rocm.rms_norm+native_attention", - "candidate": "rlkernel.rocm.deterministic_attention" - }, - "metrics": { - "dk": { - "bitwise_equal": false, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 1.4375, - "mismatch_count": 64945, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 64, - 128 - ] - }, - "dq": { - "bitwise_equal": false, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 0.92578125, - "mismatch_count": 255107, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 64, - 128 - ] - }, - "dv": { - "bitwise_equal": false, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 0.71875, - "mismatch_count": 64219, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 64, - 128 - ] - }, - "lse": { - "bitwise_equal": false, - "element_count": 2048, - "left_dtype": "torch.float32", - "max_abs": 0.41298937797546387, - "mismatch_count": 2048, - "right_dtype": "torch.float32", - "shape": [ - 1, - 32, - 64 - ] - }, - "out": { - "bitwise_equal": false, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 0.41015625, - "mismatch_count": 250177, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 64, - 128 - ] - } - }, - "outcome": "drift_detected", - "passed": true, - "probe": "qk_norm_disabled", - "realization": "Apply or bypass unit-weight PyTorch RMSNorm before the native HIP core.", - "row_id": "A2", - "sequence": 64 - }, - { - "batch": 1, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "implementations": { - "baseline": "rlkernel.rocm.deterministic_attention", - "candidate": "rlkernel.rocm.deterministic_attention" - }, - "metrics": { - "dk": { - "bitwise_equal": false, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 5.03515625, - "mismatch_count": 65433, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 64, - 128 - ] - }, - "dq": { - "bitwise_equal": false, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 3.56640625, - "mismatch_count": 256884, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 64, - 128 - ] - }, - "dv": { - "bitwise_equal": false, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 10.984375, - "mismatch_count": 65453, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 64, - 128 - ] - }, - "lse": { - "bitwise_equal": false, - "element_count": 2048, - "left_dtype": "torch.float32", - "max_abs": 6.717103481292725, - "mismatch_count": 2016, - "right_dtype": "torch.float32", - "shape": [ - 1, - 32, - 64 - ] - }, - "out": { - "bitwise_equal": false, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 3.27001953125, - "mismatch_count": 257021, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 64, - 128 - ] - } - }, - "outcome": "drift_detected", - "passed": true, - "probe": "causal_mask", - "realization": "Toggle causal masking in the native HIP core.", - "row_id": "A3", - "sequence": 64 - }, - { - "batch": 1, - "binding_gate": { - "binding_fingerprint": "291b5e9dd221cd3302bba8fe4c4ab4ae8ac6e45fce4f16ab0d2b5f77a16343ae", - "comparable": false, - "identity_fingerprint": "c2a02818c11bd195873275c75f4d94ae20b0ae2f2e9daa7fcd5f51d9b409cb4c", - "issues": [ - { - "code": "TOPOLOGY_MISMATCH", - "field": "sharding.tp_rank", - "message": "sharding.tp_rank changes TP/CP ownership; the pair is not the same local attention problem", - "rollout": 1, - "tier": "identical", - "training": 0 - }, - { - "code": "TOPOLOGY_MISMATCH", - "field": "sharding.local_q_head_start", - "message": "sharding.local_q_head_start changes TP/CP ownership; the pair is not the same local attention problem", - "rollout": 16, - "tier": "identical", - "training": 0 - }, - { - "code": "TOPOLOGY_MISMATCH", - "field": "sharding.local_kv_head_start", - "message": "sharding.local_kv_head_start changes TP/CP ownership; the pair is not the same local attention problem", - "rollout": 4, - "tier": "identical", - "training": 0 - } - ], - "passed": false, - "provenance": { - "dtype": "bf16", - "lse_domain": "attention", - "rollout": { - "backend_id": "rlkernel.rocm.deterministic_attention", - "contract": { - "batch_size": 1, - "causal": true, - "causal_offsets": [ - 0 - ], - "dtype": "bf16", - "export_lse": true, - "head_dim": 128, - "kv_cache": null, - "lse_domain": "attention", - "mode": "prefill", - "projections": { - "o_proj": { - "acc_dtype": "fp32", - "backend_policy": "native_verified_then_common_deterministic", - "deterministic_backend": "rlkernel.cuda.det_gemm", - "input_dtype": "bf16", - "k_order": "ascending", - "output_dtype": "bf16", - "qkv_split_order": [], - "require_runtime_readback": true, - "sp_backward_collective": "all_gather", - "sp_forward_collective": "reduce_scatter", - "split_kv": "disabled", - "tp_backward_dgrad_collective": "none", - "tp_forward_collective": "all_reduce" - }, - "qkv": { - "acc_dtype": "fp32", - "backend_policy": "native_verified_then_common_deterministic", - "deterministic_backend": "rlkernel.cuda.det_gemm", - "input_dtype": "bf16", - "k_order": "ascending", - "output_dtype": "bf16", - "qkv_split_order": [ - "q", - "k", - "v" - ], - "require_runtime_readback": true, - "sp_backward_collective": "reduce_scatter", - "sp_forward_collective": "all_gather", - "split_kv": "disabled", - "tp_backward_dgrad_collective": "all_reduce", - "tp_forward_collective": "none" - } - }, - "query_sequence_length": 64, - "reduction": { - "acc_dtype": "fp32", - "downcast_at": "final_write", - "engine": "in_op_reference", - "merge": "online_softmax_lse", - "order": "global_block_index" - }, - "role": "infer", - "rope": null, - "semantic_operator": "standard_softmax_attention", - "sharding": { - "cp_rank": 0, - "cp_world_size": 1, - "global_block_indices": [ - 0 - ], - "global_block_token_starts": [ - 0 - ], - "global_kv_heads": 8, - "global_q_heads": 32, - "global_sequence_length": 64, - "local_block_offsets": [ - 0, - 64 - ], - "local_kv_head_start": 4, - "local_kv_heads": 4, - "local_q_head_start": 16, - "local_q_heads": 16, - "local_sequence_length": 64, - "packed_sequence_offsets": null, - "sp_rank": 0, - "sp_world_size": 1, - "tp_rank": 1, - "tp_world_size": 2 - }, - "split_kv": { - "fixed_split_size": null, - "mode": "disabled", - "strict_consistency": true - } - }, - "recorded": { - "backend_id": "rlkernel.rocm.deterministic_attention", - "mode": "prefill", - "reduction.engine": "in_op_reference" - } - }, - "split_kv_runtime": { - "rollout": { - "batch_size": 1, - "coverage": "complete_batch_tp_cp_owner_cartesian_product", - "cp_world_size": 1, - "entries": [ - { - "actual_split_boundaries": [ - [ - 0, - 64 - ] - ], - "actual_split_kv_count": 1, - "actual_split_kv_policy": "disabled", - "actual_split_kv_size": null, - "batch_index": 0, - "cp_rank": 0, - "expected_kv_range": [ - 0, - 64 - ], - "owner_cp_rank": 0, - "requested_split_kv_policy": "disabled", - "requested_split_kv_size": null, - "split_kv_accum_dtype": "fp32", - "split_kv_backend": "rlkernel.rocm.deterministic_attention", - "split_kv_downcast_at": "final_write", - "split_kv_fallback": false, - "split_kv_fallback_reason": null, - "split_kv_merge_order": "global_block_index", - "split_kv_plan_source": "contract_exact", - "tp_rank": 0 - }, - { - "actual_split_boundaries": [ - [ - 0, - 64 - ] - ], - "actual_split_kv_count": 1, - "actual_split_kv_policy": "disabled", - "actual_split_kv_size": null, - "batch_index": 0, - "cp_rank": 0, - "expected_kv_range": [ - 0, - 64 - ], - "owner_cp_rank": 0, - "requested_split_kv_policy": "disabled", - "requested_split_kv_size": null, - "split_kv_accum_dtype": "fp32", - "split_kv_backend": "rlkernel.rocm.deterministic_attention", - "split_kv_downcast_at": "final_write", - "split_kv_fallback": false, - "split_kv_fallback_reason": null, - "split_kv_merge_order": "global_block_index", - "split_kv_plan_source": "contract_exact", - "tp_rank": 1 - } - ], - "total_kv_tokens": [ - 64 - ], - "tp_world_size": 2 - }, - "training": { - "batch_size": 1, - "coverage": "complete_batch_tp_cp_owner_cartesian_product", - "cp_world_size": 1, - "entries": [ - { - "actual_split_boundaries": [ - [ - 0, - 64 - ] - ], - "actual_split_kv_count": 1, - "actual_split_kv_policy": "disabled", - "actual_split_kv_size": null, - "batch_index": 0, - "cp_rank": 0, - "expected_kv_range": [ - 0, - 64 - ], - "owner_cp_rank": 0, - "requested_split_kv_policy": "disabled", - "requested_split_kv_size": null, - "split_kv_accum_dtype": "fp32", - "split_kv_backend": "rlkernel.rocm.deterministic_attention", - "split_kv_downcast_at": "final_write", - "split_kv_fallback": false, - "split_kv_fallback_reason": null, - "split_kv_merge_order": "global_block_index", - "split_kv_plan_source": "contract_exact", - "tp_rank": 0 - }, - { - "actual_split_boundaries": [ - [ - 0, - 64 - ] - ], - "actual_split_kv_count": 1, - "actual_split_kv_policy": "disabled", - "actual_split_kv_size": null, - "batch_index": 0, - "cp_rank": 0, - "expected_kv_range": [ - 0, - 64 - ], - "owner_cp_rank": 0, - "requested_split_kv_policy": "disabled", - "requested_split_kv_size": null, - "split_kv_accum_dtype": "fp32", - "split_kv_backend": "rlkernel.rocm.deterministic_attention", - "split_kv_downcast_at": "final_write", - "split_kv_fallback": false, - "split_kv_fallback_reason": null, - "split_kv_merge_order": "global_block_index", - "split_kv_plan_source": "contract_exact", - "tp_rank": 1 - } - ], - "total_kv_tokens": [ - 64 - ], - "tp_world_size": 2 - } - }, - "training": { - "backend_id": "rlkernel.rocm.deterministic_attention", - "contract": { - "batch_size": 1, - "causal": true, - "causal_offsets": [ - 0 - ], - "dtype": "bf16", - "export_lse": true, - "head_dim": 128, - "kv_cache": null, - "lse_domain": "attention", - "mode": "prefill", - "projections": { - "o_proj": { - "acc_dtype": "fp32", - "backend_policy": "native_verified_then_common_deterministic", - "deterministic_backend": "rlkernel.cuda.det_gemm", - "input_dtype": "bf16", - "k_order": "ascending", - "output_dtype": "bf16", - "qkv_split_order": [], - "require_runtime_readback": true, - "sp_backward_collective": "all_gather", - "sp_forward_collective": "reduce_scatter", - "split_kv": "disabled", - "tp_backward_dgrad_collective": "none", - "tp_forward_collective": "all_reduce" - }, - "qkv": { - "acc_dtype": "fp32", - "backend_policy": "native_verified_then_common_deterministic", - "deterministic_backend": "rlkernel.cuda.det_gemm", - "input_dtype": "bf16", - "k_order": "ascending", - "output_dtype": "bf16", - "qkv_split_order": [ - "q", - "k", - "v" - ], - "require_runtime_readback": true, - "sp_backward_collective": "reduce_scatter", - "sp_forward_collective": "all_gather", - "split_kv": "disabled", - "tp_backward_dgrad_collective": "all_reduce", - "tp_forward_collective": "none" - } - }, - "query_sequence_length": 64, - "reduction": { - "acc_dtype": "fp32", - "downcast_at": "final_write", - "engine": "in_op_reference", - "merge": "online_softmax_lse", - "order": "global_block_index" - }, - "role": "train", - "rope": null, - "semantic_operator": "standard_softmax_attention", - "sharding": { - "cp_rank": 0, - "cp_world_size": 1, - "global_block_indices": [ - 0 - ], - "global_block_token_starts": [ - 0 - ], - "global_kv_heads": 8, - "global_q_heads": 32, - "global_sequence_length": 64, - "local_block_offsets": [ - 0, - 64 - ], - "local_kv_head_start": 0, - "local_kv_heads": 4, - "local_q_head_start": 0, - "local_q_heads": 16, - "local_sequence_length": 64, - "packed_sequence_offsets": null, - "sp_rank": 0, - "sp_world_size": 1, - "tp_rank": 0, - "tp_world_size": 2 - }, - "split_kv": { - "fixed_split_size": null, - "mode": "disabled", - "strict_consistency": true - } - }, - "recorded": { - "backend_id": "rlkernel.rocm.deterministic_attention", - "mode": "prefill", - "reduction.engine": "in_op_reference" - } - } - }, - "recorded_differences": {}, - "reduction_fingerprint": "4bfe65d607de1dc8d1b8c04041d48e9af4e81dc38614f0340d994616da273d6c", - "schema_version": "cross_config.attention_binding.v3" - }, - "category": "comparability_gate", - "comparable": false, - "expected": "rejected", - "gate_implementation": "rl_engine.alignment.cross_config.bind_attention_contracts", - "identity_errors": [ - "sharding.tp_rank", - "sharding.local_q_head_start", - "sharding.local_kv_head_start" - ], - "metrics": { - "dk": null, - "dq": null, - "dv": null, - "lse": null, - "out": null - }, - "outcome": "rejected", - "passed": true, - "probe": "tp_head_ownership", - "realization": "Bind valid TP-rank-1 rollout and TP-rank-0 training contracts; do not run numerics.", - "row_id": "A4", - "sequence": 64 - }, - { - "batch": 1, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "implementations": { - "baseline": "rlkernel.rocm.deterministic_attention", - "candidate": "torch.rocm.index_select+native_attention" - }, - "metrics": { - "dk": { - "bitwise_equal": false, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 5.845947265625, - "mismatch_count": 65477, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 64, - 128 - ] - }, - "dq": { - "bitwise_equal": false, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 3.6943359375, - "mismatch_count": 253214, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 64, - 128 - ] - }, - "dv": { - "bitwise_equal": false, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 11.5634765625, - "mismatch_count": 65485, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 64, - 128 - ] - }, - "lse": { - "bitwise_equal": false, - "element_count": 2048, - "left_dtype": "torch.float32", - "max_abs": 3.160780906677246, - "mismatch_count": 2016, - "right_dtype": "torch.float32", - "shape": [ - 1, - 32, - 64 - ] - }, - "out": { - "bitwise_equal": false, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 6.0, - "mismatch_count": 257518, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 64, - 128 - ] - } - }, - "outcome": "drift_detected", - "passed": true, - "probe": "kv_page_order", - "realization": "Reverse four dense K/V tensor pages; this is not a paged-cache runtime.", - "row_id": "A5", - "sequence": 64 - }, - { - "batch": 1, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "implementations": { - "baseline": "torch.rocm.explicit_fp32_serial_qk_accumulator", - "candidate": "torch.rocm.explicit_bf16_serial_qk_accumulator" - }, - "metrics": { - "dk": { - "bitwise_equal": false, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 0.09375, - "mismatch_count": 57441, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 64, - 128 - ] - }, - "dq": { - "bitwise_equal": false, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0546875, - "mismatch_count": 218015, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 64, - 128 - ] - }, - "dv": { - "bitwise_equal": false, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 0.1015625, - "mismatch_count": 58082, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 64, - 128 - ] - }, - "lse": { - "bitwise_equal": false, - "element_count": 2048, - "left_dtype": "torch.float32", - "max_abs": 0.09950780868530273, - "mismatch_count": 2048, - "right_dtype": "torch.float32", - "shape": [ - 1, - 32, - 64 - ] - }, - "out": { - "bitwise_equal": false, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0859375, - "mismatch_count": 219875, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 64, - 128 - ] - } - }, - "outcome": "drift_detected", - "passed": true, - "probe": "accum_dtype", - "realization": "Use identical FP32 products/order with explicit FP32 versus BF16 accumulator state.", - "row_id": "A6", - "sequence": 64 - }, - { - "batch": 1, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "implementations": { - "baseline": "torch.rocm.fp32_chunk_merge_ascending", - "candidate": "torch.rocm.fp32_chunk_merge_descending" - }, - "metrics": { - "dk": { - "bitwise_equal": false, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 0.00048828125, - "mismatch_count": 2, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 64, - 128 - ] - }, - "dq": { - "bitwise_equal": false, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0009765625, - "mismatch_count": 10, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 64, - 128 - ] - }, - "dv": { - "bitwise_equal": true, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 64, - 128 - ] - }, - "lse": { - "bitwise_equal": false, - "element_count": 2048, - "left_dtype": "torch.float32", - "max_abs": 4.76837158203125e-07, - "mismatch_count": 62, - "right_dtype": "torch.float32", - "shape": [ - 1, - 32, - 64 - ] - }, - "out": { - "bitwise_equal": false, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0009765625, - "mismatch_count": 5, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 64, - 128 - ] - } - }, - "outcome": "drift_detected", - "passed": true, - "probe": "merge_order", - "realization": "Merge four dense chunks in opposite orders on one GPU; this is not a CP collective.", - "row_id": "A7", - "sequence": 64 - }, - { - "batch": 1, - "category": "invariant_control", - "comparable": true, - "expected": "exact_zero", - "implementations": { - "baseline": "rlkernel.rocm.deterministic_attention", - "candidate": "rlkernel.rocm.native_attention_per_tp2_head_shard" - }, - "metrics": { - "dk": { - "bitwise_equal": true, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 64, - 128 - ] - }, - "dq": { - "bitwise_equal": true, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 64, - 128 - ] - }, - "dv": { - "bitwise_equal": true, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 64, - 128 - ] - }, - "lse": { - "bitwise_equal": true, - "element_count": 2048, - "left_dtype": "torch.float32", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.float32", - "shape": [ - 1, - 32, - 64 - ] - }, - "out": { - "bitwise_equal": true, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 64, - 128 - ] - } - }, - "outcome": "matched", - "passed": true, - "probe": "tp_partition_control", - "realization": "Compare full GQA with two contiguous TP=2 head shards on one GPU.", - "row_id": "C0", - "sequence": 64 - }, - { - "batch": 1, - "category": "invariant_control", - "comparable": true, - "expected": "exact_zero", - "implementations": { - "baseline": "rlkernel.rocm.deterministic_attention", - "candidate": "rlkernel.rocm.native_attention_per_batch_row" - }, - "metrics": { - "dk": { - "bitwise_equal": true, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 64, - 128 - ] - }, - "dq": { - "bitwise_equal": true, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 64, - 128 - ] - }, - "dv": { - "bitwise_equal": true, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 64, - 128 - ] - }, - "lse": { - "bitwise_equal": true, - "element_count": 2048, - "left_dtype": "torch.float32", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.float32", - "shape": [ - 1, - 32, - 64 - ] - }, - "out": { - "bitwise_equal": true, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 64, - 128 - ] - } - }, - "outcome": "matched", - "passed": true, - "probe": "batch_composition_control", - "realization": "Compare a batch call with per-row calls to the same native HIP core.", - "row_id": "C1", - "sequence": 64 - }, - { - "batch": 1, - "category": "invariant_control", - "comparable": true, - "expected": "exact_zero", - "implementations": { - "baseline": "rlkernel.rocm.native_attention_full_prefill", - "candidate": "rlkernel.rocm.native_attention_tail" - }, - "metrics": { - "dk": { - "bitwise_equal": true, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 64, - 128 - ] - }, - "dq": { - "bitwise_equal": true, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 64, - 128 - ] - }, - "dv": { - "bitwise_equal": true, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 64, - 128 - ] - }, - "lse": { - "bitwise_equal": true, - "element_count": 32, - "left_dtype": "torch.float32", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.float32", - "shape": [ - 1, - 32, - 1 - ] - }, - "out": { - "bitwise_equal": true, - "element_count": 4096, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 1, - 128 - ] - } - }, - "outcome": "matched", - "passed": true, - "probe": "prefill_decode_tail_control", - "realization": "Compare a full-prefill tail with one trailing query over dense KV, without a serving cache.", - "row_id": "C2", - "sequence": 64 - }, - { - "batch": 1, - "category": "baseline", - "comparable": true, - "expected": "baseline", - "implementations": { - "baseline": "rlkernel.rocm.deterministic_attention", - "candidate": "rlkernel.rocm.deterministic_attention" - }, - "metrics": { - "dk": { - "bitwise_equal": true, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 128, - 128 - ] - }, - "dq": { - "bitwise_equal": true, - "element_count": 524288, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 128, - 128 - ] - }, - "dv": { - "bitwise_equal": true, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 128, - 128 - ] - }, - "lse": { - "bitwise_equal": true, - "element_count": 4096, - "left_dtype": "torch.float32", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.float32", - "shape": [ - 1, - 32, - 128 - ] - }, - "out": { - "bitwise_equal": true, - "element_count": 524288, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 128, - 128 - ] - } - }, - "outcome": "matched", - "passed": true, - "probe": null, - "realization": "Repeat the identical native HIP reference-core call.", - "row_id": "A0", - "sequence": 128 - }, - { - "batch": 1, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "implementations": { - "baseline": "rlkernel.rocm.deterministic_rope+native_attention", - "candidate": "rlkernel.rocm.deterministic_rope+native_attention" - }, - "metrics": { - "dk": { - "bitwise_equal": false, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 1.66015625, - "mismatch_count": 126148, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 128, - 128 - ] - }, - "dq": { - "bitwise_equal": false, - "element_count": 524288, - "left_dtype": "torch.bfloat16", - "max_abs": 0.8125, - "mismatch_count": 258977, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 128, - 128 - ] - }, - "dv": { - "bitwise_equal": false, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.4501953125, - "mismatch_count": 124473, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 128, - 128 - ] - }, - "lse": { - "bitwise_equal": false, - "element_count": 4096, - "left_dtype": "torch.float32", - "max_abs": 0.1391582489013672, - "mismatch_count": 2048, - "right_dtype": "torch.float32", - "shape": [ - 1, - 32, - 128 - ] - }, - "out": { - "bitwise_equal": false, - "element_count": 524288, - "left_dtype": "torch.bfloat16", - "max_abs": 0.41015625, - "mismatch_count": 258687, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 128, - 128 - ] - } - }, - "outcome": "drift_detected", - "passed": true, - "probe": "position_ids", - "realization": "Increment suffix RoPE positions while preserving Q/K/V tensors.", - "row_id": "A1", - "sequence": 128 - }, - { - "batch": 1, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "implementations": { - "baseline": "torch.rocm.rms_norm+native_attention", - "candidate": "rlkernel.rocm.deterministic_attention" - }, - "metrics": { - "dk": { - "bitwise_equal": false, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 1.453125, - "mismatch_count": 129998, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 128, - 128 - ] - }, - "dq": { - "bitwise_equal": false, - "element_count": 524288, - "left_dtype": "torch.bfloat16", - "max_abs": 0.88671875, - "mismatch_count": 514619, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 128, - 128 - ] - }, - "dv": { - "bitwise_equal": false, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.6484375, - "mismatch_count": 128904, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 128, - 128 - ] - }, - "lse": { - "bitwise_equal": false, - "element_count": 4096, - "left_dtype": "torch.float32", - "max_abs": 0.5336203575134277, - "mismatch_count": 4096, - "right_dtype": "torch.float32", - "shape": [ - 1, - 32, - 128 - ] - }, - "out": { - "bitwise_equal": false, - "element_count": 524288, - "left_dtype": "torch.bfloat16", - "max_abs": 0.38671875, - "mismatch_count": 507978, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 128, - 128 - ] - } - }, - "outcome": "drift_detected", - "passed": true, - "probe": "qk_norm_disabled", - "realization": "Apply or bypass unit-weight PyTorch RMSNorm before the native HIP core.", - "row_id": "A2", - "sequence": 128 - }, - { - "batch": 1, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "implementations": { - "baseline": "rlkernel.rocm.deterministic_attention", - "candidate": "rlkernel.rocm.deterministic_attention" - }, - "metrics": { - "dk": { - "bitwise_equal": false, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 6.15625, - "mismatch_count": 130889, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 128, - 128 - ] - }, - "dq": { - "bitwise_equal": false, - "element_count": 524288, - "left_dtype": "torch.bfloat16", - "max_abs": 3.3115234375, - "mismatch_count": 517813, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 128, - 128 - ] - }, - "dv": { - "bitwise_equal": false, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 9.1328125, - "mismatch_count": 130890, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 128, - 128 - ] - }, - "lse": { - "bitwise_equal": false, - "element_count": 4096, - "left_dtype": "torch.float32", - "max_abs": 7.67181396484375, - "mismatch_count": 4064, - "right_dtype": "torch.float32", - "shape": [ - 1, - 32, - 128 - ] - }, - "out": { - "bitwise_equal": false, - "element_count": 524288, - "left_dtype": "torch.bfloat16", - "max_abs": 3.380859375, - "mismatch_count": 518285, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 128, - 128 - ] - } - }, - "outcome": "drift_detected", - "passed": true, - "probe": "causal_mask", - "realization": "Toggle causal masking in the native HIP core.", - "row_id": "A3", - "sequence": 128 - }, - { - "batch": 1, - "binding_gate": { - "binding_fingerprint": "19a5c00a3fd47a426313eb74c50b43516adc1cba7c676476dd12ea0eca84d5a9", - "comparable": false, - "identity_fingerprint": "c2a02818c11bd195873275c75f4d94ae20b0ae2f2e9daa7fcd5f51d9b409cb4c", - "issues": [ - { - "code": "TOPOLOGY_MISMATCH", - "field": "sharding.tp_rank", - "message": "sharding.tp_rank changes TP/CP ownership; the pair is not the same local attention problem", - "rollout": 1, - "tier": "identical", - "training": 0 - }, - { - "code": "TOPOLOGY_MISMATCH", - "field": "sharding.local_q_head_start", - "message": "sharding.local_q_head_start changes TP/CP ownership; the pair is not the same local attention problem", - "rollout": 16, - "tier": "identical", - "training": 0 - }, - { - "code": "TOPOLOGY_MISMATCH", - "field": "sharding.local_kv_head_start", - "message": "sharding.local_kv_head_start changes TP/CP ownership; the pair is not the same local attention problem", - "rollout": 4, - "tier": "identical", - "training": 0 - } - ], - "passed": false, - "provenance": { - "dtype": "bf16", - "lse_domain": "attention", - "rollout": { - "backend_id": "rlkernel.rocm.deterministic_attention", - "contract": { - "batch_size": 1, - "causal": true, - "causal_offsets": [ - 0 - ], - "dtype": "bf16", - "export_lse": true, - "head_dim": 128, - "kv_cache": null, - "lse_domain": "attention", - "mode": "prefill", - "projections": { - "o_proj": { - "acc_dtype": "fp32", - "backend_policy": "native_verified_then_common_deterministic", - "deterministic_backend": "rlkernel.cuda.det_gemm", - "input_dtype": "bf16", - "k_order": "ascending", - "output_dtype": "bf16", - "qkv_split_order": [], - "require_runtime_readback": true, - "sp_backward_collective": "all_gather", - "sp_forward_collective": "reduce_scatter", - "split_kv": "disabled", - "tp_backward_dgrad_collective": "none", - "tp_forward_collective": "all_reduce" - }, - "qkv": { - "acc_dtype": "fp32", - "backend_policy": "native_verified_then_common_deterministic", - "deterministic_backend": "rlkernel.cuda.det_gemm", - "input_dtype": "bf16", - "k_order": "ascending", - "output_dtype": "bf16", - "qkv_split_order": [ - "q", - "k", - "v" - ], - "require_runtime_readback": true, - "sp_backward_collective": "reduce_scatter", - "sp_forward_collective": "all_gather", - "split_kv": "disabled", - "tp_backward_dgrad_collective": "all_reduce", - "tp_forward_collective": "none" - } - }, - "query_sequence_length": 128, - "reduction": { - "acc_dtype": "fp32", - "downcast_at": "final_write", - "engine": "in_op_reference", - "merge": "online_softmax_lse", - "order": "global_block_index" - }, - "role": "infer", - "rope": null, - "semantic_operator": "standard_softmax_attention", - "sharding": { - "cp_rank": 0, - "cp_world_size": 1, - "global_block_indices": [ - 0 - ], - "global_block_token_starts": [ - 0 - ], - "global_kv_heads": 8, - "global_q_heads": 32, - "global_sequence_length": 128, - "local_block_offsets": [ - 0, - 128 - ], - "local_kv_head_start": 4, - "local_kv_heads": 4, - "local_q_head_start": 16, - "local_q_heads": 16, - "local_sequence_length": 128, - "packed_sequence_offsets": null, - "sp_rank": 0, - "sp_world_size": 1, - "tp_rank": 1, - "tp_world_size": 2 - }, - "split_kv": { - "fixed_split_size": null, - "mode": "disabled", - "strict_consistency": true - } - }, - "recorded": { - "backend_id": "rlkernel.rocm.deterministic_attention", - "mode": "prefill", - "reduction.engine": "in_op_reference" - } - }, - "split_kv_runtime": { - "rollout": { - "batch_size": 1, - "coverage": "complete_batch_tp_cp_owner_cartesian_product", - "cp_world_size": 1, - "entries": [ - { - "actual_split_boundaries": [ - [ - 0, - 128 - ] - ], - "actual_split_kv_count": 1, - "actual_split_kv_policy": "disabled", - "actual_split_kv_size": null, - "batch_index": 0, - "cp_rank": 0, - "expected_kv_range": [ - 0, - 128 - ], - "owner_cp_rank": 0, - "requested_split_kv_policy": "disabled", - "requested_split_kv_size": null, - "split_kv_accum_dtype": "fp32", - "split_kv_backend": "rlkernel.rocm.deterministic_attention", - "split_kv_downcast_at": "final_write", - "split_kv_fallback": false, - "split_kv_fallback_reason": null, - "split_kv_merge_order": "global_block_index", - "split_kv_plan_source": "contract_exact", - "tp_rank": 0 - }, - { - "actual_split_boundaries": [ - [ - 0, - 128 - ] - ], - "actual_split_kv_count": 1, - "actual_split_kv_policy": "disabled", - "actual_split_kv_size": null, - "batch_index": 0, - "cp_rank": 0, - "expected_kv_range": [ - 0, - 128 - ], - "owner_cp_rank": 0, - "requested_split_kv_policy": "disabled", - "requested_split_kv_size": null, - "split_kv_accum_dtype": "fp32", - "split_kv_backend": "rlkernel.rocm.deterministic_attention", - "split_kv_downcast_at": "final_write", - "split_kv_fallback": false, - "split_kv_fallback_reason": null, - "split_kv_merge_order": "global_block_index", - "split_kv_plan_source": "contract_exact", - "tp_rank": 1 - } - ], - "total_kv_tokens": [ - 128 - ], - "tp_world_size": 2 - }, - "training": { - "batch_size": 1, - "coverage": "complete_batch_tp_cp_owner_cartesian_product", - "cp_world_size": 1, - "entries": [ - { - "actual_split_boundaries": [ - [ - 0, - 128 - ] - ], - "actual_split_kv_count": 1, - "actual_split_kv_policy": "disabled", - "actual_split_kv_size": null, - "batch_index": 0, - "cp_rank": 0, - "expected_kv_range": [ - 0, - 128 - ], - "owner_cp_rank": 0, - "requested_split_kv_policy": "disabled", - "requested_split_kv_size": null, - "split_kv_accum_dtype": "fp32", - "split_kv_backend": "rlkernel.rocm.deterministic_attention", - "split_kv_downcast_at": "final_write", - "split_kv_fallback": false, - "split_kv_fallback_reason": null, - "split_kv_merge_order": "global_block_index", - "split_kv_plan_source": "contract_exact", - "tp_rank": 0 - }, - { - "actual_split_boundaries": [ - [ - 0, - 128 - ] - ], - "actual_split_kv_count": 1, - "actual_split_kv_policy": "disabled", - "actual_split_kv_size": null, - "batch_index": 0, - "cp_rank": 0, - "expected_kv_range": [ - 0, - 128 - ], - "owner_cp_rank": 0, - "requested_split_kv_policy": "disabled", - "requested_split_kv_size": null, - "split_kv_accum_dtype": "fp32", - "split_kv_backend": "rlkernel.rocm.deterministic_attention", - "split_kv_downcast_at": "final_write", - "split_kv_fallback": false, - "split_kv_fallback_reason": null, - "split_kv_merge_order": "global_block_index", - "split_kv_plan_source": "contract_exact", - "tp_rank": 1 - } - ], - "total_kv_tokens": [ - 128 - ], - "tp_world_size": 2 - } - }, - "training": { - "backend_id": "rlkernel.rocm.deterministic_attention", - "contract": { - "batch_size": 1, - "causal": true, - "causal_offsets": [ - 0 - ], - "dtype": "bf16", - "export_lse": true, - "head_dim": 128, - "kv_cache": null, - "lse_domain": "attention", - "mode": "prefill", - "projections": { - "o_proj": { - "acc_dtype": "fp32", - "backend_policy": "native_verified_then_common_deterministic", - "deterministic_backend": "rlkernel.cuda.det_gemm", - "input_dtype": "bf16", - "k_order": "ascending", - "output_dtype": "bf16", - "qkv_split_order": [], - "require_runtime_readback": true, - "sp_backward_collective": "all_gather", - "sp_forward_collective": "reduce_scatter", - "split_kv": "disabled", - "tp_backward_dgrad_collective": "none", - "tp_forward_collective": "all_reduce" - }, - "qkv": { - "acc_dtype": "fp32", - "backend_policy": "native_verified_then_common_deterministic", - "deterministic_backend": "rlkernel.cuda.det_gemm", - "input_dtype": "bf16", - "k_order": "ascending", - "output_dtype": "bf16", - "qkv_split_order": [ - "q", - "k", - "v" - ], - "require_runtime_readback": true, - "sp_backward_collective": "reduce_scatter", - "sp_forward_collective": "all_gather", - "split_kv": "disabled", - "tp_backward_dgrad_collective": "all_reduce", - "tp_forward_collective": "none" - } - }, - "query_sequence_length": 128, - "reduction": { - "acc_dtype": "fp32", - "downcast_at": "final_write", - "engine": "in_op_reference", - "merge": "online_softmax_lse", - "order": "global_block_index" - }, - "role": "train", - "rope": null, - "semantic_operator": "standard_softmax_attention", - "sharding": { - "cp_rank": 0, - "cp_world_size": 1, - "global_block_indices": [ - 0 - ], - "global_block_token_starts": [ - 0 - ], - "global_kv_heads": 8, - "global_q_heads": 32, - "global_sequence_length": 128, - "local_block_offsets": [ - 0, - 128 - ], - "local_kv_head_start": 0, - "local_kv_heads": 4, - "local_q_head_start": 0, - "local_q_heads": 16, - "local_sequence_length": 128, - "packed_sequence_offsets": null, - "sp_rank": 0, - "sp_world_size": 1, - "tp_rank": 0, - "tp_world_size": 2 - }, - "split_kv": { - "fixed_split_size": null, - "mode": "disabled", - "strict_consistency": true - } - }, - "recorded": { - "backend_id": "rlkernel.rocm.deterministic_attention", - "mode": "prefill", - "reduction.engine": "in_op_reference" - } - } - }, - "recorded_differences": {}, - "reduction_fingerprint": "4bfe65d607de1dc8d1b8c04041d48e9af4e81dc38614f0340d994616da273d6c", - "schema_version": "cross_config.attention_binding.v3" - }, - "category": "comparability_gate", - "comparable": false, - "expected": "rejected", - "gate_implementation": "rl_engine.alignment.cross_config.bind_attention_contracts", - "identity_errors": [ - "sharding.tp_rank", - "sharding.local_q_head_start", - "sharding.local_kv_head_start" - ], - "metrics": { - "dk": null, - "dq": null, - "dv": null, - "lse": null, - "out": null - }, - "outcome": "rejected", - "passed": true, - "probe": "tp_head_ownership", - "realization": "Bind valid TP-rank-1 rollout and TP-rank-0 training contracts; do not run numerics.", - "row_id": "A4", - "sequence": 128 - }, - { - "batch": 1, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "implementations": { - "baseline": "rlkernel.rocm.deterministic_attention", - "candidate": "torch.rocm.index_select+native_attention" - }, - "metrics": { - "dk": { - "bitwise_equal": false, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 6.62744140625, - "mismatch_count": 130978, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 128, - 128 - ] - }, - "dq": { - "bitwise_equal": false, - "element_count": 524288, - "left_dtype": "torch.bfloat16", - "max_abs": 3.5849609375, - "mismatch_count": 514734, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 128, - 128 - ] - }, - "dv": { - "bitwise_equal": false, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 9.7392578125, - "mismatch_count": 130970, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 128, - 128 - ] - }, - "lse": { - "bitwise_equal": false, - "element_count": 4096, - "left_dtype": "torch.float32", - "max_abs": 3.14506196975708, - "mismatch_count": 4064, - "right_dtype": "torch.float32", - "shape": [ - 1, - 32, - 128 - ] - }, - "out": { - "bitwise_equal": false, - "element_count": 524288, - "left_dtype": "torch.bfloat16", - "max_abs": 4.75, - "mismatch_count": 519002, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 128, - 128 - ] - } - }, - "outcome": "drift_detected", - "passed": true, - "probe": "kv_page_order", - "realization": "Reverse four dense K/V tensor pages; this is not a paged-cache runtime.", - "row_id": "A5", - "sequence": 128 - }, - { - "batch": 1, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "implementations": { - "baseline": "torch.rocm.explicit_fp32_serial_qk_accumulator", - "candidate": "torch.rocm.explicit_bf16_serial_qk_accumulator" - }, - "metrics": { - "dk": { - "bitwise_equal": false, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0625, - "mismatch_count": 116599, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 128, - 128 - ] - }, - "dq": { - "bitwise_equal": false, - "element_count": 524288, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0546875, - "mismatch_count": 450629, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 128, - 128 - ] - }, - "dv": { - "bitwise_equal": false, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.09375, - "mismatch_count": 117447, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 128, - 128 - ] - }, - "lse": { - "bitwise_equal": false, - "element_count": 4096, - "left_dtype": "torch.float32", - "max_abs": 0.05309700965881348, - "mismatch_count": 4096, - "right_dtype": "torch.float32", - "shape": [ - 1, - 32, - 128 - ] - }, - "out": { - "bitwise_equal": false, - "element_count": 524288, - "left_dtype": "torch.bfloat16", - "max_abs": 0.09375, - "mismatch_count": 455338, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 128, - 128 - ] - } - }, - "outcome": "drift_detected", - "passed": true, - "probe": "accum_dtype", - "realization": "Use identical FP32 products/order with explicit FP32 versus BF16 accumulator state.", - "row_id": "A6", - "sequence": 128 - }, - { - "batch": 1, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "implementations": { - "baseline": "torch.rocm.fp32_chunk_merge_ascending", - "candidate": "torch.rocm.fp32_chunk_merge_descending" - }, - "metrics": { - "dk": { - "bitwise_equal": false, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.00390625, - "mismatch_count": 17, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 128, - 128 - ] - }, - "dq": { - "bitwise_equal": false, - "element_count": 524288, - "left_dtype": "torch.bfloat16", - "max_abs": 0.001953125, - "mismatch_count": 21, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 128, - 128 - ] - }, - "dv": { - "bitwise_equal": false, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.00390625, - "mismatch_count": 9, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 128, - 128 - ] - }, - "lse": { - "bitwise_equal": false, - "element_count": 4096, - "left_dtype": "torch.float32", - "max_abs": 9.5367431640625e-07, - "mismatch_count": 122, - "right_dtype": "torch.float32", - "shape": [ - 1, - 32, - 128 - ] - }, - "out": { - "bitwise_equal": false, - "element_count": 524288, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0009765625, - "mismatch_count": 12, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 128, - 128 - ] - } - }, - "outcome": "drift_detected", - "passed": true, - "probe": "merge_order", - "realization": "Merge four dense chunks in opposite orders on one GPU; this is not a CP collective.", - "row_id": "A7", - "sequence": 128 - }, - { - "batch": 1, - "category": "invariant_control", - "comparable": true, - "expected": "exact_zero", - "implementations": { - "baseline": "rlkernel.rocm.deterministic_attention", - "candidate": "rlkernel.rocm.native_attention_per_tp2_head_shard" - }, - "metrics": { - "dk": { - "bitwise_equal": true, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 128, - 128 - ] - }, - "dq": { - "bitwise_equal": true, - "element_count": 524288, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 128, - 128 - ] - }, - "dv": { - "bitwise_equal": true, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 128, - 128 - ] - }, - "lse": { - "bitwise_equal": true, - "element_count": 4096, - "left_dtype": "torch.float32", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.float32", - "shape": [ - 1, - 32, - 128 - ] - }, - "out": { - "bitwise_equal": true, - "element_count": 524288, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 128, - 128 - ] - } - }, - "outcome": "matched", - "passed": true, - "probe": "tp_partition_control", - "realization": "Compare full GQA with two contiguous TP=2 head shards on one GPU.", - "row_id": "C0", - "sequence": 128 - }, - { - "batch": 1, - "category": "invariant_control", - "comparable": true, - "expected": "exact_zero", - "implementations": { - "baseline": "rlkernel.rocm.deterministic_attention", - "candidate": "rlkernel.rocm.native_attention_per_batch_row" - }, - "metrics": { - "dk": { - "bitwise_equal": true, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 128, - 128 - ] - }, - "dq": { - "bitwise_equal": true, - "element_count": 524288, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 128, - 128 - ] - }, - "dv": { - "bitwise_equal": true, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 128, - 128 - ] - }, - "lse": { - "bitwise_equal": true, - "element_count": 4096, - "left_dtype": "torch.float32", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.float32", - "shape": [ - 1, - 32, - 128 - ] - }, - "out": { - "bitwise_equal": true, - "element_count": 524288, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 128, - 128 - ] - } - }, - "outcome": "matched", - "passed": true, - "probe": "batch_composition_control", - "realization": "Compare a batch call with per-row calls to the same native HIP core.", - "row_id": "C1", - "sequence": 128 - }, - { - "batch": 1, - "category": "invariant_control", - "comparable": true, - "expected": "exact_zero", - "implementations": { - "baseline": "rlkernel.rocm.native_attention_full_prefill", - "candidate": "rlkernel.rocm.native_attention_tail" - }, - "metrics": { - "dk": { - "bitwise_equal": true, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 128, - 128 - ] - }, - "dq": { - "bitwise_equal": true, - "element_count": 524288, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 128, - 128 - ] - }, - "dv": { - "bitwise_equal": true, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 8, - 128, - 128 - ] - }, - "lse": { - "bitwise_equal": true, - "element_count": 32, - "left_dtype": "torch.float32", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.float32", - "shape": [ - 1, - 32, - 1 - ] - }, - "out": { - "bitwise_equal": true, - "element_count": 4096, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 1, - 32, - 1, - 128 - ] - } - }, - "outcome": "matched", - "passed": true, - "probe": "prefill_decode_tail_control", - "realization": "Compare a full-prefill tail with one trailing query over dense KV, without a serving cache.", - "row_id": "C2", - "sequence": 128 - }, - { - "batch": 2, - "category": "baseline", - "comparable": true, - "expected": "baseline", - "implementations": { - "baseline": "rlkernel.rocm.deterministic_attention", - "candidate": "rlkernel.rocm.deterministic_attention" - }, - "metrics": { - "dk": { - "bitwise_equal": true, - "element_count": 32768, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 16, - 128 - ] - }, - "dq": { - "bitwise_equal": true, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 16, - 128 - ] - }, - "dv": { - "bitwise_equal": true, - "element_count": 32768, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 16, - 128 - ] - }, - "lse": { - "bitwise_equal": true, - "element_count": 1024, - "left_dtype": "torch.float32", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.float32", - "shape": [ - 2, - 32, - 16 - ] - }, - "out": { - "bitwise_equal": true, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 16, - 128 - ] - } - }, - "outcome": "matched", - "passed": true, - "probe": null, - "realization": "Repeat the identical native HIP reference-core call.", - "row_id": "A0", - "sequence": 16 - }, - { - "batch": 2, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "implementations": { - "baseline": "rlkernel.rocm.deterministic_rope+native_attention", - "candidate": "rlkernel.rocm.deterministic_rope+native_attention" - }, - "metrics": { - "dk": { - "bitwise_equal": false, - "element_count": 32768, - "left_dtype": "torch.bfloat16", - "max_abs": 1.78125, - "mismatch_count": 31944, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 16, - 128 - ] - }, - "dq": { - "bitwise_equal": false, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 1.3857421875, - "mismatch_count": 64515, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 16, - 128 - ] - }, - "dv": { - "bitwise_equal": false, - "element_count": 32768, - "left_dtype": "torch.bfloat16", - "max_abs": 0.64453125, - "mismatch_count": 31643, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 16, - 128 - ] - }, - "lse": { - "bitwise_equal": false, - "element_count": 1024, - "left_dtype": "torch.float32", - "max_abs": 0.3938255310058594, - "mismatch_count": 512, - "right_dtype": "torch.float32", - "shape": [ - 2, - 32, - 16 - ] - }, - "out": { - "bitwise_equal": false, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.671875, - "mismatch_count": 64375, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 16, - 128 - ] - } - }, - "outcome": "drift_detected", - "passed": true, - "probe": "position_ids", - "realization": "Increment suffix RoPE positions while preserving Q/K/V tensors.", - "row_id": "A1", - "sequence": 16 - }, - { - "batch": 2, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "implementations": { - "baseline": "torch.rocm.rms_norm+native_attention", - "candidate": "rlkernel.rocm.deterministic_attention" - }, - "metrics": { - "dk": { - "bitwise_equal": false, - "element_count": 32768, - "left_dtype": "torch.bfloat16", - "max_abs": 1.2265625, - "mismatch_count": 32375, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 16, - 128 - ] - }, - "dq": { - "bitwise_equal": false, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.65625, - "mismatch_count": 121092, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 16, - 128 - ] - }, - "dv": { - "bitwise_equal": false, - "element_count": 32768, - "left_dtype": "torch.bfloat16", - "max_abs": 0.455078125, - "mismatch_count": 31825, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 16, - 128 - ] - }, - "lse": { - "bitwise_equal": false, - "element_count": 1024, - "left_dtype": "torch.float32", - "max_abs": 0.4254317283630371, - "mismatch_count": 1024, - "right_dtype": "torch.float32", - "shape": [ - 2, - 32, - 16 - ] - }, - "out": { - "bitwise_equal": false, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.408203125, - "mismatch_count": 116343, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 16, - 128 - ] - } - }, - "outcome": "drift_detected", - "passed": true, - "probe": "qk_norm_disabled", - "realization": "Apply or bypass unit-weight PyTorch RMSNorm before the native HIP core.", - "row_id": "A2", - "sequence": 16 - }, - { - "batch": 2, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "implementations": { - "baseline": "rlkernel.rocm.deterministic_attention", - "candidate": "rlkernel.rocm.deterministic_attention" - }, - "metrics": { - "dk": { - "bitwise_equal": false, - "element_count": 32768, - "left_dtype": "torch.bfloat16", - "max_abs": 3.6875, - "mismatch_count": 32712, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 16, - 128 - ] - }, - "dq": { - "bitwise_equal": false, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 4.0546875, - "mismatch_count": 122436, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 16, - 128 - ] - }, - "dv": { - "bitwise_equal": false, - "element_count": 32768, - "left_dtype": "torch.bfloat16", - "max_abs": 7.96875, - "mismatch_count": 32708, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 16, - 128 - ] - }, - "lse": { - "bitwise_equal": false, - "element_count": 1024, - "left_dtype": "torch.float32", - "max_abs": 6.699570655822754, - "mismatch_count": 960, - "right_dtype": "torch.float32", - "shape": [ - 2, - 32, - 16 - ] - }, - "out": { - "bitwise_equal": false, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 4.21875, - "mismatch_count": 122371, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 16, - 128 - ] - } - }, - "outcome": "drift_detected", - "passed": true, - "probe": "causal_mask", - "realization": "Toggle causal masking in the native HIP core.", - "row_id": "A3", - "sequence": 16 - }, - { - "batch": 2, - "binding_gate": { - "binding_fingerprint": "aa9eef16d971dbbb005e00de631493bbbf265d71d4409a8517841df931c8ca4f", - "comparable": false, - "identity_fingerprint": "c2a02818c11bd195873275c75f4d94ae20b0ae2f2e9daa7fcd5f51d9b409cb4c", - "issues": [ - { - "code": "TOPOLOGY_MISMATCH", - "field": "sharding.tp_rank", - "message": "sharding.tp_rank changes TP/CP ownership; the pair is not the same local attention problem", - "rollout": 1, - "tier": "identical", - "training": 0 - }, - { - "code": "TOPOLOGY_MISMATCH", - "field": "sharding.local_q_head_start", - "message": "sharding.local_q_head_start changes TP/CP ownership; the pair is not the same local attention problem", - "rollout": 16, - "tier": "identical", - "training": 0 - }, - { - "code": "TOPOLOGY_MISMATCH", - "field": "sharding.local_kv_head_start", - "message": "sharding.local_kv_head_start changes TP/CP ownership; the pair is not the same local attention problem", - "rollout": 4, - "tier": "identical", - "training": 0 - } - ], - "passed": false, - "provenance": { - "dtype": "bf16", - "lse_domain": "attention", - "rollout": { - "backend_id": "rlkernel.rocm.deterministic_attention", - "contract": { - "batch_size": 2, - "causal": true, - "causal_offsets": [ - 0, - 0 - ], - "dtype": "bf16", - "export_lse": true, - "head_dim": 128, - "kv_cache": null, - "lse_domain": "attention", - "mode": "prefill", - "projections": { - "o_proj": { - "acc_dtype": "fp32", - "backend_policy": "native_verified_then_common_deterministic", - "deterministic_backend": "rlkernel.cuda.det_gemm", - "input_dtype": "bf16", - "k_order": "ascending", - "output_dtype": "bf16", - "qkv_split_order": [], - "require_runtime_readback": true, - "sp_backward_collective": "all_gather", - "sp_forward_collective": "reduce_scatter", - "split_kv": "disabled", - "tp_backward_dgrad_collective": "none", - "tp_forward_collective": "all_reduce" - }, - "qkv": { - "acc_dtype": "fp32", - "backend_policy": "native_verified_then_common_deterministic", - "deterministic_backend": "rlkernel.cuda.det_gemm", - "input_dtype": "bf16", - "k_order": "ascending", - "output_dtype": "bf16", - "qkv_split_order": [ - "q", - "k", - "v" - ], - "require_runtime_readback": true, - "sp_backward_collective": "reduce_scatter", - "sp_forward_collective": "all_gather", - "split_kv": "disabled", - "tp_backward_dgrad_collective": "all_reduce", - "tp_forward_collective": "none" - } - }, - "query_sequence_length": 16, - "reduction": { - "acc_dtype": "fp32", - "downcast_at": "final_write", - "engine": "in_op_reference", - "merge": "online_softmax_lse", - "order": "global_block_index" - }, - "role": "infer", - "rope": null, - "semantic_operator": "standard_softmax_attention", - "sharding": { - "cp_rank": 0, - "cp_world_size": 1, - "global_block_indices": [ - 0 - ], - "global_block_token_starts": [ - 0 - ], - "global_kv_heads": 8, - "global_q_heads": 32, - "global_sequence_length": 16, - "local_block_offsets": [ - 0, - 16 - ], - "local_kv_head_start": 4, - "local_kv_heads": 4, - "local_q_head_start": 16, - "local_q_heads": 16, - "local_sequence_length": 16, - "packed_sequence_offsets": null, - "sp_rank": 0, - "sp_world_size": 1, - "tp_rank": 1, - "tp_world_size": 2 - }, - "split_kv": { - "fixed_split_size": null, - "mode": "disabled", - "strict_consistency": true - } - }, - "recorded": { - "backend_id": "rlkernel.rocm.deterministic_attention", - "mode": "prefill", - "reduction.engine": "in_op_reference" - } - }, - "split_kv_runtime": { - "rollout": { - "batch_size": 2, - "coverage": "complete_batch_tp_cp_owner_cartesian_product", - "cp_world_size": 1, - "entries": [ - { - "actual_split_boundaries": [ - [ - 0, - 16 - ] - ], - "actual_split_kv_count": 1, - "actual_split_kv_policy": "disabled", - "actual_split_kv_size": null, - "batch_index": 0, - "cp_rank": 0, - "expected_kv_range": [ - 0, - 16 - ], - "owner_cp_rank": 0, - "requested_split_kv_policy": "disabled", - "requested_split_kv_size": null, - "split_kv_accum_dtype": "fp32", - "split_kv_backend": "rlkernel.rocm.deterministic_attention", - "split_kv_downcast_at": "final_write", - "split_kv_fallback": false, - "split_kv_fallback_reason": null, - "split_kv_merge_order": "global_block_index", - "split_kv_plan_source": "contract_exact", - "tp_rank": 0 - }, - { - "actual_split_boundaries": [ - [ - 0, - 16 - ] - ], - "actual_split_kv_count": 1, - "actual_split_kv_policy": "disabled", - "actual_split_kv_size": null, - "batch_index": 0, - "cp_rank": 0, - "expected_kv_range": [ - 0, - 16 - ], - "owner_cp_rank": 0, - "requested_split_kv_policy": "disabled", - "requested_split_kv_size": null, - "split_kv_accum_dtype": "fp32", - "split_kv_backend": "rlkernel.rocm.deterministic_attention", - "split_kv_downcast_at": "final_write", - "split_kv_fallback": false, - "split_kv_fallback_reason": null, - "split_kv_merge_order": "global_block_index", - "split_kv_plan_source": "contract_exact", - "tp_rank": 1 - }, - { - "actual_split_boundaries": [ - [ - 0, - 16 - ] - ], - "actual_split_kv_count": 1, - "actual_split_kv_policy": "disabled", - "actual_split_kv_size": null, - "batch_index": 1, - "cp_rank": 0, - "expected_kv_range": [ - 0, - 16 - ], - "owner_cp_rank": 0, - "requested_split_kv_policy": "disabled", - "requested_split_kv_size": null, - "split_kv_accum_dtype": "fp32", - "split_kv_backend": "rlkernel.rocm.deterministic_attention", - "split_kv_downcast_at": "final_write", - "split_kv_fallback": false, - "split_kv_fallback_reason": null, - "split_kv_merge_order": "global_block_index", - "split_kv_plan_source": "contract_exact", - "tp_rank": 0 - }, - { - "actual_split_boundaries": [ - [ - 0, - 16 - ] - ], - "actual_split_kv_count": 1, - "actual_split_kv_policy": "disabled", - "actual_split_kv_size": null, - "batch_index": 1, - "cp_rank": 0, - "expected_kv_range": [ - 0, - 16 - ], - "owner_cp_rank": 0, - "requested_split_kv_policy": "disabled", - "requested_split_kv_size": null, - "split_kv_accum_dtype": "fp32", - "split_kv_backend": "rlkernel.rocm.deterministic_attention", - "split_kv_downcast_at": "final_write", - "split_kv_fallback": false, - "split_kv_fallback_reason": null, - "split_kv_merge_order": "global_block_index", - "split_kv_plan_source": "contract_exact", - "tp_rank": 1 - } - ], - "total_kv_tokens": [ - 16, - 16 - ], - "tp_world_size": 2 - }, - "training": { - "batch_size": 2, - "coverage": "complete_batch_tp_cp_owner_cartesian_product", - "cp_world_size": 1, - "entries": [ - { - "actual_split_boundaries": [ - [ - 0, - 16 - ] - ], - "actual_split_kv_count": 1, - "actual_split_kv_policy": "disabled", - "actual_split_kv_size": null, - "batch_index": 0, - "cp_rank": 0, - "expected_kv_range": [ - 0, - 16 - ], - "owner_cp_rank": 0, - "requested_split_kv_policy": "disabled", - "requested_split_kv_size": null, - "split_kv_accum_dtype": "fp32", - "split_kv_backend": "rlkernel.rocm.deterministic_attention", - "split_kv_downcast_at": "final_write", - "split_kv_fallback": false, - "split_kv_fallback_reason": null, - "split_kv_merge_order": "global_block_index", - "split_kv_plan_source": "contract_exact", - "tp_rank": 0 - }, - { - "actual_split_boundaries": [ - [ - 0, - 16 - ] - ], - "actual_split_kv_count": 1, - "actual_split_kv_policy": "disabled", - "actual_split_kv_size": null, - "batch_index": 0, - "cp_rank": 0, - "expected_kv_range": [ - 0, - 16 - ], - "owner_cp_rank": 0, - "requested_split_kv_policy": "disabled", - "requested_split_kv_size": null, - "split_kv_accum_dtype": "fp32", - "split_kv_backend": "rlkernel.rocm.deterministic_attention", - "split_kv_downcast_at": "final_write", - "split_kv_fallback": false, - "split_kv_fallback_reason": null, - "split_kv_merge_order": "global_block_index", - "split_kv_plan_source": "contract_exact", - "tp_rank": 1 - }, - { - "actual_split_boundaries": [ - [ - 0, - 16 - ] - ], - "actual_split_kv_count": 1, - "actual_split_kv_policy": "disabled", - "actual_split_kv_size": null, - "batch_index": 1, - "cp_rank": 0, - "expected_kv_range": [ - 0, - 16 - ], - "owner_cp_rank": 0, - "requested_split_kv_policy": "disabled", - "requested_split_kv_size": null, - "split_kv_accum_dtype": "fp32", - "split_kv_backend": "rlkernel.rocm.deterministic_attention", - "split_kv_downcast_at": "final_write", - "split_kv_fallback": false, - "split_kv_fallback_reason": null, - "split_kv_merge_order": "global_block_index", - "split_kv_plan_source": "contract_exact", - "tp_rank": 0 - }, - { - "actual_split_boundaries": [ - [ - 0, - 16 - ] - ], - "actual_split_kv_count": 1, - "actual_split_kv_policy": "disabled", - "actual_split_kv_size": null, - "batch_index": 1, - "cp_rank": 0, - "expected_kv_range": [ - 0, - 16 - ], - "owner_cp_rank": 0, - "requested_split_kv_policy": "disabled", - "requested_split_kv_size": null, - "split_kv_accum_dtype": "fp32", - "split_kv_backend": "rlkernel.rocm.deterministic_attention", - "split_kv_downcast_at": "final_write", - "split_kv_fallback": false, - "split_kv_fallback_reason": null, - "split_kv_merge_order": "global_block_index", - "split_kv_plan_source": "contract_exact", - "tp_rank": 1 - } - ], - "total_kv_tokens": [ - 16, - 16 - ], - "tp_world_size": 2 - } - }, - "training": { - "backend_id": "rlkernel.rocm.deterministic_attention", - "contract": { - "batch_size": 2, - "causal": true, - "causal_offsets": [ - 0, - 0 - ], - "dtype": "bf16", - "export_lse": true, - "head_dim": 128, - "kv_cache": null, - "lse_domain": "attention", - "mode": "prefill", - "projections": { - "o_proj": { - "acc_dtype": "fp32", - "backend_policy": "native_verified_then_common_deterministic", - "deterministic_backend": "rlkernel.cuda.det_gemm", - "input_dtype": "bf16", - "k_order": "ascending", - "output_dtype": "bf16", - "qkv_split_order": [], - "require_runtime_readback": true, - "sp_backward_collective": "all_gather", - "sp_forward_collective": "reduce_scatter", - "split_kv": "disabled", - "tp_backward_dgrad_collective": "none", - "tp_forward_collective": "all_reduce" - }, - "qkv": { - "acc_dtype": "fp32", - "backend_policy": "native_verified_then_common_deterministic", - "deterministic_backend": "rlkernel.cuda.det_gemm", - "input_dtype": "bf16", - "k_order": "ascending", - "output_dtype": "bf16", - "qkv_split_order": [ - "q", - "k", - "v" - ], - "require_runtime_readback": true, - "sp_backward_collective": "reduce_scatter", - "sp_forward_collective": "all_gather", - "split_kv": "disabled", - "tp_backward_dgrad_collective": "all_reduce", - "tp_forward_collective": "none" - } - }, - "query_sequence_length": 16, - "reduction": { - "acc_dtype": "fp32", - "downcast_at": "final_write", - "engine": "in_op_reference", - "merge": "online_softmax_lse", - "order": "global_block_index" - }, - "role": "train", - "rope": null, - "semantic_operator": "standard_softmax_attention", - "sharding": { - "cp_rank": 0, - "cp_world_size": 1, - "global_block_indices": [ - 0 - ], - "global_block_token_starts": [ - 0 - ], - "global_kv_heads": 8, - "global_q_heads": 32, - "global_sequence_length": 16, - "local_block_offsets": [ - 0, - 16 - ], - "local_kv_head_start": 0, - "local_kv_heads": 4, - "local_q_head_start": 0, - "local_q_heads": 16, - "local_sequence_length": 16, - "packed_sequence_offsets": null, - "sp_rank": 0, - "sp_world_size": 1, - "tp_rank": 0, - "tp_world_size": 2 - }, - "split_kv": { - "fixed_split_size": null, - "mode": "disabled", - "strict_consistency": true - } - }, - "recorded": { - "backend_id": "rlkernel.rocm.deterministic_attention", - "mode": "prefill", - "reduction.engine": "in_op_reference" - } - } - }, - "recorded_differences": {}, - "reduction_fingerprint": "4bfe65d607de1dc8d1b8c04041d48e9af4e81dc38614f0340d994616da273d6c", - "schema_version": "cross_config.attention_binding.v3" - }, - "category": "comparability_gate", - "comparable": false, - "expected": "rejected", - "gate_implementation": "rl_engine.alignment.cross_config.bind_attention_contracts", - "identity_errors": [ - "sharding.tp_rank", - "sharding.local_q_head_start", - "sharding.local_kv_head_start" - ], - "metrics": { - "dk": null, - "dq": null, - "dv": null, - "lse": null, - "out": null - }, - "outcome": "rejected", - "passed": true, - "probe": "tp_head_ownership", - "realization": "Bind valid TP-rank-1 rollout and TP-rank-0 training contracts; do not run numerics.", - "row_id": "A4", - "sequence": 16 - }, - { - "batch": 2, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "implementations": { - "baseline": "rlkernel.rocm.deterministic_attention", - "candidate": "torch.rocm.index_select+native_attention" - }, - "metrics": { - "dk": { - "bitwise_equal": false, - "element_count": 32768, - "left_dtype": "torch.bfloat16", - "max_abs": 5.098358154296875, - "mismatch_count": 32726, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 16, - 128 - ] - }, - "dq": { - "bitwise_equal": false, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 3.9609375, - "mismatch_count": 114456, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 16, - 128 - ] - }, - "dv": { - "bitwise_equal": false, - "element_count": 32768, - "left_dtype": "torch.bfloat16", - "max_abs": 9.017578125, - "mismatch_count": 32739, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 16, - 128 - ] - }, - "lse": { - "bitwise_equal": false, - "element_count": 1024, - "left_dtype": "torch.float32", - "max_abs": 3.8384933471679688, - "mismatch_count": 960, - "right_dtype": "torch.float32", - "shape": [ - 2, - 32, - 16 - ] - }, - "out": { - "bitwise_equal": false, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 4.90625, - "mismatch_count": 122580, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 16, - 128 - ] - } - }, - "outcome": "drift_detected", - "passed": true, - "probe": "kv_page_order", - "realization": "Reverse four dense K/V tensor pages; this is not a paged-cache runtime.", - "row_id": "A5", - "sequence": 16 - }, - { - "batch": 2, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "implementations": { - "baseline": "torch.rocm.explicit_fp32_serial_qk_accumulator", - "candidate": "torch.rocm.explicit_bf16_serial_qk_accumulator" - }, - "metrics": { - "dk": { - "bitwise_equal": false, - "element_count": 32768, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0546875, - "mismatch_count": 27212, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 16, - 128 - ] - }, - "dq": { - "bitwise_equal": false, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0625, - "mismatch_count": 96463, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 16, - 128 - ] - }, - "dv": { - "bitwise_equal": false, - "element_count": 32768, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0703125, - "mismatch_count": 27390, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 16, - 128 - ] - }, - "lse": { - "bitwise_equal": false, - "element_count": 1024, - "left_dtype": "torch.float32", - "max_abs": 0.08946871757507324, - "mismatch_count": 1024, - "right_dtype": "torch.float32", - "shape": [ - 2, - 32, - 16 - ] - }, - "out": { - "bitwise_equal": false, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0703125, - "mismatch_count": 94303, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 16, - 128 - ] - } - }, - "outcome": "drift_detected", - "passed": true, - "probe": "accum_dtype", - "realization": "Use identical FP32 products/order with explicit FP32 versus BF16 accumulator state.", - "row_id": "A6", - "sequence": 16 - }, - { - "batch": 2, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "implementations": { - "baseline": "torch.rocm.fp32_chunk_merge_ascending", - "candidate": "torch.rocm.fp32_chunk_merge_descending" - }, - "metrics": { - "dk": { - "bitwise_equal": false, - "element_count": 32768, - "left_dtype": "torch.bfloat16", - "max_abs": 0.001953125, - "mismatch_count": 4, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 16, - 128 - ] - }, - "dq": { - "bitwise_equal": false, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.00048828125, - "mismatch_count": 3, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 16, - 128 - ] - }, - "dv": { - "bitwise_equal": true, - "element_count": 32768, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 16, - 128 - ] - }, - "lse": { - "bitwise_equal": false, - "element_count": 1024, - "left_dtype": "torch.float32", - "max_abs": 4.76837158203125e-07, - "mismatch_count": 71, - "right_dtype": "torch.float32", - "shape": [ - 2, - 32, - 16 - ] - }, - "out": { - "bitwise_equal": false, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 1.9073486328125e-06, - "mismatch_count": 2, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 16, - 128 - ] - } - }, - "outcome": "drift_detected", - "passed": true, - "probe": "merge_order", - "realization": "Merge four dense chunks in opposite orders on one GPU; this is not a CP collective.", - "row_id": "A7", - "sequence": 16 - }, - { - "batch": 2, - "category": "invariant_control", - "comparable": true, - "expected": "exact_zero", - "implementations": { - "baseline": "rlkernel.rocm.deterministic_attention", - "candidate": "rlkernel.rocm.native_attention_per_tp2_head_shard" - }, - "metrics": { - "dk": { - "bitwise_equal": true, - "element_count": 32768, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 16, - 128 - ] - }, - "dq": { - "bitwise_equal": true, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 16, - 128 - ] - }, - "dv": { - "bitwise_equal": true, - "element_count": 32768, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 16, - 128 - ] - }, - "lse": { - "bitwise_equal": true, - "element_count": 1024, - "left_dtype": "torch.float32", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.float32", - "shape": [ - 2, - 32, - 16 - ] - }, - "out": { - "bitwise_equal": true, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 16, - 128 - ] - } - }, - "outcome": "matched", - "passed": true, - "probe": "tp_partition_control", - "realization": "Compare full GQA with two contiguous TP=2 head shards on one GPU.", - "row_id": "C0", - "sequence": 16 - }, - { - "batch": 2, - "category": "invariant_control", - "comparable": true, - "expected": "exact_zero", - "implementations": { - "baseline": "rlkernel.rocm.deterministic_attention", - "candidate": "rlkernel.rocm.native_attention_per_batch_row" - }, - "metrics": { - "dk": { - "bitwise_equal": true, - "element_count": 32768, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 16, - 128 - ] - }, - "dq": { - "bitwise_equal": true, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 16, - 128 - ] - }, - "dv": { - "bitwise_equal": true, - "element_count": 32768, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 16, - 128 - ] - }, - "lse": { - "bitwise_equal": true, - "element_count": 1024, - "left_dtype": "torch.float32", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.float32", - "shape": [ - 2, - 32, - 16 - ] - }, - "out": { - "bitwise_equal": true, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 16, - 128 - ] - } - }, - "outcome": "matched", - "passed": true, - "probe": "batch_composition_control", - "realization": "Compare a batch call with per-row calls to the same native HIP core.", - "row_id": "C1", - "sequence": 16 - }, - { - "batch": 2, - "category": "invariant_control", - "comparable": true, - "expected": "exact_zero", - "implementations": { - "baseline": "rlkernel.rocm.native_attention_full_prefill", - "candidate": "rlkernel.rocm.native_attention_tail" - }, - "metrics": { - "dk": { - "bitwise_equal": true, - "element_count": 32768, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 16, - 128 - ] - }, - "dq": { - "bitwise_equal": true, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 16, - 128 - ] - }, - "dv": { - "bitwise_equal": true, - "element_count": 32768, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 16, - 128 - ] - }, - "lse": { - "bitwise_equal": true, - "element_count": 64, - "left_dtype": "torch.float32", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.float32", - "shape": [ - 2, - 32, - 1 - ] - }, - "out": { - "bitwise_equal": true, - "element_count": 8192, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 1, - 128 - ] - } - }, - "outcome": "matched", - "passed": true, - "probe": "prefill_decode_tail_control", - "realization": "Compare a full-prefill tail with one trailing query over dense KV, without a serving cache.", - "row_id": "C2", - "sequence": 16 - }, - { - "batch": 2, - "category": "baseline", - "comparable": true, - "expected": "baseline", - "implementations": { - "baseline": "rlkernel.rocm.deterministic_attention", - "candidate": "rlkernel.rocm.deterministic_attention" - }, - "metrics": { - "dk": { - "bitwise_equal": true, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 32, - 128 - ] - }, - "dq": { - "bitwise_equal": true, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 32, - 128 - ] - }, - "dv": { - "bitwise_equal": true, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 32, - 128 - ] - }, - "lse": { - "bitwise_equal": true, - "element_count": 2048, - "left_dtype": "torch.float32", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.float32", - "shape": [ - 2, - 32, - 32 - ] - }, - "out": { - "bitwise_equal": true, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 32, - 128 - ] - } - }, - "outcome": "matched", - "passed": true, - "probe": null, - "realization": "Repeat the identical native HIP reference-core call.", - "row_id": "A0", - "sequence": 32 - }, - { - "batch": 2, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "implementations": { - "baseline": "rlkernel.rocm.deterministic_rope+native_attention", - "candidate": "rlkernel.rocm.deterministic_rope+native_attention" - }, - "metrics": { - "dk": { - "bitwise_equal": false, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 2.234375, - "mismatch_count": 63791, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 32, - 128 - ] - }, - "dq": { - "bitwise_equal": false, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 1.4140625, - "mismatch_count": 129154, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 32, - 128 - ] - }, - "dv": { - "bitwise_equal": false, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 0.53125, - "mismatch_count": 63113, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 32, - 128 - ] - }, - "lse": { - "bitwise_equal": false, - "element_count": 2048, - "left_dtype": "torch.float32", - "max_abs": 0.3046402931213379, - "mismatch_count": 1024, - "right_dtype": "torch.float32", - "shape": [ - 2, - 32, - 32 - ] - }, - "out": { - "bitwise_equal": false, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 0.7109375, - "mismatch_count": 129019, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 32, - 128 - ] - } - }, - "outcome": "drift_detected", - "passed": true, - "probe": "position_ids", - "realization": "Increment suffix RoPE positions while preserving Q/K/V tensors.", - "row_id": "A1", - "sequence": 32 - }, - { - "batch": 2, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "implementations": { - "baseline": "torch.rocm.rms_norm+native_attention", - "candidate": "rlkernel.rocm.deterministic_attention" - }, - "metrics": { - "dk": { - "bitwise_equal": false, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 1.7578125, - "mismatch_count": 64890, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 32, - 128 - ] - }, - "dq": { - "bitwise_equal": false, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 0.7578125, - "mismatch_count": 250592, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 32, - 128 - ] - }, - "dv": { - "bitwise_equal": false, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 0.609375, - "mismatch_count": 64053, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 32, - 128 - ] - }, - "lse": { - "bitwise_equal": false, - "element_count": 2048, - "left_dtype": "torch.float32", - "max_abs": 0.3993661403656006, - "mismatch_count": 2048, - "right_dtype": "torch.float32", - "shape": [ - 2, - 32, - 32 - ] - }, - "out": { - "bitwise_equal": false, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 0.4453125, - "mismatch_count": 244117, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 32, - 128 - ] - } - }, - "outcome": "drift_detected", - "passed": true, - "probe": "qk_norm_disabled", - "realization": "Apply or bypass unit-weight PyTorch RMSNorm before the native HIP core.", - "row_id": "A2", - "sequence": 32 - }, - { - "batch": 2, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "implementations": { - "baseline": "rlkernel.rocm.deterministic_attention", - "candidate": "rlkernel.rocm.deterministic_attention" - }, - "metrics": { - "dk": { - "bitwise_equal": false, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 5.1640625, - "mismatch_count": 65446, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 32, - 128 - ] - }, - "dq": { - "bitwise_equal": false, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 4.1968994140625, - "mismatch_count": 252862, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 32, - 128 - ] - }, - "dv": { - "bitwise_equal": false, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 10.1328125, - "mismatch_count": 65455, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 32, - 128 - ] - }, - "lse": { - "bitwise_equal": false, - "element_count": 2048, - "left_dtype": "torch.float32", - "max_abs": 6.510496139526367, - "mismatch_count": 1984, - "right_dtype": "torch.float32", - "shape": [ - 2, - 32, - 32 - ] - }, - "out": { - "bitwise_equal": false, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 3.568359375, - "mismatch_count": 252854, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 32, - 128 - ] - } - }, - "outcome": "drift_detected", - "passed": true, - "probe": "causal_mask", - "realization": "Toggle causal masking in the native HIP core.", - "row_id": "A3", - "sequence": 32 - }, - { - "batch": 2, - "binding_gate": { - "binding_fingerprint": "5daa4663857ba9f0fd736b00ed6143ba2df4973fd0c9a24c390777b4c3719728", - "comparable": false, - "identity_fingerprint": "c2a02818c11bd195873275c75f4d94ae20b0ae2f2e9daa7fcd5f51d9b409cb4c", - "issues": [ - { - "code": "TOPOLOGY_MISMATCH", - "field": "sharding.tp_rank", - "message": "sharding.tp_rank changes TP/CP ownership; the pair is not the same local attention problem", - "rollout": 1, - "tier": "identical", - "training": 0 - }, - { - "code": "TOPOLOGY_MISMATCH", - "field": "sharding.local_q_head_start", - "message": "sharding.local_q_head_start changes TP/CP ownership; the pair is not the same local attention problem", - "rollout": 16, - "tier": "identical", - "training": 0 - }, - { - "code": "TOPOLOGY_MISMATCH", - "field": "sharding.local_kv_head_start", - "message": "sharding.local_kv_head_start changes TP/CP ownership; the pair is not the same local attention problem", - "rollout": 4, - "tier": "identical", - "training": 0 - } - ], - "passed": false, - "provenance": { - "dtype": "bf16", - "lse_domain": "attention", - "rollout": { - "backend_id": "rlkernel.rocm.deterministic_attention", - "contract": { - "batch_size": 2, - "causal": true, - "causal_offsets": [ - 0, - 0 - ], - "dtype": "bf16", - "export_lse": true, - "head_dim": 128, - "kv_cache": null, - "lse_domain": "attention", - "mode": "prefill", - "projections": { - "o_proj": { - "acc_dtype": "fp32", - "backend_policy": "native_verified_then_common_deterministic", - "deterministic_backend": "rlkernel.cuda.det_gemm", - "input_dtype": "bf16", - "k_order": "ascending", - "output_dtype": "bf16", - "qkv_split_order": [], - "require_runtime_readback": true, - "sp_backward_collective": "all_gather", - "sp_forward_collective": "reduce_scatter", - "split_kv": "disabled", - "tp_backward_dgrad_collective": "none", - "tp_forward_collective": "all_reduce" - }, - "qkv": { - "acc_dtype": "fp32", - "backend_policy": "native_verified_then_common_deterministic", - "deterministic_backend": "rlkernel.cuda.det_gemm", - "input_dtype": "bf16", - "k_order": "ascending", - "output_dtype": "bf16", - "qkv_split_order": [ - "q", - "k", - "v" - ], - "require_runtime_readback": true, - "sp_backward_collective": "reduce_scatter", - "sp_forward_collective": "all_gather", - "split_kv": "disabled", - "tp_backward_dgrad_collective": "all_reduce", - "tp_forward_collective": "none" - } - }, - "query_sequence_length": 32, - "reduction": { - "acc_dtype": "fp32", - "downcast_at": "final_write", - "engine": "in_op_reference", - "merge": "online_softmax_lse", - "order": "global_block_index" - }, - "role": "infer", - "rope": null, - "semantic_operator": "standard_softmax_attention", - "sharding": { - "cp_rank": 0, - "cp_world_size": 1, - "global_block_indices": [ - 0 - ], - "global_block_token_starts": [ - 0 - ], - "global_kv_heads": 8, - "global_q_heads": 32, - "global_sequence_length": 32, - "local_block_offsets": [ - 0, - 32 - ], - "local_kv_head_start": 4, - "local_kv_heads": 4, - "local_q_head_start": 16, - "local_q_heads": 16, - "local_sequence_length": 32, - "packed_sequence_offsets": null, - "sp_rank": 0, - "sp_world_size": 1, - "tp_rank": 1, - "tp_world_size": 2 - }, - "split_kv": { - "fixed_split_size": null, - "mode": "disabled", - "strict_consistency": true - } - }, - "recorded": { - "backend_id": "rlkernel.rocm.deterministic_attention", - "mode": "prefill", - "reduction.engine": "in_op_reference" - } - }, - "split_kv_runtime": { - "rollout": { - "batch_size": 2, - "coverage": "complete_batch_tp_cp_owner_cartesian_product", - "cp_world_size": 1, - "entries": [ - { - "actual_split_boundaries": [ - [ - 0, - 32 - ] - ], - "actual_split_kv_count": 1, - "actual_split_kv_policy": "disabled", - "actual_split_kv_size": null, - "batch_index": 0, - "cp_rank": 0, - "expected_kv_range": [ - 0, - 32 - ], - "owner_cp_rank": 0, - "requested_split_kv_policy": "disabled", - "requested_split_kv_size": null, - "split_kv_accum_dtype": "fp32", - "split_kv_backend": "rlkernel.rocm.deterministic_attention", - "split_kv_downcast_at": "final_write", - "split_kv_fallback": false, - "split_kv_fallback_reason": null, - "split_kv_merge_order": "global_block_index", - "split_kv_plan_source": "contract_exact", - "tp_rank": 0 - }, - { - "actual_split_boundaries": [ - [ - 0, - 32 - ] - ], - "actual_split_kv_count": 1, - "actual_split_kv_policy": "disabled", - "actual_split_kv_size": null, - "batch_index": 0, - "cp_rank": 0, - "expected_kv_range": [ - 0, - 32 - ], - "owner_cp_rank": 0, - "requested_split_kv_policy": "disabled", - "requested_split_kv_size": null, - "split_kv_accum_dtype": "fp32", - "split_kv_backend": "rlkernel.rocm.deterministic_attention", - "split_kv_downcast_at": "final_write", - "split_kv_fallback": false, - "split_kv_fallback_reason": null, - "split_kv_merge_order": "global_block_index", - "split_kv_plan_source": "contract_exact", - "tp_rank": 1 - }, - { - "actual_split_boundaries": [ - [ - 0, - 32 - ] - ], - "actual_split_kv_count": 1, - "actual_split_kv_policy": "disabled", - "actual_split_kv_size": null, - "batch_index": 1, - "cp_rank": 0, - "expected_kv_range": [ - 0, - 32 - ], - "owner_cp_rank": 0, - "requested_split_kv_policy": "disabled", - "requested_split_kv_size": null, - "split_kv_accum_dtype": "fp32", - "split_kv_backend": "rlkernel.rocm.deterministic_attention", - "split_kv_downcast_at": "final_write", - "split_kv_fallback": false, - "split_kv_fallback_reason": null, - "split_kv_merge_order": "global_block_index", - "split_kv_plan_source": "contract_exact", - "tp_rank": 0 - }, - { - "actual_split_boundaries": [ - [ - 0, - 32 - ] - ], - "actual_split_kv_count": 1, - "actual_split_kv_policy": "disabled", - "actual_split_kv_size": null, - "batch_index": 1, - "cp_rank": 0, - "expected_kv_range": [ - 0, - 32 - ], - "owner_cp_rank": 0, - "requested_split_kv_policy": "disabled", - "requested_split_kv_size": null, - "split_kv_accum_dtype": "fp32", - "split_kv_backend": "rlkernel.rocm.deterministic_attention", - "split_kv_downcast_at": "final_write", - "split_kv_fallback": false, - "split_kv_fallback_reason": null, - "split_kv_merge_order": "global_block_index", - "split_kv_plan_source": "contract_exact", - "tp_rank": 1 - } - ], - "total_kv_tokens": [ - 32, - 32 - ], - "tp_world_size": 2 - }, - "training": { - "batch_size": 2, - "coverage": "complete_batch_tp_cp_owner_cartesian_product", - "cp_world_size": 1, - "entries": [ - { - "actual_split_boundaries": [ - [ - 0, - 32 - ] - ], - "actual_split_kv_count": 1, - "actual_split_kv_policy": "disabled", - "actual_split_kv_size": null, - "batch_index": 0, - "cp_rank": 0, - "expected_kv_range": [ - 0, - 32 - ], - "owner_cp_rank": 0, - "requested_split_kv_policy": "disabled", - "requested_split_kv_size": null, - "split_kv_accum_dtype": "fp32", - "split_kv_backend": "rlkernel.rocm.deterministic_attention", - "split_kv_downcast_at": "final_write", - "split_kv_fallback": false, - "split_kv_fallback_reason": null, - "split_kv_merge_order": "global_block_index", - "split_kv_plan_source": "contract_exact", - "tp_rank": 0 - }, - { - "actual_split_boundaries": [ - [ - 0, - 32 - ] - ], - "actual_split_kv_count": 1, - "actual_split_kv_policy": "disabled", - "actual_split_kv_size": null, - "batch_index": 0, - "cp_rank": 0, - "expected_kv_range": [ - 0, - 32 - ], - "owner_cp_rank": 0, - "requested_split_kv_policy": "disabled", - "requested_split_kv_size": null, - "split_kv_accum_dtype": "fp32", - "split_kv_backend": "rlkernel.rocm.deterministic_attention", - "split_kv_downcast_at": "final_write", - "split_kv_fallback": false, - "split_kv_fallback_reason": null, - "split_kv_merge_order": "global_block_index", - "split_kv_plan_source": "contract_exact", - "tp_rank": 1 - }, - { - "actual_split_boundaries": [ - [ - 0, - 32 - ] - ], - "actual_split_kv_count": 1, - "actual_split_kv_policy": "disabled", - "actual_split_kv_size": null, - "batch_index": 1, - "cp_rank": 0, - "expected_kv_range": [ - 0, - 32 - ], - "owner_cp_rank": 0, - "requested_split_kv_policy": "disabled", - "requested_split_kv_size": null, - "split_kv_accum_dtype": "fp32", - "split_kv_backend": "rlkernel.rocm.deterministic_attention", - "split_kv_downcast_at": "final_write", - "split_kv_fallback": false, - "split_kv_fallback_reason": null, - "split_kv_merge_order": "global_block_index", - "split_kv_plan_source": "contract_exact", - "tp_rank": 0 - }, - { - "actual_split_boundaries": [ - [ - 0, - 32 - ] - ], - "actual_split_kv_count": 1, - "actual_split_kv_policy": "disabled", - "actual_split_kv_size": null, - "batch_index": 1, - "cp_rank": 0, - "expected_kv_range": [ - 0, - 32 - ], - "owner_cp_rank": 0, - "requested_split_kv_policy": "disabled", - "requested_split_kv_size": null, - "split_kv_accum_dtype": "fp32", - "split_kv_backend": "rlkernel.rocm.deterministic_attention", - "split_kv_downcast_at": "final_write", - "split_kv_fallback": false, - "split_kv_fallback_reason": null, - "split_kv_merge_order": "global_block_index", - "split_kv_plan_source": "contract_exact", - "tp_rank": 1 - } - ], - "total_kv_tokens": [ - 32, - 32 - ], - "tp_world_size": 2 - } - }, - "training": { - "backend_id": "rlkernel.rocm.deterministic_attention", - "contract": { - "batch_size": 2, - "causal": true, - "causal_offsets": [ - 0, - 0 - ], - "dtype": "bf16", - "export_lse": true, - "head_dim": 128, - "kv_cache": null, - "lse_domain": "attention", - "mode": "prefill", - "projections": { - "o_proj": { - "acc_dtype": "fp32", - "backend_policy": "native_verified_then_common_deterministic", - "deterministic_backend": "rlkernel.cuda.det_gemm", - "input_dtype": "bf16", - "k_order": "ascending", - "output_dtype": "bf16", - "qkv_split_order": [], - "require_runtime_readback": true, - "sp_backward_collective": "all_gather", - "sp_forward_collective": "reduce_scatter", - "split_kv": "disabled", - "tp_backward_dgrad_collective": "none", - "tp_forward_collective": "all_reduce" - }, - "qkv": { - "acc_dtype": "fp32", - "backend_policy": "native_verified_then_common_deterministic", - "deterministic_backend": "rlkernel.cuda.det_gemm", - "input_dtype": "bf16", - "k_order": "ascending", - "output_dtype": "bf16", - "qkv_split_order": [ - "q", - "k", - "v" - ], - "require_runtime_readback": true, - "sp_backward_collective": "reduce_scatter", - "sp_forward_collective": "all_gather", - "split_kv": "disabled", - "tp_backward_dgrad_collective": "all_reduce", - "tp_forward_collective": "none" - } - }, - "query_sequence_length": 32, - "reduction": { - "acc_dtype": "fp32", - "downcast_at": "final_write", - "engine": "in_op_reference", - "merge": "online_softmax_lse", - "order": "global_block_index" - }, - "role": "train", - "rope": null, - "semantic_operator": "standard_softmax_attention", - "sharding": { - "cp_rank": 0, - "cp_world_size": 1, - "global_block_indices": [ - 0 - ], - "global_block_token_starts": [ - 0 - ], - "global_kv_heads": 8, - "global_q_heads": 32, - "global_sequence_length": 32, - "local_block_offsets": [ - 0, - 32 - ], - "local_kv_head_start": 0, - "local_kv_heads": 4, - "local_q_head_start": 0, - "local_q_heads": 16, - "local_sequence_length": 32, - "packed_sequence_offsets": null, - "sp_rank": 0, - "sp_world_size": 1, - "tp_rank": 0, - "tp_world_size": 2 - }, - "split_kv": { - "fixed_split_size": null, - "mode": "disabled", - "strict_consistency": true - } - }, - "recorded": { - "backend_id": "rlkernel.rocm.deterministic_attention", - "mode": "prefill", - "reduction.engine": "in_op_reference" - } - } - }, - "recorded_differences": {}, - "reduction_fingerprint": "4bfe65d607de1dc8d1b8c04041d48e9af4e81dc38614f0340d994616da273d6c", - "schema_version": "cross_config.attention_binding.v3" - }, - "category": "comparability_gate", - "comparable": false, - "expected": "rejected", - "gate_implementation": "rl_engine.alignment.cross_config.bind_attention_contracts", - "identity_errors": [ - "sharding.tp_rank", - "sharding.local_q_head_start", - "sharding.local_kv_head_start" - ], - "metrics": { - "dk": null, - "dq": null, - "dv": null, - "lse": null, - "out": null - }, - "outcome": "rejected", - "passed": true, - "probe": "tp_head_ownership", - "realization": "Bind valid TP-rank-1 rollout and TP-rank-0 training contracts; do not run numerics.", - "row_id": "A4", - "sequence": 32 - }, - { - "batch": 2, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "implementations": { - "baseline": "rlkernel.rocm.deterministic_attention", - "candidate": "torch.rocm.index_select+native_attention" - }, - "metrics": { - "dk": { - "bitwise_equal": false, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 5.94140625, - "mismatch_count": 65470, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 32, - 128 - ] - }, - "dq": { - "bitwise_equal": false, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 4.27734375, - "mismatch_count": 245171, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 32, - 128 - ] - }, - "dv": { - "bitwise_equal": false, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 13.2333984375, - "mismatch_count": 65489, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 32, - 128 - ] - }, - "lse": { - "bitwise_equal": false, - "element_count": 2048, - "left_dtype": "torch.float32", - "max_abs": 3.504487991333008, - "mismatch_count": 1984, - "right_dtype": "torch.float32", - "shape": [ - 2, - 32, - 32 - ] - }, - "out": { - "bitwise_equal": false, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 5.21875, - "mismatch_count": 253373, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 32, - 128 - ] - } - }, - "outcome": "drift_detected", - "passed": true, - "probe": "kv_page_order", - "realization": "Reverse four dense K/V tensor pages; this is not a paged-cache runtime.", - "row_id": "A5", - "sequence": 32 - }, - { - "batch": 2, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "implementations": { - "baseline": "torch.rocm.explicit_fp32_serial_qk_accumulator", - "candidate": "torch.rocm.explicit_bf16_serial_qk_accumulator" - }, - "metrics": { - "dk": { - "bitwise_equal": false, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0625, - "mismatch_count": 56067, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 32, - 128 - ] - }, - "dq": { - "bitwise_equal": false, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0546875, - "mismatch_count": 206125, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 32, - 128 - ] - }, - "dv": { - "bitwise_equal": false, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 0.08984375, - "mismatch_count": 56748, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 32, - 128 - ] - }, - "lse": { - "bitwise_equal": false, - "element_count": 2048, - "left_dtype": "torch.float32", - "max_abs": 0.06517952680587769, - "mismatch_count": 2048, - "right_dtype": "torch.float32", - "shape": [ - 2, - 32, - 32 - ] - }, - "out": { - "bitwise_equal": false, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 0.078125, - "mismatch_count": 206686, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 32, - 128 - ] - } - }, - "outcome": "drift_detected", - "passed": true, - "probe": "accum_dtype", - "realization": "Use identical FP32 products/order with explicit FP32 versus BF16 accumulator state.", - "row_id": "A6", - "sequence": 32 - }, - { - "batch": 2, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "implementations": { - "baseline": "torch.rocm.fp32_chunk_merge_ascending", - "candidate": "torch.rocm.fp32_chunk_merge_descending" - }, - "metrics": { - "dk": { - "bitwise_equal": false, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 0.00390625, - "mismatch_count": 5, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 32, - 128 - ] - }, - "dq": { - "bitwise_equal": false, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0009765625, - "mismatch_count": 12, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 32, - 128 - ] - }, - "dv": { - "bitwise_equal": false, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 0.00390625, - "mismatch_count": 3, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 32, - 128 - ] - }, - "lse": { - "bitwise_equal": false, - "element_count": 2048, - "left_dtype": "torch.float32", - "max_abs": 4.76837158203125e-07, - "mismatch_count": 117, - "right_dtype": "torch.float32", - "shape": [ - 2, - 32, - 32 - ] - }, - "out": { - "bitwise_equal": false, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 0.00048828125, - "mismatch_count": 3, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 32, - 128 - ] - } - }, - "outcome": "drift_detected", - "passed": true, - "probe": "merge_order", - "realization": "Merge four dense chunks in opposite orders on one GPU; this is not a CP collective.", - "row_id": "A7", - "sequence": 32 - }, - { - "batch": 2, - "category": "invariant_control", - "comparable": true, - "expected": "exact_zero", - "implementations": { - "baseline": "rlkernel.rocm.deterministic_attention", - "candidate": "rlkernel.rocm.native_attention_per_tp2_head_shard" - }, - "metrics": { - "dk": { - "bitwise_equal": true, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 32, - 128 - ] - }, - "dq": { - "bitwise_equal": true, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 32, - 128 - ] - }, - "dv": { - "bitwise_equal": true, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 32, - 128 - ] - }, - "lse": { - "bitwise_equal": true, - "element_count": 2048, - "left_dtype": "torch.float32", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.float32", - "shape": [ - 2, - 32, - 32 - ] - }, - "out": { - "bitwise_equal": true, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 32, - 128 - ] - } - }, - "outcome": "matched", - "passed": true, - "probe": "tp_partition_control", - "realization": "Compare full GQA with two contiguous TP=2 head shards on one GPU.", - "row_id": "C0", - "sequence": 32 - }, - { - "batch": 2, - "category": "invariant_control", - "comparable": true, - "expected": "exact_zero", - "implementations": { - "baseline": "rlkernel.rocm.deterministic_attention", - "candidate": "rlkernel.rocm.native_attention_per_batch_row" - }, - "metrics": { - "dk": { - "bitwise_equal": true, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 32, - 128 - ] - }, - "dq": { - "bitwise_equal": true, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 32, - 128 - ] - }, - "dv": { - "bitwise_equal": true, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 32, - 128 - ] - }, - "lse": { - "bitwise_equal": true, - "element_count": 2048, - "left_dtype": "torch.float32", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.float32", - "shape": [ - 2, - 32, - 32 - ] - }, - "out": { - "bitwise_equal": true, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 32, - 128 - ] - } - }, - "outcome": "matched", - "passed": true, - "probe": "batch_composition_control", - "realization": "Compare a batch call with per-row calls to the same native HIP core.", - "row_id": "C1", - "sequence": 32 - }, - { - "batch": 2, - "category": "invariant_control", - "comparable": true, - "expected": "exact_zero", - "implementations": { - "baseline": "rlkernel.rocm.native_attention_full_prefill", - "candidate": "rlkernel.rocm.native_attention_tail" - }, - "metrics": { - "dk": { - "bitwise_equal": true, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 32, - 128 - ] - }, - "dq": { - "bitwise_equal": true, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 32, - 128 - ] - }, - "dv": { - "bitwise_equal": true, - "element_count": 65536, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 32, - 128 - ] - }, - "lse": { - "bitwise_equal": true, - "element_count": 64, - "left_dtype": "torch.float32", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.float32", - "shape": [ - 2, - 32, - 1 - ] - }, - "out": { - "bitwise_equal": true, - "element_count": 8192, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 1, - 128 - ] - } - }, - "outcome": "matched", - "passed": true, - "probe": "prefill_decode_tail_control", - "realization": "Compare a full-prefill tail with one trailing query over dense KV, without a serving cache.", - "row_id": "C2", - "sequence": 32 - }, - { - "batch": 2, - "category": "baseline", - "comparable": true, - "expected": "baseline", - "implementations": { - "baseline": "rlkernel.rocm.deterministic_attention", - "candidate": "rlkernel.rocm.deterministic_attention" - }, - "metrics": { - "dk": { - "bitwise_equal": true, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 64, - 128 - ] - }, - "dq": { - "bitwise_equal": true, - "element_count": 524288, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 64, - 128 - ] - }, - "dv": { - "bitwise_equal": true, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 64, - 128 - ] - }, - "lse": { - "bitwise_equal": true, - "element_count": 4096, - "left_dtype": "torch.float32", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.float32", - "shape": [ - 2, - 32, - 64 - ] - }, - "out": { - "bitwise_equal": true, - "element_count": 524288, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 64, - 128 - ] - } - }, - "outcome": "matched", - "passed": true, - "probe": null, - "realization": "Repeat the identical native HIP reference-core call.", - "row_id": "A0", - "sequence": 64 - }, - { - "batch": 2, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "implementations": { - "baseline": "rlkernel.rocm.deterministic_rope+native_attention", - "candidate": "rlkernel.rocm.deterministic_rope+native_attention" - }, - "metrics": { - "dk": { - "bitwise_equal": false, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 1.73046875, - "mismatch_count": 126801, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 64, - 128 - ] - }, - "dq": { - "bitwise_equal": false, - "element_count": 524288, - "left_dtype": "torch.bfloat16", - "max_abs": 1.0390625, - "mismatch_count": 258620, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 64, - 128 - ] - }, - "dv": { - "bitwise_equal": false, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.4765625, - "mismatch_count": 125563, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 64, - 128 - ] - }, - "lse": { - "bitwise_equal": false, - "element_count": 4096, - "left_dtype": "torch.float32", - "max_abs": 0.18473243713378906, - "mismatch_count": 2048, - "right_dtype": "torch.float32", - "shape": [ - 2, - 32, - 64 - ] - }, - "out": { - "bitwise_equal": false, - "element_count": 524288, - "left_dtype": "torch.bfloat16", - "max_abs": 0.40771484375, - "mismatch_count": 258446, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 64, - 128 - ] - } - }, - "outcome": "drift_detected", - "passed": true, - "probe": "position_ids", - "realization": "Increment suffix RoPE positions while preserving Q/K/V tensors.", - "row_id": "A1", - "sequence": 64 - }, - { - "batch": 2, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "implementations": { - "baseline": "torch.rocm.rms_norm+native_attention", - "candidate": "rlkernel.rocm.deterministic_attention" - }, - "metrics": { - "dk": { - "bitwise_equal": false, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 1.25, - "mismatch_count": 129907, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 64, - 128 - ] - }, - "dq": { - "bitwise_equal": false, - "element_count": 524288, - "left_dtype": "torch.bfloat16", - "max_abs": 1.015625, - "mismatch_count": 510233, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 64, - 128 - ] - }, - "dv": { - "bitwise_equal": false, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.96875, - "mismatch_count": 128678, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 64, - 128 - ] - }, - "lse": { - "bitwise_equal": false, - "element_count": 4096, - "left_dtype": "torch.float32", - "max_abs": 1.0303754806518555, - "mismatch_count": 4096, - "right_dtype": "torch.float32", - "shape": [ - 2, - 32, - 64 - ] - }, - "out": { - "bitwise_equal": false, - "element_count": 524288, - "left_dtype": "torch.bfloat16", - "max_abs": 0.490234375, - "mismatch_count": 500934, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 64, - 128 - ] - } - }, - "outcome": "drift_detected", - "passed": true, - "probe": "qk_norm_disabled", - "realization": "Apply or bypass unit-weight PyTorch RMSNorm before the native HIP core.", - "row_id": "A2", - "sequence": 64 - }, - { - "batch": 2, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "implementations": { - "baseline": "rlkernel.rocm.deterministic_attention", - "candidate": "rlkernel.rocm.deterministic_attention" - }, - "metrics": { - "dk": { - "bitwise_equal": false, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 4.3046875, - "mismatch_count": 130884, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 64, - 128 - ] - }, - "dq": { - "bitwise_equal": false, - "element_count": 524288, - "left_dtype": "torch.bfloat16", - "max_abs": 2.93408203125, - "mismatch_count": 513909, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 64, - 128 - ] - }, - "dv": { - "bitwise_equal": false, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 9.46484375, - "mismatch_count": 130910, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 64, - 128 - ] - }, - "lse": { - "bitwise_equal": false, - "element_count": 4096, - "left_dtype": "torch.float32", - "max_abs": 6.752752780914307, - "mismatch_count": 4032, - "right_dtype": "torch.float32", - "shape": [ - 2, - 32, - 64 - ] - }, - "out": { - "bitwise_equal": false, - "element_count": 524288, - "left_dtype": "torch.bfloat16", - "max_abs": 4.1802978515625, - "mismatch_count": 514012, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 64, - 128 - ] - } - }, - "outcome": "drift_detected", - "passed": true, - "probe": "causal_mask", - "realization": "Toggle causal masking in the native HIP core.", - "row_id": "A3", - "sequence": 64 - }, - { - "batch": 2, - "binding_gate": { - "binding_fingerprint": "e9c4483da82da5e16e5bdfbdbf4eeca2c280523521f62a3211bc316900df2ddb", - "comparable": false, - "identity_fingerprint": "c2a02818c11bd195873275c75f4d94ae20b0ae2f2e9daa7fcd5f51d9b409cb4c", - "issues": [ - { - "code": "TOPOLOGY_MISMATCH", - "field": "sharding.tp_rank", - "message": "sharding.tp_rank changes TP/CP ownership; the pair is not the same local attention problem", - "rollout": 1, - "tier": "identical", - "training": 0 - }, - { - "code": "TOPOLOGY_MISMATCH", - "field": "sharding.local_q_head_start", - "message": "sharding.local_q_head_start changes TP/CP ownership; the pair is not the same local attention problem", - "rollout": 16, - "tier": "identical", - "training": 0 - }, - { - "code": "TOPOLOGY_MISMATCH", - "field": "sharding.local_kv_head_start", - "message": "sharding.local_kv_head_start changes TP/CP ownership; the pair is not the same local attention problem", - "rollout": 4, - "tier": "identical", - "training": 0 - } - ], - "passed": false, - "provenance": { - "dtype": "bf16", - "lse_domain": "attention", - "rollout": { - "backend_id": "rlkernel.rocm.deterministic_attention", - "contract": { - "batch_size": 2, - "causal": true, - "causal_offsets": [ - 0, - 0 - ], - "dtype": "bf16", - "export_lse": true, - "head_dim": 128, - "kv_cache": null, - "lse_domain": "attention", - "mode": "prefill", - "projections": { - "o_proj": { - "acc_dtype": "fp32", - "backend_policy": "native_verified_then_common_deterministic", - "deterministic_backend": "rlkernel.cuda.det_gemm", - "input_dtype": "bf16", - "k_order": "ascending", - "output_dtype": "bf16", - "qkv_split_order": [], - "require_runtime_readback": true, - "sp_backward_collective": "all_gather", - "sp_forward_collective": "reduce_scatter", - "split_kv": "disabled", - "tp_backward_dgrad_collective": "none", - "tp_forward_collective": "all_reduce" - }, - "qkv": { - "acc_dtype": "fp32", - "backend_policy": "native_verified_then_common_deterministic", - "deterministic_backend": "rlkernel.cuda.det_gemm", - "input_dtype": "bf16", - "k_order": "ascending", - "output_dtype": "bf16", - "qkv_split_order": [ - "q", - "k", - "v" - ], - "require_runtime_readback": true, - "sp_backward_collective": "reduce_scatter", - "sp_forward_collective": "all_gather", - "split_kv": "disabled", - "tp_backward_dgrad_collective": "all_reduce", - "tp_forward_collective": "none" - } - }, - "query_sequence_length": 64, - "reduction": { - "acc_dtype": "fp32", - "downcast_at": "final_write", - "engine": "in_op_reference", - "merge": "online_softmax_lse", - "order": "global_block_index" - }, - "role": "infer", - "rope": null, - "semantic_operator": "standard_softmax_attention", - "sharding": { - "cp_rank": 0, - "cp_world_size": 1, - "global_block_indices": [ - 0 - ], - "global_block_token_starts": [ - 0 - ], - "global_kv_heads": 8, - "global_q_heads": 32, - "global_sequence_length": 64, - "local_block_offsets": [ - 0, - 64 - ], - "local_kv_head_start": 4, - "local_kv_heads": 4, - "local_q_head_start": 16, - "local_q_heads": 16, - "local_sequence_length": 64, - "packed_sequence_offsets": null, - "sp_rank": 0, - "sp_world_size": 1, - "tp_rank": 1, - "tp_world_size": 2 - }, - "split_kv": { - "fixed_split_size": null, - "mode": "disabled", - "strict_consistency": true - } - }, - "recorded": { - "backend_id": "rlkernel.rocm.deterministic_attention", - "mode": "prefill", - "reduction.engine": "in_op_reference" - } - }, - "split_kv_runtime": { - "rollout": { - "batch_size": 2, - "coverage": "complete_batch_tp_cp_owner_cartesian_product", - "cp_world_size": 1, - "entries": [ - { - "actual_split_boundaries": [ - [ - 0, - 64 - ] - ], - "actual_split_kv_count": 1, - "actual_split_kv_policy": "disabled", - "actual_split_kv_size": null, - "batch_index": 0, - "cp_rank": 0, - "expected_kv_range": [ - 0, - 64 - ], - "owner_cp_rank": 0, - "requested_split_kv_policy": "disabled", - "requested_split_kv_size": null, - "split_kv_accum_dtype": "fp32", - "split_kv_backend": "rlkernel.rocm.deterministic_attention", - "split_kv_downcast_at": "final_write", - "split_kv_fallback": false, - "split_kv_fallback_reason": null, - "split_kv_merge_order": "global_block_index", - "split_kv_plan_source": "contract_exact", - "tp_rank": 0 - }, - { - "actual_split_boundaries": [ - [ - 0, - 64 - ] - ], - "actual_split_kv_count": 1, - "actual_split_kv_policy": "disabled", - "actual_split_kv_size": null, - "batch_index": 0, - "cp_rank": 0, - "expected_kv_range": [ - 0, - 64 - ], - "owner_cp_rank": 0, - "requested_split_kv_policy": "disabled", - "requested_split_kv_size": null, - "split_kv_accum_dtype": "fp32", - "split_kv_backend": "rlkernel.rocm.deterministic_attention", - "split_kv_downcast_at": "final_write", - "split_kv_fallback": false, - "split_kv_fallback_reason": null, - "split_kv_merge_order": "global_block_index", - "split_kv_plan_source": "contract_exact", - "tp_rank": 1 - }, - { - "actual_split_boundaries": [ - [ - 0, - 64 - ] - ], - "actual_split_kv_count": 1, - "actual_split_kv_policy": "disabled", - "actual_split_kv_size": null, - "batch_index": 1, - "cp_rank": 0, - "expected_kv_range": [ - 0, - 64 - ], - "owner_cp_rank": 0, - "requested_split_kv_policy": "disabled", - "requested_split_kv_size": null, - "split_kv_accum_dtype": "fp32", - "split_kv_backend": "rlkernel.rocm.deterministic_attention", - "split_kv_downcast_at": "final_write", - "split_kv_fallback": false, - "split_kv_fallback_reason": null, - "split_kv_merge_order": "global_block_index", - "split_kv_plan_source": "contract_exact", - "tp_rank": 0 - }, - { - "actual_split_boundaries": [ - [ - 0, - 64 - ] - ], - "actual_split_kv_count": 1, - "actual_split_kv_policy": "disabled", - "actual_split_kv_size": null, - "batch_index": 1, - "cp_rank": 0, - "expected_kv_range": [ - 0, - 64 - ], - "owner_cp_rank": 0, - "requested_split_kv_policy": "disabled", - "requested_split_kv_size": null, - "split_kv_accum_dtype": "fp32", - "split_kv_backend": "rlkernel.rocm.deterministic_attention", - "split_kv_downcast_at": "final_write", - "split_kv_fallback": false, - "split_kv_fallback_reason": null, - "split_kv_merge_order": "global_block_index", - "split_kv_plan_source": "contract_exact", - "tp_rank": 1 - } - ], - "total_kv_tokens": [ - 64, - 64 - ], - "tp_world_size": 2 - }, - "training": { - "batch_size": 2, - "coverage": "complete_batch_tp_cp_owner_cartesian_product", - "cp_world_size": 1, - "entries": [ - { - "actual_split_boundaries": [ - [ - 0, - 64 - ] - ], - "actual_split_kv_count": 1, - "actual_split_kv_policy": "disabled", - "actual_split_kv_size": null, - "batch_index": 0, - "cp_rank": 0, - "expected_kv_range": [ - 0, - 64 - ], - "owner_cp_rank": 0, - "requested_split_kv_policy": "disabled", - "requested_split_kv_size": null, - "split_kv_accum_dtype": "fp32", - "split_kv_backend": "rlkernel.rocm.deterministic_attention", - "split_kv_downcast_at": "final_write", - "split_kv_fallback": false, - "split_kv_fallback_reason": null, - "split_kv_merge_order": "global_block_index", - "split_kv_plan_source": "contract_exact", - "tp_rank": 0 - }, - { - "actual_split_boundaries": [ - [ - 0, - 64 - ] - ], - "actual_split_kv_count": 1, - "actual_split_kv_policy": "disabled", - "actual_split_kv_size": null, - "batch_index": 0, - "cp_rank": 0, - "expected_kv_range": [ - 0, - 64 - ], - "owner_cp_rank": 0, - "requested_split_kv_policy": "disabled", - "requested_split_kv_size": null, - "split_kv_accum_dtype": "fp32", - "split_kv_backend": "rlkernel.rocm.deterministic_attention", - "split_kv_downcast_at": "final_write", - "split_kv_fallback": false, - "split_kv_fallback_reason": null, - "split_kv_merge_order": "global_block_index", - "split_kv_plan_source": "contract_exact", - "tp_rank": 1 - }, - { - "actual_split_boundaries": [ - [ - 0, - 64 - ] - ], - "actual_split_kv_count": 1, - "actual_split_kv_policy": "disabled", - "actual_split_kv_size": null, - "batch_index": 1, - "cp_rank": 0, - "expected_kv_range": [ - 0, - 64 - ], - "owner_cp_rank": 0, - "requested_split_kv_policy": "disabled", - "requested_split_kv_size": null, - "split_kv_accum_dtype": "fp32", - "split_kv_backend": "rlkernel.rocm.deterministic_attention", - "split_kv_downcast_at": "final_write", - "split_kv_fallback": false, - "split_kv_fallback_reason": null, - "split_kv_merge_order": "global_block_index", - "split_kv_plan_source": "contract_exact", - "tp_rank": 0 - }, - { - "actual_split_boundaries": [ - [ - 0, - 64 - ] - ], - "actual_split_kv_count": 1, - "actual_split_kv_policy": "disabled", - "actual_split_kv_size": null, - "batch_index": 1, - "cp_rank": 0, - "expected_kv_range": [ - 0, - 64 - ], - "owner_cp_rank": 0, - "requested_split_kv_policy": "disabled", - "requested_split_kv_size": null, - "split_kv_accum_dtype": "fp32", - "split_kv_backend": "rlkernel.rocm.deterministic_attention", - "split_kv_downcast_at": "final_write", - "split_kv_fallback": false, - "split_kv_fallback_reason": null, - "split_kv_merge_order": "global_block_index", - "split_kv_plan_source": "contract_exact", - "tp_rank": 1 - } - ], - "total_kv_tokens": [ - 64, - 64 - ], - "tp_world_size": 2 - } - }, - "training": { - "backend_id": "rlkernel.rocm.deterministic_attention", - "contract": { - "batch_size": 2, - "causal": true, - "causal_offsets": [ - 0, - 0 - ], - "dtype": "bf16", - "export_lse": true, - "head_dim": 128, - "kv_cache": null, - "lse_domain": "attention", - "mode": "prefill", - "projections": { - "o_proj": { - "acc_dtype": "fp32", - "backend_policy": "native_verified_then_common_deterministic", - "deterministic_backend": "rlkernel.cuda.det_gemm", - "input_dtype": "bf16", - "k_order": "ascending", - "output_dtype": "bf16", - "qkv_split_order": [], - "require_runtime_readback": true, - "sp_backward_collective": "all_gather", - "sp_forward_collective": "reduce_scatter", - "split_kv": "disabled", - "tp_backward_dgrad_collective": "none", - "tp_forward_collective": "all_reduce" - }, - "qkv": { - "acc_dtype": "fp32", - "backend_policy": "native_verified_then_common_deterministic", - "deterministic_backend": "rlkernel.cuda.det_gemm", - "input_dtype": "bf16", - "k_order": "ascending", - "output_dtype": "bf16", - "qkv_split_order": [ - "q", - "k", - "v" - ], - "require_runtime_readback": true, - "sp_backward_collective": "reduce_scatter", - "sp_forward_collective": "all_gather", - "split_kv": "disabled", - "tp_backward_dgrad_collective": "all_reduce", - "tp_forward_collective": "none" - } - }, - "query_sequence_length": 64, - "reduction": { - "acc_dtype": "fp32", - "downcast_at": "final_write", - "engine": "in_op_reference", - "merge": "online_softmax_lse", - "order": "global_block_index" - }, - "role": "train", - "rope": null, - "semantic_operator": "standard_softmax_attention", - "sharding": { - "cp_rank": 0, - "cp_world_size": 1, - "global_block_indices": [ - 0 - ], - "global_block_token_starts": [ - 0 - ], - "global_kv_heads": 8, - "global_q_heads": 32, - "global_sequence_length": 64, - "local_block_offsets": [ - 0, - 64 - ], - "local_kv_head_start": 0, - "local_kv_heads": 4, - "local_q_head_start": 0, - "local_q_heads": 16, - "local_sequence_length": 64, - "packed_sequence_offsets": null, - "sp_rank": 0, - "sp_world_size": 1, - "tp_rank": 0, - "tp_world_size": 2 - }, - "split_kv": { - "fixed_split_size": null, - "mode": "disabled", - "strict_consistency": true - } - }, - "recorded": { - "backend_id": "rlkernel.rocm.deterministic_attention", - "mode": "prefill", - "reduction.engine": "in_op_reference" - } - } - }, - "recorded_differences": {}, - "reduction_fingerprint": "4bfe65d607de1dc8d1b8c04041d48e9af4e81dc38614f0340d994616da273d6c", - "schema_version": "cross_config.attention_binding.v3" - }, - "category": "comparability_gate", - "comparable": false, - "expected": "rejected", - "gate_implementation": "rl_engine.alignment.cross_config.bind_attention_contracts", - "identity_errors": [ - "sharding.tp_rank", - "sharding.local_q_head_start", - "sharding.local_kv_head_start" - ], - "metrics": { - "dk": null, - "dq": null, - "dv": null, - "lse": null, - "out": null - }, - "outcome": "rejected", - "passed": true, - "probe": "tp_head_ownership", - "realization": "Bind valid TP-rank-1 rollout and TP-rank-0 training contracts; do not run numerics.", - "row_id": "A4", - "sequence": 64 - }, - { - "batch": 2, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "implementations": { - "baseline": "rlkernel.rocm.deterministic_attention", - "candidate": "torch.rocm.index_select+native_attention" - }, - "metrics": { - "dk": { - "bitwise_equal": false, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 5.31640625, - "mismatch_count": 130971, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 64, - 128 - ] - }, - "dq": { - "bitwise_equal": false, - "element_count": 524288, - "left_dtype": "torch.bfloat16", - "max_abs": 3.9140625, - "mismatch_count": 506631, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 64, - 128 - ] - }, - "dv": { - "bitwise_equal": false, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 10.416015625, - "mismatch_count": 130967, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 64, - 128 - ] - }, - "lse": { - "bitwise_equal": false, - "element_count": 4096, - "left_dtype": "torch.float32", - "max_abs": 3.361884593963623, - "mismatch_count": 4032, - "right_dtype": "torch.float32", - "shape": [ - 2, - 32, - 64 - ] - }, - "out": { - "bitwise_equal": false, - "element_count": 524288, - "left_dtype": "torch.bfloat16", - "max_abs": 5.015625, - "mismatch_count": 514873, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 64, - 128 - ] - } - }, - "outcome": "drift_detected", - "passed": true, - "probe": "kv_page_order", - "realization": "Reverse four dense K/V tensor pages; this is not a paged-cache runtime.", - "row_id": "A5", - "sequence": 64 - }, - { - "batch": 2, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "implementations": { - "baseline": "torch.rocm.explicit_fp32_serial_qk_accumulator", - "candidate": "torch.rocm.explicit_bf16_serial_qk_accumulator" - }, - "metrics": { - "dk": { - "bitwise_equal": false, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.09375, - "mismatch_count": 114541, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 64, - 128 - ] - }, - "dq": { - "bitwise_equal": false, - "element_count": 524288, - "left_dtype": "torch.bfloat16", - "max_abs": 0.09375, - "mismatch_count": 433929, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 64, - 128 - ] - }, - "dv": { - "bitwise_equal": false, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.11328125, - "mismatch_count": 115947, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 64, - 128 - ] - }, - "lse": { - "bitwise_equal": false, - "element_count": 4096, - "left_dtype": "torch.float32", - "max_abs": 0.08368206024169922, - "mismatch_count": 4096, - "right_dtype": "torch.float32", - "shape": [ - 2, - 32, - 64 - ] - }, - "out": { - "bitwise_equal": false, - "element_count": 524288, - "left_dtype": "torch.bfloat16", - "max_abs": 0.1171875, - "mismatch_count": 437879, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 64, - 128 - ] - } - }, - "outcome": "drift_detected", - "passed": true, - "probe": "accum_dtype", - "realization": "Use identical FP32 products/order with explicit FP32 versus BF16 accumulator state.", - "row_id": "A6", - "sequence": 64 - }, - { - "batch": 2, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "implementations": { - "baseline": "torch.rocm.fp32_chunk_merge_ascending", - "candidate": "torch.rocm.fp32_chunk_merge_descending" - }, - "metrics": { - "dk": { - "bitwise_equal": false, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.001953125, - "mismatch_count": 11, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 64, - 128 - ] - }, - "dq": { - "bitwise_equal": false, - "element_count": 524288, - "left_dtype": "torch.bfloat16", - "max_abs": 0.00390625, - "mismatch_count": 19, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 64, - 128 - ] - }, - "dv": { - "bitwise_equal": false, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0009765625, - "mismatch_count": 5, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 64, - 128 - ] - }, - "lse": { - "bitwise_equal": false, - "element_count": 4096, - "left_dtype": "torch.float32", - "max_abs": 4.76837158203125e-07, - "mismatch_count": 138, - "right_dtype": "torch.float32", - "shape": [ - 2, - 32, - 64 - ] - }, - "out": { - "bitwise_equal": false, - "element_count": 524288, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0009765625, - "mismatch_count": 14, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 64, - 128 - ] - } - }, - "outcome": "drift_detected", - "passed": true, - "probe": "merge_order", - "realization": "Merge four dense chunks in opposite orders on one GPU; this is not a CP collective.", - "row_id": "A7", - "sequence": 64 - }, - { - "batch": 2, - "category": "invariant_control", - "comparable": true, - "expected": "exact_zero", - "implementations": { - "baseline": "rlkernel.rocm.deterministic_attention", - "candidate": "rlkernel.rocm.native_attention_per_tp2_head_shard" - }, - "metrics": { - "dk": { - "bitwise_equal": true, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 64, - 128 - ] - }, - "dq": { - "bitwise_equal": true, - "element_count": 524288, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 64, - 128 - ] - }, - "dv": { - "bitwise_equal": true, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 64, - 128 - ] - }, - "lse": { - "bitwise_equal": true, - "element_count": 4096, - "left_dtype": "torch.float32", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.float32", - "shape": [ - 2, - 32, - 64 - ] - }, - "out": { - "bitwise_equal": true, - "element_count": 524288, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 64, - 128 - ] - } - }, - "outcome": "matched", - "passed": true, - "probe": "tp_partition_control", - "realization": "Compare full GQA with two contiguous TP=2 head shards on one GPU.", - "row_id": "C0", - "sequence": 64 - }, - { - "batch": 2, - "category": "invariant_control", - "comparable": true, - "expected": "exact_zero", - "implementations": { - "baseline": "rlkernel.rocm.deterministic_attention", - "candidate": "rlkernel.rocm.native_attention_per_batch_row" - }, - "metrics": { - "dk": { - "bitwise_equal": true, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 64, - 128 - ] - }, - "dq": { - "bitwise_equal": true, - "element_count": 524288, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 64, - 128 - ] - }, - "dv": { - "bitwise_equal": true, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 64, - 128 - ] - }, - "lse": { - "bitwise_equal": true, - "element_count": 4096, - "left_dtype": "torch.float32", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.float32", - "shape": [ - 2, - 32, - 64 - ] - }, - "out": { - "bitwise_equal": true, - "element_count": 524288, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 64, - 128 - ] - } - }, - "outcome": "matched", - "passed": true, - "probe": "batch_composition_control", - "realization": "Compare a batch call with per-row calls to the same native HIP core.", - "row_id": "C1", - "sequence": 64 - }, - { - "batch": 2, - "category": "invariant_control", - "comparable": true, - "expected": "exact_zero", - "implementations": { - "baseline": "rlkernel.rocm.native_attention_full_prefill", - "candidate": "rlkernel.rocm.native_attention_tail" - }, - "metrics": { - "dk": { - "bitwise_equal": true, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 64, - 128 - ] - }, - "dq": { - "bitwise_equal": true, - "element_count": 524288, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 64, - 128 - ] - }, - "dv": { - "bitwise_equal": true, - "element_count": 131072, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 64, - 128 - ] - }, - "lse": { - "bitwise_equal": true, - "element_count": 64, - "left_dtype": "torch.float32", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.float32", - "shape": [ - 2, - 32, - 1 - ] - }, - "out": { - "bitwise_equal": true, - "element_count": 8192, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 1, - 128 - ] - } - }, - "outcome": "matched", - "passed": true, - "probe": "prefill_decode_tail_control", - "realization": "Compare a full-prefill tail with one trailing query over dense KV, without a serving cache.", - "row_id": "C2", - "sequence": 64 - }, - { - "batch": 2, - "category": "baseline", - "comparable": true, - "expected": "baseline", - "implementations": { - "baseline": "rlkernel.rocm.deterministic_attention", - "candidate": "rlkernel.rocm.deterministic_attention" - }, - "metrics": { - "dk": { - "bitwise_equal": true, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 128, - 128 - ] - }, - "dq": { - "bitwise_equal": true, - "element_count": 1048576, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 128, - 128 - ] - }, - "dv": { - "bitwise_equal": true, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 128, - 128 - ] - }, - "lse": { - "bitwise_equal": true, - "element_count": 8192, - "left_dtype": "torch.float32", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.float32", - "shape": [ - 2, - 32, - 128 - ] - }, - "out": { - "bitwise_equal": true, - "element_count": 1048576, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 128, - 128 - ] - } - }, - "outcome": "matched", - "passed": true, - "probe": null, - "realization": "Repeat the identical native HIP reference-core call.", - "row_id": "A0", - "sequence": 128 - }, - { - "batch": 2, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "implementations": { - "baseline": "rlkernel.rocm.deterministic_rope+native_attention", - "candidate": "rlkernel.rocm.deterministic_rope+native_attention" - }, - "metrics": { - "dk": { - "bitwise_equal": false, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 1.580078125, - "mismatch_count": 252194, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 128, - 128 - ] - }, - "dq": { - "bitwise_equal": false, - "element_count": 1048576, - "left_dtype": "torch.bfloat16", - "max_abs": 1.04150390625, - "mismatch_count": 517844, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 128, - 128 - ] - }, - "dv": { - "bitwise_equal": false, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 0.3916015625, - "mismatch_count": 248733, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 128, - 128 - ] - }, - "lse": { - "bitwise_equal": false, - "element_count": 8192, - "left_dtype": "torch.float32", - "max_abs": 0.1368560791015625, - "mismatch_count": 4096, - "right_dtype": "torch.float32", - "shape": [ - 2, - 32, - 128 - ] - }, - "out": { - "bitwise_equal": false, - "element_count": 1048576, - "left_dtype": "torch.bfloat16", - "max_abs": 0.416015625, - "mismatch_count": 517567, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 128, - 128 - ] - } - }, - "outcome": "drift_detected", - "passed": true, - "probe": "position_ids", - "realization": "Increment suffix RoPE positions while preserving Q/K/V tensors.", - "row_id": "A1", - "sequence": 128 - }, - { - "batch": 2, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "implementations": { - "baseline": "torch.rocm.rms_norm+native_attention", - "candidate": "rlkernel.rocm.deterministic_attention" - }, - "metrics": { - "dk": { - "bitwise_equal": false, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 1.4921875, - "mismatch_count": 259978, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 128, - 128 - ] - }, - "dq": { - "bitwise_equal": false, - "element_count": 1048576, - "left_dtype": "torch.bfloat16", - "max_abs": 0.7890625, - "mismatch_count": 1030092, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 128, - 128 - ] - }, - "dv": { - "bitwise_equal": false, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 0.890625, - "mismatch_count": 257810, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 128, - 128 - ] - }, - "lse": { - "bitwise_equal": false, - "element_count": 8192, - "left_dtype": "torch.float32", - "max_abs": 0.3836212158203125, - "mismatch_count": 8192, - "right_dtype": "torch.float32", - "shape": [ - 2, - 32, - 128 - ] - }, - "out": { - "bitwise_equal": false, - "element_count": 1048576, - "left_dtype": "torch.bfloat16", - "max_abs": 0.58203125, - "mismatch_count": 1017881, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 128, - 128 - ] - } - }, - "outcome": "drift_detected", - "passed": true, - "probe": "qk_norm_disabled", - "realization": "Apply or bypass unit-weight PyTorch RMSNorm before the native HIP core.", - "row_id": "A2", - "sequence": 128 - }, - { - "batch": 2, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "implementations": { - "baseline": "rlkernel.rocm.deterministic_attention", - "candidate": "rlkernel.rocm.deterministic_attention" - }, - "metrics": { - "dk": { - "bitwise_equal": false, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 5.357421875, - "mismatch_count": 261780, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 128, - 128 - ] - }, - "dq": { - "bitwise_equal": false, - "element_count": 1048576, - "left_dtype": "torch.bfloat16", - "max_abs": 3.1474609375, - "mismatch_count": 1035801, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 128, - 128 - ] - }, - "dv": { - "bitwise_equal": false, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 9.1572265625, - "mismatch_count": 261805, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 128, - 128 - ] - }, - "lse": { - "bitwise_equal": false, - "element_count": 8192, - "left_dtype": "torch.float32", - "max_abs": 7.316828727722168, - "mismatch_count": 8128, - "right_dtype": "torch.float32", - "shape": [ - 2, - 32, - 128 - ] - }, - "out": { - "bitwise_equal": false, - "element_count": 1048576, - "left_dtype": "torch.bfloat16", - "max_abs": 3.4052734375, - "mismatch_count": 1036536, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 128, - 128 - ] - } - }, - "outcome": "drift_detected", - "passed": true, - "probe": "causal_mask", - "realization": "Toggle causal masking in the native HIP core.", - "row_id": "A3", - "sequence": 128 - }, - { - "batch": 2, - "binding_gate": { - "binding_fingerprint": "03ccee373f91bcf85226a86a89472e7f94b881fc788c51ffbe3d8625753af201", - "comparable": false, - "identity_fingerprint": "c2a02818c11bd195873275c75f4d94ae20b0ae2f2e9daa7fcd5f51d9b409cb4c", - "issues": [ - { - "code": "TOPOLOGY_MISMATCH", - "field": "sharding.tp_rank", - "message": "sharding.tp_rank changes TP/CP ownership; the pair is not the same local attention problem", - "rollout": 1, - "tier": "identical", - "training": 0 - }, - { - "code": "TOPOLOGY_MISMATCH", - "field": "sharding.local_q_head_start", - "message": "sharding.local_q_head_start changes TP/CP ownership; the pair is not the same local attention problem", - "rollout": 16, - "tier": "identical", - "training": 0 - }, - { - "code": "TOPOLOGY_MISMATCH", - "field": "sharding.local_kv_head_start", - "message": "sharding.local_kv_head_start changes TP/CP ownership; the pair is not the same local attention problem", - "rollout": 4, - "tier": "identical", - "training": 0 - } - ], - "passed": false, - "provenance": { - "dtype": "bf16", - "lse_domain": "attention", - "rollout": { - "backend_id": "rlkernel.rocm.deterministic_attention", - "contract": { - "batch_size": 2, - "causal": true, - "causal_offsets": [ - 0, - 0 - ], - "dtype": "bf16", - "export_lse": true, - "head_dim": 128, - "kv_cache": null, - "lse_domain": "attention", - "mode": "prefill", - "projections": { - "o_proj": { - "acc_dtype": "fp32", - "backend_policy": "native_verified_then_common_deterministic", - "deterministic_backend": "rlkernel.cuda.det_gemm", - "input_dtype": "bf16", - "k_order": "ascending", - "output_dtype": "bf16", - "qkv_split_order": [], - "require_runtime_readback": true, - "sp_backward_collective": "all_gather", - "sp_forward_collective": "reduce_scatter", - "split_kv": "disabled", - "tp_backward_dgrad_collective": "none", - "tp_forward_collective": "all_reduce" - }, - "qkv": { - "acc_dtype": "fp32", - "backend_policy": "native_verified_then_common_deterministic", - "deterministic_backend": "rlkernel.cuda.det_gemm", - "input_dtype": "bf16", - "k_order": "ascending", - "output_dtype": "bf16", - "qkv_split_order": [ - "q", - "k", - "v" - ], - "require_runtime_readback": true, - "sp_backward_collective": "reduce_scatter", - "sp_forward_collective": "all_gather", - "split_kv": "disabled", - "tp_backward_dgrad_collective": "all_reduce", - "tp_forward_collective": "none" - } - }, - "query_sequence_length": 128, - "reduction": { - "acc_dtype": "fp32", - "downcast_at": "final_write", - "engine": "in_op_reference", - "merge": "online_softmax_lse", - "order": "global_block_index" - }, - "role": "infer", - "rope": null, - "semantic_operator": "standard_softmax_attention", - "sharding": { - "cp_rank": 0, - "cp_world_size": 1, - "global_block_indices": [ - 0 - ], - "global_block_token_starts": [ - 0 - ], - "global_kv_heads": 8, - "global_q_heads": 32, - "global_sequence_length": 128, - "local_block_offsets": [ - 0, - 128 - ], - "local_kv_head_start": 4, - "local_kv_heads": 4, - "local_q_head_start": 16, - "local_q_heads": 16, - "local_sequence_length": 128, - "packed_sequence_offsets": null, - "sp_rank": 0, - "sp_world_size": 1, - "tp_rank": 1, - "tp_world_size": 2 - }, - "split_kv": { - "fixed_split_size": null, - "mode": "disabled", - "strict_consistency": true - } - }, - "recorded": { - "backend_id": "rlkernel.rocm.deterministic_attention", - "mode": "prefill", - "reduction.engine": "in_op_reference" - } - }, - "split_kv_runtime": { - "rollout": { - "batch_size": 2, - "coverage": "complete_batch_tp_cp_owner_cartesian_product", - "cp_world_size": 1, - "entries": [ - { - "actual_split_boundaries": [ - [ - 0, - 128 - ] - ], - "actual_split_kv_count": 1, - "actual_split_kv_policy": "disabled", - "actual_split_kv_size": null, - "batch_index": 0, - "cp_rank": 0, - "expected_kv_range": [ - 0, - 128 - ], - "owner_cp_rank": 0, - "requested_split_kv_policy": "disabled", - "requested_split_kv_size": null, - "split_kv_accum_dtype": "fp32", - "split_kv_backend": "rlkernel.rocm.deterministic_attention", - "split_kv_downcast_at": "final_write", - "split_kv_fallback": false, - "split_kv_fallback_reason": null, - "split_kv_merge_order": "global_block_index", - "split_kv_plan_source": "contract_exact", - "tp_rank": 0 - }, - { - "actual_split_boundaries": [ - [ - 0, - 128 - ] - ], - "actual_split_kv_count": 1, - "actual_split_kv_policy": "disabled", - "actual_split_kv_size": null, - "batch_index": 0, - "cp_rank": 0, - "expected_kv_range": [ - 0, - 128 - ], - "owner_cp_rank": 0, - "requested_split_kv_policy": "disabled", - "requested_split_kv_size": null, - "split_kv_accum_dtype": "fp32", - "split_kv_backend": "rlkernel.rocm.deterministic_attention", - "split_kv_downcast_at": "final_write", - "split_kv_fallback": false, - "split_kv_fallback_reason": null, - "split_kv_merge_order": "global_block_index", - "split_kv_plan_source": "contract_exact", - "tp_rank": 1 - }, - { - "actual_split_boundaries": [ - [ - 0, - 128 - ] - ], - "actual_split_kv_count": 1, - "actual_split_kv_policy": "disabled", - "actual_split_kv_size": null, - "batch_index": 1, - "cp_rank": 0, - "expected_kv_range": [ - 0, - 128 - ], - "owner_cp_rank": 0, - "requested_split_kv_policy": "disabled", - "requested_split_kv_size": null, - "split_kv_accum_dtype": "fp32", - "split_kv_backend": "rlkernel.rocm.deterministic_attention", - "split_kv_downcast_at": "final_write", - "split_kv_fallback": false, - "split_kv_fallback_reason": null, - "split_kv_merge_order": "global_block_index", - "split_kv_plan_source": "contract_exact", - "tp_rank": 0 - }, - { - "actual_split_boundaries": [ - [ - 0, - 128 - ] - ], - "actual_split_kv_count": 1, - "actual_split_kv_policy": "disabled", - "actual_split_kv_size": null, - "batch_index": 1, - "cp_rank": 0, - "expected_kv_range": [ - 0, - 128 - ], - "owner_cp_rank": 0, - "requested_split_kv_policy": "disabled", - "requested_split_kv_size": null, - "split_kv_accum_dtype": "fp32", - "split_kv_backend": "rlkernel.rocm.deterministic_attention", - "split_kv_downcast_at": "final_write", - "split_kv_fallback": false, - "split_kv_fallback_reason": null, - "split_kv_merge_order": "global_block_index", - "split_kv_plan_source": "contract_exact", - "tp_rank": 1 - } - ], - "total_kv_tokens": [ - 128, - 128 - ], - "tp_world_size": 2 - }, - "training": { - "batch_size": 2, - "coverage": "complete_batch_tp_cp_owner_cartesian_product", - "cp_world_size": 1, - "entries": [ - { - "actual_split_boundaries": [ - [ - 0, - 128 - ] - ], - "actual_split_kv_count": 1, - "actual_split_kv_policy": "disabled", - "actual_split_kv_size": null, - "batch_index": 0, - "cp_rank": 0, - "expected_kv_range": [ - 0, - 128 - ], - "owner_cp_rank": 0, - "requested_split_kv_policy": "disabled", - "requested_split_kv_size": null, - "split_kv_accum_dtype": "fp32", - "split_kv_backend": "rlkernel.rocm.deterministic_attention", - "split_kv_downcast_at": "final_write", - "split_kv_fallback": false, - "split_kv_fallback_reason": null, - "split_kv_merge_order": "global_block_index", - "split_kv_plan_source": "contract_exact", - "tp_rank": 0 - }, - { - "actual_split_boundaries": [ - [ - 0, - 128 - ] - ], - "actual_split_kv_count": 1, - "actual_split_kv_policy": "disabled", - "actual_split_kv_size": null, - "batch_index": 0, - "cp_rank": 0, - "expected_kv_range": [ - 0, - 128 - ], - "owner_cp_rank": 0, - "requested_split_kv_policy": "disabled", - "requested_split_kv_size": null, - "split_kv_accum_dtype": "fp32", - "split_kv_backend": "rlkernel.rocm.deterministic_attention", - "split_kv_downcast_at": "final_write", - "split_kv_fallback": false, - "split_kv_fallback_reason": null, - "split_kv_merge_order": "global_block_index", - "split_kv_plan_source": "contract_exact", - "tp_rank": 1 - }, - { - "actual_split_boundaries": [ - [ - 0, - 128 - ] - ], - "actual_split_kv_count": 1, - "actual_split_kv_policy": "disabled", - "actual_split_kv_size": null, - "batch_index": 1, - "cp_rank": 0, - "expected_kv_range": [ - 0, - 128 - ], - "owner_cp_rank": 0, - "requested_split_kv_policy": "disabled", - "requested_split_kv_size": null, - "split_kv_accum_dtype": "fp32", - "split_kv_backend": "rlkernel.rocm.deterministic_attention", - "split_kv_downcast_at": "final_write", - "split_kv_fallback": false, - "split_kv_fallback_reason": null, - "split_kv_merge_order": "global_block_index", - "split_kv_plan_source": "contract_exact", - "tp_rank": 0 - }, - { - "actual_split_boundaries": [ - [ - 0, - 128 - ] - ], - "actual_split_kv_count": 1, - "actual_split_kv_policy": "disabled", - "actual_split_kv_size": null, - "batch_index": 1, - "cp_rank": 0, - "expected_kv_range": [ - 0, - 128 - ], - "owner_cp_rank": 0, - "requested_split_kv_policy": "disabled", - "requested_split_kv_size": null, - "split_kv_accum_dtype": "fp32", - "split_kv_backend": "rlkernel.rocm.deterministic_attention", - "split_kv_downcast_at": "final_write", - "split_kv_fallback": false, - "split_kv_fallback_reason": null, - "split_kv_merge_order": "global_block_index", - "split_kv_plan_source": "contract_exact", - "tp_rank": 1 - } - ], - "total_kv_tokens": [ - 128, - 128 - ], - "tp_world_size": 2 - } - }, - "training": { - "backend_id": "rlkernel.rocm.deterministic_attention", - "contract": { - "batch_size": 2, - "causal": true, - "causal_offsets": [ - 0, - 0 - ], - "dtype": "bf16", - "export_lse": true, - "head_dim": 128, - "kv_cache": null, - "lse_domain": "attention", - "mode": "prefill", - "projections": { - "o_proj": { - "acc_dtype": "fp32", - "backend_policy": "native_verified_then_common_deterministic", - "deterministic_backend": "rlkernel.cuda.det_gemm", - "input_dtype": "bf16", - "k_order": "ascending", - "output_dtype": "bf16", - "qkv_split_order": [], - "require_runtime_readback": true, - "sp_backward_collective": "all_gather", - "sp_forward_collective": "reduce_scatter", - "split_kv": "disabled", - "tp_backward_dgrad_collective": "none", - "tp_forward_collective": "all_reduce" - }, - "qkv": { - "acc_dtype": "fp32", - "backend_policy": "native_verified_then_common_deterministic", - "deterministic_backend": "rlkernel.cuda.det_gemm", - "input_dtype": "bf16", - "k_order": "ascending", - "output_dtype": "bf16", - "qkv_split_order": [ - "q", - "k", - "v" - ], - "require_runtime_readback": true, - "sp_backward_collective": "reduce_scatter", - "sp_forward_collective": "all_gather", - "split_kv": "disabled", - "tp_backward_dgrad_collective": "all_reduce", - "tp_forward_collective": "none" - } - }, - "query_sequence_length": 128, - "reduction": { - "acc_dtype": "fp32", - "downcast_at": "final_write", - "engine": "in_op_reference", - "merge": "online_softmax_lse", - "order": "global_block_index" - }, - "role": "train", - "rope": null, - "semantic_operator": "standard_softmax_attention", - "sharding": { - "cp_rank": 0, - "cp_world_size": 1, - "global_block_indices": [ - 0 - ], - "global_block_token_starts": [ - 0 - ], - "global_kv_heads": 8, - "global_q_heads": 32, - "global_sequence_length": 128, - "local_block_offsets": [ - 0, - 128 - ], - "local_kv_head_start": 0, - "local_kv_heads": 4, - "local_q_head_start": 0, - "local_q_heads": 16, - "local_sequence_length": 128, - "packed_sequence_offsets": null, - "sp_rank": 0, - "sp_world_size": 1, - "tp_rank": 0, - "tp_world_size": 2 - }, - "split_kv": { - "fixed_split_size": null, - "mode": "disabled", - "strict_consistency": true - } - }, - "recorded": { - "backend_id": "rlkernel.rocm.deterministic_attention", - "mode": "prefill", - "reduction.engine": "in_op_reference" - } - } - }, - "recorded_differences": {}, - "reduction_fingerprint": "4bfe65d607de1dc8d1b8c04041d48e9af4e81dc38614f0340d994616da273d6c", - "schema_version": "cross_config.attention_binding.v3" - }, - "category": "comparability_gate", - "comparable": false, - "expected": "rejected", - "gate_implementation": "rl_engine.alignment.cross_config.bind_attention_contracts", - "identity_errors": [ - "sharding.tp_rank", - "sharding.local_q_head_start", - "sharding.local_kv_head_start" - ], - "metrics": { - "dk": null, - "dq": null, - "dv": null, - "lse": null, - "out": null - }, - "outcome": "rejected", - "passed": true, - "probe": "tp_head_ownership", - "realization": "Bind valid TP-rank-1 rollout and TP-rank-0 training contracts; do not run numerics.", - "row_id": "A4", - "sequence": 128 - }, - { - "batch": 2, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "implementations": { - "baseline": "rlkernel.rocm.deterministic_attention", - "candidate": "torch.rocm.index_select+native_attention" - }, - "metrics": { - "dk": { - "bitwise_equal": false, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 5.84375, - "mismatch_count": 261932, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 128, - 128 - ] - }, - "dq": { - "bitwise_equal": false, - "element_count": 1048576, - "left_dtype": "torch.bfloat16", - "max_abs": 4.1328125, - "mismatch_count": 1029699, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 128, - 128 - ] - }, - "dv": { - "bitwise_equal": false, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 9.7470703125, - "mismatch_count": 261935, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 128, - 128 - ] - }, - "lse": { - "bitwise_equal": false, - "element_count": 8192, - "left_dtype": "torch.float32", - "max_abs": 3.6807479858398438, - "mismatch_count": 8128, - "right_dtype": "torch.float32", - "shape": [ - 2, - 32, - 128 - ] - }, - "out": { - "bitwise_equal": false, - "element_count": 1048576, - "left_dtype": "torch.bfloat16", - "max_abs": 5.421875, - "mismatch_count": 1038063, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 128, - 128 - ] - } - }, - "outcome": "drift_detected", - "passed": true, - "probe": "kv_page_order", - "realization": "Reverse four dense K/V tensor pages; this is not a paged-cache runtime.", - "row_id": "A5", - "sequence": 128 - }, - { - "batch": 2, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "implementations": { - "baseline": "torch.rocm.explicit_fp32_serial_qk_accumulator", - "candidate": "torch.rocm.explicit_bf16_serial_qk_accumulator" - }, - "metrics": { - "dk": { - "bitwise_equal": false, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 0.078125, - "mismatch_count": 232744, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 128, - 128 - ] - }, - "dq": { - "bitwise_equal": false, - "element_count": 1048576, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0703125, - "mismatch_count": 900263, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 128, - 128 - ] - }, - "dv": { - "bitwise_equal": false, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 0.09375, - "mismatch_count": 235097, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 128, - 128 - ] - }, - "lse": { - "bitwise_equal": false, - "element_count": 8192, - "left_dtype": "torch.float32", - "max_abs": 0.08346295356750488, - "mismatch_count": 8190, - "right_dtype": "torch.float32", - "shape": [ - 2, - 32, - 128 - ] - }, - "out": { - "bitwise_equal": false, - "element_count": 1048576, - "left_dtype": "torch.bfloat16", - "max_abs": 0.078125, - "mismatch_count": 909350, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 128, - 128 - ] - } - }, - "outcome": "drift_detected", - "passed": true, - "probe": "accum_dtype", - "realization": "Use identical FP32 products/order with explicit FP32 versus BF16 accumulator state.", - "row_id": "A6", - "sequence": 128 - }, - { - "batch": 2, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "implementations": { - "baseline": "torch.rocm.fp32_chunk_merge_ascending", - "candidate": "torch.rocm.fp32_chunk_merge_descending" - }, - "metrics": { - "dk": { - "bitwise_equal": false, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0009765625, - "mismatch_count": 17, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 128, - 128 - ] - }, - "dq": { - "bitwise_equal": false, - "element_count": 1048576, - "left_dtype": "torch.bfloat16", - "max_abs": 0.00390625, - "mismatch_count": 43, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 128, - 128 - ] - }, - "dv": { - "bitwise_equal": false, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 0.001953125, - "mismatch_count": 10, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 128, - 128 - ] - }, - "lse": { - "bitwise_equal": false, - "element_count": 8192, - "left_dtype": "torch.float32", - "max_abs": 9.5367431640625e-07, - "mismatch_count": 279, - "right_dtype": "torch.float32", - "shape": [ - 2, - 32, - 128 - ] - }, - "out": { - "bitwise_equal": false, - "element_count": 1048576, - "left_dtype": "torch.bfloat16", - "max_abs": 0.001953125, - "mismatch_count": 26, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 128, - 128 - ] - } - }, - "outcome": "drift_detected", - "passed": true, - "probe": "merge_order", - "realization": "Merge four dense chunks in opposite orders on one GPU; this is not a CP collective.", - "row_id": "A7", - "sequence": 128 - }, - { - "batch": 2, - "category": "invariant_control", - "comparable": true, - "expected": "exact_zero", - "implementations": { - "baseline": "rlkernel.rocm.deterministic_attention", - "candidate": "rlkernel.rocm.native_attention_per_tp2_head_shard" - }, - "metrics": { - "dk": { - "bitwise_equal": true, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 128, - 128 - ] - }, - "dq": { - "bitwise_equal": true, - "element_count": 1048576, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 128, - 128 - ] - }, - "dv": { - "bitwise_equal": true, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 128, - 128 - ] - }, - "lse": { - "bitwise_equal": true, - "element_count": 8192, - "left_dtype": "torch.float32", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.float32", - "shape": [ - 2, - 32, - 128 - ] - }, - "out": { - "bitwise_equal": true, - "element_count": 1048576, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 128, - 128 - ] - } - }, - "outcome": "matched", - "passed": true, - "probe": "tp_partition_control", - "realization": "Compare full GQA with two contiguous TP=2 head shards on one GPU.", - "row_id": "C0", - "sequence": 128 - }, - { - "batch": 2, - "category": "invariant_control", - "comparable": true, - "expected": "exact_zero", - "implementations": { - "baseline": "rlkernel.rocm.deterministic_attention", - "candidate": "rlkernel.rocm.native_attention_per_batch_row" - }, - "metrics": { - "dk": { - "bitwise_equal": true, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 128, - 128 - ] - }, - "dq": { - "bitwise_equal": true, - "element_count": 1048576, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 128, - 128 - ] - }, - "dv": { - "bitwise_equal": true, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 128, - 128 - ] - }, - "lse": { - "bitwise_equal": true, - "element_count": 8192, - "left_dtype": "torch.float32", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.float32", - "shape": [ - 2, - 32, - 128 - ] - }, - "out": { - "bitwise_equal": true, - "element_count": 1048576, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 128, - 128 - ] - } - }, - "outcome": "matched", - "passed": true, - "probe": "batch_composition_control", - "realization": "Compare a batch call with per-row calls to the same native HIP core.", - "row_id": "C1", - "sequence": 128 - }, - { - "batch": 2, - "category": "invariant_control", - "comparable": true, - "expected": "exact_zero", - "implementations": { - "baseline": "rlkernel.rocm.native_attention_full_prefill", - "candidate": "rlkernel.rocm.native_attention_tail" - }, - "metrics": { - "dk": { - "bitwise_equal": true, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 128, - 128 - ] - }, - "dq": { - "bitwise_equal": true, - "element_count": 1048576, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 128, - 128 - ] - }, - "dv": { - "bitwise_equal": true, - "element_count": 262144, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 8, - 128, - 128 - ] - }, - "lse": { - "bitwise_equal": true, - "element_count": 64, - "left_dtype": "torch.float32", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.float32", - "shape": [ - 2, - 32, - 1 - ] - }, - "out": { - "bitwise_equal": true, - "element_count": 8192, - "left_dtype": "torch.bfloat16", - "max_abs": 0.0, - "mismatch_count": 0, - "right_dtype": "torch.bfloat16", - "shape": [ - 2, - 32, - 1, - 128 - ] - } - }, - "outcome": "matched", - "passed": true, - "probe": "prefill_decode_tail_control", - "realization": "Compare a full-prefill tail with one trailing query over dense KV, without a serving cache.", - "row_id": "C2", - "sequence": 128 - } - ], - "command": [ - "/opt/venv/bin/python", - "benchmarks/benchmark_rocm_attention_ablation.py", - "--device", - "0", - "--output-dir", - "benchmarks/results/pr230_rocm_mi300x_ablation" - ], - "configuration": { - "dtype": "bfloat16", - "head_dim": 128, - "kv_heads": 8, - "q_heads": 32, - "seed": 230, - "shapes": [ - [ - 1, - 16 - ], - [ - 1, - 32 - ], - [ - 1, - 64 - ], - [ - 1, - 128 - ], - [ - 2, - 16 - ], - [ - 2, - 32 - ], - [ - 2, - 64 - ], - [ - 2, - 128 - ] - ] - }, - "created_at": "2026-09-03T10:42:33.755824+00:00", - "environment": { - "architecture": "gfx942:sramecc+:xnack-", - "device_index": 0, - "device_name": "AMD Instinct MI300X VF", - "execution_kind": "operator_only_rocm_reference", - "gpu_count": 1, - "hip_runtime": "7.14.60850", - "primary_backend_id": "rlkernel.rocm.deterministic_attention", - "primary_core_id": "rlkernel.attention.deterministic_core.v1", - "primary_production_ready": false, - "primary_reference_only": true, - "primary_schedule": "single_batch_single_query_global_kv_blocks", - "python": "3.12.3", - "pytorch": "2.12.0+rocm7.14.0a20260608" - }, - "matrix": [ - { - "case_count": 8, - "category": "baseline", - "comparable": true, - "expected": "baseline", - "id": "A0", - "label": "Strict replay baseline", - "metrics": { - "dk": { - "all_bitwise_equal": true, - "total_mismatch_count": 0, - "worst_max_abs": 0.0 - }, - "dq": { - "all_bitwise_equal": true, - "total_mismatch_count": 0, - "worst_max_abs": 0.0 - }, - "dv": { - "all_bitwise_equal": true, - "total_mismatch_count": 0, - "worst_max_abs": 0.0 - }, - "lse": { - "all_bitwise_equal": true, - "total_mismatch_count": 0, - "worst_max_abs": 0.0 - }, - "out": { - "all_bitwise_equal": true, - "total_mismatch_count": 0, - "worst_max_abs": 0.0 - } - }, - "passed": true, - "probe": null, - "root_cause_axis": null - }, - { - "case_count": 8, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "id": "A1", - "label": "Position / RoPE", - "metrics": { - "dk": { - "all_bitwise_equal": false, - "total_mismatch_count": 712216, - "worst_max_abs": 2.234375 - }, - "dq": { - "all_bitwise_equal": false, - "total_mismatch_count": 1455200, - "worst_max_abs": 1.4140625 - }, - "dv": { - "all_bitwise_equal": false, - "total_mismatch_count": 703681, - "worst_max_abs": 0.783203125 - }, - "lse": { - "all_bitwise_equal": false, - "total_mismatch_count": 11520, - "worst_max_abs": 0.3938255310058594 - }, - "out": { - "all_bitwise_equal": false, - "total_mismatch_count": 1453811, - "worst_max_abs": 0.7109375 - } - }, - "passed": true, - "probe": "position_ids", - "root_cause_axis": "position_rope" - }, - { - "case_count": 8, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "id": "A2", - "label": "Q/K preprocessing", - "metrics": { - "dk": { - "all_bitwise_equal": false, - "total_mismatch_count": 730740, - "worst_max_abs": 1.7578125 - }, - "dq": { - "all_bitwise_equal": false, - "total_mismatch_count": 2867769, - "worst_max_abs": 1.015625 - }, - "dv": { - "all_bitwise_equal": false, - "total_mismatch_count": 723462, - "worst_max_abs": 0.96875 - }, - "lse": { - "all_bitwise_equal": false, - "total_mismatch_count": 23040, - "worst_max_abs": 1.0303754806518555 - }, - "out": { - "all_bitwise_equal": false, - "total_mismatch_count": 2818071, - "worst_max_abs": 0.58203125 - } - }, - "passed": true, - "probe": "qk_norm_disabled", - "root_cause_axis": "qk_preprocessing" - }, - { - "case_count": 8, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "id": "A3", - "label": "Mask / sequence boundary", - "metrics": { - "dk": { - "all_bitwise_equal": false, - "total_mismatch_count": 736222, - "worst_max_abs": 6.15625 - }, - "dq": { - "all_bitwise_equal": false, - "total_mismatch_count": 2887461, - "worst_max_abs": 4.1968994140625 - }, - "dv": { - "all_bitwise_equal": false, - "total_mismatch_count": 736298, - "worst_max_abs": 10.984375 - }, - "lse": { - "all_bitwise_equal": false, - "total_mismatch_count": 22656, - "worst_max_abs": 7.67181396484375 - }, - "out": { - "all_bitwise_equal": false, - "total_mismatch_count": 2888765, - "worst_max_abs": 4.21875 - } - }, - "passed": true, - "probe": "causal_mask", - "root_cause_axis": "mask_sequence_boundary" - }, - { - "case_count": 8, - "category": "comparability_gate", - "comparable": false, - "expected": "rejected", - "id": "A4", - "label": "Topology / head ownership", - "metrics": { - "dk": null, - "dq": null, - "dv": null, - "lse": null, - "out": null - }, - "passed": true, - "probe": "tp_head_ownership", - "root_cause_axis": "topology_head_ownership" - }, - { - "case_count": 8, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "id": "A5", - "label": "KV-cache identity / layout", - "metrics": { - "dk": { - "all_bitwise_equal": false, - "total_mismatch_count": 736656, - "worst_max_abs": 6.62744140625 - }, - "dq": { - "all_bitwise_equal": false, - "total_mismatch_count": 2843694, - "worst_max_abs": 4.27734375 - }, - "dv": { - "all_bitwise_equal": false, - "total_mismatch_count": 736681, - "worst_max_abs": 13.2333984375 - }, - "lse": { - "all_bitwise_equal": false, - "total_mismatch_count": 22656, - "worst_max_abs": 4.578392028808594 - }, - "out": { - "all_bitwise_equal": false, - "total_mismatch_count": 2893400, - "worst_max_abs": 6.0 - } - }, - "passed": true, - "probe": "kv_page_order", - "root_cause_axis": "kv_cache_identity_layout" - }, - { - "case_count": 8, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "id": "A6", - "label": "Numerical policy", - "metrics": { - "dk": { - "all_bitwise_equal": false, - "total_mismatch_count": 646044, - "worst_max_abs": 0.09375 - }, - "dq": { - "all_bitwise_equal": false, - "total_mismatch_count": 2456076, - "worst_max_abs": 0.09375 - }, - "dv": { - "all_bitwise_equal": false, - "total_mismatch_count": 652656, - "worst_max_abs": 0.11328125 - }, - "lse": { - "all_bitwise_equal": false, - "total_mismatch_count": 23038, - "worst_max_abs": 0.09950780868530273 - }, - "out": { - "all_bitwise_equal": false, - "total_mismatch_count": 2473674, - "worst_max_abs": 0.1171875 - } - }, - "passed": true, - "probe": "accum_dtype", - "root_cause_axis": "numerical_policy" - }, - { - "case_count": 8, - "category": "root_cause", - "comparable": true, - "expected": "diagnostic", - "id": "A7", - "label": "Distributed schedule", - "metrics": { - "dk": { - "all_bitwise_equal": false, - "total_mismatch_count": 60, - "worst_max_abs": 0.00390625 - }, - "dq": { - "all_bitwise_equal": false, - "total_mismatch_count": 116, - "worst_max_abs": 0.00390625 - }, - "dv": { - "all_bitwise_equal": false, - "total_mismatch_count": 30, - "worst_max_abs": 0.00390625 - }, - "lse": { - "all_bitwise_equal": false, - "total_mismatch_count": 860, - "worst_max_abs": 9.5367431640625e-07 - }, - "out": { - "all_bitwise_equal": false, - "total_mismatch_count": 66, - "worst_max_abs": 0.001953125 - } - }, - "passed": true, - "probe": "merge_order", - "root_cause_axis": "distributed_schedule" - }, - { - "case_count": 8, - "category": "invariant_control", - "comparable": true, - "expected": "exact_zero", - "id": "C0", - "label": "Invariant control", - "metrics": { - "dk": { - "all_bitwise_equal": true, - "total_mismatch_count": 0, - "worst_max_abs": 0.0 - }, - "dq": { - "all_bitwise_equal": true, - "total_mismatch_count": 0, - "worst_max_abs": 0.0 - }, - "dv": { - "all_bitwise_equal": true, - "total_mismatch_count": 0, - "worst_max_abs": 0.0 - }, - "lse": { - "all_bitwise_equal": true, - "total_mismatch_count": 0, - "worst_max_abs": 0.0 - }, - "out": { - "all_bitwise_equal": true, - "total_mismatch_count": 0, - "worst_max_abs": 0.0 - } - }, - "passed": true, - "probe": "tp_partition_control", - "root_cause_axis": null - }, - { - "case_count": 8, - "category": "invariant_control", - "comparable": true, - "expected": "exact_zero", - "id": "C1", - "label": "Invariant control", - "metrics": { - "dk": { - "all_bitwise_equal": true, - "total_mismatch_count": 0, - "worst_max_abs": 0.0 - }, - "dq": { - "all_bitwise_equal": true, - "total_mismatch_count": 0, - "worst_max_abs": 0.0 - }, - "dv": { - "all_bitwise_equal": true, - "total_mismatch_count": 0, - "worst_max_abs": 0.0 - }, - "lse": { - "all_bitwise_equal": true, - "total_mismatch_count": 0, - "worst_max_abs": 0.0 - }, - "out": { - "all_bitwise_equal": true, - "total_mismatch_count": 0, - "worst_max_abs": 0.0 - } - }, - "passed": true, - "probe": "batch_composition_control", - "root_cause_axis": null - }, - { - "case_count": 8, - "category": "invariant_control", - "comparable": true, - "expected": "exact_zero", - "id": "C2", - "label": "Invariant control", - "metrics": { - "dk": { - "all_bitwise_equal": true, - "total_mismatch_count": 0, - "worst_max_abs": 0.0 - }, - "dq": { - "all_bitwise_equal": true, - "total_mismatch_count": 0, - "worst_max_abs": 0.0 - }, - "dv": { - "all_bitwise_equal": true, - "total_mismatch_count": 0, - "worst_max_abs": 0.0 - }, - "lse": { - "all_bitwise_equal": true, - "total_mismatch_count": 0, - "worst_max_abs": 0.0 - }, - "out": { - "all_bitwise_equal": true, - "total_mismatch_count": 0, - "worst_max_abs": 0.0 - } - }, - "passed": true, - "probe": "prefill_decode_tail_control", - "root_cause_axis": null - } - ], - "matrix_manifest": { - "baseline_row": "A0", - "cartesian_product": false, - "method": "fixed_replay_one_at_a_time", - "metrics": [ - "train_rollout_logprob_abs_diff", - "mismatch_kl", - "mismatch_k3_kl", - "out_max_abs", - "lse_max_abs", - "dq_max_abs", - "dk_max_abs", - "dv_max_abs" - ], - "replay_identity": "same checkpoint, token IDs, selected-token IDs, masks, positions, cache metadata, and pre-update model state", - "row_baseline": "each diagnostic row is compared with its own phase-local A0 baseline", - "rows": [ - { - "category": "baseline", - "expected": "baseline", - "id": "A0", - "label": "Strict replay baseline", - "probe": null, - "root_cause_axis": null - }, - { - "category": "root_cause", - "expected": "diagnostic", - "id": "A1", - "label": "Position / RoPE", - "probe": "position_ids", - "root_cause_axis": "position_rope" - }, - { - "category": "root_cause", - "expected": "diagnostic", - "id": "A2", - "label": "Q/K preprocessing", - "probe": "qk_norm_disabled", - "root_cause_axis": "qk_preprocessing" - }, - { - "category": "root_cause", - "expected": "diagnostic", - "id": "A3", - "label": "Mask / sequence boundary", - "probe": "causal_mask", - "root_cause_axis": "mask_sequence_boundary" - }, - { - "category": "comparability_gate", - "expected": "rejected", - "id": "A4", - "label": "Topology / head ownership", - "probe": "tp_head_ownership", - "root_cause_axis": "topology_head_ownership" - }, - { - "category": "root_cause", - "expected": "diagnostic", - "id": "A5", - "label": "KV-cache identity / layout", - "probe": "kv_page_order", - "root_cause_axis": "kv_cache_identity_layout" - }, - { - "category": "root_cause", - "expected": "diagnostic", - "id": "A6", - "label": "Numerical policy", - "probe": "accum_dtype", - "root_cause_axis": "numerical_policy" - }, - { - "category": "root_cause", - "expected": "diagnostic", - "id": "A7", - "label": "Distributed schedule", - "probe": "merge_order", - "root_cause_axis": "distributed_schedule" - }, - { - "category": "invariant_control", - "expected": "exact_zero", - "id": "C0", - "label": "Invariant control", - "probe": "tp_partition_control", - "root_cause_axis": null - }, - { - "category": "invariant_control", - "expected": "exact_zero", - "id": "C1", - "label": "Invariant control", - "probe": "batch_composition_control", - "root_cause_axis": null - }, - { - "category": "invariant_control", - "expected": "exact_zero", - "id": "C2", - "label": "Invariant control", - "probe": "prefill_decode_tail_control", - "root_cause_axis": null - } - ], - "schema_version": "rlkernel.attention.debug_matrix.v1", - "topology_gate": [ - "checkpoint/model/token identity", - "TP head ownership", - "CP sequence ownership", - "actual Split-KV plan" - ] - }, - "schema_version": "rlkernel.rocm.attention_ablation_microprobe.v1", - "scope": { - "covered_metrics": [ - "out_max_abs", - "lse_max_abs", - "dq_max_abs", - "dk_max_abs", - "dv_max_abs" - ], - "excluded_metrics": [ - "train_rollout_logprob_abs_diff", - "mismatch_kl", - "mismatch_k3_kl" - ], - "frozen_rollout_replay": false, - "kind": "operator_microprobe", - "model_or_serving_execution": false, - "pr230_row_taxonomy": true - }, - "source_provenance": { - "revision": "2ea63b22b74feb6e5a748d09780fa075d5e644ed", - "script_matches_head": true, - "script_path": "benchmarks/benchmark_rocm_attention_ablation.py", - "script_sha256": "0139a15ee6bae14e9ee8ca7cc32146956b10d70d201d37086a0a45315f5816ac", - "tracked_diff_sha256": null, - "tracked_dirty": false - } -} diff --git a/examples/vime_qwen3_8b_rocm_ablation/README.md b/examples/vime_qwen3_8b_rocm_ablation/README.md new file mode 100644 index 00000000..31674e3e --- /dev/null +++ b/examples/vime_qwen3_8b_rocm_ablation/README.md @@ -0,0 +1,93 @@ +# Vime Qwen3-8B ROCm Attention ablation + +This is the ROCm end-to-end counterpart of PR230's production/RL-Kernel +operator matrix. It launches the real Vime orchestration once per Attention +cell and requires runtime evidence from both sides: + +| Case | Megatron training | vLLM rollout | +|---|---|---| +| `P/P` | framework-native | framework-native | +| `P/R` | framework-native | RL-Kernel AITER/CK | +| `R/P` | RL-Kernel AITER/CK | framework-native | +| `R/R` | RL-Kernel AITER/CK | RL-Kernel AITER/CK | + +FFN and Logp remain fixed at `P/P`, so only the Attention implementation +changes. Each cell starts in a fresh process and inherits the same model, +checkpoint, prompt data, seeds, token limits, and one-rollout pre-update state. + +This is not an operator microbenchmark. The subprocess must run both vLLM +rollout and Megatron training. A return code of zero is insufficient: the +runner fails the cell if either framework emitted no executed Attention +readback, selected the wrong P/R route, reported fallback, or failed to prove +the strict ROCm runtime on an R side. + +## Required host state + +- A ROCm PyTorch build with visible AMD GPUs. +- AITER with `aiter.ops.mha.mha_fwd` and `mha_bwd` available. +- Vime, Megatron-LM, vLLM and RL-Kernel importable by every Ray worker. +- A frozen Qwen3 model/checkpoint and prompt file. +- A Vime launcher that honors `RL_KERNEL_ABLATION_OUTPUT_DIR` for case-local + output, so one cell cannot update the input checkpoint used by the next. +- Megatron startup wired through + `rl_engine.integrations.megatron_runtime.initialize_from_environment`, so + the training worker installs the selected P/R plan and emits its readback. + +The executable run requires these immutable input variables: + +```bash +export MODEL_ROOT=/models/Qwen3-8B +export TORCH_DIST_ROOT=/models/Qwen3-8B_torch_dist +export VIME_CKPT=/checkpoints/qwen3-8b-pre-update +export PROMPT_DATA=/data/dapo-math-17k.jsonl +export NUM_ROLLOUT=1 +export TRAIN_SEED=1234 +export ROLLOUT_SEED=42 +``` + +The parent launcher must propagate the P/R and readback variables into Ray +workers. `rocm_python_entrypoint.sh` is provided for launchers that replace +their Python executable. Point `RL_KERNEL_REAL_PYTHON` at the real interpreter +and configure Vime to invoke this wrapper. + +## Review the launch contract + +Without `--run`, the runner writes only a review summary and does not require a +ROCm host: + +```bash +python examples/vime_qwen3_8b_rocm_ablation/run.py \ + --output-dir /tmp/rocm-attention-ablation \ + -- bash /path/to/vime/scripts/run-qwen3-8B-rocm.sh +``` + +## Execute the full matrix + +```bash +python examples/vime_qwen3_8b_rocm_ablation/run.py \ + --run \ + --output-dir /tmp/rocm-attention-ablation \ + -- bash /path/to/vime/scripts/run-qwen3-8B-rocm.sh +``` + +Use `--case R/R` (repeatable) to run a subset while debugging. The final +acceptance run should execute all four cells. + +## Evidence and pass boundary + +Each case directory contains the combined process log and the unmodified JSON +readbacks emitted by `FrameworkOperatorIntegration`. The aggregate is a human- +readable `summary.md`; no generated result JSON is checked into the repository. + +For each R side, accepted evidence includes: + +- semantic backend `rlkernel.attention.deterministic.v1`; +- `runtime_platform=rocm`; +- actual runtime `rlkernel.rocm.attention.aiter_ck_ag_rs.v1`; +- AITER/CK fixed no-Split-KV schedule; +- no native, PyTorch-reference, Triton, or fallback execution. + +The rollout route consumes vLLM's paged cache, reconstructs logical KV order, +and invokes the same strict AITER/CK core as the training route. The training +route binds the Megatron CP process group to the RCCL AG/RS transport and +preserves the explicit global position order used by the PR230 contract. diff --git a/examples/vime_qwen3_8b_rocm_ablation/rocm_python_entrypoint.sh b/examples/vime_qwen3_8b_rocm_ablation/rocm_python_entrypoint.sh new file mode 100755 index 00000000..c957274f --- /dev/null +++ b/examples/vime_qwen3_8b_rocm_ablation/rocm_python_entrypoint.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -euo pipefail + +REAL_PYTHON="${RL_KERNEL_REAL_PYTHON:?RL_KERNEL_REAL_PYTHON must name the real Python executable}" +RL_KERNEL_ROOT="${RL_KERNEL_ROOT:-$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)}" +export RL_KERNEL_ROOT +export PYTHONPATH="${RL_KERNEL_ROOT}:${PYTHONPATH:-}" + +if [[ "${1:-}" == "train.py" || "${1:-}" == */train.py ]]; then + : "${RL_KERNEL_ATTENTION_CASE:?the ablation runner must select an Attention case}" + : "${RL_KERNEL_FFN_CASE:?the ablation runner must freeze the FFN case}" + : "${RL_KERNEL_LOGP_CASE:?the ablation runner must freeze the Logp case}" + : "${RL_KERNEL_READBACK_DIR:?the ablation runner must provide a readback directory}" + + export RL_KERNEL_VLLM_INTEGRATION=1 + export RL_KERNEL_PLATFORM=rocm + export RL_KERNEL_ROCM_STRICT_ATTENTION=1 + export RL_KERNEL_ROUTE_REPORT=1 + + exec "${REAL_PYTHON}" "$@" \ + --seed "${TRAIN_SEED:-1234}" \ + --rollout-seed "${ROLLOUT_SEED:-42}" \ + --vllm-enable-deterministic-inference \ + --vllm-attention-backend rocm_aiter_fa \ + --vllm-disable-custom-all-reduce \ + --deterministic-mode \ + --accumulate-allreduce-grads-in-fp32 +fi + +exec "${REAL_PYTHON}" "$@" diff --git a/examples/vime_qwen3_8b_rocm_ablation/run.py b/examples/vime_qwen3_8b_rocm_ablation/run.py new file mode 100644 index 00000000..21f5daea --- /dev/null +++ b/examples/vime_qwen3_8b_rocm_ablation/run.py @@ -0,0 +1,81 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""Launch the PR230 Attention P/R matrix through a real Vime ROCm job.""" + +from __future__ import annotations + +import argparse +import os +from pathlib import Path + +from rl_engine.integrations.rocm_ablation import ( + ROCM_ATTENTION_CASE_IDS, + run_rocm_attention_ablation, +) + + +def _case_id(value: str) -> str: + normalized = value.strip().upper() + if normalized not in ROCM_ATTENTION_CASE_IDS: + raise argparse.ArgumentTypeError( + f"case must be one of {', '.join(ROCM_ATTENTION_CASE_IDS)}" + ) + return normalized + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--output-dir", + type=Path, + default=Path("runs/rocm-attention-ablation"), + help="case logs, framework readbacks, and summary location", + ) + parser.add_argument( + "--case", + action="append", + type=_case_id, + dest="cases", + help="run only one matrix cell; repeat to select multiple cells", + ) + parser.add_argument( + "--run", + action="store_true", + help="execute the orchestration command (default: review-only dry run)", + ) + parser.add_argument( + "command", + nargs=argparse.REMAINDER, + help="Vime command after '--', for example: -- bash scripts/run-qwen3.sh", + ) + args = parser.parse_args(argv) + if args.command[:1] == ["--"]: + args.command = args.command[1:] + if not args.command: + parser.error("a Vime orchestration command is required after '--'") + return args + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + results = run_rocm_attention_ablation( + args.command, + output_dir=args.output_dir.resolve(), + base_environment=os.environ, + case_ids=args.cases or ROCM_ATTENTION_CASE_IDS, + execute=args.run, + ) + for result in results: + print( + f"[{result.status.upper()}] Attention={result.case_id} " + f"log={result.log_path} readbacks={result.readback_dir}" + ) + for error in result.errors: + print(f" - {error}") + print(f"summary={args.output_dir.resolve() / 'summary.md'}") + return 1 if any(result.status == "failed" for result in results) else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/rl_engine/integrations/__init__.py b/rl_engine/integrations/__init__.py index 49b7193d..fc1a229e 100644 --- a/rl_engine/integrations/__init__.py +++ b/rl_engine/integrations/__init__.py @@ -14,6 +14,13 @@ ) from rl_engine.integrations.megatron import MegatronIntegration from rl_engine.integrations.megatron_runtime import install_megatron_integration +from rl_engine.integrations.rocm_ablation import ( + ROCM_ATTENTION_CASE_IDS, + RocmAblationCaseResult, + RocmAttentionAblationCase, + rocm_attention_ablation_matrix, + run_rocm_attention_ablation, +) from rl_engine.integrations.vllm import VllmIntegration from rl_engine.integrations.vllm_runtime import configure_vllm_environment @@ -22,6 +29,9 @@ "IntegrationPlan", "MegatronIntegration", "OperatorAblationCase", + "ROCM_ATTENTION_CASE_IDS", + "RocmAblationCaseResult", + "RocmAttentionAblationCase", "VllmIntegration", "configure_integration_environment", "configure_vllm_environment", @@ -29,4 +39,6 @@ "integration_plan_from_environment", "operator_ablation_case", "operator_ablation_cases", + "rocm_attention_ablation_matrix", + "run_rocm_attention_ablation", ] diff --git a/rl_engine/integrations/framework_operators.py b/rl_engine/integrations/framework_operators.py index b1b5f970..3f19816b 100644 --- a/rl_engine/integrations/framework_operators.py +++ b/rl_engine/integrations/framework_operators.py @@ -22,6 +22,8 @@ from rl_engine.kernels.attention_contract import ( STRICT_ATTENTION_FA4_SCHEDULE_ID, STRICT_ATTENTION_PRODUCTION_CORE_ID, + STRICT_ATTENTION_ROCM_PRODUCTION_CORE_ID, + STRICT_ATTENTION_ROCM_SCHEDULE_ID, AttentionContract, AttentionDType, AttentionMode, @@ -120,6 +122,30 @@ def _require_nvidia_cuda(tensor: torch.Tensor, module: str) -> None: raise RuntimeError(f"strict {module} R/R requires NVIDIA CUDA tensors") +def _require_attention_accelerator(tensor: torch.Tensor) -> str: + """Return the real Attention platform behind PyTorch's CUDA device API.""" + + if tensor.device.type != "cuda": + raise RuntimeError("strict Attention R/R requires CUDA or ROCm GPU tensors") + return "rocm" if torch.version.hip is not None else "cuda" + + +def _strict_attention_platform_contract(platform: str) -> tuple[str, str, str]: + if platform == "rocm": + return ( + STRICT_ATTENTION_ROCM_PRODUCTION_CORE_ID, + STRICT_ATTENTION_ROCM_SCHEDULE_ID, + "rccl_ag_rs", + ) + if platform == "cuda": + return ( + STRICT_ATTENTION_PRODUCTION_CORE_ID, + STRICT_ATTENTION_FA4_SCHEDULE_ID, + "cuda_ag_rs", + ) + raise RuntimeError(f"unsupported strict Attention platform {platform!r}") + + class SemanticOperatorHandle: """Resolve one exact semantic backend once for one framework process.""" @@ -478,7 +504,10 @@ def __call__( if query.ndim != expected_ndim or key.ndim != expected_ndim or value.ndim != expected_ndim: layout = "[T, H, D]" if packed_seq_params is not None else "[S, B, H, D]" raise RuntimeError(f"Megatron Attention Q/K/V must use {layout}") - _require_nvidia_cuda(query, "Attention") + runtime_platform = _require_attention_accelerator(query) + strict_core_id, strict_schedule, communication_backend = ( + _strict_attention_platform_contract(runtime_platform) + ) parallel_state = _megatron_parallel_state() cp_world = int(parallel_state.get_context_parallel_world_size()) @@ -494,7 +523,7 @@ def __call__( "context_parallel_size": cp_world, }, ) - operator.bind_cuda_runtime(process_group=cp_group) + operator.bind_accelerator_runtime(query, process_group=cp_group) scale = float(getattr(module, "softmax_scale", query.size(-1) ** -0.5)) def execute_sequence( @@ -533,11 +562,11 @@ def execute_sequence( local_block_offsets=block_offsets, ), config=AttentionAblationConfig( - strict_core_id=STRICT_ATTENTION_PRODUCTION_CORE_ID, - strict_schedule=STRICT_ATTENTION_FA4_SCHEDULE_ID, + strict_core_id=strict_core_id, + strict_schedule=strict_schedule, ), return_lse=True, - communication_backend="cuda_ag_rs" if cp_world > 1 else "none", + communication_backend=communication_backend if cp_world > 1 else "none", query_position_ids=position_ids, key_position_ids=position_ids, scale=scale, @@ -632,10 +661,10 @@ def execute_sequence( if packed_seq_params is not None else "megatron_sbh_zigzag_cp" ), - "materialization": "owner_local_zigzag_cuda_ag_rs", + "materialization": f"owner_local_zigzag_{communication_backend}", "cp_world_size": cp_world, "tp_world_size": tp_world, - "runtime_platform": "cuda", + "runtime_platform": runtime_platform, "triton_used": False, **execution_provenance, } @@ -748,16 +777,68 @@ def _vllm_kv_cache_views( kv_cache: torch.Tensor, *, head_size: int, + num_kv_heads: int | None = None, + platform: str | None = None, ) -> tuple[torch.Tensor, torch.Tensor]: - """Return vLLM's paged cache as [blocks, block, kv_heads, head].""" + """Return CUDA and ROCm vLLM caches as [blocks, block, kv_heads, head].""" + + def normalize(plane: torch.Tensor) -> torch.Tensor: + if plane.ndim != 4 or plane.size(-1) != head_size: + raise RuntimeError("vLLM K/V cache planes must be 4-D with a head-size tail") + if num_kv_heads is None or plane.size(2) == num_kv_heads: + # [blocks, block, heads, head] (LBHNC after selecting K/V). + return plane + if plane.size(1) == num_kv_heads: + # [blocks, heads, block, head]. + return plane.permute(0, 2, 1, 3) + if plane.size(0) == num_kv_heads: + # [heads, blocks, block, head] (LHBNC after selecting K/V). + return plane.permute(1, 2, 0, 3) + raise RuntimeError( + "vLLM K/V cache layout does not expose the declared number of KV heads" + ) + if ( + kv_cache.ndim == 4 + and platform == "rocm" + and kv_cache.size(1) == 2 + and num_kv_heads is not None + and kv_cache.size(-1) == num_kv_heads * head_size + ): + key_cache, value_cache = kv_cache.unbind(1) + blocks, block_size, _packed_heads = key_cache.shape + return ( + key_cache.view(blocks, block_size, num_kv_heads, head_size), + value_cache.view(blocks, block_size, num_kv_heads, head_size), + ) if kv_cache.ndim == 4 and kv_cache.size(-1) == 2 * head_size: return kv_cache.transpose(1, 2).split(head_size, dim=-1) + # The K/V axis is leading in CUDA FlashAttention caches and follows the + # block axis in the ROCm AITER layouts. Prefer the platform convention in + # the ambiguous two-block case where both dimensions happen to equal two. + if kv_cache.ndim == 5 and platform == "rocm" and kv_cache.size(1) == 2: + key_cache, value_cache = kv_cache.unbind(1) + return normalize(key_cache), normalize(value_cache) if kv_cache.ndim == 5 and kv_cache.size(0) == 2: key_cache, value_cache = kv_cache.unbind(0) - return key_cache, value_cache + return normalize(key_cache), normalize(value_cache) + if kv_cache.ndim == 5 and kv_cache.size(1) == 2: + key_cache, value_cache = kv_cache.unbind(1) + return normalize(key_cache), normalize(value_cache) + if ( + kv_cache.ndim == 4 + and kv_cache.size(1) == 2 + and num_kv_heads is not None + and kv_cache.size(-1) == num_kv_heads * head_size + ): + key_cache, value_cache = kv_cache.unbind(1) + blocks, block_size, _packed_heads = key_cache.shape + return ( + key_cache.view(blocks, block_size, num_kv_heads, head_size), + value_cache.view(blocks, block_size, num_kv_heads, head_size), + ) raise RuntimeError( - "vLLM FlashAttention KV cache must use " "[blocks, kv_heads, block, 2 * head_size]" + "vLLM Attention KV cache does not match a supported CUDA/ROCm paged layout" ) @@ -931,10 +1012,12 @@ def __call__( return output.zero_() if query.ndim != 3: raise RuntimeError("vLLM query must use [tokens, heads, head_dim]") - _require_nvidia_cuda(query, "Attention") + runtime_platform = _require_attention_accelerator(query) key_cache, value_cache = _vllm_kv_cache_views( kv_cache, head_size=int(impl.head_size), + num_kv_heads=int(impl.num_kv_heads), + platform=runtime_platform, ) if key_cache.dtype != query.dtype or value_cache.dtype != query.dtype: raise RuntimeError("strict vLLM Attention requires an unquantized KV cache") @@ -964,7 +1047,7 @@ def __call__( "context_parallel_size": 1, }, ) - runtime = operator.bind_cuda_runtime() + runtime = operator.bind_accelerator_runtime(query) groups, metadata_summary = self._materialization_groups( attn_metadata, query=query, @@ -1013,10 +1096,14 @@ def __call__( last_operator_provenance = _compact_attention_provenance(result.provenance) self._last_provenance = { "framework_layout": "vllm_paged_kv", - "materialization": "direct_paged_fa4", + "materialization": ( + "logical_paged_kv_to_aiter_ck_dense" + if runtime_platform == "rocm" + else "direct_paged_fa4" + ), "tp_world_size": tp_world, "tp_group_bound": tp_group is not None, - "runtime_platform": "cuda", + "runtime_platform": runtime_platform, "triton_used": False, "direct_output_buffer": direct_output_buffer, **metadata_summary, diff --git a/rl_engine/integrations/rocm_ablation.py b/rl_engine/integrations/rocm_ablation.py new file mode 100644 index 00000000..4a1fb040 --- /dev/null +++ b/rl_engine/integrations/rocm_ablation.py @@ -0,0 +1,588 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +"""End-to-end ROCm rollout/training Attention ablation runner. + +PR230 defines a four-cell production/RL-Kernel matrix. This module executes +that matrix against a real orchestration command (normally Vime), one fresh +process per cell, and validates the runtime readbacks emitted independently by +Megatron training and vLLM rollout workers. It deliberately does not execute +synthetic tensors or manufacture a checked-in result payload. + +The replay identity is frozen before the first case. Only case selection and +case-local artifact paths may differ between subprocesses. A successful exit +without both framework readbacks is a failure, as is an R-side readback that +does not prove ROCm execution through the strict AITER/CK runtime. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import subprocess +from dataclasses import dataclass, field +from pathlib import Path +from types import MappingProxyType +from typing import Any, Iterable, Mapping, Sequence + +from rl_engine.integrations.ablation import Implementation, IntegrationPlan +from rl_engine.kernels.attention_contract import ( + STRICT_ATTENTION_ROCM_PRODUCTION_CORE_ID, + STRICT_ATTENTION_ROCM_SCHEDULE_ID, +) + +ROCM_ABLATION_SCHEMA_VERSION = "rlkernel.rocm.e2e_attention_ablation.v1" +ROCM_ATTENTION_CASE_IDS = ("P/P", "P/R", "R/P", "R/R") +STRICT_ROCM_ATTENTION_RUNTIME = "rlkernel.rocm.attention.aiter_ck_ag_rs.v1" +STRICT_ROCM_ATTENTION_CORE = STRICT_ATTENTION_ROCM_PRODUCTION_CORE_ID +STRICT_ROCM_ATTENTION_SCHEDULE = STRICT_ATTENTION_ROCM_SCHEDULE_ID + +# These values identify the immutable input and sampling stream. They must be +# present for an actual matrix execution; a dry run may omit them so launchers +# can be reviewed before expensive GPU allocation. +REQUIRED_REPLAY_ENV = ( + "MODEL_ROOT", + "TORCH_DIST_ROOT", + "PROMPT_DATA", +) +FROZEN_REPLAY_ENV = ( + *REQUIRED_REPLAY_ENV, + "VIME_CKPT", + "TOKENIZER_PATH", + "DATASET_SEED", + "PYTHONHASHSEED", + "ROLLOUT_SEED", + "TRAIN_SEED", + "NUM_ROLLOUT", + "ROLLOUT_BATCH_SIZE", + "N_SAMPLES_PER_PROMPT", + "MAX_PROMPT_LENGTH", + "MAX_RESPONSE_LENGTH", +) + +_FRAMEWORK_TARGETS = MappingProxyType( + { + "training": ("megatron", "training"), + "rollout": ("vllm", "rollout"), + } +) + + +@dataclass(frozen=True) +class RocmAttentionAblationCase: + """One PR230 Attention implementation pairing.""" + + case_id: str + plan: IntegrationPlan + + def __post_init__(self) -> None: + if self.case_id not in ROCM_ATTENTION_CASE_IDS: + raise ValueError(f"unknown ROCm Attention ablation case {self.case_id!r}") + if self.plan.cases["attention"].case_id != self.case_id: + raise ValueError("case_id must match the Attention case in the integration plan") + if self.plan.cases["ffn"].case_id != "P/P" or self.plan.cases["logp"].case_id != "P/P": + raise ValueError("the Attention matrix must hold FFN and Logp at P/P") + + @property + def slug(self) -> str: + return self.case_id.lower().replace("/", "-") + + def implementation_for(self, target: str) -> Implementation: + return self.plan.implementation_for("attention", target) + + +@dataclass(frozen=True) +class FrameworkRouteEvidence: + """Aggregated route evidence from one framework side of one case.""" + + framework: str + target: str + implementation: str + backend_ids: tuple[str, ...] + call_count: int + readback_count: int + runtime_platforms: tuple[str, ...] + actual_backends: tuple[str, ...] + + +@dataclass(frozen=True) +class RocmAblationCaseResult: + """Outcome of one real Vime subprocess.""" + + case_id: str + status: str + returncode: int | None + log_path: Path + readback_dir: Path + routes: Mapping[str, FrameworkRouteEvidence] = field(default_factory=dict) + errors: tuple[str, ...] = () + + def __post_init__(self) -> None: + if self.status not in {"not_run", "passed", "failed"}: + raise ValueError(f"unknown case status {self.status!r}") + object.__setattr__(self, "routes", MappingProxyType(dict(self.routes))) + + +def rocm_attention_ablation_matrix( + case_ids: Iterable[str] = ROCM_ATTENTION_CASE_IDS, +) -> tuple[RocmAttentionAblationCase, ...]: + """Build the PR230 matrix while freezing non-Attention modules.""" + + normalized = tuple(str(case_id).strip().upper() for case_id in case_ids) + if not normalized: + raise ValueError("at least one Attention ablation case is required") + if len(set(normalized)) != len(normalized): + raise ValueError("Attention ablation case IDs must be unique") + unknown = [case_id for case_id in normalized if case_id not in ROCM_ATTENTION_CASE_IDS] + if unknown: + raise ValueError(f"unknown Attention ablation cases: {', '.join(unknown)}") + return tuple( + RocmAttentionAblationCase( + case_id=case_id, + plan=IntegrationPlan.from_case_ids( + attention=case_id, + ffn="P/P", + logp="P/P", + ), + ) + for case_id in normalized + ) + + +def validate_rocm_host() -> dict[str, Any]: + """Verify that a real ROCm device is available before allocating a run.""" + + import torch + + hip = getattr(torch.version, "hip", None) + if hip is None: + raise RuntimeError("end-to-end ROCm ablation requires a ROCm PyTorch build") + if not torch.cuda.is_available() or torch.cuda.device_count() <= 0: + raise RuntimeError("end-to-end ROCm ablation requires at least one visible AMD GPU") + devices = tuple(torch.cuda.get_device_name(index) for index in range(torch.cuda.device_count())) + return { + "hip_runtime": str(hip), + "pytorch": str(torch.__version__), + "device_count": len(devices), + "devices": devices, + } + + +def validate_replay_environment(environment: Mapping[str, str]) -> None: + """Reject a run that cannot identify its immutable replay inputs.""" + + missing = [name for name in REQUIRED_REPLAY_ENV if not str(environment.get(name, "")).strip()] + if missing: + raise RuntimeError("missing frozen replay environment: " + ", ".join(missing)) + num_rollout = str(environment.get("NUM_ROLLOUT", "1")).strip() + try: + count = int(num_rollout) + except ValueError as exc: + raise RuntimeError("NUM_ROLLOUT must be an integer") from exc + if count != 1: + raise RuntimeError( + "the ablation matrix requires NUM_ROLLOUT=1 so every cell starts from " + "the same pre-update state" + ) + + +def replay_identity( + command: Sequence[str], + environment: Mapping[str, str], +) -> dict[str, Any]: + """Return one stable identity shared by every matrix cell.""" + + if not command or any(not isinstance(item, str) or not item for item in command): + raise ValueError("command must contain non-empty argument strings") + frozen = { + name: str(environment[name]) + for name in FROZEN_REPLAY_ENV + if str(environment.get(name, "")).strip() + } + payload = { + "command": list(command), + "environment": frozen, + } + canonical = json.dumps(payload, sort_keys=True, separators=(",", ":")) + payload["sha256"] = hashlib.sha256(canonical.encode("utf-8")).hexdigest() + return payload + + +def build_case_environment( + base_environment: Mapping[str, str], + case: RocmAttentionAblationCase, + *, + case_dir: Path, +) -> dict[str, str]: + """Materialize one worker environment without mutating the parent process.""" + + environment = {str(key): str(value) for key, value in base_environment.items()} + readback_dir = case_dir / "readbacks" + environment.update( + { + "RL_KERNEL_ATTENTION_CASE": case.case_id, + "RL_KERNEL_FFN_CASE": "P/P", + "RL_KERNEL_LOGP_CASE": "P/P", + "RL_KERNEL_VLLM_INTEGRATION": "1", + "RL_KERNEL_READBACK_DIR": str(readback_dir), + "RL_KERNEL_ROUTE_REPORT": "1", + "RL_KERNEL_PLATFORM": "rocm", + "RL_KERNEL_ROCM_STRICT_ATTENTION": "1", + "VLLM_ATTENTION_BACKEND": "ROCM_AITER_FA", + "RL_KERNEL_ABLATION_CASE": case.case_id, + "RL_KERNEL_ABLATION_OUTPUT_DIR": str(case_dir), + "NUM_ROLLOUT": "1", + } + ) + return environment + + +def load_runtime_readbacks(directory: Path) -> list[dict[str, Any]]: + """Load framework-owned evidence and reject malformed artifacts.""" + + payloads: list[dict[str, Any]] = [] + if not directory.is_dir(): + return payloads + for path in sorted(directory.glob("*.json")): + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ValueError(f"invalid runtime readback {path}: {exc}") from exc + if not isinstance(value, dict): + raise ValueError(f"runtime readback must contain an object: {path}") + value["_source_path"] = str(path) + payloads.append(value) + return payloads + + +def _nested_strings(value: Any, key: str) -> set[str]: + found: set[str] = set() + if isinstance(value, Mapping): + direct = value.get(key) + if isinstance(direct, str) and direct.strip(): + found.add(direct.strip()) + for item in value.values(): + found.update(_nested_strings(item, key)) + elif isinstance(value, (list, tuple)): + for item in value: + found.update(_nested_strings(item, key)) + return found + + +def _contains_truthy_fallback(value: Any) -> bool: + if isinstance(value, Mapping): + for key, item in value.items(): + if str(key).strip().lower() in { + "fallback", + "fallback_used", + "used_fallback", + "split_kv_fallback", + } and item not in (False, None, "", 0): + return True + if _contains_truthy_fallback(item): + return True + elif isinstance(value, (list, tuple)): + return any(_contains_truthy_fallback(item) for item in value) + return False + + +def _contains_triton(value: Any) -> bool: + if isinstance(value, str): + return "triton" in value.lower() + if isinstance(value, Mapping): + for key, item in value.items(): + if str(key).strip().lower() == "triton_used" and item is True: + return True + if _contains_triton(item): + return True + elif isinstance(value, (list, tuple)): + return any(_contains_triton(item) for item in value) + return False + + +def _attention_plan_case(payload: Mapping[str, Any]) -> str | None: + plan = payload.get("plan") + cases = plan.get("cases") if isinstance(plan, Mapping) else None + attention = cases.get("attention") if isinstance(cases, Mapping) else None + value = attention.get("case_id") if isinstance(attention, Mapping) else None + return str(value) if isinstance(value, str) else None + + +def _validate_route( + case: RocmAttentionAblationCase, + payloads: Sequence[Mapping[str, Any]], + *, + side: str, +) -> tuple[FrameworkRouteEvidence | None, list[str]]: + framework, target = _FRAMEWORK_TARGETS[side] + expected = case.implementation_for(target) + label = f"{framework}/{target}" + matching = [ + payload + for payload in payloads + if payload.get("framework") == framework and payload.get("target") == target + ] + if not matching: + return None, [f"missing {label} runtime readback"] + + errors: list[str] = [] + records: list[Mapping[str, Any]] = [] + for payload in matching: + if _attention_plan_case(payload) != case.case_id: + errors.append(f"{label} readback used a different Attention case") + if payload.get("fallbacks"): + errors.append(f"{label} recorded fallback: {payload['fallbacks']}") + operators = payload.get("operators") + record = operators.get("attention") if isinstance(operators, Mapping) else None + if isinstance(record, Mapping): + try: + call_count = int(record.get("call_count", 0)) + except (TypeError, ValueError): + errors.append(f"{label} Attention has an invalid call count") + continue + if call_count > 0: + records.append(record) + if not records: + errors.append(f"{label} Attention had zero executed calls") + return None, errors + + implementations = {str(record.get("implementation", "")) for record in records} + if implementations != {expected.value}: + errors.append( + f"{label} implementation mismatch: expected {expected.value}, " + f"observed {sorted(implementations)}" + ) + backend_ids = tuple(sorted({str(record.get("backend_id", "")) for record in records})) + provenance = [record.get("provenance", {}) for record in records] + runtime_platforms = tuple( + sorted( + set().union( + *(_nested_strings(item, "runtime_platform") for item in provenance) + ) + ) + ) + actual_backends = tuple( + sorted( + set().union(*(_nested_strings(item, "actual_backend") for item in provenance)) + ) + ) + strict_core_ids = set().union( + *(_nested_strings(item, "strict_core_id") for item in provenance) + ) + strict_schedules = set().union( + *(_nested_strings(item, "strict_schedule") for item in provenance) + ) + + if any(_contains_truthy_fallback(item) for item in provenance): + errors.append(f"{label} Attention provenance contains a fallback") + if expected is Implementation.PRODUCTION: + native_backend = f"{framework}.production.attention" + if any(backend != native_backend for backend in backend_ids): + errors.append( + f"{label} did not execute framework-native Attention: {backend_ids}" + ) + else: + if any(not backend.startswith("rlkernel.attention.") for backend in backend_ids): + errors.append(f"{label} did not execute the RL-Kernel Attention wrapper") + if runtime_platforms != ("rocm",): + errors.append(f"{label} did not prove ROCm execution: {runtime_platforms}") + if STRICT_ROCM_ATTENTION_RUNTIME not in actual_backends: + errors.append( + f"{label} did not prove strict AITER/CK runtime " + f"{STRICT_ROCM_ATTENTION_RUNTIME!r}" + ) + if STRICT_ROCM_ATTENTION_CORE not in strict_core_ids: + errors.append( + f"{label} did not prove strict AITER/CK core " + f"{STRICT_ROCM_ATTENTION_CORE!r}" + ) + if STRICT_ROCM_ATTENTION_SCHEDULE not in strict_schedules: + errors.append( + f"{label} did not prove fixed no-Split-KV schedule " + f"{STRICT_ROCM_ATTENTION_SCHEDULE!r}" + ) + if any(_contains_triton(item) for item in provenance): + errors.append(f"{label} strict ROCm Attention used Triton") + + evidence = FrameworkRouteEvidence( + framework=framework, + target=target, + implementation=expected.value, + backend_ids=backend_ids, + call_count=sum(int(record.get("call_count", 0)) for record in records), + readback_count=len(matching), + runtime_platforms=runtime_platforms, + actual_backends=actual_backends, + ) + return evidence, errors + + +def validate_case_readbacks( + case: RocmAttentionAblationCase, + payloads: Sequence[Mapping[str, Any]], +) -> tuple[dict[str, FrameworkRouteEvidence], tuple[str, ...]]: + """Require executed and correctly routed training and rollout evidence.""" + + routes: dict[str, FrameworkRouteEvidence] = {} + errors: list[str] = [] + for side in _FRAMEWORK_TARGETS: + route, route_errors = _validate_route(case, payloads, side=side) + if route is not None: + routes[side] = route + errors.extend(route_errors) + return routes, tuple(errors) + + +def run_rocm_attention_ablation( + command: Sequence[str], + *, + output_dir: Path, + base_environment: Mapping[str, str] | None = None, + case_ids: Iterable[str] = ROCM_ATTENTION_CASE_IDS, + execute: bool = False, +) -> tuple[RocmAblationCaseResult, ...]: + """Execute each case in a fresh process and validate its real readbacks.""" + + matrix = rocm_attention_ablation_matrix(case_ids) + environment = dict(os.environ if base_environment is None else base_environment) + identity = replay_identity(command, environment) + if execute: + validate_replay_environment(environment) + occupied = [output_dir / case.slug for case in matrix if (output_dir / case.slug).exists()] + if occupied: + joined = ", ".join(str(path) for path in occupied) + raise FileExistsError( + "refusing to mix ROCm ablation evidence with existing case directories: " + + joined + ) + validate_rocm_host() + output_dir.mkdir(parents=True, exist_ok=True) + + results: list[RocmAblationCaseResult] = [] + for case in matrix: + case_dir = output_dir / case.slug + readback_dir = case_dir / "readbacks" + log_path = case_dir / "run.log" + case_environment = build_case_environment(environment, case, case_dir=case_dir) + if not execute: + results.append( + RocmAblationCaseResult( + case_id=case.case_id, + status="not_run", + returncode=None, + log_path=log_path, + readback_dir=readback_dir, + ) + ) + continue + + case_dir.mkdir(parents=True, exist_ok=False) + readback_dir.mkdir(parents=True, exist_ok=False) + with log_path.open("w", encoding="utf-8") as log_handle: + process = subprocess.run( + list(command), + env=case_environment, + stdout=log_handle, + stderr=subprocess.STDOUT, + check=False, + ) + try: + payloads = load_runtime_readbacks(readback_dir) + routes, validation_errors = validate_case_readbacks(case, payloads) + except ValueError as exc: + routes = {} + validation_errors = (str(exc),) + errors = list(validation_errors) + if process.returncode != 0: + errors.insert(0, f"orchestration command exited with {process.returncode}") + results.append( + RocmAblationCaseResult( + case_id=case.case_id, + status="passed" if not errors else "failed", + returncode=process.returncode, + log_path=log_path, + readback_dir=readback_dir, + routes=routes, + errors=tuple(errors), + ) + ) + + write_markdown_summary( + output_dir / "summary.md", + results, + replay=identity, + executed=execute, + ) + return tuple(results) + + +def write_markdown_summary( + path: Path, + results: Sequence[RocmAblationCaseResult], + *, + replay: Mapping[str, Any], + executed: bool, +) -> None: + """Write a compact human-readable report; raw JSON remains worker evidence.""" + + lines = [ + "# ROCm rollout/training Attention ablation", + "", + f"- Schema: `{ROCM_ABLATION_SCHEMA_VERSION}`", + f"- Replay identity: `{replay['sha256']}`", + f"- Executed: `{'yes' if executed else 'no'}`", + "- FFN / Logp: fixed at `P/P`", + "", + "| Attention | Megatron training | vLLM rollout | Calls | Result |", + "|---|---|---|---:|---|", + ] + for result in results: + training = result.routes.get("training") + rollout = result.routes.get("rollout") + calls = sum(route.call_count for route in result.routes.values()) + lines.append( + "| {case} | {training} | {rollout} | {calls} | {status} |".format( + case=result.case_id, + training="—" if training is None else training.implementation, + rollout="—" if rollout is None else rollout.implementation, + calls=calls, + status=result.status, + ) + ) + failures = [result for result in results if result.errors] + if failures: + lines.extend(["", "## Failures", ""]) + for result in failures: + lines.append(f"- `{result.case_id}`: {'; '.join(result.errors)}") + lines.extend( + [ + "", + "The report is derived from Megatron and vLLM runtime readbacks. A zero-exit", + "command with missing route evidence is reported as failed.", + "", + ] + ) + path.write_text("\n".join(lines), encoding="utf-8") + + +__all__ = [ + "FROZEN_REPLAY_ENV", + "REQUIRED_REPLAY_ENV", + "ROCM_ABLATION_SCHEMA_VERSION", + "ROCM_ATTENTION_CASE_IDS", + "STRICT_ROCM_ATTENTION_CORE", + "STRICT_ROCM_ATTENTION_RUNTIME", + "STRICT_ROCM_ATTENTION_SCHEDULE", + "RocmAblationCaseResult", + "RocmAttentionAblationCase", + "build_case_environment", + "load_runtime_readbacks", + "replay_identity", + "rocm_attention_ablation_matrix", + "run_rocm_attention_ablation", + "validate_case_readbacks", + "validate_replay_environment", + "validate_rocm_host", + "write_markdown_summary", +] diff --git a/rl_engine/integrations/runtime.py b/rl_engine/integrations/runtime.py index db651bce..ae65a2f5 100644 --- a/rl_engine/integrations/runtime.py +++ b/rl_engine/integrations/runtime.py @@ -251,8 +251,8 @@ def assert_strict_ready(self) -> None: wrong_backends.append(f"{module}={operator['backend_id']}") elif _contains_triton(operator): wrong_backends.append(f"{module}=triton") - elif _runtime_platform(operator.get("provenance")) != "cuda": - wrong_backends.append(f"{module}=non-cuda") + elif _runtime_platform(operator.get("provenance")) not in {"cuda", "rocm"}: + wrong_backends.append(f"{module}=non-cuda/rocm") failures: list[str] = [] if missing_hooks: failures.append("missing hooks: " + ", ".join(missing_hooks)) diff --git a/rl_engine/integrations/vllm_runtime.py b/rl_engine/integrations/vllm_runtime.py index 6351f0ab..37e2c521 100644 --- a/rl_engine/integrations/vllm_runtime.py +++ b/rl_engine/integrations/vllm_runtime.py @@ -105,7 +105,9 @@ def configure_vllm_environment(plan: IntegrationPlan, *, readback_dir: str | Non os.environ["RL_KERNEL_VLLM_INTEGRATION"] = "1" configure_integration_environment(plan, readback_dir=readback_dir) if plan.implementation_for("attention", "rollout") is Implementation.RL_KERNEL: - os.environ["VLLM_ATTENTION_BACKEND"] = "FLASH_ATTN" + os.environ["VLLM_ATTENTION_BACKEND"] = ( + "ROCM_AITER_FA" if torch.version.hip is not None else "FLASH_ATTN" + ) if plan.implementation_for("logp", "rollout") is Implementation.RL_KERNEL: real_vocab = os.getenv("RL_KERNEL_VLLM_REAL_VOCAB_SIZE", "").strip() padded_vocab = os.getenv("RL_KERNEL_VLLM_PADDED_VOCAB_SIZE", "").strip() @@ -602,19 +604,39 @@ def native(_sampler: Any, *call_args: Any, **call_kwargs: Any) -> Any: def _register_attention_backend(integration: VllmIntegration) -> None: global _RLK_ATTENTION_BACKEND, _RLK_ATTENTION_BUILDER, _RLK_ATTENTION_IMPL - from vllm.v1.attention.backends.flash_attn import ( - FlashAttentionBackend, - FlashAttentionImpl, - FlashAttentionMetadataBuilder, - ) from vllm.v1.attention.backends.registry import AttentionBackendEnum, register_backend + if torch.version.hip is not None: + from vllm.v1.attention.backends.rocm_aiter_fa import ( + AiterFlashAttentionBackend as PlatformAttentionBackend, + ) + from vllm.v1.attention.backends.rocm_aiter_fa import ( + AiterFlashAttentionImpl as PlatformAttentionImpl, + ) + from vllm.v1.attention.backends.rocm_aiter_fa import ( + AiterFlashAttentionMetadataBuilder as PlatformAttentionMetadataBuilder, + ) + + selected_backend = AttentionBackendEnum.ROCM_AITER_FA + else: + from vllm.v1.attention.backends.flash_attn import ( + FlashAttentionBackend as PlatformAttentionBackend, + ) + from vllm.v1.attention.backends.flash_attn import ( + FlashAttentionImpl as PlatformAttentionImpl, + ) + from vllm.v1.attention.backends.flash_attn import ( + FlashAttentionMetadataBuilder as PlatformAttentionMetadataBuilder, + ) + + selected_backend = AttentionBackendEnum.FLASH_ATTN + operator: VllmAttentionOperator | None = None if integration.plan.implementation_for("attention", "rollout") is Implementation.RL_KERNEL: operator = VllmAttentionOperator() integration.install_operator("attention", operator) - class RlKernelFlashAttentionImpl(FlashAttentionImpl): + class RlKernelAttentionImpl(PlatformAttentionImpl): def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) if operator is not None: @@ -626,42 +648,41 @@ def forward(self, *args: Any, **kwargs: Any) -> Any: raise RuntimeError("vLLM Attention executed without its installed integration") def native(_impl: Any, *call_args: Any, **call_kwargs: Any) -> Any: - return FlashAttentionImpl.forward(self, *call_args, **call_kwargs) + return PlatformAttentionImpl.forward(self, *call_args, **call_kwargs) return integration.execute("attention", native, self, *args, **kwargs) - class RlKernelFlashAttentionMetadataBuilder(FlashAttentionMetadataBuilder): + class RlKernelAttentionMetadataBuilder(PlatformAttentionMetadataBuilder): pass - class RlKernelFlashAttentionBackend(FlashAttentionBackend): + class RlKernelAttentionBackend(PlatformAttentionBackend): @staticmethod def get_impl_cls() -> type[Any]: - return RlKernelFlashAttentionImpl + return RlKernelAttentionImpl @staticmethod def get_builder_cls() -> type[Any]: - return RlKernelFlashAttentionMetadataBuilder - - RlKernelFlashAttentionImpl.__module__ = __name__ - RlKernelFlashAttentionImpl.__qualname__ = "RlKernelFlashAttentionImpl" - RlKernelFlashAttentionMetadataBuilder.__module__ = __name__ - RlKernelFlashAttentionMetadataBuilder.__qualname__ = "RlKernelFlashAttentionMetadataBuilder" - RlKernelFlashAttentionBackend.__module__ = __name__ - RlKernelFlashAttentionBackend.__qualname__ = "RlKernelFlashAttentionBackend" - _RLK_ATTENTION_IMPL = RlKernelFlashAttentionImpl - _RLK_ATTENTION_BUILDER = RlKernelFlashAttentionMetadataBuilder - _RLK_ATTENTION_BACKEND = RlKernelFlashAttentionBackend - globals()["RlKernelFlashAttentionImpl"] = RlKernelFlashAttentionImpl - globals()["RlKernelFlashAttentionMetadataBuilder"] = RlKernelFlashAttentionMetadataBuilder - globals()["RlKernelFlashAttentionBackend"] = RlKernelFlashAttentionBackend - # vLLM 0.27 selects FLASH_ATTN for Qwen on CUDA before custom third-party - # names are considered. Override the selected enum in-place so the launcher - # does not need to edit vLLM source or pass version-specific CLI flags. + return RlKernelAttentionMetadataBuilder + + RlKernelAttentionImpl.__module__ = __name__ + RlKernelAttentionImpl.__qualname__ = "RlKernelAttentionImpl" + RlKernelAttentionMetadataBuilder.__module__ = __name__ + RlKernelAttentionMetadataBuilder.__qualname__ = "RlKernelAttentionMetadataBuilder" + RlKernelAttentionBackend.__module__ = __name__ + RlKernelAttentionBackend.__qualname__ = "RlKernelAttentionBackend" + _RLK_ATTENTION_IMPL = RlKernelAttentionImpl + _RLK_ATTENTION_BUILDER = RlKernelAttentionMetadataBuilder + _RLK_ATTENTION_BACKEND = RlKernelAttentionBackend + globals()["RlKernelAttentionImpl"] = RlKernelAttentionImpl + globals()["RlKernelAttentionMetadataBuilder"] = RlKernelAttentionMetadataBuilder + globals()["RlKernelAttentionBackend"] = RlKernelAttentionBackend + # Override the platform-selected enum so vLLM keeps its native metadata and + # cache update path while RL-Kernel owns the attention arithmetic call. register_backend( - AttentionBackendEnum.FLASH_ATTN, - f"{__name__}.RlKernelFlashAttentionBackend", + selected_backend, + f"{__name__}.RlKernelAttentionBackend", ) - integration.record_installed_hook("attention", f"{__name__}.RlKernelFlashAttentionBackend") + integration.record_installed_hook("attention", f"{__name__}.RlKernelAttentionBackend") def install_vllm_integration(plan: IntegrationPlan) -> VllmIntegration: diff --git a/rl_engine/kernels/ops/pytorch/attention/ablation.py b/rl_engine/kernels/ops/pytorch/attention/ablation.py index d594c164..72342686 100644 --- a/rl_engine/kernels/ops/pytorch/attention/ablation.py +++ b/rl_engine/kernels/ops/pytorch/attention/ablation.py @@ -163,27 +163,74 @@ def __init__( # fallback to the PyTorch reference when AG/RS is required. self.cp_backend = cp_backend self.communication_backend = communication_backend.strip() - self._cuda_runtime_group: Any = None - self._cuda_runtime_bound = False - - def bind_cuda_runtime(self, *, process_group: Any = None) -> Any: - """Bind the shared production CUDA core/transport once per process.""" - - if self._cuda_runtime_bound: - if process_group is not self._cuda_runtime_group: - raise AttentionContractError( - "Attention CUDA runtime is already bound to another process group" - ) - return self.core - from rl_engine.kernels.ops.cuda.attention.strict_runtime import StrictCUDAAttentionRuntime - - runtime = StrictCUDAAttentionRuntime(process_group=process_group) - self.core = runtime - self.cp_backend = runtime - self._cuda_runtime_group = process_group - self._cuda_runtime_bound = True - return runtime - + self._runtime_group: Any = None + self._runtime_platform: str | None = None + + def bind_accelerator_runtime( + self, + tensor: Tensor, + *, + process_group: Any = None, + ) -> Any: + """Bind the strict runtime matching a CUDA or ROCm tensor. + + PyTorch intentionally exposes AMD GPUs through the ``cuda`` device + type. The HIP build marker, rather than ``tensor.device.type``, is + therefore the platform discriminator used by every framework bridge. + """ + + if tensor.device.type != "cuda": + raise AttentionContractError( + "production Attention requires a CUDA or ROCm accelerator tensor" + ) + platform = "rocm" if torch.version.hip is not None else "cuda" + return self.bind_runtime(platform=platform, process_group=process_group) + + def bind_runtime(self, *, platform: str, process_group: Any = None) -> Any: + """Bind one platform runtime and reject process-local identity drift.""" + + normalized = platform.strip().lower() + if normalized not in {"cuda", "rocm"}: + raise AttentionContractError("Attention runtime platform must be 'cuda' or 'rocm'") + if self._runtime_platform is not None: + if normalized != self._runtime_platform: + raise AttentionContractError( + "Attention runtime is already bound to another accelerator platform" + ) + if process_group is not self._runtime_group: + raise AttentionContractError( + "Attention runtime is already bound to another process group" + ) + return self.core + + if normalized == "rocm": + from rl_engine.kernels.ops.rocm.attention.strict_runtime import ( + StrictRocmAttentionRuntime, + ) + + runtime = StrictRocmAttentionRuntime(process_group=process_group) + else: + from rl_engine.kernels.ops.cuda.attention.strict_runtime import ( + StrictCUDAAttentionRuntime, + ) + + runtime = StrictCUDAAttentionRuntime(process_group=process_group) + self.core = runtime + self.cp_backend = runtime + self._runtime_group = process_group + self._runtime_platform = normalized + return runtime + + def bind_cuda_runtime(self, *, process_group: Any = None) -> Any: + """Compatibility entry point for CUDA-only callers.""" + + return self.bind_runtime(platform="cuda", process_group=process_group) + + def bind_rocm_runtime(self, *, process_group: Any = None) -> Any: + """Bind the AITER/CK + RCCL production runtime once per process.""" + + return self.bind_runtime(platform="rocm", process_group=process_group) + def __call__( self, q: Tensor, @@ -395,13 +442,17 @@ def _select_backend( ) return self.cp_backend, _callable_backend_id(self.cp_backend) return self._reference_backend(), REFERENCE_BACKEND_ID - if q.device.type != "cuda" or torch.version.hip is not None: - return self._reference_backend(), REFERENCE_BACKEND_ID - if self.core is None: - from rl_engine.kernels.ops.cuda.attention.deterministic_attn import ( - DeterministicAttentionOp, - ) - + if q.device.type != "cuda": + return self._reference_backend(), REFERENCE_BACKEND_ID + if torch.version.hip is not None: + if self.core is None: + self.bind_rocm_runtime() + return self.core, _callable_backend_id(self.core) + if self.core is None: + from rl_engine.kernels.ops.cuda.attention.deterministic_attn import ( + DeterministicAttentionOp, + ) + self.core = DeterministicAttentionOp() return self.core, BACKEND_ID diff --git a/rl_engine/kernels/registry.py b/rl_engine/kernels/registry.py index e60edb27..23286d94 100644 --- a/rl_engine/kernels/registry.py +++ b/rl_engine/kernels/registry.py @@ -264,7 +264,14 @@ def _default_semantic_descriptors() -> tuple[OperatorBackendDescriptor, ...]: "deterministic": True, "split_kv": "contract_bound", "reduction_order": "global_block_index", - "strict_schedule": "single_batch_single_query_global_kv_blocks", + "strict_schedules": { + "cuda": "single_batch_fa4_kernel_sm100_no_splitkv", + "rocm": "single_batch_aiter_ck_dense_mha_no_splitkv", + }, + "platform_runtimes": { + "cuda": "rlkernel.cuda.attention.fa4_ag_rs.v1", + "rocm": "rlkernel.rocm.attention.aiter_ck_ag_rs.v1", + }, "strict_observable": True, }, lifecycle=OperatorLifecycle.ENGINE_CONSTRUCTION, @@ -272,7 +279,7 @@ def _default_semantic_descriptors() -> tuple[OperatorBackendDescriptor, ...]: "rl_engine.kernels.ops.pytorch.attention.ablation.AttentionAblationOp" ), fallback_policy=OperatorFallbackPolicy.ERROR, - version_or_build_fingerprint="AttentionAblationOp-bitwise-v2", + version_or_build_fingerprint="AttentionAblationOp-platform-runtime-v3", ), OperatorBackendDescriptor( semantic_op="attention", diff --git a/tests/test_attention_ablation.py b/tests/test_attention_ablation.py index 3edb0a43..ce9ac462 100644 --- a/tests/test_attention_ablation.py +++ b/tests/test_attention_ablation.py @@ -139,6 +139,45 @@ def forward_with_lse(self, q, k, v, *, causal, scale): assert torch.equal(result.out, q) +def test_attention_wrapper_binds_rocm_runtime_once(monkeypatch): + import rl_engine.kernels.ops.rocm.attention.strict_runtime as rocm_runtime + + calls = [] + + class Runtime: + def __init__(self, *, process_group=None): + calls.append(process_group) + + monkeypatch.setattr(rocm_runtime, "StrictRocmAttentionRuntime", Runtime) + group = object() + operator = AttentionAblationOp() + + first = operator.bind_rocm_runtime(process_group=group) + second = operator.bind_runtime(platform="rocm", process_group=group) + + assert first is second + assert calls == [group] + assert operator.core is first and operator.cp_backend is first + + +def test_attention_wrapper_rejects_runtime_platform_or_group_drift(monkeypatch): + import rl_engine.kernels.ops.rocm.attention.strict_runtime as rocm_runtime + + class Runtime: + def __init__(self, *, process_group=None): + self.process_group = process_group + + monkeypatch.setattr(rocm_runtime, "StrictRocmAttentionRuntime", Runtime) + group = object() + operator = AttentionAblationOp() + operator.bind_rocm_runtime(process_group=group) + + with pytest.raises(AttentionContractError, match="platform"): + operator.bind_cuda_runtime(process_group=group) + with pytest.raises(AttentionContractError, match="process group"): + operator.bind_rocm_runtime(process_group=object()) + + def test_wrapper_owned_deterministic_core_does_not_require_external_provenance(): q, k, v = _qkv() diff --git a/tests/test_framework_runtime_adapters.py b/tests/test_framework_runtime_adapters.py index 27d54eed..5e140b2a 100644 --- a/tests/test_framework_runtime_adapters.py +++ b/tests/test_framework_runtime_adapters.py @@ -5,10 +5,11 @@ import ast import os +import sys from collections import namedtuple from dataclasses import dataclass from pathlib import Path -from types import SimpleNamespace +from types import ModuleType, SimpleNamespace import pytest import torch @@ -22,6 +23,7 @@ from rl_engine.integrations.framework_operators import ( MegatronAttentionOperator, SemanticOperatorHandle, + VllmAttentionOperator, VllmLogpOperator, _megatron_zigzag_layout, _packed_local_sequence_layout, @@ -35,6 +37,7 @@ from rl_engine.integrations.state import clear_active_integration from rl_engine.integrations.vllm_runtime import ( _patch_qwen3_strict_model, + _register_attention_backend, configure_vllm_environment, ) from rl_engine.kernels.attention_contract import ( @@ -129,6 +132,79 @@ def test_vllm_rlkernel_attention_overrides_selected_flash_attn_backend( assert os.environ["VLLM_ATTENTION_BACKEND"] == "FLASH_ATTN" +def test_vllm_rocm_attention_selects_aiter_metadata_backend(monkeypatch): + monkeypatch.delenv("VLLM_ATTENTION_BACKEND", raising=False) + monkeypatch.setattr(torch.version, "hip", "7.1") + plan = IntegrationPlan.from_case_ids(attention="P/R") + + configure_vllm_environment(plan) + + assert os.environ["VLLM_ATTENTION_BACKEND"] == "ROCM_AITER_FA" + + +def test_vllm_rocm_registration_wraps_aiter_backend(monkeypatch): + selected = [] + + class BackendEnum: + ROCM_AITER_FA = "ROCM_AITER_FA" + FLASH_ATTN = "FLASH_ATTN" + + class AiterImpl: + def __init__(self, *args, **kwargs): + del args, kwargs + + def forward(self, *args, **kwargs): + return args, kwargs + + class AiterBuilder: + pass + + class AiterBackend: + pass + + registry = ModuleType("vllm.v1.attention.backends.registry") + registry.AttentionBackendEnum = BackendEnum + registry.register_backend = lambda backend, path: selected.append((backend, path)) + aiter = ModuleType("vllm.v1.attention.backends.rocm_aiter_fa") + aiter.AiterFlashAttentionBackend = AiterBackend + aiter.AiterFlashAttentionImpl = AiterImpl + aiter.AiterFlashAttentionMetadataBuilder = AiterBuilder + monkeypatch.setitem(sys.modules, registry.__name__, registry) + monkeypatch.setitem(sys.modules, aiter.__name__, aiter) + monkeypatch.setattr(torch.version, "hip", "7.1") + + plan = IntegrationPlan.from_case_ids(attention="P/R") + + class Integration: + def __init__(self): + self.plan = plan + self.installed = {} + self.hooks = [] + + def install_operator(self, module, operator): + self.installed[module] = operator + + def record_installed_hook(self, module, hook): + self.hooks.append((module, hook)) + + integration = Integration() + _register_attention_backend(integration) + + assert selected == [ + ( + BackendEnum.ROCM_AITER_FA, + "rl_engine.integrations.vllm_runtime.RlKernelAttentionBackend", + ) + ] + assert "attention" in integration.installed + assert integration.hooks == [ + ( + "attention", + "rl_engine.integrations.vllm_runtime.RlKernelAttentionBackend", + ) + ] + + def test_megatron_install_is_idempotent_in_one_actor(): class Attention: def forward(self, value): @@ -187,7 +263,8 @@ def test_megatron_packed_attention_runs_each_sequence_in_thd_order(monkeypatch): calls: list[dict[str, object]] = [] class Operator: - def bind_cuda_runtime(self, *, process_group=None): + def bind_accelerator_runtime(self, tensor, *, process_group=None): + assert tensor is query assert process_group == "cp-group" def __call__(self, q, k, v, **kwargs): @@ -219,7 +296,11 @@ def get(self, tensor, *, topology): get_context_parallel_group=lambda: "cp-group", ) monkeypatch.setattr(framework_operators, "_megatron_parallel_state", lambda: parallel_state) - monkeypatch.setattr(framework_operators, "_require_nvidia_cuda", lambda tensor, module: None) + monkeypatch.setattr( + framework_operators, + "_require_attention_accelerator", + lambda tensor: "cuda", + ) packed = SimpleNamespace( qkv_format="thd", cu_seqlens_q=torch.tensor([0, 8, 16], dtype=torch.int32), @@ -246,7 +327,117 @@ def get(self, tensor, *, topology): assert calls[0]["query_position_ids"].tolist() == [ [0, 1, 6, 7], [0, 1, 6, 7], - ] + ] + + +def test_megatron_attention_binds_rocm_core_and_schedule(monkeypatch): + calls = [] + + class Operator: + def bind_accelerator_runtime(self, tensor, *, process_group=None): + calls.append((tensor, process_group)) + + def __call__(self, q, k, v, **kwargs): + del k, v + calls.append(kwargs) + return SimpleNamespace( + out=q.clone(), + provenance={ + "actual_backend": "rlkernel.rocm.attention.aiter_ck_ag_rs.v1", + "fallback": False, + }, + ) + + operator = Operator() + + class Handle: + provenance = {} + + def get(self, tensor, *, topology): + assert topology["context_parallel_size"] == 1 + return operator + + parallel_state = SimpleNamespace( + get_context_parallel_world_size=lambda: 1, + get_context_parallel_rank=lambda: 0, + get_tensor_model_parallel_world_size=lambda: 2, + get_tensor_model_parallel_rank=lambda: 0, + ) + monkeypatch.setattr(framework_operators, "_megatron_parallel_state", lambda: parallel_state) + monkeypatch.setattr( + framework_operators, + "_require_attention_accelerator", + lambda tensor: "rocm", + ) + query = torch.zeros(4, 1, 2, 8, dtype=torch.bfloat16) + key = torch.zeros(4, 1, 1, 8, dtype=torch.bfloat16) + adapter = MegatronAttentionOperator(handle=Handle()) + + output = adapter(SimpleNamespace(softmax_scale=0.25), query, key, key, None) + + assert output.shape == (4, 1, 16) + assert calls[0] == (query, None) + config = calls[1]["config"] + assert config.strict_core_id == "rlkernel.attention.rocm.aiter_ck_dense_mha.v1" + assert config.strict_schedule == "single_batch_aiter_ck_dense_mha_no_splitkv" + assert calls[1]["communication_backend"] == "none" + assert adapter.provenance["execution"]["runtime_platform"] == "rocm" + + +def test_vllm_attention_routes_paged_cache_to_rocm_runtime(monkeypatch): + runtime_calls = [] + + class Runtime: + def forward_paged_with_lse(self, q, k, v, **kwargs): + runtime_calls.append((q, k, v, kwargs)) + return SimpleNamespace( + out=q.clone(), + lse=torch.zeros(q.shape[:-1], dtype=torch.float32), + provenance={ + "actual_backend": "rlkernel.rocm.attention.aiter_ck_ag_rs.v1", + "fallback": False, + }, + ) + + class Operator: + def bind_accelerator_runtime(self, tensor, *, process_group=None): + assert process_group is None + return Runtime() + + class Handle: + provenance = {} + + def get(self, tensor, *, topology): + assert topology["context_parallel_size"] == 1 + return Operator() + + monkeypatch.setattr( + framework_operators, + "_require_attention_accelerator", + lambda tensor: "rocm", + ) + query = torch.zeros(1, 2, 8, dtype=torch.bfloat16) + kv_cache = torch.zeros(2, 1, 4, 16, dtype=torch.bfloat16) + metadata = SimpleNamespace( + block_table=torch.tensor([[0]], dtype=torch.int32), + query_start_loc=torch.tensor([0, 1], dtype=torch.int32), + seq_lens=torch.tensor([1], dtype=torch.int32), + num_actual_tokens=1, + max_seq_len=1, + ) + impl = SimpleNamespace(head_size=8, num_heads=2, num_kv_heads=1, scale=8**-0.5) + adapter = VllmAttentionOperator(handle=Handle()) + + output = adapter(impl, object(), query, query, query, kv_cache, metadata) + + assert output.shape == (1, 16) + assert len(runtime_calls) == 1 + assert runtime_calls[0][0].shape == (1, 2, 1, 8) + assert adapter.provenance["execution"]["runtime_platform"] == "rocm" + assert ( + adapter.provenance["execution"]["materialization"] + == "logical_paged_kv_to_aiter_ck_dense" + ) def test_vllm_current_flash_attention_kv_cache_layout_is_materialized(): @@ -260,6 +451,49 @@ def test_vllm_current_flash_attention_kv_cache_layout_is_materialized(): assert torch.equal(value, cache.transpose(1, 2)[..., 5:]) +def test_vllm_rocm_kv_cache_pair_axis_is_materialized(): + cache = torch.arange(3 * 2 * 4 * 1 * 5).reshape(3, 2, 4, 1, 5) + + key, value = _vllm_kv_cache_views( + cache, + head_size=5, + num_kv_heads=1, + platform="rocm", + ) + + assert key.shape == (3, 4, 1, 5) + assert value.shape == (3, 4, 1, 5) + assert torch.equal(key, cache[:, 0]) + assert torch.equal(value, cache[:, 1]) + + +def test_vllm_rocm_lhbnc_kv_cache_is_normalized_to_block_major(): + cache = torch.arange(2 * 1 * 3 * 4 * 5).reshape(2, 1, 3, 4, 5) + + key, value = _vllm_kv_cache_views(cache, head_size=5, num_kv_heads=1) + + assert key.shape == (3, 4, 1, 5) + assert value.shape == (3, 4, 1, 5) + assert torch.equal(key, cache[0].permute(1, 2, 0, 3)) + assert torch.equal(value, cache[1].permute(1, 2, 0, 3)) + + +def test_vllm_rocm_flattened_kv_cache_is_unpacked(): + cache = torch.arange(3 * 2 * 4 * 10).reshape(3, 2, 4, 10) + + key, value = _vllm_kv_cache_views( + cache, + head_size=5, + num_kv_heads=2, + platform="rocm", + ) + + assert key.shape == (3, 4, 2, 5) + assert value.shape == (3, 4, 2, 5) + assert torch.equal(key.flatten(2), cache[:, 0]) + assert torch.equal(value.flatten(2), cache[:, 1]) + + def test_megatron_strict_attention_projections_install_without_debug_environment( monkeypatch, ): @@ -519,3 +753,25 @@ def test_strict_readback_accepts_cuda_without_triton(): integration.execute("attention", lambda value: value, "x") integration.assert_strict_ready() + + +def test_strict_readback_accepts_rocm_without_triton(): + plan = IntegrationPlan.from_case_ids(attention="R/R") + integration = FrameworkOperatorIntegration( + framework="megatron", + target="training", + plan=plan, + rl_kernel_operators={ + "attention": _ReadbackOperator( + { + "runtime_platform": "rocm", + "actual_backend": "rlkernel.rocm.attention.aiter_ck_ag_rs.v1", + "triton_used": False, + } + ) + }, + ) + integration.record_installed_hook("attention", "test.attention") + integration.execute("attention", lambda value: value, "x") + + integration.assert_strict_ready() diff --git a/tests/test_rocm_attention_ablation_benchmark.py b/tests/test_rocm_attention_ablation_benchmark.py deleted file mode 100644 index 974fe9ac..00000000 --- a/tests/test_rocm_attention_ablation_benchmark.py +++ /dev/null @@ -1,309 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 - -from __future__ import annotations - -import copy -import importlib.util -import json -import math -import sys -from pathlib import Path - -import pytest -import torch - -ROOT = Path(__file__).parents[1] -SCRIPT = ROOT / "benchmarks" / "benchmark_rocm_attention_ablation.py" -CHECKED_IN_RESULT = ROOT / "benchmarks" / "results" / "pr230_rocm_mi300x_ablation" / "results.json" -SPEC = importlib.util.spec_from_file_location("rocm_attention_ablation", SCRIPT) -assert SPEC is not None and SPEC.loader is not None -MODULE = importlib.util.module_from_spec(SPEC) -sys.modules[SPEC.name] = MODULE -SPEC.loader.exec_module(MODULE) - - -def _metric(*, drift: bool, shape: list[int], dtype: str): - return { - "max_abs": 0.5 if drift else 0.0, - "mismatch_count": 1 if drift else 0, - "element_count": math.prod(shape), - "bitwise_equal": not drift, - "left_dtype": dtype, - "right_dtype": dtype, - "shape": shape, - } - - -def _valid_payload(): - shapes = [list(shape) for shape in MODULE.DEFAULT_SHAPES] - cases = [] - for batch, sequence in MODULE.DEFAULT_SHAPES: - for matrix_row in MODULE.attention_debug_matrix()["rows"]: - row_id = matrix_row["id"] - common = { - "row_id": row_id, - "batch": batch, - "sequence": sequence, - "category": matrix_row["category"], - "probe": matrix_row["probe"], - "expected": matrix_row["expected"], - "passed": True, - "realization": MODULE.ROW_REALIZATIONS[row_id], - } - if row_id == "A4": - binding = MODULE._topology_gate(batch=batch, sequence=sequence) - cases.append( - { - **common, - "comparable": False, - "outcome": "rejected", - "gate_implementation": ( - "rl_engine.alignment.cross_config.bind_attention_contracts" - ), - "identity_errors": [issue["field"] for issue in binding["issues"]], - "binding_gate": binding, - "metrics": {name: None for name in MODULE.METRIC_NAMES}, - } - ) - continue - drift = matrix_row["expected"] == "diagnostic" - implementations = MODULE.ROW_IMPLEMENTATIONS[row_id] - metric_shapes = { - "out": [ - batch, - MODULE.QWEN3_Q_HEADS, - 1 if row_id == "C2" else sequence, - MODULE.QWEN3_HEAD_DIM, - ], - "lse": [batch, MODULE.QWEN3_Q_HEADS, 1 if row_id == "C2" else sequence], - "dq": [batch, MODULE.QWEN3_Q_HEADS, sequence, MODULE.QWEN3_HEAD_DIM], - "dk": [batch, MODULE.QWEN3_KV_HEADS, sequence, MODULE.QWEN3_HEAD_DIM], - "dv": [batch, MODULE.QWEN3_KV_HEADS, sequence, MODULE.QWEN3_HEAD_DIM], - } - cases.append( - { - **common, - "comparable": True, - "outcome": "drift_detected" if drift else "matched", - "implementations": { - "baseline": implementations[0], - "candidate": implementations[1], - }, - "metrics": { - name: _metric( - drift=drift, - shape=metric_shapes[name], - dtype="torch.float32" if name == "lse" else "torch.bfloat16", - ) - for name in MODULE.METRIC_NAMES - }, - } - ) - return { - "schema_version": MODULE.RESULT_SCHEMA, - "scope": MODULE.RESULT_SCOPE, - "command": ["python", "benchmarks/benchmark_rocm_attention_ablation.py"], - "source_provenance": { - "revision": "a" * 40, - "tracked_dirty": False, - "tracked_diff_sha256": None, - "script_path": "benchmarks/benchmark_rocm_attention_ablation.py", - "script_sha256": "b" * 64, - "script_matches_head": True, - }, - "matrix_manifest": MODULE.attention_debug_matrix(), - "environment": { - "python": "3.10.0", - "pytorch": "2.12.0+rocm7.0", - "hip_runtime": "7.0", - "device_index": 0, - "device_name": "AMD Instinct MI300X", - "architecture": "gfx942:sramecc+:xnack-", - "gpu_count": 1, - "primary_backend_id": MODULE.ROCM_REFERENCE_BACKEND_ID, - "primary_core_id": MODULE.STRICT_ATTENTION_REFERENCE_CORE_ID, - "primary_schedule": MODULE.STRICT_ATTENTION_SCHEDULE_ID, - "primary_reference_only": True, - "primary_production_ready": False, - "execution_kind": "operator_only_rocm_reference", - }, - "configuration": { - "seed": 230, - "dtype": "bfloat16", - "q_heads": MODULE.QWEN3_Q_HEADS, - "kv_heads": MODULE.QWEN3_KV_HEADS, - "head_dim": MODULE.QWEN3_HEAD_DIM, - "shapes": shapes, - }, - "cases": cases, - "matrix": MODULE._aggregate(cases), - } - - -def test_metric_records_bitwise_and_numerical_drift(): - same = MODULE._metric(torch.tensor([1.0]), torch.tensor([1.0])) - drift = MODULE._metric(torch.tensor([1.0]), torch.tensor([1.5])) - - assert same == { - "max_abs": 0.0, - "mismatch_count": 0, - "element_count": 1, - "bitwise_equal": True, - "left_dtype": "torch.float32", - "right_dtype": "torch.float32", - "shape": [1], - } - assert drift["max_abs"] == 0.5 - assert drift["mismatch_count"] == 1 - assert drift["bitwise_equal"] is False - - -def test_metric_uses_raw_bits_and_requires_matching_dtype(): - signed_zero = MODULE._metric(torch.tensor([0.0]), torch.tensor([-0.0])) - mixed_dtype = MODULE._metric( - torch.tensor([1.0], dtype=torch.bfloat16), - torch.tensor([1.0], dtype=torch.float32), - ) - - assert signed_zero["max_abs"] == 0.0 - assert signed_zero["mismatch_count"] == 1 - assert signed_zero["bitwise_equal"] is False - assert mixed_dtype["mismatch_count"] == 1 - assert mixed_dtype["left_dtype"] != mixed_dtype["right_dtype"] - - -def test_chunk_merge_probe_has_attention_shapes_and_finite_values(): - generator = torch.Generator().manual_seed(230) - q = torch.randn(1, 4, 8, 128, dtype=torch.bfloat16, generator=generator) - k = torch.randn(1, 1, 8, 128, dtype=torch.bfloat16, generator=generator) - v = torch.randn(1, 1, 8, 128, dtype=torch.bfloat16, generator=generator) - dout = torch.randn(q.shape, dtype=torch.bfloat16, generator=generator) - dense = MODULE._evaluate( - q, - k, - v, - dout, - MODULE._dense_attention(accumulator_dtype=torch.float32), - ) - - for order in ("ascending", "descending"): - chunked = MODULE._evaluate(q, k, v, dout, MODULE._chunked_attention(order)) - assert chunked.out.shape == q.shape - assert chunked.lse.shape == q.shape[:-1] - assert all(torch.isfinite(getattr(chunked, name)).all() for name in MODULE.METRIC_NAMES) - maximums = { - name: MODULE._metric(getattr(dense, name), getattr(chunked, name))["max_abs"] - for name in MODULE.METRIC_NAMES - } - assert maximums["lse"] <= 1.0e-5 - assert all(maximums[name] <= 0.015625 for name in ("out", "dq", "dk", "dv")) - - -def test_explicit_accumulator_probe_changes_all_five_metrics(): - generator = torch.Generator().manual_seed(231) - q = torch.randn(1, 4, 4, 128, dtype=torch.bfloat16, generator=generator) - k = torch.randn(1, 1, 4, 128, dtype=torch.bfloat16, generator=generator) - v = torch.randn(1, 1, 4, 128, dtype=torch.bfloat16, generator=generator) - dout = torch.randn(q.shape, dtype=torch.bfloat16, generator=generator) - fp32 = MODULE._evaluate(q, k, v, dout, MODULE._dense_attention(accumulator_dtype=torch.float32)) - bf16 = MODULE._evaluate( - q, k, v, dout, MODULE._dense_attention(accumulator_dtype=torch.bfloat16) - ) - - assert all( - MODULE._metric(getattr(fp32, name), getattr(bf16, name))["mismatch_count"] > 0 - for name in MODULE.METRIC_NAMES - ) - - -def test_topology_probe_uses_binding_gate_and_isolates_ownership_mismatch(): - binding = MODULE._topology_gate(batch=2, sequence=32) - - assert binding["comparable"] is False - assert binding["passed"] is False - assert {issue["code"] for issue in binding["issues"]} == {"TOPOLOGY_MISMATCH"} - assert {issue["field"] for issue in binding["issues"]} == { - "sharding.tp_rank", - "sharding.local_q_head_start", - "sharding.local_kv_head_start", - } - - -def test_payload_validator_accepts_complete_pr230_rocm_evidence(): - payload = _valid_payload() - MODULE.validate_payload(payload) - - -def _invent_a4_issue(payload): - case = payload["cases"][4] - case["binding_gate"]["issues"] = [ - { - "code": "TOPOLOGY_MISMATCH", - "tier": "identical", - "field": "invented", - "rollout": 1, - "training": 0, - } - ] - case["identity_errors"] = ["invented"] - - -@pytest.mark.parametrize( - ("mutate", "message"), - [ - (lambda payload: payload.update(cases=[]), "cover every PR230 row"), - ( - lambda payload: payload["cases"][1]["metrics"]["out"].update(max_abs=float("nan")), - "finite and non-negative", - ), - ( - lambda payload: payload["cases"][4]["binding_gate"].update(comparable=True), - "topology-gate rejection", - ), - (_invent_a4_issue, "topology-gate rejection"), - ( - lambda payload: payload["cases"][0]["metrics"]["out"].update( - shape=[1], element_count=1 - ), - "incompatible dtype or shape", - ), - ( - lambda payload: payload["configuration"]["shapes"].pop(), - "exact eight-shape", - ), - ( - lambda payload: payload["configuration"].pop("seed"), - "configuration.seed", - ), - ( - lambda payload: payload["matrix"][0].update(case_count=0), - "does not reproduce", - ), - ( - lambda payload: payload["environment"].update(architecture="sm_90"), - "gfx942 ROCm", - ), - ( - lambda payload: payload["environment"].pop("device_name"), - "environment.device_name", - ), - ], -) -def test_payload_validator_rejects_incomplete_or_fabricated_evidence(mutate, message): - payload = copy.deepcopy(_valid_payload()) - mutate(payload) - with pytest.raises(ValueError, match=message): - MODULE.validate_payload(payload) - - -def test_repository_provenance_rejects_unbacked_hashes(): - with pytest.raises(ValueError, match="not backed"): - MODULE.validate_repository_provenance(_valid_payload()) - - -def test_checked_in_mi300x_matrix_is_complete_and_source_backed(): - payload = json.loads(CHECKED_IN_RESULT.read_text(encoding="utf-8")) - - MODULE.validate_payload(payload) - MODULE.validate_repository_provenance(payload) - assert all(row["passed"] for row in payload["matrix"]) diff --git a/tests/test_rocm_e2e_ablation.py b/tests/test_rocm_e2e_ablation.py new file mode 100644 index 00000000..295936af --- /dev/null +++ b/tests/test_rocm_e2e_ablation.py @@ -0,0 +1,303 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 RL-Kernel Contributors + +from __future__ import annotations + +import json +from pathlib import Path +from types import SimpleNamespace + +import pytest + +import rl_engine.integrations.rocm_ablation as rocm_ablation +from rl_engine.integrations.ablation import Implementation +from rl_engine.integrations.rocm_ablation import ( + STRICT_ROCM_ATTENTION_CORE, + STRICT_ROCM_ATTENTION_RUNTIME, + STRICT_ROCM_ATTENTION_SCHEDULE, + build_case_environment, + replay_identity, + rocm_attention_ablation_matrix, + run_rocm_attention_ablation, + validate_case_readbacks, + validate_replay_environment, +) + + +def _environment() -> dict[str, str]: + return { + "MODEL_ROOT": "/models/Qwen3-8B", + "TORCH_DIST_ROOT": "/models/Qwen3-8B_torch_dist", + "VIME_CKPT": "/checkpoints/pre-update", + "PROMPT_DATA": "/data/prompts.jsonl", + "NUM_ROLLOUT": "1", + "TRAIN_SEED": "1234", + "ROLLOUT_SEED": "42", + } + + +def _readback(case, *, side: str, calls: int = 1, strict: bool | None = None): + if side == "training": + framework, target = "megatron", "training" + else: + framework, target = "vllm", "rollout" + implementation = case.implementation_for(target) + is_strict = implementation is Implementation.RL_KERNEL if strict is None else strict + backend = ( + "rlkernel.attention.deterministic.v1" + if is_strict + else f"{framework}.production.attention" + ) + provenance = ( + { + "execution": { + "runtime_platform": "rocm", + "operator": { + "actual_backend": STRICT_ROCM_ATTENTION_RUNTIME, + "strict_core_id": STRICT_ROCM_ATTENTION_CORE, + "strict_schedule": STRICT_ROCM_ATTENTION_SCHEDULE, + "fallback": False, + "triton_used": False, + }, + } + } + if is_strict + else {} + ) + return { + "framework": framework, + "target": target, + "plan": case.plan.to_dict(), + "installed_hooks": {"attention": f"{framework}.attention"}, + "fallbacks": [], + "operators": { + "attention": { + "implementation": implementation.value, + "backend_id": backend, + "call_count": calls, + "provenance": provenance, + } + }, + } + + +def test_matrix_is_pr230_four_cell_attention_matrix(): + matrix = rocm_attention_ablation_matrix() + + assert tuple(case.case_id for case in matrix) == ("P/P", "P/R", "R/P", "R/R") + assert [ + ( + case.implementation_for("training"), + case.implementation_for("rollout"), + ) + for case in matrix + ] == [ + (Implementation.PRODUCTION, Implementation.PRODUCTION), + (Implementation.PRODUCTION, Implementation.RL_KERNEL), + (Implementation.RL_KERNEL, Implementation.PRODUCTION), + (Implementation.RL_KERNEL, Implementation.RL_KERNEL), + ] + assert all(case.plan.cases["ffn"].case_id == "P/P" for case in matrix) + assert all(case.plan.cases["logp"].case_id == "P/P" for case in matrix) + + +def test_matrix_rejects_unknown_and_duplicate_cells(): + with pytest.raises(ValueError, match="unknown"): + rocm_attention_ablation_matrix(["R/X"]) + with pytest.raises(ValueError, match="unique"): + rocm_attention_ablation_matrix(["R/R", "R/R"]) + + +def test_replay_identity_changes_only_when_frozen_inputs_change(): + environment = _environment() + first = replay_identity(["bash", "run.sh"], environment) + environment["RL_KERNEL_ATTENTION_CASE"] = "R/R" + second = replay_identity(["bash", "run.sh"], environment) + environment["PROMPT_DATA"] = "/data/other.jsonl" + changed = replay_identity(["bash", "run.sh"], environment) + + assert first == second + assert changed["sha256"] != first["sha256"] + + +def test_executable_replay_requires_inputs_and_single_rollout(): + environment = _environment() + validate_replay_environment(environment) + environment.pop("PROMPT_DATA") + with pytest.raises(RuntimeError, match="PROMPT_DATA"): + validate_replay_environment(environment) + environment = _environment() + environment["NUM_ROLLOUT"] = "2" + with pytest.raises(RuntimeError, match="NUM_ROLLOUT=1"): + validate_replay_environment(environment) + + +def test_case_environment_propagates_plan_to_megatron_vllm_and_ray(tmp_path): + case = rocm_attention_ablation_matrix(["P/R"])[0] + environment = build_case_environment(_environment(), case, case_dir=tmp_path / "p-r") + + assert environment["RL_KERNEL_ATTENTION_CASE"] == "P/R" + assert environment["RL_KERNEL_FFN_CASE"] == "P/P" + assert environment["RL_KERNEL_LOGP_CASE"] == "P/P" + assert environment["RL_KERNEL_VLLM_INTEGRATION"] == "1" + assert environment["RL_KERNEL_PLATFORM"] == "rocm" + assert environment["VLLM_ATTENTION_BACKEND"] == "ROCM_AITER_FA" + assert environment["RL_KERNEL_READBACK_DIR"].endswith("p-r/readbacks") + assert environment["NUM_ROLLOUT"] == "1" + + +@pytest.mark.parametrize("case_id", ["P/P", "P/R", "R/P", "R/R"]) +def test_case_readbacks_validate_both_real_framework_sides(case_id): + case = rocm_attention_ablation_matrix([case_id])[0] + routes, errors = validate_case_readbacks( + case, + [ + _readback(case, side="training", calls=11), + _readback(case, side="rollout", calls=17), + ], + ) + + assert errors == () + assert set(routes) == {"training", "rollout"} + assert routes["training"].call_count == 11 + assert routes["rollout"].call_count == 17 + + +def test_strict_side_requires_rocm_runtime_identity(): + case = rocm_attention_ablation_matrix(["R/R"])[0] + bad = _readback(case, side="rollout") + bad["operators"]["attention"]["provenance"]["execution"]["operator"][ + "actual_backend" + ] = "rlkernel.cuda.attention.fa4_ag_rs.v1" + + _routes, errors = validate_case_readbacks( + case, + [_readback(case, side="training"), bad], + ) + + assert any("strict AITER/CK runtime" in error for error in errors) + + +def test_strict_side_rejects_triton_and_missing_fixed_schedule(): + case = rocm_attention_ablation_matrix(["P/R"])[0] + bad = _readback(case, side="rollout") + operator = bad["operators"]["attention"]["provenance"]["execution"]["operator"] + operator["triton_used"] = True + operator["core_actual_backends"] = ["triton.attention"] + operator.pop("strict_schedule") + + _routes, errors = validate_case_readbacks( + case, + [_readback(case, side="training"), bad], + ) + + assert any("fixed no-Split-KV schedule" in error for error in errors) + assert any("used Triton" in error for error in errors) + + +def test_production_side_requires_framework_native_backend_identity(): + case = rocm_attention_ablation_matrix(["P/P"])[0] + bad = _readback(case, side="training") + bad["operators"]["attention"]["backend_id"] = ( + "rlkernel.attention.deterministic.v1" + ) + + _routes, errors = validate_case_readbacks( + case, + [bad, _readback(case, side="rollout")], + ) + + assert any("framework-native Attention" in error for error in errors) + + +def test_zero_exit_without_framework_evidence_is_a_failure(): + case = rocm_attention_ablation_matrix(["R/R"])[0] + _routes, errors = validate_case_readbacks(case, []) + + assert errors == ( + "missing megatron/training runtime readback", + "missing vllm/rollout runtime readback", + ) + + +def test_malformed_call_count_is_a_validation_error(): + case = rocm_attention_ablation_matrix(["P/P"])[0] + bad = _readback(case, side="training") + bad["operators"]["attention"]["call_count"] = "not-an-integer" + + _routes, errors = validate_case_readbacks( + case, + [bad, _readback(case, side="rollout")], + ) + + assert any("invalid call count" in error for error in errors) + + +def test_dry_run_writes_human_summary_without_fabricating_results(tmp_path): + results = run_rocm_attention_ablation( + ["bash", "run.sh"], + output_dir=tmp_path, + base_environment={}, + case_ids=["P/P", "R/R"], + execute=False, + ) + + assert [result.status for result in results] == ["not_run", "not_run"] + summary = (tmp_path / "summary.md").read_text(encoding="utf-8") + assert "Executed: `no`" in summary + assert "| P/P |" in summary and "| R/R |" in summary + assert not list(tmp_path.glob("*.json")) + + +def test_execute_rejects_stale_case_evidence_before_gpu_probe(monkeypatch, tmp_path): + (tmp_path / "r-r").mkdir() + probed = False + + def probe(): + nonlocal probed + probed = True + + monkeypatch.setattr(rocm_ablation, "validate_rocm_host", probe) + + with pytest.raises(FileExistsError, match="existing case directories"): + run_rocm_attention_ablation( + ["bash", "run.sh"], + output_dir=tmp_path, + base_environment=_environment(), + case_ids=["R/R"], + execute=True, + ) + + assert probed is False + + +def test_execute_runs_each_case_fresh_and_checks_emitted_readbacks(monkeypatch, tmp_path): + invocations: list[str] = [] + monkeypatch.setattr(rocm_ablation, "validate_rocm_host", lambda: {"hip_runtime": "7.1"}) + + def fake_run(command, *, env, stdout, stderr, check): + del command, stdout, stderr, check + case = rocm_attention_ablation_matrix([env["RL_KERNEL_ATTENTION_CASE"]])[0] + readback_dir = Path(env["RL_KERNEL_READBACK_DIR"]) + invocations.append(case.case_id) + for side in ("training", "rollout"): + payload = _readback(case, side=side) + (readback_dir / f"{side}.json").write_text( + json.dumps(payload), + encoding="utf-8", + ) + return SimpleNamespace(returncode=0) + + monkeypatch.setattr(rocm_ablation.subprocess, "run", fake_run) + results = run_rocm_attention_ablation( + ["bash", "run.sh"], + output_dir=tmp_path, + base_environment=_environment(), + case_ids=["P/R", "R/P"], + execute=True, + ) + + assert invocations == ["P/R", "R/P"] + assert [result.status for result in results] == ["passed", "passed"] + assert all(result.log_path.is_file() for result in results) + assert "Executed: `yes`" in (tmp_path / "summary.md").read_text(encoding="utf-8")