From 00b37e0fe12eed11232e79f032cae3af9d740873 Mon Sep 17 00:00:00 2001 From: zhshgmail Date: Tue, 28 Apr 2026 16:14:32 -0700 Subject: [PATCH] =?UTF-8?q?benchmark:=20conv-family=20ops=20accept=20`seed?= =?UTF-8?q?`=20to=20make=20ref=20=E2=86=94=20cand=20verification=20possibl?= =?UTF-8?q?e?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Affects level3/6_ConvStandard1d.py, 7_ConvStandard2d.py, 8_ConvStandard3d.py, 9_ConvDepthwise2d.py, 10_ConvTranspose2d.py. ## Problem All five conv ops have `Model.forward()` that constructs `nn.ConvNd(...)` inline on every call. The verification harness `utils/verification_ascendc.py:306` does `torch.manual_seed(0)` ONCE before the per-case for-loop. Inside each iteration, it calls `ref_model.forward(*args)` then `cand_model.forward(*args)` back-to-back. Because `nn.ConvNd.__init__` draws random weights from the global RNG, and the RNG state advances after the ref call, **cand draws different weights than ref**. Verification then compares two different random outputs → ref ≠ cand by construction, regardless of whether the candidate kernel is correct. This makes the conv family structurally unverifiable. Hits any worker porting `nn.Conv1d/2d/3d/Depthwise/Transpose2d` to a candidate kernel: even an exact re-implementation in PyTorch (e.g. `F.conv1d(x, conv.weight, conv.bias, ...)`) would diverge from the ref because the ref's `conv.weight` ≠ the ones the cand sees. ## Fix Add a new optional `seed` kwarg (default 0) to each conv `forward()`. At the start of `forward()`, call `torch.manual_seed(seed)` so the inline conv construction draws weights from a fixed RNG state. Both ref and cand pass the same seed → both construct conv with identical weights → comparison works. Also adds `.to(x.device)` after construction so the conv's weights track the input device (handles the existing CPU-vs-NPU device-mismatch error that the inline-construction pattern triggers when called with NPU input). `get_input_groups()` extracts `seed` from the JSON case attrs (defaulting to 0) so existing JSON cases continue to work unchanged. Adding a `seed` entry to existing cases is optional — the default of 0 is sufficient for verification. ## Verification Tested on A5 NPU (Ascend950PR_9589, container a5ops_dev3): - ref_preflight on op#6 ConvStandard1d: verdict=RUNNABLE (was FAILED with "Expected all tensors to be on the same device" before this fix) - Two consecutive `Model().forward(x, ..., seed=42)` calls produce bit-exact identical output for op#6/7/9/10. Verified by `torch.equal(out1, out2)`. (op#8 ConvStandard3d skipped in test due to different x rank, but fix is identical pattern.) ## Backward compat - `seed` is optional with default 0. Existing JSON cases that don't include `{"name": "seed", ...}` continue to work — `attr_inputs.get("seed", 0)` returns 0. - ModelNew / candidate implementations that don't pass seed will get seed=0 by default. They MUST also seed before any RNG-dependent construction (or extract weights from the seed-0 conv) to match ref. ## Why now Caught during op-gen cold-start of op#6 ConvStandard1d on Ascend950PR (2026-04-28). Worker repeatedly hit the structural unverifiability before the harness behavior was understood; downstream fix in worker model.py (workspace-local OL-89 prose-spec extension with `torch.manual_seed`) unblocks individual workers but the upstream fix unblocks every future cold-start at no cost. --- .../NPUKernelBench/level3/10_ConvTranspose2d.py | 13 +++++++++---- .../NPUKernelBench/level3/6_ConvStandard1d.py | 15 +++++++++++---- .../NPUKernelBench/level3/7_ConvStandard2d.py | 13 +++++++++---- .../NPUKernelBench/level3/8_ConvStandard3d.py | 13 +++++++++---- .../NPUKernelBench/level3/9_ConvDepthwise2d.py | 13 +++++++++---- 5 files changed, 47 insertions(+), 20 deletions(-) diff --git a/benchmarks/NPUKernelBench/level3/10_ConvTranspose2d.py b/benchmarks/NPUKernelBench/level3/10_ConvTranspose2d.py index 952678e7..d27ff81e 100644 --- a/benchmarks/NPUKernelBench/level3/10_ConvTranspose2d.py +++ b/benchmarks/NPUKernelBench/level3/10_ConvTranspose2d.py @@ -10,7 +10,7 @@ class Model(nn.Module): def __init__(self): super(Model, self).__init__() - def forward(self, x: torch.Tensor, in_channels, out_channels, kernel_size, stride=1, padding=0, bias=True) -> torch.Tensor: + def forward(self, x: torch.Tensor, in_channels, out_channels, kernel_size, stride=1, padding=0, bias=True, seed=0) -> torch.Tensor: """ Applies transpose 2D convolution to the input tensor. @@ -22,11 +22,15 @@ def forward(self, x: torch.Tensor, in_channels, out_channels, kernel_size, strid stride (int or tuple, optional): Stride of the convolution. Default: 1. padding (int or tuple, optional): Zero-padding added to both sides of the input. Default: 0. bias (bool, optional): If True, adds a learnable bias to the output. Default: True. + seed (int, optional): RNG seed for deterministic conv weight init. Required so + ref Model.forward and candidate ModelNew.forward produce IDENTICAL random + weights when invoked back-to-back inside one verification case. Default: 0. Returns: torch.Tensor: Output tensor after performing nn.ConvTranspose2d. """ - conv = nn.ConvTranspose2d(in_channels, out_channels, kernel_size, stride=stride, padding=padding, bias=bias) + torch.manual_seed(seed) + conv = nn.ConvTranspose2d(in_channels, out_channels, kernel_size, stride=stride, padding=padding, bias=bias).to(x.device) return conv(x) @@ -57,8 +61,9 @@ def get_input_groups(): stride = attr_inputs.get("stride", 1) padding = attr_inputs.get("padding", 0) bias = attr_inputs.get("bias", True) - - input_groups.append([x, in_channels, out_channels, kernel_size, stride, padding, bias]) + seed = attr_inputs.get("seed", 0) + + input_groups.append([x, in_channels, out_channels, kernel_size, stride, padding, bias, seed]) return input_groups diff --git a/benchmarks/NPUKernelBench/level3/6_ConvStandard1d.py b/benchmarks/NPUKernelBench/level3/6_ConvStandard1d.py index cf74fe6c..83a6fd80 100644 --- a/benchmarks/NPUKernelBench/level3/6_ConvStandard1d.py +++ b/benchmarks/NPUKernelBench/level3/6_ConvStandard1d.py @@ -10,7 +10,7 @@ class Model(nn.Module): def __init__(self): super(Model, self).__init__() - def forward(self, x: torch.Tensor, in_channels, out_channels, kernel_size, stride=1, padding=0, dilation=1, groups=1, bias=True) -> torch.Tensor: + def forward(self, x: torch.Tensor, in_channels, out_channels, kernel_size, stride=1, padding=0, dilation=1, groups=1, bias=True, seed=0) -> torch.Tensor: """ Applies standard 1D convolution to the input tensor. @@ -24,11 +24,17 @@ def forward(self, x: torch.Tensor, in_channels, out_channels, kernel_size, strid dilation (int or tuple, optional): Spacing between kernel elements. Default: 1. groups (int, optional): Number of blocked connections from input channels to output channels. Default: 1. bias (bool, optional): If True, adds a learnable bias to the output. Default: True. + seed (int, optional): RNG seed used to initialize conv weights deterministically. + Required so that ref Model.forward and candidate ModelNew.forward produce + IDENTICAL random weights when the verification harness invokes both + back-to-back inside one test case. Without this, RNG advances between the + two calls and ref ≠ cand by construction. Default: 0. Returns: torch.Tensor: Output tensor after performing nn.Conv1d. """ - conv = nn.Conv1d(in_channels, out_channels, kernel_size, stride=stride, padding=padding, dilation=dilation, groups=groups, bias=bias) + torch.manual_seed(seed) + conv = nn.Conv1d(in_channels, out_channels, kernel_size, stride=stride, padding=padding, dilation=dilation, groups=groups, bias=bias).to(x.device) return conv(x) @@ -61,8 +67,9 @@ def get_input_groups(): dilation = attr_inputs.get("dilation", 1) groups = attr_inputs.get("groups", 1) bias = attr_inputs.get("bias", True) - - input_groups.append([x, in_channels, out_channels, kernel_size, stride, padding, dilation, groups, bias]) + seed = attr_inputs.get("seed", 0) + + input_groups.append([x, in_channels, out_channels, kernel_size, stride, padding, dilation, groups, bias, seed]) return input_groups diff --git a/benchmarks/NPUKernelBench/level3/7_ConvStandard2d.py b/benchmarks/NPUKernelBench/level3/7_ConvStandard2d.py index 4a6763b0..3f26238a 100644 --- a/benchmarks/NPUKernelBench/level3/7_ConvStandard2d.py +++ b/benchmarks/NPUKernelBench/level3/7_ConvStandard2d.py @@ -10,7 +10,7 @@ class Model(nn.Module): def __init__(self): super(Model, self).__init__() - def forward(self, x: torch.Tensor, in_channels, out_channels, kernel_size, stride=1, padding=0, dilation=1, groups=1, bias=True) -> torch.Tensor: + def forward(self, x: torch.Tensor, in_channels, out_channels, kernel_size, stride=1, padding=0, dilation=1, groups=1, bias=True, seed=0) -> torch.Tensor: """ Applies standard 2D convolution to the input tensor. @@ -24,11 +24,15 @@ def forward(self, x: torch.Tensor, in_channels, out_channels, kernel_size, strid dilation (int or tuple, optional): Spacing between kernel elements. Default: 1. groups (int, optional): Number of blocked connections from input channels to output channels. Default: 1. bias (bool, optional): If True, adds a learnable bias to the output. Default: True. + seed (int, optional): RNG seed for deterministic conv weight init. Required so + ref Model.forward and candidate ModelNew.forward produce IDENTICAL random + weights when invoked back-to-back inside one verification case. Default: 0. Returns: torch.Tensor: Output tensor after performing nn.Conv2d. """ - conv = nn.Conv2d(in_channels, out_channels, (kernel_size, kernel_size), stride=stride, padding=padding, dilation=dilation, groups=groups, bias=bias) + torch.manual_seed(seed) + conv = nn.Conv2d(in_channels, out_channels, (kernel_size, kernel_size), stride=stride, padding=padding, dilation=dilation, groups=groups, bias=bias).to(x.device) return conv(x) @@ -61,8 +65,9 @@ def get_input_groups(): dilation = attr_inputs.get("dilation", 1) groups = attr_inputs.get("groups", 1) bias = attr_inputs.get("bias", True) - - input_groups.append([x, in_channels, out_channels, kernel_size, stride, padding, dilation, groups, bias]) + seed = attr_inputs.get("seed", 0) + + input_groups.append([x, in_channels, out_channels, kernel_size, stride, padding, dilation, groups, bias, seed]) return input_groups diff --git a/benchmarks/NPUKernelBench/level3/8_ConvStandard3d.py b/benchmarks/NPUKernelBench/level3/8_ConvStandard3d.py index b27d34eb..7ce6dac3 100644 --- a/benchmarks/NPUKernelBench/level3/8_ConvStandard3d.py +++ b/benchmarks/NPUKernelBench/level3/8_ConvStandard3d.py @@ -10,7 +10,7 @@ class Model(nn.Module): def __init__(self): super(Model, self).__init__() - def forward(self, x: torch.Tensor, in_channels, out_channels, kernel_size, stride=1, padding=0, dilation=1, groups=1, bias=True) -> torch.Tensor: + def forward(self, x: torch.Tensor, in_channels, out_channels, kernel_size, stride=1, padding=0, dilation=1, groups=1, bias=True, seed=0) -> torch.Tensor: """ Applies standard 3D convolution to the input tensor. @@ -24,11 +24,15 @@ def forward(self, x: torch.Tensor, in_channels, out_channels, kernel_size, strid dilation (int or tuple, optional): Spacing between kernel elements. Default: 1. groups (int, optional): Number of blocked connections from input channels to output channels. Default: 1. bias (bool, optional): If True, adds a learnable bias to the output. Default: True. + seed (int, optional): RNG seed for deterministic conv weight init. Required so + ref Model.forward and candidate ModelNew.forward produce IDENTICAL random + weights when invoked back-to-back inside one verification case. Default: 0. Returns: torch.Tensor: Output tensor after performing nn.Conv3d. """ - conv = nn.Conv3d(in_channels, out_channels, (kernel_size, kernel_size, 1), stride=stride, padding=padding, dilation=dilation, groups=groups, bias=bias) + torch.manual_seed(seed) + conv = nn.Conv3d(in_channels, out_channels, (kernel_size, kernel_size, 1), stride=stride, padding=padding, dilation=dilation, groups=groups, bias=bias).to(x.device) return conv(x) @@ -61,8 +65,9 @@ def get_input_groups(): dilation = attr_inputs.get("dilation", 1) groups = attr_inputs.get("groups", 1) bias = attr_inputs.get("bias", True) - - input_groups.append([x, in_channels, out_channels, kernel_size, stride, padding, dilation, groups, bias]) + seed = attr_inputs.get("seed", 0) + + input_groups.append([x, in_channels, out_channels, kernel_size, stride, padding, dilation, groups, bias, seed]) return input_groups diff --git a/benchmarks/NPUKernelBench/level3/9_ConvDepthwise2d.py b/benchmarks/NPUKernelBench/level3/9_ConvDepthwise2d.py index a61c0232..70557e47 100644 --- a/benchmarks/NPUKernelBench/level3/9_ConvDepthwise2d.py +++ b/benchmarks/NPUKernelBench/level3/9_ConvDepthwise2d.py @@ -10,7 +10,7 @@ class Model(nn.Module): def __init__(self): super(Model, self).__init__() - def forward(self, x: torch.Tensor, in_channels, kernel_size, stride=1, padding=0, bias=True) -> torch.Tensor: + def forward(self, x: torch.Tensor, in_channels, kernel_size, stride=1, padding=0, bias=True, seed=0) -> torch.Tensor: """ Applies depthwise 2D convolution to the input tensor. @@ -21,11 +21,15 @@ def forward(self, x: torch.Tensor, in_channels, kernel_size, stride=1, padding=0 stride (int or tuple, optional): Stride of the convolution. Default: 1. padding (int or tuple, optional): Zero-padding added to both sides of the input. Default: 0. bias (bool, optional): If True, adds a learnable bias to the output. Default: True. + seed (int, optional): RNG seed for deterministic conv weight init. Required so + ref Model.forward and candidate ModelNew.forward produce IDENTICAL random + weights when invoked back-to-back inside one verification case. Default: 0. Returns: torch.Tensor: Output tensor of shape (batch, in_channels, height, width) after performing depthwise nn.Conv2d. Depthwise convolution processes each input channel independently, so output channels equal input channels. """ - conv = nn.Conv2d(in_channels, in_channels, kernel_size=(kernel_size, kernel_size), stride=stride, padding=padding, groups=in_channels, bias=bias) + torch.manual_seed(seed) + conv = nn.Conv2d(in_channels, in_channels, kernel_size=(kernel_size, kernel_size), stride=stride, padding=padding, groups=in_channels, bias=bias).to(x.device) return conv(x) @@ -55,8 +59,9 @@ def get_input_groups(): stride = attr_inputs.get("stride", 1) padding = attr_inputs.get("padding", 0) bias = attr_inputs.get("bias", True) - - input_groups.append([x, in_channels, kernel_size, stride, padding, bias]) + seed = attr_inputs.get("seed", 0) + + input_groups.append([x, in_channels, kernel_size, stride, padding, bias, seed]) return input_groups