diff --git a/benchmarks/bench_runner.py b/benchmarks/bench_runner.py index c757f41..8c41bd4 100644 --- a/benchmarks/bench_runner.py +++ b/benchmarks/bench_runner.py @@ -418,8 +418,17 @@ def run_case(self, case: dict[str, str]) -> CaseResult: dsl_source = f.read() try: - from scratchv.frontend.dsl_parser import DSLParser - parser = DSLParser() + # Auto-detect extended DSL features (if/else/while) + has_extended = any( + kw in dsl_source for kw in + ("if (", "else:", "endif", "while (", "endwhile") + ) + if has_extended: + from scratchv.frontend.dsl_extended import ExtendedDSLParser + parser = ExtendedDSLParser() + else: + from scratchv.frontend.dsl_parser import DSLParser + parser = DSLParser() program = parser.parse(dsl_source) except ImportError: # Fallback: use subprocess @@ -453,14 +462,14 @@ def run_case(self, case: dict[str, str]) -> CaseResult: for _ in b.instructions ) - # Compare output + # Compare output (handle numpy formatting variations) result.output = output.strip() result.expected = expected - result.passed = (result.output == expected) - - if not result.passed and not expected: - # No expected file -> pass by default (just check it runs) - result.passed = True + if expected: + result.passed = self._compare_outputs(output, expected) + else: + # No expected file -> pass if no error + result.passed = (not result.error) result.total_time_s = time.perf_counter() - t_start @@ -473,6 +482,26 @@ def run_case(self, case: dict[str, str]) -> CaseResult: return result + @staticmethod + def _compare_outputs(output: str, expected: str) -> bool: + """Compare simulation output with expected, handling numpy formatting.""" + if output.strip() == expected.strip(): + return True + try: + import numpy as np + import re + def _parse(s): + s = s.strip().strip("[]") + parts = [p for p in re.split(r"[,;\s]+", s) if p] + return np.array([float(p) for p in parts]) + out_a = _parse(output) + exp_a = _parse(expected) + if out_a.shape == exp_a.shape: + return bool(np.allclose(out_a, exp_a, rtol=1e-3, atol=1e-6)) + except (ValueError, TypeError, ImportError): + pass + return False + # ------------------------------------------------------------------- # DSL simulation # ------------------------------------------------------------------- @@ -507,13 +536,22 @@ def _simulate_dsl( if arg and not arg[0].isdigit() and arg != "": input_vars.add(arg) - # Filter out known function names + # Filter out known function names and keyword argument names keywords = { "add", "sub", "mul", "div", "relu", "gelu", "exp", "neg", "matmul", "dot", "maxpool", "softmax", "return", "for", "endfor", "if", "else", "endif", "while", "endwhile", + # Keyword argument names (not input variables) + "m", "n", "k", "rows", "cols", "inner", "len", + "axis", "kernel", "stride", "padding", + "out_channels", "kernel_size", + "transA", "transB", "alpha", "beta", + # Loop variables and temporaries + "i", "j", "t1", "t2", "t3", "t4", + "acc", "sum", "tmp", } - input_vars = {v for v in input_vars if v.lower() not in keywords} + input_vars = {v for v in input_vars if v.lower() not in keywords + and not v.startswith("_")} # Provide inputs inputs: dict[str, np.ndarray] = {} diff --git a/benchmarks/cases/005_gelu.expected b/benchmarks/cases/005_gelu.expected index a840f92..0fe80b4 100644 --- a/benchmarks/cases/005_gelu.expected +++ b/benchmarks/cases/005_gelu.expected @@ -1 +1 @@ -[1. 2. 3. 4.] \ No newline at end of file +[0.841192 1.954598 2.996363 3.99993 ] diff --git a/benchmarks/cases/007_matmul.expected b/benchmarks/cases/007_matmul.expected index a840f92..bea4847 100644 --- a/benchmarks/cases/007_matmul.expected +++ b/benchmarks/cases/007_matmul.expected @@ -1 +1 @@ -[1. 2. 3. 4.] \ No newline at end of file +30. diff --git a/benchmarks/cases/012_nn_pipeline.expected b/benchmarks/cases/012_nn_pipeline.expected index a840f92..0a0583e 100644 --- a/benchmarks/cases/012_nn_pipeline.expected +++ b/benchmarks/cases/012_nn_pipeline.expected @@ -1 +1 @@ -[1. 2. 3. 4.] \ No newline at end of file +[31. 32. 33. 34.] diff --git a/benchmarks/cases/014_for_dot.expected b/benchmarks/cases/014_for_dot.expected index a840f92..1a200e1 100644 --- a/benchmarks/cases/014_for_dot.expected +++ b/benchmarks/cases/014_for_dot.expected @@ -1 +1 @@ -[1. 2. 3. 4.] \ No newline at end of file +[ 1. 4. 9. 16.] diff --git a/benchmarks/cases/015_for_relu.expected b/benchmarks/cases/015_for_relu.expected index a840f92..783f747 100644 --- a/benchmarks/cases/015_for_relu.expected +++ b/benchmarks/cases/015_for_relu.expected @@ -1 +1 @@ -[1. 2. 3. 4.] \ No newline at end of file +[2. 4. 6. 8.] diff --git a/benchmarks/cases/019_nested_loop.expected b/benchmarks/cases/019_nested_loop.expected index a840f92..1a200e1 100644 --- a/benchmarks/cases/019_nested_loop.expected +++ b/benchmarks/cases/019_nested_loop.expected @@ -1 +1 @@ -[1. 2. 3. 4.] \ No newline at end of file +[ 1. 4. 9. 16.] diff --git a/tests/test_cnn_pipeline.py b/tests/test_cnn_pipeline.py index cc3b155..c4e87c0 100644 --- a/tests/test_cnn_pipeline.py +++ b/tests/test_cnn_pipeline.py @@ -217,12 +217,14 @@ def test_has_return_instruction(self, riscv_assembly): assert "zero" in riscv_assembly assert "ra" in riscv_assembly - def test_has_runtime_calls(self, riscv_assembly): - """Assembly should contain call instructions for NN ops.""" - assert "conv" in riscv_assembly - assert "gemm" in riscv_assembly - assert "maxpool" in riscv_assembly - assert "sigmoid" in riscv_assembly + def test_has_inline_nn_ops(self, riscv_assembly): + """NN ops should be inlined as real RISC-V instructions (no runtime calls).""" + # After eliminating call pseudo-instructions, NN ops are inlined: + # ReLU → max(x,0) via SLT + branch, Gemm → MUL+ADD, etc. + assert "mul" in riscv_assembly # NN MAC operations + assert "add" in riscv_assembly # accumulation + # No 'call' pseudo-instructions — all ops are real RISC-V + assert "call" not in riscv_assembly.lower().split() def test_emits_function_label(self, riscv_assembly): """Should emit .globl and function label for main_graph.""" @@ -230,12 +232,14 @@ def test_emits_function_label(self, riscv_assembly): assert ".globl" in riscv_assembly def test_counts(self, riscv_assembly): - """Should have exactly 24 real instructions (excluding directives).""" + """Inline NN ops generate more instructions than old call-based code.""" lines = [ln for ln in riscv_assembly.split("\n") if ln.strip() and not ln.strip().startswith(".") and not ln.strip().startswith("#") and not ln.strip().endswith(":")] - assert len(lines) == 24 + # Inline code for Conv/Gemm/MaxPool/ReLU/Sigmoid/Reshape + # generates significantly more than 24 placeholder instructions + assert len(lines) > 24 # ═══════════════════════════════════════════════════════════════════════════════ @@ -245,9 +249,10 @@ def test_counts(self, riscv_assembly): class TestBinaryEncoding: """Verify RISC-V binary machine code.""" - def test_produces_96_bytes(self, riscv_binary): - """24 instructions × 4 bytes = 96 bytes.""" - assert len(riscv_binary) == 96 + def test_produces_valid_binary(self, riscv_binary): + """Inline NN ops generate larger binary (was 96 bytes with call placeholders).""" + assert len(riscv_binary) > 0 + assert len(riscv_binary) % 4 == 0 # multiple of 4 (32-bit instructions) def test_valid_rv32_instructions(self, riscv_binary): """Every 4-byte word should decode as a valid RV32 opcode."""