From 2b5052e8b8e7c2e92e0be74e04d065f4952b8776 Mon Sep 17 00:00:00 2001 From: Seven Gao <799889633@qq.com> Date: Mon, 14 Sep 2026 01:52:04 +0800 Subject: [PATCH 1/5] feat(topic27): full RV32 benchmark with honest provenance and budget controls --- scratchv/standalone/bench_report.py | 423 +++++++ scratchv/standalone/rv32_bench.py | 1579 +++++++++++++++++++++------ tests/test_rv32_bench.py | 458 ++++++++ 3 files changed, 2127 insertions(+), 333 deletions(-) create mode 100644 tests/test_rv32_bench.py diff --git a/scratchv/standalone/bench_report.py b/scratchv/standalone/bench_report.py index 9dfb533..1dbc6ad 100644 --- a/scratchv/standalone/bench_report.py +++ b/scratchv/standalone/bench_report.py @@ -17,6 +17,7 @@ import json import os +import re import sys import time from typing import Any @@ -438,6 +439,428 @@ def generate_github_summary( return "\n".join(lines) +# ═══════════════════════════════════════════════════════════════════════════ +# Schema v2 renderers (rv32_bench.py reports) +# ═══════════════════════════════════════════════════════════════════════════ + +_MISSING = object() + + +def _dig(data: Any, path: str) -> Any: + node: Any = data + for part in path.split("."): + if not isinstance(node, dict) or part not in node: + return _MISSING + node = node[part] + return node + + +def _fmt(value: Any) -> str: + if value is None: + return "—" + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, float): + return f"{value:g}" + return str(value) + + +def _shape(shape: Any) -> str: + if not shape: + return "?" + return "×".join(str(d) for d in shape) + + +def render_markdown(report: dict) -> str: + """Render a schema v2 report as Markdown with provenance tags.""" + model = report.get("model") or {} + env = report.get("environment") or {} + targets = report.get("targets") or {} + sv = report.get("scratchv") or {} + llvm = report.get("llvm") or {} + cmp_ = report.get("comparison") or {} + + sv_compile = sv.get("compile") or {} + sv_dyn = sv.get("dynamic") or {} + sv_mix = sv.get("static_instruction_mix") or {} + sv_out = sv.get("output") or {} + ll_compile = llvm.get("compile") or {} + ll_dyn = llvm.get("dynamic") or {} + tgt_sv = targets.get("scratchv") or {} + tgt_ll = targets.get("llvm") or {} + + sha = str(model.get("sha256") or "") + completion = sv_dyn.get("completion") + dyn_rows = [ + ("source", _fmt(sv_dyn.get("source")), _fmt(ll_dyn.get("source"))), + ("completion", _fmt(completion), _fmt(ll_dyn.get("completion"))), + ("executed", _fmt(sv_dyn.get("executed")), _fmt(ll_dyn.get("executed"))), + ] + sv_ops = sv_dyn.get("ops") or {} + for key in ("total", "load", "store", "mul", "add", "madd", "branch"): + dyn_rows.append( + (f"ops.{key}", _fmt(sv_ops.get(key)), _fmt(None)), + ) + + lines = [] + lines.append(f"# RV32 Benchmark Report — {_fmt(model.get('path'))}") + lines.append("") + lines.append(f"- Generated: {_fmt(report.get('generated_at'))} | " + f"schema: {_fmt(report.get('schema_version'))}") + lines.append( + f"- Model: {_fmt(model.get('input_name'))}{_shape(model.get('input_shape'))}" + f" → {_fmt(model.get('output_name'))}{_shape(model.get('output_shape'))}" + f" | sha256={sha[:12]} | bytes={_fmt(model.get('bytes'))}" + ) + lines.append( + f"- Targets: ScratchV {_fmt(tgt_sv.get('isa'))}/{_fmt(tgt_sv.get('numeric_format'))}" + f" | LLVM {_fmt(tgt_ll.get('isa'))}/{_fmt(tgt_ll.get('numeric_format'))}" + f" (opt={_fmt(tgt_ll.get('opt_level'))})" + ) + lines.append( + f"- Environment: python={_fmt(env.get('python'))} " + f"numpy={_fmt(env.get('numpy'))} tinyfive={_fmt(env.get('tinyfive'))} " + f"llvmlite={_fmt(env.get('llvmlite'))}" + ) + lines.append("") + + lines.append("## 1. Compilation [static]") + lines.append("") + lines.append("| Metric | ScratchV | LLVM |") + lines.append("|--------|----------|------|") + lines.append(f"| status | {_fmt(sv_compile.get('status'))} | {_fmt(ll_compile.get('status'))} |") + lines.append(f"| code bytes | {_fmt(sv_compile.get('code_bytes'))} | — |") + lines.append(f"| data offset | {_fmt(sv_compile.get('data_offset'))} | — |") + lines.append(f"| data bytes | {_fmt(sv_compile.get('data_bytes'))} | — |") + lines.append( + f"| static insns [asm_scan] | {_fmt(sv_compile.get('static_insns'))} " + f"| {_fmt(ll_compile.get('static_insns'))} |" + ) + lines.append("") + lines.append("### Static instruction mix [static]") + lines.append("") + lines.append("| class | count |") + lines.append("|-------|-------|") + for key in ("load", "store", "mul", "add", "madd", "branch", "other"): + lines.append(f"| {key} | {_fmt(sv_mix.get(key))} |") + lines.append("") + + lines.append("## 2. Dynamic Execution [measured]") + lines.append("") + lines.append("| Metric | ScratchV | LLVM |") + lines.append("|--------|----------|------|") + for label, sv_cell, ll_cell in dyn_rows: + lines.append(f"| {label} | {sv_cell} | {ll_cell} |") + lines.append("") + if completion == "budget_exhausted": + lines.append( + f"> [measured/budget] budget exhausted at {sv_dyn.get('limit')} " + "instructions; dynamic counts are partial and excluded from " + "comparison." + ) + lines.append("") + elif completion == "timeout": + lines.append( + f"> [measured/timeout] wall-clock timeout after " + f"{_fmt(sv_dyn.get('elapsed_s'))}s; dynamic counts are partial." + ) + lines.append("") + elif sv_dyn.get("source") == "unavailable": + lines.append( + f"> [unavailable] ScratchV dynamic section omitted: " + f"{_fmt(sv_dyn.get('reason'))}" + ) + lines.append("") + if ll_dyn.get("source") == "unavailable": + lines.append( + f"> [unavailable] LLVM dynamic section omitted: " + f"{_fmt(ll_dyn.get('reason'))}" + ) + lines.append("") + if sv_dyn.get("source") == "simulated": + lines.append( + f"> Simulated by {_fmt(sv_dyn.get('simulator'))} " + f"{_fmt(sv_dyn.get('simulator_version'))} | " + f"completion={_fmt(completion)} | executed={_fmt(sv_dyn.get('executed'))} " + f"| limit={_fmt(sv_dyn.get('limit'))} | " + f"memory={_fmt(sv_dyn.get('memory_size_bytes'))} " + f"| seed={_fmt(sv_dyn.get('input_seed'))} " + f"| halt=0x{int(sv_dyn.get('halt_addr') or 0):x}" + ) + lines.append("") + + lines.append("## 3. Comparison [measured]") + lines.append("") + ratio = cmp_.get("dynamic_instruction_ratio") + if ratio is None: + lines.append( + f"- comparison ratio: **null** — " + f"{_fmt(cmp_.get('incomparable_reason'))}" + ) + else: + lines.append( + f"- dynamic_instruction_ratio: **{ratio:g}** " + "(ScratchV / LLVM, both sides halted)" + ) + lines.append("") + + lines.append("## 4. Analytical Warnings [estimated]") + lines.append("") + warnings = report.get("warnings") or [] + if warnings: + lines.extend(f"- {w}" for w in warnings) + else: + lines.append("- none") + lines.append("") + errors = report.get("errors") or [] + if errors: + lines.append("## 5. Errors") + lines.append("") + lines.extend(f"- {e}" for e in errors) + lines.append("") + + lines.append("## Provenance") + lines.append("") + lines.append(f"- model sha256={sha}") + lines.append( + f"- binary sha256={_fmt(sv_compile.get('binary_sha256'))} " + f"| data_offset={_fmt(sv_compile.get('data_offset'))} " + f"({_fmt(sv_compile.get('data_offset_source'))})" + ) + lines.append( + f"- static_source={_fmt(sv_compile.get('static_source'))} " + f"| llvm static_source={_fmt(ll_compile.get('static_source'))}" + ) + lines.append( + f"- output: {_fmt(sv_out.get('raw_hex'))} " + f"(addr=0x{int(sv_out.get('addr') or 0):x}, " + f"elements={_fmt(sv_out.get('elements'))})" + ) + return "\n".join(lines) + + +def _md_to_html(md: str) -> str: + from html import escape + lines = md.splitlines() + out: list[str] = [] + in_table = False + in_list = False + + def close_blocks(): + nonlocal in_table, in_list + if in_table: + out.append("") + in_table = False + if in_list: + out.append("") + in_list = False + + def inline(text: str) -> str: + text = escape(text) + text = re.sub(r"\*\*(.+?)\*\*", r"\1", text) + text = re.sub(r"`(.+?)`", r"\1", text) + return text + + for line in lines: + stripped = line.strip() + if stripped.startswith("|"): + cells = [c.strip() for c in stripped.strip("|").split("|")] + if all(set(c) <= {"-", " "} and c for c in cells): + continue + if not in_table: + close_blocks() + out.append("") + in_table = True + tag = "th" if not any(mark.startswith("") for mark in out[-2:]) else "td" + out.append("" + "".join( + f"<{tag}>{inline(c)}" for c in cells) + "") + continue + if stripped.startswith("- "): + if not in_list: + close_blocks() + out.append("
"); in_table = False - l = l.replace("

", "").rstrip() - out.append(f"

{l}

") - elif l.startswith("# "): - l = l.replace("# ", "").rstrip() - out.append(f"

{l}

") - elif l.startswith("|"): - if not in_table: out.append(""); in_table = True - cells = l.split("|")[1:-1] - if all(c.strip().startswith("--") for c in cells): - continue # skip separator - tag = "th" if out and out[-1] == "
" else "td" - cls = ' class="n"' if tag == "td" else "" - row = "" + "".join(f"<{tag}{cls}>{c.strip()}" for c in cells) + "" - out.append(row) - elif l.startswith("**") and l.endswith("**"): - out.append(f"

{l[2:-2]}

") - elif l.startswith("*All metrics"): - out.append(f"

{l[1:]}

") - elif l.startswith("*ScratchV"): - out.append(f"

{l[1:]}

") - elif l.strip(): - # Convert inline **bold** and `code` - l = re.sub(r'\*\*(.+?)\*\*', r'\1', l) - l = re.sub(r'`(.+?)`', r'\1', l) - out.append(f"

{l}

") - if in_table: out.append("
") - - body = "\n".join(out) - return f"RV32 BenchmarkScratchV RV32IM LLVM RV32IMF\n{body}\n" + t0 = time.perf_counter() + limit = None if max_instructions in (0, None) else int(max_instructions) + if limit is not None and limit < 0: + raise ValueError(f"max_instructions must be >= 0, got {max_instructions}") + if chunk_instructions <= 0: + raise ValueError(f"chunk_instructions must be > 0, got {chunk_instructions}") + + binary = Path(binary_path).read_bytes() + data_size = len(binary) - data_offset + layout = compute_layout( + data_offset=data_offset, data_size=data_size, + workspace_bytes=workspace_bytes, input_elements=input_elements, + output_elements=output_elements, mem_size=mem_size, + ) + halt_addr = layout["halt_addr"] + + m = ProfiledMachine(mem_size=mem_size) + if not m.available: + return { + "dynamic": _unavailable_dynamic( + reason="tinyfive not installed (ProfiledMachine.available=False)", + mem_size=mem_size, timeout_s=timeout_s, input_seed=input_seed, + input_elements=input_elements, halt_addr=halt_addr, + ), + "output": { + "addr": OUTPUT_ADDR, "elements": output_elements, + "raw_hex": None, "q16_16": None, + }, + } + _install_tinyfive_compat(m, halt_addr) + + try: + code_words, weights = load_scratchv_image(binary_path, data_offset) + except LayoutError as exc: + return { + "dynamic": _unavailable_dynamic( + reason=f"image_load_failed: {exc}", mem_size=mem_size, + timeout_s=timeout_s, input_seed=input_seed, + input_elements=input_elements, halt_addr=halt_addr, + ), + "output": { + "addr": OUTPUT_ADDR, "elements": output_elements, + "raw_hex": None, "q16_16": None, + }, + } + + unsupported = check_mnemonics(code_words) + if unsupported: + dyn = _unavailable_dynamic( + reason="unsupported_mnemonics: " + ", ".join(unsupported[:8]), + mem_size=mem_size, timeout_s=timeout_s, input_seed=input_seed, + input_elements=input_elements, halt_addr=halt_addr, + ) + return { + "dynamic": dyn, + "output": { + "addr": OUTPUT_ADDR, "elements": output_elements, + "raw_hex": None, "q16_16": None, + }, + } + + m.load_binary(code_words, origin=0) + m.load_data(weights, data_offset) + input_blob = build_input_q16(input_elements, input_seed) + if input_blob: + m.load_data(input_blob, INPUT_ADDR) + m.set_reg(2, layout["sp"]) + m.set_reg(10, layout["input_addr"]) + m.set_reg(11, layout["output_addr"]) + m.set_reg(1, halt_addr) + + _timed_out = False + old_handler = None + use_alarm = hasattr(signal, "SIGALRM") and timeout_s is not None and timeout_s > 0 + if use_alarm: + try: + old_handler = signal.signal(signal.SIGALRM, _alarm_handler) + except (ValueError, OSError): # not in main thread + use_alarm = False + deadline = ( + time.monotonic() + float(timeout_s) + if timeout_s and timeout_s > 0 else float("inf") + ) + + executed = 0 + completion = "error" + error_msg: str | None = None + devnull = open(os.devnull, "w") + try: + while True: + pc = _read_pc(m) + if limit is not None and executed >= limit: + completion = "budget_exhausted" + break + if pc == halt_addr: + completion = "halted" + break + remaining = deadline - time.monotonic() + if remaining <= 0: + completion = "timeout" + break + chunk = chunk_instructions + if limit is not None: + chunk = min(chunk, limit - executed) + if chunk <= 0: + completion = "budget_exhausted" + break + + if use_alarm: + signal.setitimer(signal.ITIMER_REAL, max(remaining, 1e-6)) + try: + with contextlib.redirect_stdout(devnull): + m.run(instructions=chunk, start=pc, strict=True) + except RuntimeError as exc: + if _timed_out: + completion = "timeout" + else: + completion = "error" + error_msg = m.last_error or str(exc)[:300] + break + finally: + if use_alarm: + signal.setitimer(signal.ITIMER_REAL, 0.0) + + after = _perf_total(m) + if after == executed and _read_pc(m) != halt_addr: + completion = "error" + error_msg = ( + f"simulation stalled at pc=0x{pc:x} " + "(unsupported instruction or wedged decoder)" + ) + break + executed = after + finally: + if use_alarm: + signal.setitimer(signal.ITIMER_REAL, 0.0) + if old_handler is not None: + signal.signal(signal.SIGALRM, old_handler) + devnull.close() + + elapsed = time.perf_counter() - t0 + perf = m.get_perf() + executed = _perf_total(m) + x_used, x_total, f_used = _read_register_usage(m) + + if completion == "error": + dyn = _unavailable_dynamic( + reason=error_msg or m.last_error or "simulation error", + mem_size=mem_size, timeout_s=timeout_s, input_seed=input_seed, + input_elements=input_elements, halt_addr=halt_addr, + ) + dyn["completion"] = "error" + dyn["executed"] = executed + dyn["elapsed_s"] = elapsed + dyn["last_error"] = m.last_error + return { + "dynamic": dyn, + "output": { + "addr": OUTPUT_ADDR, "elements": output_elements, + "raw_hex": None, "q16_16": None, + }, + } + + ops = {key: int(perf.get(key, 0)) for key in OPS_KEYS} + return { + "dynamic": { + "source": "simulated", + "simulator": "tinyfive", + "simulator_version": get_environment().get("tinyfive"), + "completion": completion, + "limit": limit, + "executed": executed, + "timeout_s": float(timeout_s), + "elapsed_s": elapsed, + "memory_size_bytes": mem_size, + "input_seed": input_seed, + "input_elements": input_elements, + "halt_addr": halt_addr, + "ops": ops, + "x_registers_used": x_used, + "x_usage_total": x_total, + "f_registers_used": f_used, + "per_label": None, + "per_label_note": "tinyfive exe() exposes no per-PC trace", + "last_error": m.last_error, + }, + "output": _read_output(m, layout["output_addr"], output_elements), + } + + +def _perf_total(machine) -> int: + try: + return int(machine.get_perf().get("total", 0)) + except Exception: + return int(getattr(machine, "instr_count", 0)) + + +# ═══════════════════════════════════════════════════════════════════════════ +# Report assembly and audit +# ═══════════════════════════════════════════════════════════════════════════ + +def _simulated_halted(side: dict, name: str): + dyn = side.get("dynamic") or {} + if dyn.get("source") != "simulated": + return f"{name}.dynamic.source!='simulated'" + if dyn.get("completion") != "halted": + return f"{name}.completion=='{dyn.get('completion')}'" + ops = dyn.get("ops") + if not isinstance(ops, dict) or ops.get("total") is None: + return f"{name}.dynamic.ops missing" + return True + + +def _comparison(scratchv: dict, llvm: dict) -> dict: + reasons = [] + sv_state = _simulated_halted(scratchv, "scratchv") + ll_state = _simulated_halted(llvm, "llvm") + if sv_state is not True: + reasons.append(sv_state) + if ll_state is not True: + reasons.append(ll_state) + if reasons: + return { + "dynamic_instruction_ratio": None, + "incomparable_reason": "; ".join(reasons), + } + sv_total = scratchv["dynamic"]["ops"]["total"] + ll_total = llvm["dynamic"]["ops"]["total"] + if ll_total <= 0: + return { + "dynamic_instruction_ratio": None, + "incomparable_reason": "llvm.dynamic.ops.total==0", + } + return { + "dynamic_instruction_ratio": round(sv_total / ll_total, 6), + "incomparable_reason": None, + } + + +def build_report(model: dict, environment: dict, scratchv: dict, llvm: dict) -> dict: + """Assemble the schema v2 report dictionary.""" + return { + "schema_version": SCHEMA_VERSION, + "generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "generator": {"script": "rv32_bench.py"}, + "model": {key: model.get(key) for key in MODEL_FIELDS}, + "environment": environment, + "targets": TARGETS, + "scratchv": scratchv, + "llvm": llvm, + "comparison": _comparison(scratchv, llvm), + "warnings": [], + "errors": [], + } + + +def audit_provenance(report: dict) -> list[str]: + """Return the list of honesty violations; empty means the report is clean.""" + violations: list[str] = [] + + if report.get("schema_version") != SCHEMA_VERSION: + violations.append( + f"schema_version must be {SCHEMA_VERSION!r}, got " + f"{report.get('schema_version')!r}" + ) + + model = report.get("model") or {} + sha = str(model.get("sha256") or "") + if not re.fullmatch(r"[0-9a-f]{64}", sha): + violations.append("model.sha256 missing or not a 64-char hex digest") + if not model.get("path"): + violations.append("model.path missing") + + env = report.get("environment") or {} + for key in ("python", "numpy", "tinyfive", "llvmlite"): + if key not in env: + violations.append(f"environment.{key} missing") + + targets = report.get("targets") or {} + for side in ("scratchv", "llvm"): + target = targets.get(side) or {} + if not target.get("isa"): + violations.append(f"targets.{side}.isa missing") + + for side_name in ("scratchv", "llvm"): + side = report.get(side_name) or {} + compile_info = side.get("compile") or {} + if compile_info.get("status") == "success" and \ + compile_info.get("static_source") != "asm_scan": + violations.append( + f"{side_name}.compile.static_source must be 'asm_scan'" + ) + dyn = side.get("dynamic") + if dyn is None: + violations.append(f"{side_name}.dynamic missing") + continue + source = dyn.get("source") + if source == "simulated": + for key in ("simulator", "simulator_version", "executed", + "memory_size_bytes", "input_seed"): + if dyn.get(key) is None: + violations.append( + f"{side_name}.dynamic.{key} missing for simulated data" + ) + if dyn.get("completion") not in ("halted", "budget_exhausted", "timeout"): + violations.append( + f"{side_name}.dynamic.completion invalid: " + f"{dyn.get('completion')!r}" + ) + ops = dyn.get("ops") + if not isinstance(ops, dict): + violations.append(f"{side_name}.dynamic.ops missing") + else: + for key in OPS_KEYS: + if not isinstance(ops.get(key), int): + violations.append( + f"{side_name}.dynamic.ops.{key} must be an int" + ) + if dyn.get("completion") == "halted" and \ + dyn.get("limit") is not None and \ + dyn.get("executed") == dyn.get("limit"): + violations.append( + f"{side_name}.dynamic: halted but executed==limit==" + f"{dyn.get('limit')} (unverified halt)" + ) + elif source == "unavailable": + if not dyn.get("reason"): + violations.append(f"{side_name}.dynamic.reason missing") + if dyn.get("ops") is not None: + violations.append( + f"{side_name}.dynamic.ops must be null when unavailable" + ) + else: + violations.append( + f"{side_name}.dynamic.source invalid: {source!r}" + ) + + comparison = report.get("comparison") or {} + ratio = comparison.get("dynamic_instruction_ratio") + if ratio is not None: + if not isinstance(ratio, (int, float)): + violations.append("comparison.dynamic_instruction_ratio not numeric") + for side_name in ("scratchv", "llvm"): + dyn = (report.get(side_name) or {}).get("dynamic") or {} + if dyn.get("source") != "simulated" or \ + dyn.get("completion") != "halted": + violations.append( + "comparison.dynamic_instruction_ratio computed from an " + f"incomplete/non-simulated side ({side_name})" + ) + break + elif not comparison.get("incomparable_reason"): + violations.append("comparison.incomparable_reason missing when ratio is null") + + for key in ("warnings", "errors"): + if not isinstance(report.get(key), list): + violations.append(f"{key} must be a list") + + return violations # ═══════════════════════════════════════════════════════════════════════════ -# Main +# CLI # ═══════════════════════════════════════════════════════════════════════════ -def main(): +def build_parser(): import argparse - p = argparse.ArgumentParser(description="RV32 Benchmark: ScratchV vs LLVM on TinyFive") + p = argparse.ArgumentParser( + description="RV32 full benchmark: ScratchV vs LLVM on TinyFive " + "(honest provenance, explicit budgets)" + ) p.add_argument("model", help="Path to ONNX model") p.add_argument("--output-dir", default="benchmark_reports") p.add_argument("--html", default="rv32_bench.html") p.add_argument("--json", default="rv32_bench.json") p.add_argument("--md", default="rv32_bench.md") - a = p.parse_args() - - out = Path(a.output_dir); out.mkdir(parents=True, exist_ok=True) - model = a.model - - print(f"RV32 Benchmark: {model}", file=sys.stderr) - print(f"{'='*60}", file=sys.stderr) - - # 1. Compile ScratchV - print("\n[1/4] ScratchV compilation (RV32IM, Q16.16)...", file=sys.stderr) - sv = BenchResult(name="ScratchV", isa="RV32IM (Q16.16)") - sv.compile = compile_scratchv(model, str(out / "_sv.bin"), str(out / "_sv.s")) - - # 2. Compile LLVM → RV32IMF - print("[2/4] LLVM compilation (RV32IMF, float32)...", file=sys.stderr) - ll = BenchResult(name="LLVM", isa="RV32IMF (float32)") - ll.compile = compile_llvm_rv32(model, str(out / "_ll_rv32.s")) - - # 3. TinyFive simulation - print("[3/4] TinyFive simulation...", file=sys.stderr) - sv.tinyfive = run_tinyfive(str(out / "_sv.s"), n_instructions=5000) - ll.tinyfive = run_tinyfive(str(out / "_ll_rv32.s"), n_instructions=5000) if ll.compile["status"] == "success" else {} + p.add_argument("--max-instructions", type=int, default=0, + help="Instruction budget; 0 means full simulation (default: 0)") + p.add_argument("--full", action="store_true", + help="Explicitly request a full (untruncated) simulation") + p.add_argument("--mem-size", type=int, default=DEFAULT_MEM_SIZE, + help="TinyFive memory size in bytes (default: 268435456)") + p.add_argument("--timeout", type=float, default=DEFAULT_TIMEOUT_S, + help="Wall-clock timeout in seconds (default: 900)") + p.add_argument("--chunk-instructions", type=int, + default=DEFAULT_CHUNK_INSTRUCTIONS, + help="Instructions per simulation chunk (default: 10000000)") + p.add_argument("--input-seed", type=int, default=DEFAULT_INPUT_SEED) + p.add_argument("--skip-llvm", action="store_true") + p.add_argument("--allow-missing-simulator", action="store_true") + p.add_argument("--fail-on-incomplete", action="store_true") + p.add_argument("--quiet", action="store_true") + return p + + +def _unavailable_llvm(reason: str) -> dict: + return { + "compile": { + "status": "skipped" if reason == "--skip-llvm" else "unavailable", + "reason": reason, + "isa_detected": None, + "isa_mismatch": False, + "static_insns": 0, + "static_source": "asm_scan", + "elapsed_s": 0.0, + }, + "dynamic": { + "source": "unavailable", + "simulator": "tinyfive", + "completion": "not_run", + "reason": reason, + "ops": None, + }, + } + + +def main(argv: list[str] | None = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + + if args.full and args.max_instructions > 0: + parser.error( + "--full conflicts with --max-instructions N>0: choose one" + ) + if args.max_instructions < 0: + parser.error("--max-instructions must be >= 0") + if args.chunk_instructions <= 0: + parser.error("--chunk-instructions must be > 0") + + out = Path(args.output_dir) + out.mkdir(parents=True, exist_ok=True) + effective_limit = 0 if args.full else args.max_instructions + warnings: list[str] = [] + errors: list[str] = [] + + if not hasattr(signal, "SIGALRM") and effective_limit == 0 and \ + not args.allow_missing_simulator: + parser.error( + "this platform has no SIGALRM timeout; " + "provide --max-instructions N or --allow-missing-simulator" + ) - # 4. Report - print("[4/4] Generating report...", file=sys.stderr) - md = generate_report(sv, ll) - html = generate_html_report(sv, ll, md) + try: + model_info = get_model_info(args.model) + except Exception as exc: + print(f"error: model_parse_failed: {exc}", file=sys.stderr) + return EXIT_LAYOUT + + out_bytes = model_info["output_elements"] * 4 + min_mem = OUTPUT_ADDR + out_bytes + OUTPUT_GUARD_BYTES + if args.mem_size < min_mem: + print( + f"error: memory_layout_invalid: need mem_size >= {min_mem} bytes " + f"(192MiB output base + {out_bytes} bytes output + " + f"{OUTPUT_GUARD_BYTES} bytes guard); got {args.mem_size}", + file=sys.stderr, + ) + return EXIT_LAYOUT - with open(out / a.md, "w") as f: f.write(md) - with open(out / a.html, "w") as f: f.write(html) - with open(out / a.json, "w") as f: - json.dump({"scratchv": asdict(sv), "llvm": asdict(ll)}, f, indent=2, default=str) + print(f"RV32 benchmark: {args.model}", file=sys.stderr) + print(" [1/4] ScratchV compilation (RV32IM, Q16.16)", file=sys.stderr) + sv_compile = compile_scratchv( + args.model, str(out / "_sv.bin"), str(out / "_sv.s"), + ) + if sv_compile["status"] != "success": + print( + f"error: scratchv_compile_failed: " + f"{sv_compile.get('reason') or sv_compile.get('error')}", + file=sys.stderr, + ) + return EXIT_LAYOUT + + asm_text = (out / "_sv.s").read_text() + static_mix = static_instruction_mix(asm_text) + + if args.skip_llvm: + llvm = _unavailable_llvm("--skip-llvm") + else: + print(" [2/4] LLVM compilation (RV32IMF, float32)", file=sys.stderr) + llvm_compile = compile_llvm_rv32(args.model, str(out / "_ll_rv32.s")) + llvm = { + "compile": llvm_compile, + "dynamic": { + "source": "unavailable", + "simulator": "tinyfive", + "completion": "not_run", + "reason": ( + "llvm executable image pipeline not implemented " + "(topic 25 boundary)" + ), + "ops": None, + }, + } + if llvm_compile["status"] == "skipped": + warnings.append( + f"LLVM side skipped: {llvm_compile.get('reason')}" + ) + elif llvm_compile.get("isa_mismatch"): + warnings.append( + "LLVM assembly contains RV64-only mnemonics; dynamic " + "comparison disabled" + ) + + print(" [3/4] TinyFive simulation", file=sys.stderr) + try: + sim = run_simulation( + asm_path=str(out / "_sv.s"), + binary_path=str(out / "_sv.bin"), + data_offset=sv_compile["data_offset"], + workspace_bytes=sv_compile["workspace_bytes"], + input_elements=model_info["input_elements"], + output_elements=model_info["output_elements"], + max_instructions=effective_limit, + mem_size=args.mem_size, + timeout_s=args.timeout, + chunk_instructions=args.chunk_instructions, + input_seed=args.input_seed, + ) + except LayoutError as exc: + print(f"error: memory_layout_invalid: {exc}", file=sys.stderr) + return EXIT_LAYOUT + + dynamic = sim["dynamic"] + if dynamic["source"] == "unavailable" and \ + dynamic["completion"] == "not_run": + if dynamic.get("reason", "").startswith("unsupported_mnemonics") or \ + dynamic.get("reason", "").startswith("image_load_failed"): + print(f"error: {dynamic['reason']}", file=sys.stderr) + return EXIT_LAYOUT + if not args.allow_missing_simulator: + print( + f"error: simulator_unavailable: {dynamic['reason']} " + "(use --allow-missing-simulator for a static-only report)", + file=sys.stderr, + ) + return EXIT_NO_SIMULATOR + warnings.append( + "simulator unavailable; dynamic section omitted " + f"({dynamic['reason']})" + ) + elif dynamic.get("completion") == "budget_exhausted": + warnings.append( + f"budget exhausted at {dynamic['limit']} instructions; " + "dynamic counts are partial" + ) + elif dynamic.get("completion") == "timeout": + warnings.append( + f"wall-clock timeout after {dynamic['elapsed_s']:.1f}s; " + "dynamic counts are partial" + ) + elif dynamic.get("completion") == "error": + errors.append(dynamic.get("reason") or "simulation error") + + scratchv = { + "compile": sv_compile, + "static_instruction_mix": static_mix, + "dynamic": dynamic, + "output": sim["output"], + } + + print(" [4/4] Report", file=sys.stderr) + report = build_report(model_info, get_environment(), scratchv, llvm) + report["warnings"] = warnings + report["errors"] = errors + + violations = audit_provenance(report) + if violations: + for violation in violations: + print(f"provenance_violation: {violation}", file=sys.stderr) + return EXIT_ERROR + + markdown = bench_report.render_markdown(report) + (out / args.md).write_text(markdown, encoding="utf-8") + (out / args.html).write_text(bench_report.render_html(report), encoding="utf-8") + (out / args.json).write_text(bench_report.render_bench_json(report), encoding="utf-8") + if not args.quiet: + print(markdown) + print( + f"Reports: {out / args.html} | {out / args.md} | {out / args.json}", + file=sys.stderr, + ) - print(f"\nReports: {out/a.html} | {out/a.md} | {out/a.json}", file=sys.stderr) - print(md) + if args.fail_on_incomplete and dynamic.get("completion") != "halted": + return EXIT_INCOMPLETE + if dynamic.get("completion") == "error": + return EXIT_ERROR + return EXIT_OK if __name__ == "__main__": diff --git a/tests/test_rv32_bench.py b/tests/test_rv32_bench.py new file mode 100644 index 0000000..4614f08 --- /dev/null +++ b/tests/test_rv32_bench.py @@ -0,0 +1,458 @@ +"""Tests for the RV32 full benchmark driver (topic 27). + +Dynamic tests run only tiny synthetic models with explicit budgets; the big +``cnn.onnx`` full simulation is deliberately never executed here. +""" + +import json +from copy import deepcopy + +import pytest + +from scratchv.standalone import bench_report, rv32_bench + + +# ─────────────────────────────────────────────────────────────────────────── +# Fixtures / helpers +# ─────────────────────────────────────────────────────────────────────────── + +def _build_mini_onnx(path): + onnx = pytest.importorskip("onnx") + import numpy as np + from onnx import TensorProto, helper, numpy_helper + + rng = np.random.RandomState(0) + inp = helper.make_tensor_value_info("input", TensorProto.FLOAT, [1, 1, 8, 8]) + out = helper.make_tensor_value_info("output", TensorProto.FLOAT, [1, 1, 6, 6]) + weight = numpy_helper.from_array( + (rng.randn(1, 1, 3, 3).astype(np.float32) * 0.1), "W", + ) + bias = numpy_helper.from_array(np.zeros(1, np.float32), "B") + node = helper.make_node( + "Conv", ["input", "W", "B"], ["output"], + kernel_shape=[3, 3], pads=[0, 0, 0, 0], strides=[1, 1], + ) + graph = helper.make_graph([node], "mini", [inp], [out], [weight, bias]) + model = helper.make_model( + graph, opset_imports=[helper.make_opsetid("", 13)], + ) + onnx.save(model, str(path)) + + +@pytest.fixture(scope="module") +def mini_model(tmp_path_factory): + path = tmp_path_factory.mktemp("mini_rv32") / "mini.onnx" + _build_mini_onnx(path) + return path + + +@pytest.fixture(scope="module") +def mini_compiled(mini_model, tmp_path_factory): + out = tmp_path_factory.mktemp("mini_build") + info = rv32_bench.compile_scratchv( + str(mini_model), str(out / "_sv.bin"), str(out / "_sv.s"), + ) + assert info["status"] == "success", info + return {"onnx": mini_model, "out": out, "info": info} + + +def _fake_model(): + return { + "path": "mini.onnx", "sha256": "0" * 64, "bytes": 241, + "input_name": "input", "input_shape": [1, 1, 8, 8], + "output_name": "output", "output_shape": [1, 1, 6, 6], + "initializer_count": 2, "weight_bytes": 40, + } + + +def _fake_env(): + return { + "python": "3.11.0", "numpy": "2.0.0", + "tinyfive": "1.0.0", "llvmlite": None, + } + + +def _fake_scratchv(*, completion="halted", limit=None, executed=100, + ops=None): + if ops is None: + ops = { + "total": executed, "load": 7, "store": 3, "mul": 5, + "add": 10, "madd": 0, "branch": 4, + } + return { + "compile": { + "status": "success", "binary": "_sv.bin", "binary_bytes": 680, + "binary_sha256": "1" * 64, "code_bytes": 640, "data_offset": 640, + "data_offset_source": "compiler_stdout", "data_bytes": 40, + "workspace_bytes": 400, "static_insns": 160, + "static_source": "asm_scan", "elapsed_s": 1.0, + }, + "static_instruction_mix": { + "source": "asm_scan", "load": 7, "store": 3, "mul": 5, + "add": 10, "madd": 0, "branch": 4, "other": 0, + }, + "dynamic": { + "source": "simulated", "simulator": "tinyfive", + "simulator_version": "1.0.0", "completion": completion, + "limit": limit, "executed": executed, "timeout_s": 900.0, + "elapsed_s": 1.0, "memory_size_bytes": 268435456, + "input_seed": 42, "input_elements": 64, "halt_addr": 688, + "ops": ops, "x_registers_used": 12, "x_usage_total": 30, + "f_registers_used": 0, "per_label": None, + "per_label_note": "tinyfive exe() exposes no per-PC trace", + "last_error": None, + }, + "output": { + "addr": 201326592, "elements": 36, + "raw_hex": "0x" + "00" * 144, "q16_16": [0.0] * 36, + }, + } + + +def _unavailable_llvm(reason="llvm executable image pipeline not " + "implemented (topic 25 boundary)"): + return { + "compile": { + "status": "skipped", "reason": "llvmlite not available", + "isa_detected": None, "isa_mismatch": False, + "static_insns": 0, "static_source": "asm_scan", "elapsed_s": 0.0, + }, + "dynamic": { + "source": "unavailable", "simulator": "tinyfive", + "completion": "not_run", "reason": reason, "ops": None, + }, + } + + +def _valid_fake_report(): + return rv32_bench.build_report( + _fake_model(), _fake_env(), _fake_scratchv(), _unavailable_llvm(), + ) + + +# ─────────────────────────────────────────────────────────────────────────── +# T0: defaults / budget semantics +# ─────────────────────────────────────────────────────────────────────────── + +def test_cli_defaults_to_full_simulation_no_hidden_truncation(): + args = rv32_bench.build_parser().parse_args(["model.onnx"]) + assert args.max_instructions == 0 + assert args.full is False + assert args.mem_size == 268435456 + assert args.chunk_instructions == 10_000_000 + + with pytest.raises(SystemExit) as excinfo: + rv32_bench.main(["model.onnx", "--full", "--max-instructions", "5"]) + assert excinfo.value.code == rv32_bench.EXIT_USAGE + + +# ─────────────────────────────────────────────────────────────────────────── +# T1: full run of a tiny model matches an independent emulator +# ─────────────────────────────────────────────────────────────────────────── + +def test_full_run_small_model_matches_reference(tmp_path, mini_model, + mini_compiled): + pytest.importorskip("tinyfive") + out = tmp_path / "full" + rc = rv32_bench.main([ + str(mini_model), "--full", "--quiet", "--output-dir", str(out), + "--timeout", "120", "--chunk-instructions", "4096", + ]) + assert rc == rv32_bench.EXIT_OK + + report = json.loads((out / "rv32_bench.json").read_text()) + dyn = report["scratchv"]["dynamic"] + assert dyn["completion"] == "halted" + assert dyn["executed"] > 0 + assert dyn["ops"]["total"] == dyn["executed"] + assert rv32_bench.audit_provenance(report) == [] + assert bench_report.validate_report_schema(report) == [] + + from scratchv.standalone.benchmark import RV32EmulatorFast + + model_info = rv32_bench.get_model_info(str(mini_model)) + binary = (out / "_sv.bin").read_bytes() + data_offset = report["scratchv"]["compile"]["data_offset"] + emu = RV32EmulatorFast(mem_size_mb=256) + emu.load_unified_binary(binary, data_offset, load_addr=0) + emu.regs[2] = rv32_bench.SP_ADDR + emu.regs[10] = rv32_bench.INPUT_ADDR + emu.regs[11] = rv32_bench.OUTPUT_ADDR + blob = rv32_bench.build_input_q16(model_info["input_elements"], 42) + emu.mem[rv32_bench.INPUT_ADDR:rv32_bench.INPUT_ADDR + len(blob)] = blob + perf = emu.run(max_instr=2_000_000_000) + + assert perf.total == dyn["executed"] + assert perf.load_count == dyn["ops"]["load"] + assert perf.store_count == dyn["ops"]["store"] + assert perf.branch_total == dyn["ops"]["branch"] + + raw_reference = "".join( + f"{emu.read_mem_i32(rv32_bench.OUTPUT_ADDR + 4 * i) & 0xFFFFFFFF:08x}" + for i in range(model_info["output_elements"]) + ) + assert report["scratchv"]["output"]["raw_hex"] == "0x" + raw_reference + + rerun = rv32_bench.run_simulation( + asm_path=str(out / "_sv.s"), + binary_path=str(out / "_sv.bin"), + data_offset=data_offset, + workspace_bytes=report["scratchv"]["compile"]["workspace_bytes"], + input_elements=model_info["input_elements"], + output_elements=model_info["output_elements"], + max_instructions=0, mem_size=268435456, timeout_s=60.0, + chunk_instructions=4096, input_seed=42, + ) + assert rerun["dynamic"]["completion"] == "halted" + assert rerun["dynamic"]["executed"] == dyn["executed"] + assert rerun["dynamic"]["ops"] == dyn["ops"] + assert rerun["output"]["raw_hex"] == report["scratchv"]["output"]["raw_hex"] + + +# ─────────────────────────────────────────────────────────────────────────── +# T2: budget truncation is labeled, never presented as full +# ─────────────────────────────────────────────────────────────────────────── + +def test_budget_exhausted_is_labeled(tmp_path, mini_model): + pytest.importorskip("tinyfive") + out = tmp_path / "budget" + rc = rv32_bench.main([ + str(mini_model), "--max-instructions", "1000", "--quiet", + "--output-dir", str(out), "--timeout", "120", + "--chunk-instructions", "4096", + ]) + assert rc == rv32_bench.EXIT_OK + + report = json.loads((out / "rv32_bench.json").read_text()) + dyn = report["scratchv"]["dynamic"] + assert dyn["completion"] == "budget_exhausted" + assert dyn["limit"] == 1000 + assert dyn["executed"] == 1000 + assert dyn["ops"]["total"] == 1000 + assert report["comparison"]["dynamic_instruction_ratio"] is None + assert "budget_exhausted" in report["comparison"]["incomparable_reason"] + assert rv32_bench.audit_provenance(report) == [] + assert bench_report.validate_report_schema(report) == [] + + markdown = (out / "rv32_bench.md").read_text() + assert "[measured/budget]" in markdown + assert "dynamic instruction ratio" not in markdown + + rc_strict = rv32_bench.main([ + str(mini_model), "--max-instructions", "1000", "--quiet", + "--fail-on-incomplete", "--output-dir", str(tmp_path / "budget2"), + "--timeout", "120", "--chunk-instructions", "4096", + ]) + assert rc_strict == rv32_bench.EXIT_INCOMPLETE + + +# ─────────────────────────────────────────────────────────────────────────── +# T3: missing simulator never fabricates dynamic data +# ─────────────────────────────────────────────────────────────────────────── + +class _UnavailableProfiledMachine: + available = False + + def __init__(self, mem_size=0): + self.mem_size = mem_size + + +def test_report_requires_provenance(tmp_path, mini_model, monkeypatch): + monkeypatch.setattr( + rv32_bench, "ProfiledMachine", _UnavailableProfiledMachine, + ) + out = tmp_path / "static" + rc = rv32_bench.main([ + str(mini_model), "--allow-missing-simulator", "--quiet", + "--output-dir", str(out), + ]) + assert rc == rv32_bench.EXIT_OK + + report = json.loads((out / "rv32_bench.json").read_text()) + dyn = report["scratchv"]["dynamic"] + assert dyn["source"] == "unavailable" + assert dyn["ops"] is None + assert dyn["completion"] == "not_run" + assert dyn["reason"] + assert report["scratchv"]["compile"]["static_insns"] > 0 + assert report["scratchv"]["compile"]["static_source"] == "asm_scan" + assert report["comparison"]["dynamic_instruction_ratio"] is None + assert rv32_bench.audit_provenance(report) == [] + assert bench_report.validate_report_schema(report) == [] + + markdown = (out / "rv32_bench.md").read_text() + assert "[static]" in markdown + assert "[unavailable]" in markdown + + rc_no_flag = rv32_bench.main([ + str(mini_model), "--quiet", "--output-dir", str(tmp_path / "static2"), + ]) + assert rc_no_flag == rv32_bench.EXIT_NO_SIMULATOR + + +# ─────────────────────────────────────────────────────────────────────────── +# T4: memory layout validation refuses impossible budgets +# ─────────────────────────────────────────────────────────────────────────── + +def test_memory_layout_validation(tmp_path, mini_model, capsys): + out = tmp_path / "memfail" + rc = rv32_bench.main([ + str(mini_model), "--mem-size", "1048576", "--output-dir", str(out), + ]) + assert rc == rv32_bench.EXIT_LAYOUT + stderr = capsys.readouterr().err + assert "memory_layout_invalid" in stderr + assert "need mem_size >=" in stderr + assert not (out / "rv32_bench.json").exists() + assert not (out / "rv32_bench.md").exists() + assert not (out / "rv32_bench.html").exists() + + +# ─────────────────────────────────────────────────────────────────────────── +# T5: labels cover all branch targets; size cross-check is strict +# ─────────────────────────────────────────────────────────────────────────── + +def test_parsed_labels_cover_all_branches(mini_compiled): + asm = (mini_compiled["out"] / "_sv.s").read_text() + data_offset = mini_compiled["info"]["data_offset"] + labels = rv32_bench.parse_labels(asm, data_offset) + + assert labels.get(0) == "_start" + assert "_done" in labels.values() + assert mini_compiled["info"]["static_insns"] == data_offset // 4 + + for _pc, op, operands in rv32_bench._iter_asm_lines(asm): + if op in ("beq", "bne", "blt", "bge", "bltu", "bgeu", "j", "jal"): + assert operands[-1] in labels or operands[-1].lstrip("+-").isdigit() + + with pytest.raises(rv32_bench.LabelParseError): + rv32_bench.parse_labels( + "_start:\n bne t0, zero, +8\n addi t0, t0, 1\n", 8, + ) + with pytest.raises(rv32_bench.LabelParseError): + rv32_bench.parse_labels("_start:\n addi t0, t0, 1\n", 8) + + +# ─────────────────────────────────────────────────────────────────────────── +# T6: LLVM RV64 output is flagged as incomparable +# ─────────────────────────────────────────────────────────────────────────── + +def test_llvm_riscv64_flagged_isa_mismatch(): + asm = ( + " ld a0, 0(a1)\n" + " sd a0, 8(a1)\n" + " addiw a0, a0, 1\n" + " ret\n" + ) + assert set(rv32_bench.detect_isa_mismatch(asm)) == {"ld", "sd", "addiw"} + + llvm = { + "compile": { + "status": "success", + "reason": "rv64 mnemonics detected: addiw, ld, sd", + "isa_detected": "riscv64", "isa_mismatch": True, + "static_insns": 3, "static_source": "asm_scan", "elapsed_s": 0.0, + }, + "dynamic": { + "source": "unavailable", "simulator": "tinyfive", + "completion": "not_run", + "reason": "isa mismatch: rv64 mnemonics detected", + "ops": None, + }, + } + report = rv32_bench.build_report( + _fake_model(), _fake_env(), _fake_scratchv(), llvm, + ) + assert report["llvm"]["compile"]["isa_detected"] == "riscv64" + assert report["llvm"]["compile"]["isa_mismatch"] is True + assert report["llvm"]["dynamic"]["source"] == "unavailable" + assert report["comparison"]["dynamic_instruction_ratio"] is None + assert "llvm" in report["comparison"]["incomparable_reason"] + markdown = bench_report.render_markdown(report) + assert "RV32IMF" not in markdown + + +# ─────────────────────────────────────────────────────────────────────────── +# T7: the provenance audit rejects static counts masquerading as dynamic +# ─────────────────────────────────────────────────────────────────────────── + +def test_audit_provenance_rejects_static_fallback(): + clean = _valid_fake_report() + assert rv32_bench.audit_provenance(clean) == [] + + static_masquerade = deepcopy(clean) + static_masquerade["scratchv"]["dynamic"] = { + "source": "simulated", + "completion": "halted", + "ops": {"total": 3841}, + } + violations = rv32_bench.audit_provenance(static_masquerade) + assert violations + assert any("simulator" in v for v in violations) + assert any("executed" in v for v in violations) + + ratio_from_truncated = deepcopy(clean) + for side in ("scratchv", "llvm"): + ratio_from_truncated[side]["dynamic"] = { + "source": "simulated", "simulator": "tinyfive", + "simulator_version": "1.0.0", "completion": "budget_exhausted", + "limit": 1000, "executed": 1000, "memory_size_bytes": 268435456, + "input_seed": 42, + "ops": {"total": 1000, "load": 0, "store": 0, "mul": 0, + "add": 0, "madd": 0, "branch": 0}, + } + ratio_from_truncated["comparison"] = { + "dynamic_instruction_ratio": 0.31, "incomparable_reason": None, + } + assert rv32_bench.audit_provenance(ratio_from_truncated) + + forged_halt = deepcopy(clean) + forged_halt["scratchv"]["dynamic"].update( + {"completion": "halted", "limit": 5000, "executed": 5000, "ops": { + "total": 5000, "load": 0, "store": 0, "mul": 0, + "add": 0, "madd": 0, "branch": 0, + }}, + ) + assert any( + "unverified halt" in v + for v in rv32_bench.audit_provenance(forged_halt) + ) + + +# ─────────────────────────────────────────────────────────────────────────── +# T8: schema validation covers every required provenance key +# ─────────────────────────────────────────────────────────────────────────── + +def test_bench_report_schema_required_keys(): + report = _valid_fake_report() + assert bench_report.validate_report_schema(report) == [] + + empty_errors = bench_report.validate_report_schema({}) + joined = "\n".join(empty_errors) + for key in ( + "schema_version", "model.sha256", "environment", "targets", + "scratchv.compile", "scratchv.dynamic", "llvm.compile", + "llvm.dynamic", "comparison", + ): + assert key in joined, f"{key} missing from {empty_errors}" + + no_static_source = deepcopy(report) + del no_static_source["scratchv"]["compile"]["static_source"] + assert any( + "scratchv.compile.static_source" in e + for e in bench_report.validate_report_schema(no_static_source) + ) + + no_sha = deepcopy(report) + del no_sha["model"]["sha256"] + assert any( + "model.sha256" in e + for e in bench_report.validate_report_schema(no_sha) + ) + + bad_ratio = deepcopy(report) + bad_ratio["comparison"] = {"dynamic_instruction_ratio": 1.23} + assert any( + "comparison.incomparable_reason" in e + for e in bench_report.validate_report_schema(bad_ratio) + ) From ef1ee278ab230fc951d8ef44fec95c4846deea32 Mon Sep 17 00:00:00 2001 From: Seven Gao <799889633@qq.com> Date: Mon, 14 Sep 2026 21:21:51 +0800 Subject: [PATCH 2/5] docs(topic27): add design and development documents --- ...00\345\217\221\346\226\207\346\241\243.md" | 447 +++++++++++++++ ...76\350\256\241\346\226\207\346\241\243.md" | 514 ++++++++++++++++++ 2 files changed, 961 insertions(+) create mode 100644 "docs/topics/27-RV32\345\205\250\351\207\217Benchmark-\345\274\200\345\217\221\346\226\207\346\241\243.md" create mode 100644 "docs/topics/27-RV32\345\205\250\351\207\217Benchmark-\350\256\276\350\256\241\346\226\207\346\241\243.md" diff --git "a/docs/topics/27-RV32\345\205\250\351\207\217Benchmark-\345\274\200\345\217\221\346\226\207\346\241\243.md" "b/docs/topics/27-RV32\345\205\250\351\207\217Benchmark-\345\274\200\345\217\221\346\226\207\346\241\243.md" new file mode 100644 index 0000000..35f2557 --- /dev/null +++ "b/docs/topics/27-RV32\345\205\250\351\207\217Benchmark-\345\274\200\345\217\221\346\226\207\346\241\243.md" @@ -0,0 +1,447 @@ +# 课题 27：RV32 全量 Benchmark 开发文档 + +> 文档版本：v1.0 +> 编写日期：2026-09-14 +> 配套设计：同目录《设计文档.md》（术语、全量定义、诚实报告规范以设计文档为准） +> 涉及文件：`scratchv/standalone/rv32_bench.py`、`scratchv/standalone/bench_report.py`、`tests/test_rv32_bench.py`（新增） +> 只读依赖：`scratchv/simulator/tinyfive.py`、`scratchv/standalone/benchmark.py`、`scratchv/standalone/onnx_to_riscv_standalone.py`、`scratchv/backend/riscv_encoder.py` + +--- + +## 一、目标与范围 + +本课题把 `rv32_bench.py` 从“5000 条指令 + 64KB 内存 + 静态计数兜底”改造成“全量或明确预算 + 真实装载 + 诚实报告”的 RV32 统一基准驱动，并让 `bench_report.py` 按 schema v2 渲染带 provenance 的报告。 + +范围边界（与设计文档 4.9 一致）： + +- **不改** `onnx_to_riscv_standalone.py` 代码生成/ABI/内存布局（只读 stdout 与 `.bin/.s`）。 +- **不改** TinyFive 适配器公共接口（只使用既有 `load_binary/load_data/run(strict=True)/pc/get_perf`）。 +- **不改** Spike（课题 24）、基准套件（课题 06）、`cache_model.py`（课题 23）、`llvm_cache_compare.py`（课题 25）。 +- **LLVM 口径**：本课题只做“ISA 标注 + 统一 RV32 请求 + 跨 ISA 检测拒绝”；LLVM 侧可执行镜像（汇编→链接→ABI→数据段）与实测对比属课题 25/27 交界，本次以 `unavailable` 如实标注，不用解析估算补位。 + +--- + +## 二、`rv32_bench.py` 改动清单 + +### 2.1 改动总表 + +| # | 现行位置 | 改动 | 目的 | +|---|----------|------|------| +| C1 | `:118` `run_tinyfive(..., n_instructions: int = 10000)` | 替换为 `run_simulation(...)`，`max_instructions` 默认 `0`（全量语义） | 去除默认截断 | +| C2 | `:410-412` 调用点 `n_instructions=5000` | 删除该实参，改为 CLI `--max-instructions/--full` 驱动 | 同上 | +| C3 | `:124` `ProfiledMachine(mem_size=65536)` | 改为 `ProfiledMachine(mem_size=args.mem_size)`（默认 256MiB）+ `compute_layout()` 校验 | 内存容量正确 | +| C4 | 新增 | `parse_data_offset()` / `parse_workspace_bytes()` 解析编译器 stdout（`:2738,:2794`） | 拿到 `data_offset/workspace_size` | +| C5 | 新增 | `load_scratchv_image()`：`load_binary(code_words, 0)` + `load_data(weights, data_offset)` + `load_data(input_blob, 160MiB)` | 权重与输入真实装载 | +| C6 | `:160-184` `_prepare_asm_for_tinyfive` | 删除；新增 `parse_labels()`，标签不再被过滤，`.s` 只用于标签映射与静态计数，不用于装载代码 | 修复分支自跳转 | +| C7 | `:187-234` `_tinyfive_static_fallback` | 删除；静态计数改由 `static_instruction_mix()` 产出，字段独立为 `static_instruction_mix` | 静态数不冒充动态数 | +| C8 | `:118-157` `run_tinyfive` | 替换为分块执行器：`chunk` 循环 + `m.pc == halt_addr` 停机检测 + `SIGALRM` 超时 + 完整 provenance 输出 | 停机/预算/超时可控 | +| C9 | `:58-104` `compile_llvm_rv32` | llvmlite 可用时 `llmod.triple="riscv32-unknown-elf"`；新增 `detect_isa_mismatch()`；静态计数只扫 `.text`；失败原因入档 | 统一 RV32 口径与诚实降级 | +| C10 | `:241-312` `BenchResult` / `generate_report` | 改为 `build_report()`（schema v2 dict）+ `bench_report.render_*(report)`；删除硬编码模型描述、无依据 ratio 与 “No analytical estimates” 文案 | 诚实报告 | +| C11 | `:383-425` `main` | 新增 CLI 参数、退出码、`audit_provenance` 写盘前检查、`--quiet` | 接口规格化 | +| C12 | `bench_report.py` 全文 | 新增 `render_markdown/render_html/render_github_summary/render_bench_json/validate_report_schema`；旧 `generate_*` 保留兼容 | 报告字段与模板 | + +### 2.2 去默认截断 / 改可配（C1、C2、C11） + +现行代码： + +```python +def run_tinyfive(asm_path: str, n_instructions: int = 10000) -> dict: # :118 +... +sv.tinyfive = run_tinyfive(str(out / "_sv.s"), n_instructions=5000) # :411 +``` + +改为： + +```python +def run_simulation( + *, + asm_path: str, + binary_path: str, + max_instructions: int = 0, # 0 = 全量 + mem_size: int = 268435456, + timeout_s: float = 900.0, + chunk_instructions: int = 10_000_000, + input_seed: int = 42, + input_elements: int, + output_elements: int, + halt_addr: int, +) -> dict: ... +``` + +调用点： + +```python +sv.dynamic = run_simulation(..., max_instructions=args.max_instructions, ...) +``` + +CLI 语义：`--max-instructions 0`（默认）与 `--full` 等价；两者与 `--max-instructions N>0` 冲突时 `parser.error(...)` → exit 2。`--full` 存在的意义是抵抗“上游/CI 默认值”污染，显式声明无截断。 + +### 2.3 内存与数据装载（C3、C4、C5） + +1. **stdout 解析**（在 `compile_scratchv` 内，`rc==0` 时）： + ```python + DATA_OFFSET_RE = re.compile(r"Data offset:\s*0x([0-9A-Fa-f]+)") + WORKSPACE_RE = re.compile(r"Workspace:\s*([\d,]+)\s+bytes") + CODE_SIZE_RE = re.compile(r"Code size:\s*([\d,]+)\s+bytes") + ``` + 三个正则分别对应 `onnx_to_riscv_standalone.py:2794`、`:2738`、`:2729`。`compile.scratchv` 返回： + `{status, binary, binary_bytes, binary_sha256, code_bytes, data_offset, data_offset_source:"compiler_stdout", data_bytes, workspace_bytes, static_insns, static_source:"asm_scan", elapsed_s}`。 + 任一正则失配 → `status="failed"`, `error="binary_layout_unparsed"` → main exit 4（未显式给预算时）；禁止“猜 `data_offset = len(code_bytes)`”。 +2. **镜像装载**： + ```python + binary = Path(binary_path).read_bytes() + assert data_offset % 4 == 0 and 0 < data_offset < len(binary) + code_words = [int.from_bytes(binary[i:i+4], "little") + for i in range(0, data_offset, 4)] + weights = binary[data_offset:] + m = ProfiledMachine(mem_size=mem_size) + m.load_binary(code_words, origin=0) + m.load_data(weights, data_offset) + ``` +3. **输入构造**（一次性）：`random.Random(input_seed)`，每个元素 `int((r.random() - 0.5) * 0.2 * 65536)`，`struct.pack(f"<{n}i", *vals)`，`m.load_data(blob, 160*1024*1024)`。禁止逐元素 `write_mem_i32`（慢且易错）。 +4. **寄存器初始化**：`m.set_reg(2, 128*1024*1024)`、`m.set_reg(10, 160*1024*1024)`、`m.set_reg(11, 192*1024*1024)`、`m.set_reg(1, halt_addr)`；`gp` 由代码自身 `_start` 的 AUIPC 补丁设置，harness 不写。 +5. **布局校验**（`compute_layout()`，设计文档 2.1.2 公式；失败抛 `LayoutError` → exit 4，消息含所需最小字节数）。 +6. `halt_addr = align_up(binary_bytes, 16)`；要求 `halt_addr + 4 <= mem_size`。 + +### 2.4 标签解析（C6） + +`.s` 格式：标签独占一行（列 0，`name:`），指令行缩进两格并可能带 `# 注释`（`RISCVEmitter.disassemble()`，`onnx_to_riscv_standalone.py:1291-1305`）。 + +```python +def parse_labels(asm_text: str) -> dict[int, str]: + pc, labels, seen_tokens = 0, {}, [] + for raw in asm_text.splitlines(): + line = raw.split("#", 1)[0].rstrip() + if not line.strip(): + continue + if line.endswith(":") and "(" not in line: + labels[pc] = line.strip()[:-1] + continue + if line.strip().startswith("."): # 理论不存在；防御性跳过 + continue + pc += 4 + seen_tokens.append(line.strip()) + if pc != expected_code_bytes: # 与 data_offset 交叉校验 + raise LabelParseError(f"asm/code-size mismatch: {pc} != {expected_code_bytes}") + return labels +``` + +关键点： + +- **保留标签**（现行 `:174-176` 丢弃标签，`label_map` 未用，导致 `riscv_encoder.py:502` 的 `labels.get(label, current_idx)` 把分支解析为自跳转）。 +- 代码装载走 `load_binary`（编译器自己的二进制），不再把 `.s` 喂给 `assemble_to_binary`；`.s` 仅作标签映射与静态 scan。 +- `_start` 必须映射 `pc=0`；`_done` 必须存在；`static_insns == data_offset // 4` 作为不变量断言。 + +### 2.5 失败路径（C7、C11） + +| 失败 | 检测点 | 行为 | +|------|--------|------| +| TinyFive 未安装 | `m.available is False` | 无 `--allow-missing-simulator` → exit 3；有 → `dynamic.source="unavailable"`、`ops=null`、`completion="not_run"`，报告只含 static | +| 内存布局不满足 | `compute_layout()` | exit 4，`errors=["memory_layout_invalid: need >= N bytes"]`，不落盘动态报告 | +| `data_offset` 解析失败 | `compile_scratchv` stdout 正则 | exit 4，`reason="binary_layout_unparsed"` | +| 标签/代码尺寸不一致 | `parse_labels` | 抛 `LabelParseError` → exit 4 | +| 助记符预检失败 | 子集扫描 `.s` vs TinyFive 支持表 | `completion="not_run"`，exit 4，列出不支持助记符 | +| 预算耗尽 | 循环额度判断 | `completion="budget_exhausted"`，`executed==limit`，ratio `null`，exit 0（显式预算属正常） | +| 超时 | `SIGALRM` + `_timed_out` 标志 | `completion="timeout"`，保留部分 ops，exit 0（`--fail-on-incomplete` 时 exit 7） | +| `m.last_error` 非空且非超时 | 适配器 `strict=True` 抛错 | `completion="error"`，`dynamic=null`，exit 1 | +| 未完成且要求严格 | `--fail-on-incomplete` | exit 7 | + +静态兜底已删除：`static_instruction_mix` 只写入 `scratchv.static_instruction_mix`，字段名带 `static_`，永不出现在 `dynamic.ops`。 + +### 2.6 LLVM 侧改动（C9） + +1. `binding.Target.from_triple("riscv32-unknown-elf")`；`ImportError`/`RuntimeError` → `status="skipped"`，`reason` 记录原始异常。llvmlite 当前环境未安装，该分支即 `skipped`。 +2. 成功路径。解析 IR 文本后，用 llvmlite API 覆盖模块头： + ```python + llmod = binding.parse_assembly(ir_text) + llmod.triple = "riscv32-unknown-elf" # 覆盖 onnx_to_llvm_standalone.py:446 的 riscv64 头 + llmod.data_layout = str(tm.target_data) # 与目标机一致，避免指针宽度假设漂移 + llmod.verify() + asm = tm.emit_assembly(llmod) + ``` +3. `detect_isa_mismatch(asm_text) -> list[str]`：扫描助记符集合 `{ld, sd, lwu, addw, subw, addiw, sllw, srlw, sraw, slliw, srliw, sraiw, mulw, divw, divuw, remw, remuw, fld, fsd, fcvt.l.s, fcvt.s.l}`；命中 → `isa_detected="riscv64"`、`isa_mismatch=true`，`llvm.dynamic` 不运行。 +4. 静态计数：遇到 `.section .rodata`（权重数组，`onnx_to_llvm_standalone.py:132-157` 的 `private constant [N x float]`）停止计数；`.word/.long/.byte/.float` 不算指令。`static_source="asm_scan"`。 +5. 不构造 LLVM 侧可执行镜像；`llvm.dynamic={source:"unavailable", completion:"not_run", reason:"llvm executable image pipeline not implemented (topic 25 boundary)", ops:null}`。 + +--- + +## 三、`bench_report.py` 报告字段与模板 + +### 3.1 新增函数 + +| 函数 | 签名 | 说明 | +|------|------|------| +| `render_markdown` | `(report: dict) -> str` | schema v2 → Markdown | +| `render_html` | `(report: dict) -> str` | Markdown 内容包进既有 CSS 壳 | +| `render_bench_json` | `(report: dict) -> str` | 规范化 JSON（`sort_keys=False, indent=2`） | +| `render_github_summary` | `(report: dict) -> str` | CI 摘要，只引用 measured/static 分区 | +| `validate_report_schema` | `(report: dict) -> list[str]` | 返回缺失/类型错误列表；空列表为通过 | + +旧 `generate_html_report/generate_json_report/generate_github_summary` 保留为兼容壳（内部转调 `render_*` 或维持原行为），避免影响 `onnx_to_riscv_standalone.py --report` 与 CI（课题 06/30）。 + +### 3.2 Markdown 模板（骨架） + +```markdown +# RV32 Benchmark Report — {model.path} +- Generated: {generated_at} | schema: rv32-bench/2 +- Model: sha256={model.sha256[:12]} | bytes={model.bytes} | input={model.input_shape} +- Targets: ScratchV {targets.scratchv.isa}/{targets.scratchv.numeric_format} + LLVM {targets.llvm.isa}/{targets.llvm.numeric_format} (opt={targets.llvm.opt_level}) +- Environment: python={environment.python} numpy={environment.numpy} + tinyfive={environment.tinyfive} llvmlite={environment.llvmlite} + +## 1. Compilation [static] +| Metric | ScratchV | LLVM | +| status | … | … | +| code bytes | … | … | +| static insns [asm_scan] | … | … | + +## 2. Dynamic Execution [measured] +| Metric | ScratchV | LLVM | +| completion | halted | not_run | +| executed | … | — | +| ops.total | … | — | +| load/store/mul/add/madd/branch | … | — | +> LLVM dynamic unavailable: {llvm.dynamic.reason} + +## 3. Comparison [measured] +- dynamic_instruction_ratio: **null** — {comparison.incomparable_reason} +（仅当两侧 source=="simulated" 且 completion=="halted" 时打印比值行，否则打印 null + 原因） + +## 4. Analytical Warnings [estimated] +- {warnings[*]} + +## Provenance +Simulated by tinyfive {environment.tinyfive} | completion={completion} | executed={executed} +| limit={limit} | memory={memory_size_bytes} | seed={input_seed} | halt=0x{halt_addr:x} +| model sha256={model.sha256} | binary sha256={scratchv.compile.binary_sha256} +``` + +渲染规则： + +- 每个表标题必带 `[measured]/[static]/[estimated]/[unavailable]` 之一；同一表格中不得混用来源类别。 +- 页脚逐字取 provenance 字段，禁止出现“all metrics from simulation / no analytical estimates”之类无法由字段证明的断言。 +- 模型名与层结构只能来自 `model.*`；删除现行 `rv32_bench.py:268` 的硬编码描述。 +- `ops` 任一计数为 0 时照实显示 `0`，不得显示 `—` 掩盖“未跑到该类别”。 + +### 3.3 报告字段总表（dotted path） + +| 字段 | 类型 | 来源分类 | 说明 | +|------|------|----------|------| +| `schema_version` | str | — | 固定 `"rv32-bench/2"` | +| `generated_at` | str | — | UTC ISO8601 | +| `generator.script` | str | — | `"rv32_bench.py"` | +| `model.path/sha256/bytes/input_name/input_shape/output_name/output_shape/initializer_count/weight_bytes` | — | static | 模型身份 | +| `environment.python/numpy/tinyfive/llvmlite` | str\|null | — | 依赖版本 | +| `targets.scratchv.{isa,abi,numeric_format}` | str | — | `rv32im/ilp32/q16.16` | +| `targets.llvm.{isa,abi,numeric_format,triple,opt_level}` | — | — | `rv32imf/ilp32/float32` | +| `scratchv.compile.status/binary/binary_bytes/binary_sha256/code_bytes/data_offset/data_offset_source/data_bytes/workspace_bytes/static_insns/static_source/elapsed_s` | — | static | 编译与镜像 | +| `scratchv.static_instruction_mix.{source,load,store,mul,add,madd,branch,other}` | int | static | 静态助记符分布 | +| `scratchv.dynamic.{source,simulator,simulator_version,completion,limit,executed,timeout_s,elapsed_s,memory_size_bytes,input_seed,input_elements,halt_addr,ops{total,load,store,mul,add,madd,branch},x_registers_used,x_usage_total,f_registers_used,per_label,per_label_note,last_error}` | — | measured | 动态执行 | +| `scratchv.output.{addr,elements,raw_hex,q16_16}` | — | measured | 输出读出 | +| `llvm.compile.{status,reason,isa_detected,isa_mismatch,static_insns,static_source,elapsed_s}` | — | static/unavailable | LLVM 侧 | +| `llvm.dynamic.{source,simulator,completion,reason,ops}` | — | unavailable | 失败必须 `ops=null` | +| `comparison.{dynamic_instruction_ratio,incomparable_reason}` | float\|null | measured 派生 | 比值规则见设计文档 2.1.1 | +| `warnings[]` / `errors[]` | list[str] | — | 预估告警与错误 | + +--- + +## 四、与 `benchmark.py` / `ProfiledMachine` 的接口 + +### 4.1 `ProfiledMachine`（`scratchv/simulator/tinyfive.py`，只读使用） + +| 用法 | 契约 | +|------|------| +| `ProfiledMachine(mem_size)` | `mem_size` 为字节数；构造即 `np.zeros(mem_size, uint8)`；256MiB 默认值可接受（本机 numpy 1.24.4） | +| `.available` | `False` 时只能走 `unavailable` 路径，不得继续 `load_*` | +| `load_binary(words, origin=0)` | `words: list[int]` 32 位词；越界抛 `ValueError` | +| `load_data(data: bytes, addr)` | `np.frombuffer` 切片赋值；用于权重（26MB 一次性）与输入 | +| `run(instructions=N, strict=True)` | 适配器内部 `finally` 更新 `instr_count`；异常包装为 `RuntimeError(last_error)` 继续抛出，ops 保留 | +| `.pc` / `.get_perf()` | `get_perf()` 返回 `{total,load,store,mul,add,madd,branch}` 累计值；分块循环用 `pc` 判停机 | +| `.set_reg(idx, value)` | `0 < idx < 32`；本课题写 x1/x2/x10/x11 | +| `.read_mem_i32(addr)` | 输出读取；不越界返回 0 | + +不使用：`load_asm`（标签语义不可靠）、`print_perf`、`StubProfiledMachine`。 + +### 4.2 `benchmark.py`（只读使用） + +- `estimate_cnn_model(model_spec=None) -> dict`：仅用于 `warnings` 与“预计墙钟”提示；字段进入 `estimated` 分类，前缀 `estimated_`。不得写入 `dynamic`。 +- `RV32EmulatorFast` / `run_benchmark`：仅测试用例 1 作为独立功能对照（`load_unified_binary(binary, code_size_base=..., load_addr=0)` + `run(max_instr=...)`）；其计数分类（`Cat_*`）与 TinyFive 不同，只对照 `total/load_count/store_count/branch_total`。若未来作为正式数据源，须以 `simulator="rv32_emulator_fast"` 独立字段呈现，不与 TinyFive 混算。 +- `estimate_cnn_instructions` 的 per-MAC 常量（`CONV_INSNS_PER_MAC=8` 等）属解析模型，禁止用于 `measured`。 + +### 4.3 `cache_model.py`（不在运行时链路） + +`CacheSim`/`create_cache_pair` 为分析模型；本课题报告不包含 cache 指标。若后续引入，字段必须放 `estimated` 分类并标注 `model="cache_model"`。 + +--- + +## 五、接口契约 + +### 5.1 CLI（精确名称） + +``` +python scratchv/standalone/rv32_bench.py MODEL + [--output-dir DIR] # 默认 benchmark_reports + [--html FILE] [--json FILE] [--md FILE] # 默认 rv32_bench.{html,json,md} + [--max-instructions N] # int，默认 0；0=全量，>0=预算 + [--full] # 无截断；与 N>0 互斥 + [--mem-size BYTES] # int，默认 268435456 + [--timeout SECONDS] # float，默认 900.0 + [--chunk-instructions N] # int，默认 10000000 + [--input-seed N] # int，默认 42 + [--skip-llvm] # flag + [--allow-missing-simulator] # flag + [--fail-on-incomplete] # flag + [--quiet] # flag +``` + +退出码：`0` 成功；`1` 未预期错误/审计失败；`2` 参数冲突；`3` 仿真器不可用；`4` 布局/镜像/预检失败；`7` 未完成且 `--fail-on-incomplete`。 + +### 5.2 函数（精确签名） + +```python +# rv32_bench.py +SCHEMA_VERSION = "rv32-bench/2" +EXIT_OK, EXIT_ERROR, EXIT_USAGE, EXIT_NO_SIMULATOR, EXIT_LAYOUT, EXIT_INCOMPLETE = 0, 1, 2, 3, 4, 7 + +class LayoutError(RuntimeError): ... +class LabelParseError(RuntimeError): ... +class SimulationTimeout(RuntimeError): ... + +def sha256_file(path: str | Path) -> str +def parse_data_offset(stdout: str) -> int | None +def parse_workspace_bytes(stdout: str) -> int | None +def parse_labels(asm_text: str, expected_code_bytes: int) -> dict[int, str] +def static_instruction_mix(asm_text: str) -> dict # 键: source,load,store,mul,add,madd,branch,other +def compute_layout(*, data_offset: int, data_size: int, workspace_bytes: int, + input_elements: int, output_elements: int, + mem_size: int) -> dict # 键: sp,input_addr,output_addr,halt_addr,mem_size +def load_scratchv_image(binary_path: str, data_offset: int) -> tuple[list[int], bytes] +def build_input_q16(elements: int, seed: int) -> bytes +def run_simulation(*, asm_path: str, binary_path: str, data_offset: int, + workspace_bytes: int, input_elements: int, output_elements: int, + max_instructions: int = 0, mem_size: int = 268435456, + timeout_s: float = 900.0, chunk_instructions: int = 10_000_000, + input_seed: int = 42) -> dict # -> scratchv.dynamic + output +def compile_scratchv(onnx_path: str, output_bin: str, output_asm: str, + timeout_s: float = 120.0) -> dict +def compile_llvm_rv32(onnx_path: str, output_asm: str, *, + triple: str = "riscv32-unknown-elf", opt_level: int = 2) -> dict +def detect_isa_mismatch(asm_text: str) -> list[str] +def build_report(model: dict, environment: dict, scratchv: dict, llvm: dict) -> dict +def audit_provenance(report: dict) -> list[str] +def main(argv: list[str] | None = None) -> int + +# bench_report.py +def render_markdown(report: dict) -> str +def render_html(report: dict) -> str +def render_bench_json(report: dict) -> str +def render_github_summary(report: dict) -> str +def validate_report_schema(report: dict) -> list[str] +``` + +### 5.3 报告字段（精确名称） + +一级键：`schema_version, generated_at, generator, model, environment, targets, scratchv, llvm, comparison, warnings, errors`。 + +二级/三级键以设计文档 5.1 与本文 3.3 为准，关键枚举： + +- `scratchv.dynamic.source ∈ {"simulated","unavailable"}` +- `scratchv.dynamic.completion ∈ {"halted","budget_exhausted","timeout","error","not_run"}` +- `llvm.dynamic.source ∈ {"simulated","unavailable"}` +- `comparison.dynamic_instruction_ratio: float | null`（仅两侧 `simulated`+`halted`） +- `*.static_source == "asm_scan"` + +--- + +## 六、测试文件与用例 + +测试文件：`tests/test_rv32_bench.py`（新增）。 + +| 用例 | 名称 | 要点 | +|------|------|------| +| T1 | `test_full_run_small_model_matches_reference` | `onnx.helper` 造迷你 CNN → 全量 `halted`；与 `RV32EmulatorFast` 对照 `total/load/store/branch` 精确相等；重跑确定性 | +| T2 | `test_budget_exhausted_is_labeled` | `--max-instructions 1000` → `budget_exhausted`、`executed==limit==1000`、ratio `null`、`--fail-on-incomplete` 退出 7 | +| T3 | `test_report_requires_provenance` | monkeypatch TinyFive 不可用 + `--allow-missing-simulator` → `dynamic.source=="unavailable"`、`ops is null`、static 分区存在；无 flag 时退出 3 | +| T4 | `test_memory_layout_validation` | `--mem-size 1048576` → exit 4，stderr 含 `memory_layout_invalid`，不落动态报告 | +| T5 | `test_parsed_labels_cover_all_branches` | 分支/跳转目标全部有标签；`static_insns == data_offset//4`；缺标签抛 `LabelParseError` | +| T6 | `test_llvm_riscv64_flagged_isa_mismatch` | 含 `ld/sd/addiw` 的 fixture → `isa_mismatch=true`、ratio `null` | +| T7 | `test_audit_provenance_rejects_static_fallback` | 构造非法报告（设计文档 5.3 两例）→ `audit_provenance()` 非空 | +| T8 | `test_bench_report_schema_required_keys` | `validate_report_schema()` 对必填键逐一断言（无需 TinyFive，不得 skip） | + +运行： + +```bash +python -m pytest tests/test_rv32_bench.py -q +make test +``` + +无 TinyFive 环境下 T1/T2 走 `pytest.importorskip("tinyfive")`；T3–T8 必须运行。 + +--- + +## 七、验收标准 + +1. 全量路径：`python scratchv/standalone/rv32_bench.py models/graph/cnn.onnx --full --mem-size 268435456 --timeout 1800 --output-dir /tmp/rv32_full` + - 成功产出 3 格式报告；`scratchv.dynamic.completion ∈ {"halted","timeout"}`； + - 若 `halted`，`executed` 为全量真实数（不受 5000/10000 限制）；若 `timeout`，报告明示部分轨迹且 `comparison=null`； + - `audit_provenance(report) == []`；`validate_report_schema(report) == []`。 +2. 预算路径：`--max-instructions 1000000` → `completion=="budget_exhausted"`、`limit==executed==1000000`、ratio `null`；报告含 `[measured/budget]` 标识。 +3. 任何输出（Markdown/HTML/JSON）包含：`model.sha256`、`environment.tinyfive`、`input_seed`、`memory_size_bytes`、`completion`、`static_source`；不再出现 “All metrics sourced from TinyFive … No analytical estimates” 这类无字段支撑的断言。 +4. LLVM 侧：无 llvmlite → `status=="skipped"`、`dynamic.source=="unavailable"`、报告不打印比值数字；有 llvmlite → 模块 triple 为 `riscv32-unknown-elf`，`.s` 若含 RV64 助记符则 `isa_mismatch==true`。 +5. 布局失败：`--mem-size` 不足时 exit 4 且不产生虚假动态数据。 +6. 回归：`make test` 全绿；`python .claude/harness/verify/run.py --level L2` 通过；未改动编译器/Spike/基准套件代码。 +7. 文档一致性：报告字段与本文 3.3/5.3 完全同名；CLI 与 5.1 完全同名。 + +--- + +## 八、风险与回退 + +| # | 风险 | 影响 | 缓解/回退 | +|---|------|------|-----------| +| R1 | TinyFive 吞吐低（Python 解释执行），全量 18.5 亿指令可能需数小时 | 全量不可行 | `--max-instructions` 预算并如实标注；先跑 1M 探测得到 MIPS 预估墙钟，写入 `warnings` | +| R2 | 256MiB numpy 内存 + 26MB 权重 | 内存压力 | 布局不可压缩（ABI 地址固定）；内存不足只能拒绝并提示，禁止缩容假装成功 | +| R3 | `ra` 被生成代码内部 `jal ra, …` 覆盖，`halt_addr` 永不命中 | 无法 `halted` | `completion` 如实标 `budget_exhausted/timeout`；`--fail-on-incomplete` 供 CI | +| R4 | 平台无 `SIGALRM` | 超时保护缺失 | 退化为要求 `--max-instructions`；否则拒绝启动 | +| R5 | TinyFive 不支持某助记符导致 PC 不前进（卡死） | 挂起 | 启动前助记符白名单预检，`not_run` + exit 4 | +| R6 | `.s` 标签格式未来漂移 | 标签解析失败 | `parse_labels` 与 `data_offset//4` 交叉校验，失败即抛错；T5 固定回归 | +| R7 | llvmlite 未安装/目标不支持 | LLVM 侧不可测 | `skipped`+`unavailable` 如实标注；不引入解析估算 | +| R8 | LLVM IR 为 riscv64 假设（指针宽度/全局） | 强行 RV32 代码错误 | `llmod.triple/data_layout` 覆盖 + `verify()` + RV64 助记符检测；失败标 `isa_mismatch` | +| R9 | 工作区 `[sp, sp+workspace)` 与 input@160MiB 冲突 | 静默数据损坏 | `compute_layout()` 的 GUARD 校验，不满足 exit 4 | +| R10 | 静态兜底被误用回动态展示 | 报告失真 | 字段拆分为 `static_instruction_mix`；`audit_provenance()` 阻断；T7 回归 | + +回退策略：若不满足验收（如全量在可接受时间内无法完成），保留本设计的所有“诚实标注”改动，仅把默认改为 `--max-instructions` 显式预算模式，并在报告与 CI 摘要中显著标注 `budget_exhausted`；任何情况下不得恢复“静态计数 + 无 provenance”的报告形态。 + +--- + +## 实现结果（2026-09-14 集成） + +> **集成 commit**：`12685db`（`feat(topic27): full RV32 benchmark with honest provenance and budget controls`） +> **集成位置**：`Seven_big_summary` 上第 7 个 topic commit（顺序 … → 24 → **27** → 10 → …） +> **集成后全量**：`PYTHONPATH=. python3.11 -m pytest tests/ -q` → **1011 passed / 13 xfailed / 20 xpassed / 0 failed** + +### 实现文件与要点 + +| 文件 | 要点 | +|------|------| +| `scratchv/standalone/rv32_bench.py` | 重写：默认全量 + `--max-instructions` 预算、`audit_provenance`、`compute_layout` 的 GUARD 校验 | +| `scratchv/standalone/bench_report.py` | 报告 schema v2（`model.sha256` / `environment.tinyfive` / `completion` / `static_source` 等） | +| `tests/test_rv32_bench.py` | 9 个新用例 | + +### 测试数字 + +| 口径 | 结果 | +|------|------| +| 定向（`tests/test_rv32_bench.py`） | 9 passed | +| 分支全量（cherry-pick 前） | 574 passed | +| 集成后全量 | 1011 passed / 13 xfailed / 20 xpassed / 0 failed | + +### 与本文档的偏差 / 未完成项 + +- cnn 全量实测**未跑**（TinyFive 约 7 万 instr/s，预计约 7 小时），只做预算中断干跑（`budget_exhausted`）。 + +### 已知限制 + +- 对 TinyFive 机器实例的 NumPy 2.x `LW/LH` 兼容 shim 属课题 26 追修（`tinyfive.py` 未改）。 +- 错误路径统一用 `source=unavailable` 表达（而非 `dynamic=null`）。 +- LLVM 侧无 llvmlite 时 `status=skipped`、不打印比值数字。 diff --git "a/docs/topics/27-RV32\345\205\250\351\207\217Benchmark-\350\256\276\350\256\241\346\226\207\346\241\243.md" "b/docs/topics/27-RV32\345\205\250\351\207\217Benchmark-\350\256\276\350\256\241\346\226\207\346\241\243.md" new file mode 100644 index 0000000..234bde1 --- /dev/null +++ "b/docs/topics/27-RV32\345\205\250\351\207\217Benchmark-\350\256\276\350\256\241\346\226\207\346\241\243.md" @@ -0,0 +1,514 @@ +# 课题 27：RV32 全量 Benchmark 设计文档 + +> 文档版本：v1.0 +> 编写日期：2026-09-14 +> 涉及模块：`scratchv/standalone/rv32_bench.py`（RV32 统一对比驱动）、`scratchv/standalone/bench_report.py`（报告渲染）、`scratchv/simulator/tinyfive.py`（`ProfiledMachine` 适配器，只读接口）、`scratchv/standalone/benchmark.py`（解析估算，仅作告警） +> 功能范围：RV32IMF 统一目标下的 ScratchV/LLVM 对比基准；全量指令仿真与预算控制；诚实报告规范（provenance 字段、ISA 标注、模型身份）；CLI 参数规格；报告 JSON schema v2 + +--- + +## 一、功能介绍 + +### 1.1 功能概述 + +课题 27 的目标是让 `rv32_bench.py` 真正执行“全量或明确预算并如实标注”的 RV32 模拟基准，而不是当前的“5000 条指令 + 64KB 内存 + 无权重”的伪全量对比。现状问题如下（均已核对源码与运行记录）： + +1. **默认截断**：`rv32_bench.py:118` 默认 `n_instructions: int = 10000`，调用点 `rv32_bench.py:410-412` 实际传 `n_instructions=5000`。任何一次运行中，`ops.total` 恒 ≤ 5000，仅覆盖 `_start` 与输入拷贝循环的开头，与“动态指令数”语义无关。 +2. **内存容量错误**：`rv32_bench.py:124` 使用 `ProfiledMachine(mem_size=65536)`（64KB）。而 ScratchV 编译器约定的裸机 ABI 是 `sp=128MiB`、`a0=160MiB`（input）、`a1=192MiB`（output）、权重紧跟在代码之后（`data_offset`，`onnx_to_riscv_standalone.py:2548-2557`）。`models/graph/cnn.onnx` 为 27,677,152 字节（约 26.4MB 权重），64KB 内存连权重区都容纳不下。 +3. **权重数据未装载**：编译器只把权重写进 `.bin`（`binary = code_bytes + weight_data`，`onnx_to_riscv_standalone.py:2791`）；`.s` 仅是代码反汇编（`onnx_to_riscv_standalone.py:2802-2807`），不含 26MB 权重。`run_tinyfive` 只把 `.s` 文本喂给 `load_asm`，权重恒为 0。 +4. **标签解析失效**：`_prepare_asm_for_tinyfive`（`rv32_bench.py:160-184`）构造了 `label_map` 却从不使用，标签行被丢弃；随后 `load_asm` 经 `RISCVAEncoder` 组装，而未解析标签走 `self.labels.get(label, current_idx)`（`riscv_encoder.py:502`），分支全部退化成“跳到自己”。同时 `.word` 数据行被当指令跳过。 +5. **静态计数冒充动态计数**：TinyFive 缺失时 `_tinyfive_static_fallback`（`rv32_bench.py:187-234`）返回 `ops` 字典，其 `total = sum(静态计数)`，并写入 `instr_count=n_instr`；报告把该结果与真实仿真结果同表渲染。 +6. **报告声明失真**：`generate_report` 硬编码模型描述（`rv32_bench.py:268` 写死 `cnn.onnx (3×Conv + …)`），输出 `Dynamic instruction ratio`（`rv32_bench.py:303`），页脚宣称 “All metrics sourced from TinyFive ProfiledMachine simulation. No analytical estimates.”（`rv32_bench.py:310`）。 +7. **LLVM 侧跨 ISA 与估算**：LLVM IR 生成器固定输出 `riscv64-unknown-elf` 模块头（`onnx_to_llvm_standalone.py:446`），而 `rv32_bench.py:82` 却向 llvmlite 请求 `riscv32` 目标机；两者不一致。llvmlite 未安装时 `status="skipped"`，但报告仍渲染对照表并保留 ratio 行。周边工具 `llvm_cache_compare.py` 的“动态指令数”是解析估算（该文件第 9 行自述），不属于本课题的实测口径。 + +本课题的功能定义： + +- **全量仿真（full simulation）**：从入口 PC=0（`_start`）执行到 `_done: ret` 的停机地址，不设指令数截断；内存按裸机 ABI 尺寸分配；`.bin` 的代码段与权重段分别装载；输入按确定性种子生成。 +- **明确预算（budgeted simulation）**：允许 `--max-instructions N` 主动截断；截断必须写入 `completion="budget_exhausted"`，且所有依赖完整轨迹的派生指标（比值、per-MAC）置空。 +- **诚实报告（honest reporting）**：每个数字携带来源分类（measured/static/estimated/unavailable）与 provenance；动态指标只能来自真实仿真；ISA 与数值格式显式标注；模型身份由内容哈希与维度定义；不可复现的旧文案全部删除。 + +### 1.2 设计目标 + +- **真实性**：报告中的每一列都有 `source` 与 provenance；`audit_provenance()` 空违规列表是产出报告的前置条件。 +- **可复现**：相同的模型、种子、参数、依赖版本产出相同的计数（时间类字段除外）；模型 `sha256`、输入种子、内存布局全部入档。 +- **全量优先、预算可选**：默认全量；截断是显式选择且显式标注；禁止把截断结果当作全量结果展示。 +- **可比性**：两侧统一标注 `rv32/imelf` 与数值格式（Q16.16 vs float32）；不可比时 ratio 为 `null` 并给出 `incomparable_reason`。 +- **失败可见**：TinyFive 缺失、内存不足、标签解析失败、ISA 不匹配都必须以非零退出码或显式状态呈现，绝不静默降级。 +- **零新依赖**：仅使用 `tinyfive`、`numpy`、标准库；llvmlite 保持可选。 +- **兼容性**：保留 `--output-dir/--html/--json/--md` 既有 CLI 语义，既有测试不回归。 + +--- + +## 二、设计规范 + +### 2.1 全量仿真定义 + +#### 2.1.1 指令上限策略 + +用运行模式与完成状态替代“隐藏的 10000/5000 常量”： + +``` +run_mode ::= full | budgeted(N) | (缺省 = full) +full ::= "--full" | "--max-instructions 0" +budgeted(N) ::= "--max-instructions" N ; N > 0 +completion ::= "halted" | "budget_exhausted" | "timeout" | "error" | "not_run" +``` + +| 运行模式 | TinyFive 调用 | 停止条件 | `completion` | +|----------|---------------|----------|--------------| +| full（默认） | `m.run(instructions=chunk)` 分块循环，累计直到 `pc == halt_addr` | PC 命中停机地址；或墙钟超时 | `halted` / `timeout` | +| budgeted(N) | 同上，剩余额度 = N − executed | 额度耗尽；或 PC 命中停机地址 | `budget_exhausted` / `halted` | +| 参数冲突 | `--full` 与 `--max-instructions N>0` 同时给出 | 启动前拒绝 | 退出码 2（usage） | + +规则： + +- `--max-instructions 0` 与 `--full` 等价，均为无截断语义；`limit=null`、`executed` 为真实执行数。 +- 分块循环使用 `--chunk-instructions`（默认 10,000,000）作为每块大小；块间检查停机与超时。这是在不修改 `ProfiledMachine` 公共接口的前提下获得“停机检测 + 进度 + 超时”的唯一手段。 +- `budget_exhausted` 时 `executed == limit` 必须为等式不变量；`halted` 时 `limit` 可为 `null`（full）或任意（budgeted 提前停机）。 +- 超时只在 `--full` 模式下产生 `timeout`；`budgeted` 模式下先到限额即结束，超时属于异常保护（同样标 `timeout`）。 +- 派生指标约束：`comparison.dynamic_instruction_ratio` 仅当两侧 `source=="simulated"` 且 `completion=="halted"` 时计算；任一 `budget_exhausted/timeout` 一律 `null` 并写 `incomparable_reason`。 + +#### 2.1.2 内存布局与容量 + +内存布局沿用编译器自检脚本的裸机约定（`onnx_to_riscv_standalone.py:2548-2557`），并补充停机字： + +| 区域 | 起址 | 大小 | 来源 | +|------|------|------|------| +| code | `0x00000000` | `data_offset` | `.bin[0:data_offset]` | +| weights | `data_offset` | `binary_bytes − data_offset` | `.bin[data_offset:]` | +| HALT（保留 4B） | `halt_addr = align_up(binary_bytes, 16)` | 4 | 本课题注入，不作为指令执行 | +| workspace | `sp` 向上 | `workspace_size` | 编译器 stdout `Workspace: N bytes` | +| stack | `sp` 向下 | 预留 | 由 `sp` 指向栈顶 | +| sp | `128 MiB` | — | ABI 约定 | +| input | `160 MiB` | `input_elements × 4` | Q16.16，确定性种子 | +| output | `192 MiB` | `output_elements × 4` | 仿真结束后读取 | + +容量约束（启动前校验，不满足即 `exit 4`）： + +``` +data_offset % 4 == 0 +halt_addr + 4 <= mem_size +192 MiB + out_bytes + 4096 <= mem_size # output 区必须可用 +sp + workspace_size + GUARD <= 160 MiB # 工作区不得撞 input 区，GUARD = 1 MiB +``` + +默认 `--mem-size 268435456`（256MiB）。若模型或工作区要求超过该值，必须显式调大而不能静默截断；若小于 `192MiB + out_bytes`，直接拒绝运行（因为 ABI 地址不可压缩）。 + +#### 2.1.3 权重数据装载 + +``` +image ::= code_bytes (已 4 字节对齐) ‖ weight_bytes +data_offset ::= len(code_bytes) +data_size ::= len(weight_bytes) = len(.bin) − data_offset +``` + +- `data_offset` 从编译器 stdout 行 ` Data offset: 0x{hex} ({n} bytes)`（`onnx_to_riscv_standalone.py:2794`）解析，正则 `r"Data offset:\s*0x([0-9A-Fa-f]+)"`；`workspace_size` 从 ` Workspace: {n} bytes`（`:2738`）解析。解析失败 → `exit 4`，`reason="binary_layout_unparsed"`。 +- 装载顺序：`ProfiledMachine(mem_size)` → `m.load_binary(code_words, origin=0)` → `m.load_data(weight_bytes, data_offset)` → `m.set_reg(2, 128MiB)` / `set_reg(10, 160MiB)` / `set_reg(11, 192MiB)` / `set_reg(1, halt_addr)`。 +- `gp` 由代码自身在 `_start` 通过打补丁的 `auipc/addi` 设置（`onnx_to_riscv_standalone.py:2757-2789`），harness **不得**另写 `gp`。 +- 输入一次性构造后 `m.load_data(input_blob, 160MiB)`，不得逐元素 `write_mem_i32`（性能与确定性双重原因）。Q16.16 生成算法与编译器自检一致：`random.seed(seed)`、`val = int((random.random() − 0.5) × 0.2 × 65536)`（`onnx_to_riscv_standalone.py:2564-2567`），默认 `seed=42`。 + +#### 2.1.4 停机条件 + +- 生成代码结尾为 `_done: ret`（`jalr x0, x1, 0`，`onnx_to_riscv_standalone.py:1553-1556`）。harness 在运行前设 `x1 = halt_addr`，当 PC 到达 `halt_addr` 即判定 `halted`。TinyFive 的 `exe(start, end)` 原生支持按 end 地址停止；`ProfiledMachine` 未暴露该参数，故采用 2.1.1 的分块 `instructions=` 循环 + 每块后检查 `m.pc`。 +- 若生成代码在返回前用 `jal ra, …` 覆盖了 `ra`，或跳转路径异常，PC 永远不会命中 `halt_addr`：`budgeted` 记 `budget_exhausted`，`full` 记 `timeout`。不得谎报 `halted`。 +- TinyFive 遇到不支持的指令时打印错误且 PC 不前进（`dec()` 无匹配分支时不调用 `ipc()`），会表现为“卡死”。启动前必须做**助记符白名单预检**：用 ScratchV 的 `_disasm_one` 解析每个 code word，若出现 TinyFive 不支持/无法识别的助记符 → `completion="not_run"`，`exit 4`，禁止开跑。 +- `m.last_error` 非空（适配器捕获到异常）→ `completion="error"`，`dynamic` 整体置 `null`。 + +#### 2.1.5 超时与预算保护 + +- `--timeout SECONDS`（默认 900）为单侧仿真墙钟上限。实现：主线程 `signal.setitimer(ITIMER_REAL, remaining)` + `SimulationTimeout` 处理器；`m.run(..., strict=True)` 使适配器 `finally` 仍在异常路径更新 `instr_count`，ops 计数器保留部分值。 +- 超时判定不依赖异常类型（适配器会把异常包装成 `RuntimeError`）：置模块级 `_timed_out` 标志，捕获后据此写 `completion="timeout"`，并把 `elapsed_s`、`executed` 落盘。 +- 平台不支持 `SIGALRM`（如 Windows）时：`--timeout` 退化为告警，要求用户必须给 `--max-instructions`；否则拒绝启动并提示替代方案。 +- **预算预估仅作告警**：`benchmark.estimate_cnn_model()` 的输出只能进入 `warnings` 与 “预计墙钟” 提示，字段名带 `estimated_` 前缀，绝不填入 `dynamic`。 + +### 2.2 诚实报告规范 + +#### 2.2.1 字段来源分类 + +报告所有数值字段必须可归入以下四类之一，渲染时以标签区分： + +| 分类 | 允许的字段 | 硬性要求 | +|------|-----------|----------| +| `measured` | `*.dynamic.*`（ops、executed、输出值） | 必须来自 TinyFive 真实执行；`source="simulated"`，且 `completion ∈ {halted, budget_exhausted, timeout}` | +| `static` | `*.compile.static_insns`、`*.static_instruction_mix.*` | `source="asm_scan"`；只统计 `.text` 助记符行；数据指令（`.word/.long/.byte/.float`）不计 | +| `estimated` | 预估值、`est_hw_time_*`、cache 模型输出（如引用） | 字段名含 `estimated`/`model`；不得进入 `dynamic` | +| `unavailable` | 失败/跳过侧 | `source="unavailable"`，`ops=null`，必须有 `reason` | + +#### 2.2.2 ISA 标注与统一 RV32 + +- `targets.scratchv = {isa:"rv32im", abi:"ilp32", numeric_format:"q16.16"}`。 +- `targets.llvm = {isa:"rv32imf", abi:"ilp32", numeric_format:"float32", triple:"riscv32-unknown-elf", opt_level:2}`。 +- 统一口径动作：llvmlite 可用时，解析 IR 后显式设置 `llmod.triple = "riscv32-unknown-elf"` 再 `emit_assembly`；不可用则 `status="skipped"`。 +- **跨 ISA 检测**：对 LLVM 侧 `.s` 做助记符扫描，命中 RV64-only 集合（`ld, sd, lwu, addw, subw, addiw, sllw, srlw, sraw, slliw, srliw, sraiw, mulw, divw, divuw, remw, remuw, fld, fsd, fcvt.l.s, fcvt.s.l` 等）时置 `isa_mismatch=true`、`isa_detected="riscv64"`，禁用该侧动态对照。 +- 两侧 ISA 或数值格式不同的比值一律 `null` + `incomparable_reason`；ISA 相同也不得用估算数替代。 +- **边界**：LLVM 侧真实可执行镜像（汇编→链接→ABI→输入输出）属于课题 25/27 交界；本课题只负责“标注与统一 RV32 口径”，不实现 LLVM 侧符号重定位与运行时。 + +#### 2.2.3 模型身份与维度 + +报告必须包含且只依据以下事实描述模型： + +``` +model.path, model.sha256, model.bytes, +model.input_name, model.input_shape, model.output_name, model.output_shape, +model.initializer_count, model.weight_bytes +scratchv.compile.binary_sha256, scratchv.compile.data_offset, scratchv.compile.data_bytes +environment.python, environment.numpy, environment.tinyfive, environment.llvmlite +``` + +禁止在报告模板中硬编码任何具体模型名或层结构描述（现行 `rv32_bench.py:268` 的 “cnn.onnx (3×Conv + …)” 必须删除）。 + +#### 2.2.4 不可复现数据的禁用 + +以下做法在 schema v2 中一律违规，由 `audit_provenance()` 检出并阻断： + +1. 静态计数标为 `simulated`，或任何 `dynamic` 字段缺少 `simulator/simulator_version/executed/memory_size_bytes/input_seed`。 +2. `instr_count = limit` 式伪造：`completion=="halted"` 但 `executed == limit` 且 `limit != null`（未验证停机）。 +3. 截断/超时结果参与比值计算。 +4. 报告声明与 provenance 不符（例如页脚宣称 “all metrics from simulation” 而存在 `estimated` 列）。 +5. 把 `llvm_cache_compare.py` 的解析估算、`benchmark.estimate_cnn_model()` 的结果、`cache_model.py` 的命中率填入 `measured` 分类。 +6. 硬编码模型描述与 `model.path/sha256` 不一致。 +7. 只记录时间戳不记录哈希/种子/版本。 + +页脚文案由 provenance 动态生成，例如： +`Simulated by tinyfive {version} | completion={completion} | executed={executed} | limit={limit} | model sha256={sha256[:12]} | seed={seed}`。 + +### 2.3 CLI 参数规格 + +| 参数 | 类型/默认 | 语义 | 约束 | +|------|-----------|------|------| +| `model`（位置参数） | path 必填 | ONNX 模型 | 存在且可读 | +| `--output-dir` | path，`benchmark_reports` | 产出目录 | 自动创建 | +| `--html` / `--json` / `--md` | filename，`rv32_bench.{html,json,md}` | 三种格式文件名 | 相对 `--output-dir` | +| `--max-instructions` | int，`0` | `>0` 为预算；`0` 为全量 | 与 `--full` 互斥（N>0 时） | +| `--full` | flag，默认关 | 等价 `--max-instructions 0` | 与 N>0 同时出现 → exit 2 | +| `--mem-size` | int 字节，`268435456` | TinyFive 内存容量 | 必须满足 2.1.2 约束，否则 exit 4 | +| `--timeout` | float 秒，`900` | 单侧仿真墙钟上限 | POSIX `SIGALRM`；否则需预算模式 | +| `--chunk-instructions` | int，`10000000` | 分块粒度 | `>0` | +| `--input-seed` | int，`42` | 输入生成种子 | 与编译器自检同算法 | +| `--skip-llvm` | flag | 跳过 LLVM 编译 | LLVM 侧整体 `not_run` | +| `--allow-missing-simulator` | flag | TinyFive 缺失时输出纯静态报告 | 报告 `dynamic.source="unavailable"`，exit 0 | +| `--fail-on-incomplete` | flag | `completion != "halted"` 时 exit 7 | 供 CI 使用 | +| `--quiet` | flag | 不向 stdout 打印 Markdown | 报告仍落盘 | + +退出码：`0` 成功；`1` 未预期错误；`2` 用法错误；`3` 仿真器不可用（未加 `--allow-missing-simulator`）；`4` 模型/镜像/内存布局校验失败；`7` 未完成且指定 `--fail-on-incomplete`。 + +### 2.4 合法/非法报告示例 + +#### 2.4.1 合法示例 A：全量完成 + +```json +{ + "schema_version": "rv32-bench/2", + "model": {"path": "models/graph/cnn.onnx", "sha256": "0123…cdef", "bytes": 27677152}, + "scratchv": { + "compile": {"status": "success", "data_offset": 24336, "data_bytes": 26700000, + "static_insns": 6084, "static_source": "asm_scan"}, + "dynamic": {"source": "simulated", "simulator": "tinyfive", "simulator_version": "1.0.0", + "completion": "halted", "limit": null, "executed": 1844674407, + "memory_size_bytes": 268435456, "input_seed": 42, + "ops": {"total": 1844674407, "load": 1, "store": 1, "mul": 1, + "add": 1, "madd": 0, "branch": 1}} + }, + "comparison": {"dynamic_instruction_ratio": null, + "incomparable_reason": "llvm.dynamic.source!='simulated'"} +} +``` +（数值为占位示例；关键点是全字段 provenance 齐备。） + +#### 2.4.2 合法示例 B：预算截断（如实标注） + +```json +{ + "scratchv": { + "dynamic": {"source": "simulated", "simulator": "tinyfive", "simulator_version": "1.0.0", + "completion": "budget_exhausted", "limit": 1000000, "executed": 1000000, + "memory_size_bytes": 268435456, "input_seed": 42, + "ops": {"total": 1000000, "load": 0, "store": 0, "mul": 0, + "add": 0, "madd": 0, "branch": 0}} + }, + "comparison": {"dynamic_instruction_ratio": null, + "incomparable_reason": "scratchv.completion=='budget_exhausted'"} +} +``` + +#### 2.4.3 非法示例（应被 `audit_provenance()` 拒绝） + +| # | 片段 | 违反条款 | +|---|------|----------| +| 1 | `"dynamic":{"source":"simulated","ops":{"total":3841}}`，无 simulator/version/executed | 2.2.4-1 | +| 2 | `"completion":"halted","limit":5000,"executed":5000` | 2.2.4-2（5000 是截断值） | +| 3 | `"comparison":{"dynamic_instruction_ratio":0.74}` 两侧均 `budget_exhausted` | 2.2.4-3 | +| 4 | 页脚 “All metrics sourced from TinyFive … No analytical estimates” 而 `estimated_hw_time` 列存在 | 2.2.4-4 | +| 5 | `"model":{"path":"resnet18.onnx"}` 而正文描述为 “cnn.onnx (3×Conv + …)” | 2.2.4-6 | + +--- + +## 三、测试设计 + +### 测试用例 1：小模型全量仿真计数正确性 + +- **文件**：`tests/test_rv32_bench.py::test_full_run_small_model_matches_reference` +- **输入**：测试内用 `onnx.helper` 构造的迷你模型（如输入 `1×1×8×8`、单层 `Conv 3×3 → 1×1×6×6`，无 FC），写成临时 `.onnx`；随后调用 `compile_scratchv()` + `run_simulation(max_instructions=0, mem_size=268435456, timeout_s=60)`（内存仍按 192MiB 输出区约定，不可缩小）。 +- **预期输出**：`scratchv.dynamic.completion == "halted"`；`executed > 0`；`ops.total == executed`；随后用独立功能仿真器 `benchmark.RV32EmulatorFast`（`load_unified_binary` + `run(max_instr=2_000_000_000)`）跑同一 `.bin` 与同一输入/指针布局。 +- **验证点**：`tinyfive.total == emulator.total`、`load` 与 `load_count`、`store` 与 `store_count`、`branch` 与 `branch_total` 逐一相等；输出 `Q16.16` 值一致；对同一命令重跑一次，计数完全一致（确定性）。任何不等即失败，禁止容差放过。 + +### 测试用例 2：预算超限行为 + +- **文件**：`tests/test_rv32_bench.py::test_budget_exhausted_is_labeled` +- **输入**：对同一迷你模型（或 `cnn.onnx` 若存在）执行 `run_simulation(max_instructions=1000, ...)`。 +- **预期输出**：`completion=="budget_exhausted"`、`limit==1000`、`executed==1000`、`ops.total==1000`；`comparison.dynamic_instruction_ratio is null` 且 `incomparable_reason` 含 `budget_exhausted`；Markdown 中出现 `[measured/budget]` 标签而非 “dynamic instruction ratio” 正文。 +- **验证点**：`audit_provenance(report) == []`；把该 JSON 传给 `bench_report.validate_report_schema()` 通过；`--fail-on-incomplete` 下退出码为 7。 + +### 测试用例 3：报告字段与 provenance 校验 + +- **文件**：`tests/test_rv32_bench.py::test_report_requires_provenance` +- **输入**：monkeypatch `ProfiledMachine` 为不可用（模拟未安装 TinyFive），命令行加 `--allow-missing-simulator`。 +- **预期输出**：`scratchv.dynamic.source=="unavailable"`、`ops is null`、`completion=="not_run"`、`reason` 非空；`scratchv.compile.static_insns > 0` 且 `static_source=="asm_scan"`；无 `dynamic_instruction_ratio`；页面含 “static” 标签与显式不可用提示。 +- **验证点**：`validate_report_schema()` 对必填 provenance 键（`schema_version/model.sha256/environment/targets/*.compile.static_source/comparison.incomparable_reason`）逐项断言；未加 `--allow-missing-simulator` 时退出码为 3。 + +### 测试用例 4：内存布局校验 + +- **文件**：`tests/test_rv32_bench.py::test_memory_layout_validation` +- **输入**：`--mem-size 1048576`（1MiB，小于 output 区 192MiB）。 +- **预期输出**：退出码 4，stderr 含 `memory_layout_invalid` 与所需最小字节数；不产生任何带 `dynamic.ops` 的报告文件。 +- **验证点**：`write_report` 未被调用（临时目录为空）；错误信息给出可操作建议（明确 “需要 ≥ 192MiB + 输出区 + 4096 字节” 的具体数值）。 + +### 测试用例 5：标签解析不产生自跳转 + +- **文件**：`tests/test_rv32_bench.py::test_parsed_labels_cover_all_branches` +- **输入**：迷你模型产出的 `.s`；`parse_labels(asm_text)` 的结果与 `data_offset/4` 的关系校验。 +- **预期输出**：`len(labels) > 0`；`_start` 映射 0；`_done` 在文件末尾附近；每个分支/跳转目标标签都能在 labels 中找到；`static_insns == data_offset // 4`。 +- **验证点**：若任一分支目标缺失，函数抛 `LabelParseError`（不允许 `labels.get(target, pc)` 回退为自跳转）；该断言直接回归现行 `riscv_encoder.py:502` 的静默语义。 + +### 测试用例 6：LLVM 跨 ISA 标注 + +- **文件**：`tests/test_rv32_bench.py::test_llvm_riscv64_flagged_isa_mismatch` +- **输入**：含 `ld/sd/addiw` 的 RV64 汇编 fixture（或跳过 llvmlite 时的真实 `_ll_rv32.s`）。 +- **预期输出**：`llvm.compile.isa_detected=="riscv64"`、`isa_mismatch==true`、`llvm.dynamic.source=="unavailable"`、`comparison.dynamic_instruction_ratio is null`。 +- **验证点**：报告不出现 “RV32IMF” 对照列；`incomparable_reason` 指名 ISA 不匹配。 + +--- + +## 四、修改模块与实现步骤 + +### 4.1 涉及文件 + +| 文件 | 角色 | 改动性质 | +|------|------|----------| +| `scratchv/standalone/rv32_bench.py` | 主驱动：编译、装载、仿真、组装报告 | 大幅重构（详见开发文档改动清单） | +| `scratchv/standalone/bench_report.py` | Markdown/HTML/JSON 渲染 | 新增 `render_*(report)` 与 schema 校验；旧函数保留兼容壳 | +| `tests/test_rv32_bench.py` | 新增测试 | 新建 | +| `tests/fixtures/`（或测试内生成） | 迷你 ONNX 与 RV64 汇编 fixture | 新建（测试内用 `onnx.helper` 生成优先） | +| `scratchv/standalone/onnx_to_riscv_standalone.py` | 编译器 | **不改**（只读取 stdout 与 `.bin/.s`） | +| `scratchv/simulator/tinyfive.py` | 仿真适配器 | **不改公共接口**（只使用 `load_binary/load_data/run(strict=True)/pc/get_perf`） | +| `scratchv/standalone/llvm_cache_compare.py` / `cache_model.py` / Spike / `benchmarks/` | 其他课题资产 | **不改**（边界见 4.9） | + +（注：上表路径为本仓库真实路径；若后续目录重构，以 `scratchv/standalone/` 下同名文件为准。） + +### 4.2 镜像装载与布局解析 + +1. `compile_scratchv()` 保留子进程调用（`--asm` 输出同目录 `.s`），新增解析 stdout： + - `Data offset: 0x…` → `data_offset`；`Workspace: N bytes` → `workspace_size`；`Code size: N bytes` → 校验值。 +2. `load_scratchv_image(binary_path)` 返回 `(code_words, weight_bytes, data_offset)`： + - 校验 `data_offset % 4 == 0`、`data_offset ≤ len(binary)`、`len(binary) − data_offset > 0`； + - `code_size = data_offset`，`binary_bytes = len(binary)`，`static_insns` 由 `.s` 扫描并与 `data_offset // 4` 交叉校验。 +3. `halt_addr = align_up(binary_bytes, 16)`；在 TinyFive `mem` 中 `halt_addr` 处无需写指令（停止发生在取指前），但要求 `halt_addr + 4 ≤ mem_size`。 + +### 4.3 内存与输入初始化 + +1. `compute_layout()` 按 2.1.2 公式校验并返回全部地址；失败抛 `LayoutError`，`main` 转 exit 4。 +2. `ProfiledMachine(mem_size=args.mem_size)`；`available` 为假时按 `--allow-missing-simulator` 决策（exit 3 或静态报告）。 +3. `load_binary(code_words, 0)` → `load_data(weight_bytes, data_offset)` → `load_data(input_blob, INPUT_ADDR)`。 +4. `set_reg(2, SP_ADDR=128MiB)`、`set_reg(10, 160MiB)`、`set_reg(11, 192MiB)`、`set_reg(1, halt_addr)`；`gp` 留空由代码设置。 +5. 输出读取：`read_mem_i32(192MiB)`（单元素）或按 `output_elements × 4` 读字节；记录 `output.raw_hex` 与 `output.q16_16`。 + +### 4.4 仿真执行器 + +1. `run_simulation(...)` 分块循环： + +``` +executed = 0; timed_out = False +while True: + 若 budgeted 且 executed == limit: completion = budget_exhausted; break + 若 m.pc == halt_addr: completion = halted; break + 若 墙钟超时: completion = timeout; break + chunk = min(chunk_instructions, limit - executed) 若 budgeted 否则 chunk_instructions + 设置 SIGALRM(剩余墙钟)；m.run(instructions=chunk, strict=True)；清除 SIGALRM + executed = get_perf()["total"] +``` + +2. 停机/超时后从 `m.get_perf()` 取 ops、从 `(m._machine.x_usage > 0).sum()` 取 `x_registers_used`、`.sum()` 取 `x_usage_total`；`m.last_error` 非空且非超时 → `completion="error"`。 +3. 启动前执行助记符白名单预检（2.1.4）；不通过 → `not_run` + exit 4。 +4. 整个执行器不写 `scratchv/standalone/` 之外的任何文件；临时产物仅 `--output-dir`。 + +### 4.5 标签解析与静态统计 + +1. `parse_labels(asm_text)` 两遍扫描：`.s` 中标签独占一行（`RISCVEmitter.disassemble()`，`:1291-1305`），每遇一行缩进的指令行 word index +1；返回 `{pc: label}`。 +2. `static_instruction_mix(asm_text)` 按助记符表分类（复用 `rv32_bench.py:206-220` 的 opcode→类别映射），但输出字段名为 `static_instruction_mix`（`source="asm_scan"`），不与 `dynamic.ops` 同名同表。 +3. 若解析出的指令数 ≠ `data_offset // 4`，抛 `LabelParseError`，禁止“尽力而为”。 + +### 4.6 报告组装与渲染 + +1. `build_report()` 产出 schema v2 字典（详细字段见开发文档《接口契约》）。 +2. `audit_provenance(report)` 返回违规列表；`main` 在写盘前调用，非空则打印并 exit 1。 +3. `bench_report.render_markdown/html/github_summary(report)` 负责三格式渲染；所有表格列头或行内带 `[measured]/[static]/[estimated]/[unavailable]` 标签；页脚由 provenance 动态生成。 +4. `bench_report.validate_report_schema(report)` 供测试与 CI 做 JSON 结构断言。 + +### 4.7 LLVM 侧：标注与统一 RV32 + +1. `compile_llvm_rv32()`：`binding.Target.from_triple("riscv32-unknown-elf")`；llvmlite 缺失/目标不可用 → `status="skipped"` + `reason`，`dynamic.source="unavailable"`。 +2. 成功的路径上：解析 IR → 显式 `llmod.triple = "riscv32-unknown-elf"`（覆盖 `onnx_to_llvm_standalone.py:446` 的 riscv64 模块头）→ `verify()` → `emit_assembly()`。 +3. 对输出的 `.s` 做 `detect_isa_mismatch()`；命中 RV64-only 助记符 → `isa_mismatch=true`，动态对照关闭。 +4. 静态计数只扫 `.text` 段：遇到 `.section .rodata`（权重 `.word/.long`）即暂停计数；`static_source="asm_scan"`。 +5. 不实现 LLVM 侧可执行镜像（符号重定位、`a0/a1` 约定、数据段加载）——属课题 25 边界；未实现即 `unavailable`，不得用 `llvm_cache_compare.py` 的估算补位。 + +### 4.8 集成与回归测试 + +- 新增 `tests/test_rv32_bench.py`（第三部分 6 个用例），全部可在 `make test` 下运行；无 TinyFive 的环境自动 skip 动态类用例（用 `pytest.importorskip("tinyfive")` 模式，与 `tests/test_simulator.py` 一致），但不允许 skip 掉 provenance 校验类用例。 +- 回归：`python .claude/harness/verify/run.py --level L2`；确认 `tests/test_simulator.py`、`tests/test_bench_runner.py`、`tests/test_cnn_pipeline.py` 不回归。 +- 手工验收命令（详见开发文档 7 节）：全量（或超时标注）、预算截断、内存拒绝、TinyFive 缺失四条路径各跑一次并人工检查报告。 + +### 4.9 范围边界 + +- **不改** `scratchv/standalone/onnx_to_riscv_standalone.py` 的代码生成、ABI 与内存布局；只读取其 stdout 与输出文件。 +- **不改** Spike 相关（课题 24：`spike_sim.py`、`run_spike_bench.py`）与基准套件（课题 06：`benchmarks/`、`bench_runner.py`）。 +- **不改** `tinyfive.py` 公共接口；若未来需要 `end=` 精确停机，作为独立后续课题，附 ABI 兼容性说明。 +- **LLVM 实测对比**：`llvmlite` 目标机切换、RV64→RV32 差异修复、LLVM 侧可执行镜像的构建属于课题 25/27 交界；本课题只做 ISA 标注与 RV32 统一请求，`llvm_cache_compare.py` 的解析估算继续留在课题 25，不得进入本报告 `measured` 分类。 +- **cache 指标**：`cache_model.py` 为分析模型（课题 23）；本课题不引入 cache 命中率类指标，若后续引入必须标 `estimated`。 + +--- + +## 五、附录 + +### 5.1 报告 JSON 示例（全量完成，schema v2） + +> 以下数值为结构示例（占位），字段名与嵌套关系为规范值。 + +```json +{ + "schema_version": "rv32-bench/2", + "generated_at": "2026-09-14T00:00:00Z", + "generator": {"script": "rv32_bench.py"}, + "model": { + "path": "models/graph/cnn.onnx", + "sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "bytes": 27677152, + "input_name": "input.1", + "input_shape": [1, 3, 250, 250], + "output_name": "output", + "output_shape": [1, 1], + "initializer_count": 16, + "weight_bytes": 26700000 + }, + "environment": { + "python": "3.8.10", "numpy": "1.24.4", + "tinyfive": "1.0.0", "llvmlite": null + }, + "targets": { + "scratchv": {"isa": "rv32im", "abi": "ilp32", "numeric_format": "q16.16"}, + "llvm": {"isa": "rv32imf", "abi": "ilp32", "numeric_format": "float32", + "triple": "riscv32-unknown-elf", "opt_level": 2} + }, + "scratchv": { + "compile": { + "status": "success", "binary": "_sv.bin", "binary_bytes": 26724336, + "binary_sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "code_bytes": 24336, "data_offset": 24336, "data_offset_source": "compiler_stdout", + "data_bytes": 26700000, "static_insns": 6084, "static_source": "asm_scan", + "elapsed_s": 12.3 + }, + "static_instruction_mix": { + "source": "asm_scan", + "load": 0, "store": 0, "mul": 0, "add": 0, "madd": 0, "branch": 0, "other": 0 + }, + "dynamic": { + "source": "simulated", + "simulator": "tinyfive", "simulator_version": "1.0.0", + "completion": "halted", "limit": null, "executed": 1844674407, + "timeout_s": 900.0, "elapsed_s": 3210.5, + "memory_size_bytes": 268435456, + "input_seed": 42, "input_elements": 187500, + "halt_addr": 26724336, + "ops": {"total": 1844674407, "load": 0, "store": 0, "mul": 0, + "add": 0, "madd": 0, "branch": 0}, + "x_registers_used": 31, "x_usage_total": 123456789, + "f_registers_used": 0, + "per_label": null, + "per_label_note": "tinyfive exe() exposes no per-PC trace", + "last_error": null + }, + "output": {"addr": 201326592, "elements": 1, "raw_hex": "0x00018000", "q16_16": 1.5} + }, + "llvm": { + "compile": {"status": "skipped", "reason": "llvmlite not available", + "isa_detected": null, "isa_mismatch": false, + "static_insns": 0, "static_source": "asm_scan", "elapsed_s": 0.0}, + "dynamic": {"source": "unavailable", "simulator": "tinyfive", "completion": "not_run", + "reason": "llvm executable image pipeline not implemented (topic 25 boundary)", + "ops": null} + }, + "comparison": { + "dynamic_instruction_ratio": null, + "incomparable_reason": "llvm.dynamic.source=='unavailable'" + }, + "warnings": [ + "estimated dynamic instructions (analytical): 1.85e9 — used for wall-clock warning only" + ], + "errors": [] +} +``` + +### 5.2 报告 JSON 示例（预算截断，schema v2） + +```json +{ + "schema_version": "rv32-bench/2", + "model": {"path": "models/graph/cnn.onnx", "sha256": "0123…cdef"}, + "scratchv": { + "compile": {"status": "success", "data_offset": 24336, "data_bytes": 26700000, + "static_insns": 6084, "static_source": "asm_scan"}, + "dynamic": { + "source": "simulated", "simulator": "tinyfive", "simulator_version": "1.0.0", + "completion": "budget_exhausted", "limit": 1000000, "executed": 1000000, + "timeout_s": 900.0, "elapsed_s": 42.0, "memory_size_bytes": 268435456, + "input_seed": 42, "halt_addr": 26724336, + "ops": {"total": 1000000, "load": 0, "store": 0, "mul": 0, + "add": 0, "madd": 0, "branch": 0}, + "x_registers_used": 12, "x_usage_total": 3456, "f_registers_used": 0, + "per_label": null, "per_label_note": "tinyfive exe() exposes no per-PC trace" + } + }, + "comparison": { + "dynamic_instruction_ratio": null, + "incomparable_reason": "scratchv.completion=='budget_exhausted'" + }, + "warnings": ["budget exhausted at 1000000 instructions; dynamic counts are partial"], + "errors": [] +} +``` + +### 5.3 非法报告片段与拒绝理由（对照 2.2.4） + +```json +{"scratchv": {"dynamic": {"source": "simulated", "completion": "halted", + "ops": {"total": 3841}}, + "compile": {"_note": "fallback: static counts only"}}} +``` +→ 违规：缺 `simulator/simulator_version/executed/memory_size_bytes/input_seed`；`3841` 是静态计数。 + +```json +{"comparison": {"dynamic_instruction_ratio": 0.31}, + "scratchv": {"dynamic": {"completion": "budget_exhausted"}}, + "llvm": {"dynamic": {"completion": "budget_exhausted"}}} +``` +→ 违规：截断轨迹不可比。 + +### 5.4 参考资料 + +- 设计文档模板：`/root/Lab/ScratchV/设计文档模板.md` +- 课题 27 背景：`docs/topics/27-RV32全量Benchmark.md` +- TinyFive 适配器：`scratchv/simulator/tinyfive.py`（`run(instructions=…, strict=True)`、`get_perf`、`pc`） +- 编译器 ABI/布局：`scratchv/standalone/onnx_to_riscv_standalone.py:2548-2557, 2729-2794, 2802-2807` +- LLVM 模块头：`scratchv/standalone/onnx_to_llvm_standalone.py:446` +- 编码器标签回退语义：`scratchv/backend/riscv_encoder.py:502` +- 相邻课题：课题 06（基准套件）、课题 23（Cache 模型）、课题 24（Spike）、课题 25（LLVM 对比工具）、课题 26（TinyFive 对比） From 133437b7ebf3349f7e28f0071cc012ad4a412bb6 Mon Sep 17 00:00:00 2001 From: Seven Gao <799889633@qq.com> Date: Mon, 14 Sep 2026 22:46:41 +0800 Subject: [PATCH 3/5] fix(topic27): tag partial outputs, surface LLVM failures, tighten audit - mark scratchv.output as partial (plus completion) whenever the run does not halt, force q16_16 to always be a list, and render [measured/partial] in the Markdown/HTML provenance section - validate the output block in audit_provenance and validate_report_schema; add executed==ops.total and executed<=limit invariants to the provenance audit - surface llvm.compile.status=='failed' as a warning carrying the real reason, propagate it into llvm.dynamic.reason, render LLVM static insns as em-dash for non-success compiles, and emit an explicit no-auto-estimate warning for full runs - tests: wall-clock timeout path, partial output, LLVM failure, audit counter forgeries and q16_16 type stability --- scratchv/standalone/bench_report.py | 57 ++++++++- scratchv/standalone/rv32_bench.py | 138 +++++++++++++++++----- tests/test_rv32_bench.py | 173 +++++++++++++++++++++++++++- 3 files changed, 334 insertions(+), 34 deletions(-) diff --git a/scratchv/standalone/bench_report.py b/scratchv/standalone/bench_report.py index 1dbc6ad..05533f6 100644 --- a/scratchv/standalone/bench_report.py +++ b/scratchv/standalone/bench_report.py @@ -532,9 +532,13 @@ def render_markdown(report: dict) -> str: lines.append(f"| code bytes | {_fmt(sv_compile.get('code_bytes'))} | — |") lines.append(f"| data offset | {_fmt(sv_compile.get('data_offset'))} | — |") lines.append(f"| data bytes | {_fmt(sv_compile.get('data_bytes'))} | — |") + ll_static = ( + _fmt(ll_compile.get("static_insns")) + if ll_compile.get("status") == "success" else "—" + ) lines.append( f"| static insns [asm_scan] | {_fmt(sv_compile.get('static_insns'))} " - f"| {_fmt(ll_compile.get('static_insns'))} |" + f"| {ll_static} |" ) lines.append("") lines.append("### Static instruction mix [static]") @@ -631,10 +635,17 @@ def render_markdown(report: dict) -> str: f"- static_source={_fmt(sv_compile.get('static_source'))} " f"| llvm static_source={_fmt(ll_compile.get('static_source'))}" ) + if sv_dyn.get("source") != "simulated": + out_tag = "[unavailable]" + elif sv_out.get("partial"): + out_tag = "[measured/partial]" + else: + out_tag = "[measured]" lines.append( - f"- output: {_fmt(sv_out.get('raw_hex'))} " + f"- output {out_tag}: {_fmt(sv_out.get('raw_hex'))} " f"(addr=0x{int(sv_out.get('addr') or 0):x}, " - f"elements={_fmt(sv_out.get('elements'))})" + f"elements={_fmt(sv_out.get('elements'))}, " + f"completion={_fmt(sv_out.get('completion'))})" ) return "\n".join(lines) @@ -846,6 +857,46 @@ def is_int(value) -> bool: f"{side}.dynamic.ops must be null when unavailable" ) + out = _dig(report, "scratchv.output") + if out is _MISSING or not isinstance(out, dict): + errors.append("missing required field: scratchv.output") + else: + if not isinstance(out.get("partial"), bool): + errors.append( + f"invalid scratchv.output.partial: {out.get('partial')!r}" + ) + if not isinstance(out.get("completion"), str) or \ + not out.get("completion"): + errors.append( + "invalid scratchv.output.completion: " + f"{out.get('completion')!r}" + ) + if not is_int(out.get("elements")): + errors.append( + f"invalid scratchv.output.elements: " + f"{out.get('elements')!r}" + ) + q16 = out.get("q16_16") + if out.get("partial") is False: + if out.get("raw_hex") is None: + errors.append( + "missing required field: scratchv.output.raw_hex" + ) + if not isinstance(q16, list): + errors.append( + f"invalid scratchv.output.q16_16: {q16!r}" + ) + elif q16 is not None and not isinstance(q16, list): + errors.append( + f"invalid scratchv.output.q16_16: {q16!r}" + ) + if isinstance(q16, list) and is_int(out.get("elements")) and \ + len(q16) != out["elements"]: + errors.append( + f"invalid scratchv.output.q16_16 length: {len(q16)} " + f"!= elements {out['elements']}" + ) + comparison = _dig(report, "comparison") if comparison is _MISSING or not isinstance(comparison, dict): errors.append("missing required field: comparison") diff --git a/scratchv/standalone/rv32_bench.py b/scratchv/standalone/rv32_bench.py index bab33d6..bd139d7 100644 --- a/scratchv/standalone/rv32_bench.py +++ b/scratchv/standalone/rv32_bench.py @@ -722,6 +722,18 @@ def _unavailable_dynamic(*, reason: str, mem_size: int, timeout_s: float, } +def _unavailable_output(elements: int, completion: str) -> dict: + """Output block for runs that produced no (or no complete) values.""" + return { + "addr": OUTPUT_ADDR, + "elements": elements, + "raw_hex": None, + "q16_16": None, + "completion": completion, + "partial": completion != "halted", + } + + def _read_output(machine, addr: int, elements: int) -> dict: values = [] if machine.available and elements > 0: @@ -735,7 +747,7 @@ def _read_output(machine, addr: int, elements: int) -> dict: "addr": addr, "elements": elements, "raw_hex": raw_hex, - "q16_16": q16[0] if len(q16) == 1 else q16, + "q16_16": q16, } @@ -773,10 +785,7 @@ def run_simulation(*, asm_path: str, binary_path: str, data_offset: int, mem_size=mem_size, timeout_s=timeout_s, input_seed=input_seed, input_elements=input_elements, halt_addr=halt_addr, ), - "output": { - "addr": OUTPUT_ADDR, "elements": output_elements, - "raw_hex": None, "q16_16": None, - }, + "output": _unavailable_output(output_elements, "not_run"), } _install_tinyfive_compat(m, halt_addr) @@ -789,10 +798,7 @@ def run_simulation(*, asm_path: str, binary_path: str, data_offset: int, timeout_s=timeout_s, input_seed=input_seed, input_elements=input_elements, halt_addr=halt_addr, ), - "output": { - "addr": OUTPUT_ADDR, "elements": output_elements, - "raw_hex": None, "q16_16": None, - }, + "output": _unavailable_output(output_elements, "not_run"), } unsupported = check_mnemonics(code_words) @@ -804,10 +810,7 @@ def run_simulation(*, asm_path: str, binary_path: str, data_offset: int, ) return { "dynamic": dyn, - "output": { - "addr": OUTPUT_ADDR, "elements": output_elements, - "raw_hex": None, "q16_16": None, - }, + "output": _unavailable_output(output_elements, "not_run"), } m.load_binary(code_words, origin=0) @@ -906,13 +909,13 @@ def run_simulation(*, asm_path: str, binary_path: str, data_offset: int, dyn["last_error"] = m.last_error return { "dynamic": dyn, - "output": { - "addr": OUTPUT_ADDR, "elements": output_elements, - "raw_hex": None, "q16_16": None, - }, + "output": _unavailable_output(output_elements, "error"), } ops = {key: int(perf.get(key, 0)) for key in OPS_KEYS} + output = _read_output(m, layout["output_addr"], output_elements) + output["completion"] = completion + output["partial"] = completion != "halted" return { "dynamic": { "source": "simulated", @@ -935,7 +938,7 @@ def run_simulation(*, asm_path: str, binary_path: str, data_offset: int, "per_label_note": "tinyfive exe() exposes no per-PC trace", "last_error": m.last_error, }, - "output": _read_output(m, layout["output_addr"], output_elements), + "output": output, } @@ -1046,6 +1049,7 @@ def audit_provenance(report: dict) -> list[str]: violations.append(f"{side_name}.dynamic missing") continue source = dyn.get("source") + completion = dyn.get("completion") if source == "simulated": for key in ("simulator", "simulator_version", "executed", "memory_size_bytes", "input_seed"): @@ -1053,7 +1057,7 @@ def audit_provenance(report: dict) -> list[str]: violations.append( f"{side_name}.dynamic.{key} missing for simulated data" ) - if dyn.get("completion") not in ("halted", "budget_exhausted", "timeout"): + if completion not in ("halted", "budget_exhausted", "timeout"): violations.append( f"{side_name}.dynamic.completion invalid: " f"{dyn.get('completion')!r}" @@ -1067,12 +1071,26 @@ def audit_provenance(report: dict) -> list[str]: violations.append( f"{side_name}.dynamic.ops.{key} must be an int" ) - if dyn.get("completion") == "halted" and \ - dyn.get("limit") is not None and \ - dyn.get("executed") == dyn.get("limit"): + executed = dyn.get("executed") + if isinstance(ops.get("total"), int) and \ + isinstance(executed, int) and ops["total"] != executed: + violations.append( + f"{side_name}.dynamic: executed={executed} != " + f"ops.total={ops['total']} (inconsistent counters)" + ) + limit = dyn.get("limit") + executed = dyn.get("executed") + if isinstance(limit, int) and isinstance(executed, int) and \ + executed > limit: + violations.append( + f"{side_name}.dynamic: executed={executed} > " + f"limit={limit} (budget overrun)" + ) + if completion == "halted" and limit is not None and \ + executed == limit: violations.append( f"{side_name}.dynamic: halted but executed==limit==" - f"{dyn.get('limit')} (unverified halt)" + f"{limit} (unverified halt)" ) elif source == "unavailable": if not dyn.get("reason"): @@ -1086,6 +1104,41 @@ def audit_provenance(report: dict) -> list[str]: f"{side_name}.dynamic.source invalid: {source!r}" ) + out = side.get("output") + if out is None: + if side_name == "scratchv": + violations.append("scratchv.output missing") + elif not isinstance(out, dict): + violations.append(f"{side_name}.output must be a dict") + else: + partial = out.get("partial") + if not isinstance(partial, bool): + violations.append(f"{side_name}.output.partial must be a bool") + elif isinstance(completion, str) and \ + partial != (completion != "halted"): + violations.append( + f"{side_name}.output.partial={partial} inconsistent with " + f"dynamic.completion={completion!r}" + ) + out_completion = out.get("completion") + if not isinstance(out_completion, str) or not out_completion: + violations.append(f"{side_name}.output.completion missing") + if partial is False and out.get("raw_hex") is None: + violations.append( + f"{side_name}.output.raw_hex missing for a complete result" + ) + q16 = out.get("q16_16") + if q16 is not None and not isinstance(q16, list): + violations.append( + f"{side_name}.output.q16_16 must be a list or null" + ) + elif isinstance(q16, list) and isinstance(out.get("elements"), int) \ + and len(q16) != out["elements"]: + violations.append( + f"{side_name}.output.q16_16 has {len(q16)} elements, " + f"expected {out['elements']}" + ) + comparison = report.get("comparison") or {} ratio = comparison.get("dynamic_instruction_ratio") if ratio is not None: @@ -1229,20 +1282,41 @@ def main(argv: list[str] | None = None) -> int: else: print(" [2/4] LLVM compilation (RV32IMF, float32)", file=sys.stderr) llvm_compile = compile_llvm_rv32(args.model, str(out / "_ll_rv32.s")) + if llvm_compile["status"] == "failed": + llvm_reason = ( + "llvm compile failed: " + f"{llvm_compile.get('reason') or llvm_compile.get('error')}" + ) + elif llvm_compile["status"] == "skipped": + llvm_reason = ( + f"llvm compile skipped: {llvm_compile.get('reason')}" + ) + elif llvm_compile.get("isa_mismatch"): + llvm_reason = ( + f"llvm isa mismatch: {llvm_compile.get('reason')} " + "(dynamic comparison disabled)" + ) + else: + llvm_reason = ( + "llvm executable image pipeline not implemented " + "(topic 25 boundary)" + ) llvm = { "compile": llvm_compile, "dynamic": { "source": "unavailable", "simulator": "tinyfive", "completion": "not_run", - "reason": ( - "llvm executable image pipeline not implemented " - "(topic 25 boundary)" - ), + "reason": llvm_reason, "ops": None, }, } - if llvm_compile["status"] == "skipped": + if llvm_compile["status"] == "failed": + warnings.append( + "LLVM compilation failed: " + f"{llvm_compile.get('reason') or llvm_compile.get('error')}" + ) + elif llvm_compile["status"] == "skipped": warnings.append( f"LLVM side skipped: {llvm_compile.get('reason')}" ) @@ -1252,6 +1326,14 @@ def main(argv: list[str] | None = None) -> int: "comparison disabled" ) + if effective_limit == 0: + warnings.append( + "full simulation requested; no automatic instruction-count or " + "wall-clock estimate is available (use --max-instructions N to " + "calibrate); results are marked partial if the wall-clock " + "timeout fires" + ) + print(" [3/4] TinyFive simulation", file=sys.stderr) try: sim = run_simulation( diff --git a/tests/test_rv32_bench.py b/tests/test_rv32_bench.py index 4614f08..7a40729 100644 --- a/tests/test_rv32_bench.py +++ b/tests/test_rv32_bench.py @@ -105,6 +105,7 @@ def _fake_scratchv(*, completion="halted", limit=None, executed=100, "output": { "addr": 201326592, "elements": 36, "raw_hex": "0x" + "00" * 144, "q16_16": [0.0] * 36, + "completion": completion, "partial": completion != "halted", }, } @@ -165,6 +166,13 @@ def test_full_run_small_model_matches_reference(tmp_path, mini_model, assert dyn["completion"] == "halted" assert dyn["executed"] > 0 assert dyn["ops"]["total"] == dyn["executed"] + assert any( + "no automatic" in w for w in report["warnings"] + ), report["warnings"] + out_block = report["scratchv"]["output"] + assert out_block["completion"] == "halted" + assert out_block["partial"] is False + assert isinstance(out_block["q16_16"], list) assert rv32_bench.audit_provenance(report) == [] assert bench_report.validate_report_schema(report) == [] @@ -231,11 +239,16 @@ def test_budget_exhausted_is_labeled(tmp_path, mini_model): assert dyn["ops"]["total"] == 1000 assert report["comparison"]["dynamic_instruction_ratio"] is None assert "budget_exhausted" in report["comparison"]["incomparable_reason"] + out_block = report["scratchv"]["output"] + assert out_block["completion"] == "budget_exhausted" + assert out_block["partial"] is True + assert isinstance(out_block["q16_16"], list) assert rv32_bench.audit_provenance(report) == [] assert bench_report.validate_report_schema(report) == [] markdown = (out / "rv32_bench.md").read_text() assert "[measured/budget]" in markdown + assert "[measured/partial]" in markdown assert "dynamic instruction ratio" not in markdown rc_strict = rv32_bench.main([ @@ -246,6 +259,49 @@ def test_budget_exhausted_is_labeled(tmp_path, mini_model): assert rc_strict == rv32_bench.EXIT_INCOMPLETE +# ─────────────────────────────────────────────────────────────────────────── +# T2b: wall-clock timeout keeps partial counters and partial output +# ─────────────────────────────────────────────────────────────────────────── + +def test_wall_clock_timeout_is_labeled_partial(tmp_path, mini_model): + pytest.importorskip("tinyfive") + out = tmp_path / "timeout" + rc = rv32_bench.main([ + str(mini_model), "--quiet", "--output-dir", str(out), + "--timeout", "0.05", "--chunk-instructions", "4096", + ]) + assert rc == rv32_bench.EXIT_OK + + report = json.loads((out / "rv32_bench.json").read_text()) + dyn = report["scratchv"]["dynamic"] + assert dyn["source"] == "simulated" + assert dyn["completion"] == "timeout" + assert dyn["limit"] is None + assert dyn["executed"] > 0 + assert dyn["ops"]["total"] == dyn["executed"] + assert report["comparison"]["dynamic_instruction_ratio"] is None + assert "timeout" in report["comparison"]["incomparable_reason"] + assert any("timeout" in w for w in report["warnings"]) + + out_block = report["scratchv"]["output"] + assert out_block["completion"] == "timeout" + assert out_block["partial"] is True + assert isinstance(out_block["q16_16"], list) + assert rv32_bench.audit_provenance(report) == [] + assert bench_report.validate_report_schema(report) == [] + + markdown = (out / "rv32_bench.md").read_text() + assert "[measured/timeout]" in markdown + assert "[measured/partial]" in markdown + + rc_strict = rv32_bench.main([ + str(mini_model), "--quiet", "--fail-on-incomplete", + "--output-dir", str(tmp_path / "timeout_strict"), + "--timeout", "0.05", "--chunk-instructions", "4096", + ]) + assert rc_strict == rv32_bench.EXIT_INCOMPLETE + + # ─────────────────────────────────────────────────────────────────────────── # T3: missing simulator never fabricates dynamic data # ─────────────────────────────────────────────────────────────────────────── @@ -277,6 +333,11 @@ def test_report_requires_provenance(tmp_path, mini_model, monkeypatch): assert report["scratchv"]["compile"]["static_insns"] > 0 assert report["scratchv"]["compile"]["static_source"] == "asm_scan" assert report["comparison"]["dynamic_instruction_ratio"] is None + out_block = report["scratchv"]["output"] + assert out_block["completion"] == "not_run" + assert out_block["partial"] is True + assert out_block["raw_hex"] is None + assert out_block["q16_16"] is None assert rv32_bench.audit_provenance(report) == [] assert bench_report.validate_report_schema(report) == [] @@ -372,6 +433,47 @@ def test_llvm_riscv64_flagged_isa_mismatch(): assert "RV32IMF" not in markdown +# ─────────────────────────────────────────────────────────────────────────── +# T6b: a failed LLVM compile is surfaced, not silently swallowed +# ─────────────────────────────────────────────────────────────────────────── + +def test_llvm_compile_failure_is_surfaced(tmp_path, mini_model, monkeypatch): + failure_reason = "RuntimeError: no rv32 target available" + + def _failed(*_args, **_kwargs): + return { + "status": "failed", "reason": failure_reason, + "isa_detected": None, "isa_mismatch": False, + "static_insns": 0, "static_source": "asm_scan", "elapsed_s": 0.0, + } + + monkeypatch.setattr(rv32_bench, "compile_llvm_rv32", _failed) + out = tmp_path / "llvmfail" + rc = rv32_bench.main([ + str(mini_model), "--quiet", "--output-dir", str(out), + "--max-instructions", "500", "--timeout", "60", + "--chunk-instructions", "4096", + ]) + assert rc == rv32_bench.EXIT_OK + + report = json.loads((out / "rv32_bench.json").read_text()) + assert report["llvm"]["compile"]["status"] == "failed" + assert any( + "LLVM compilation failed" in w and failure_reason in w + for w in report["warnings"] + ), report["warnings"] + assert failure_reason in report["llvm"]["dynamic"]["reason"] + assert report["errors"] == [] + + markdown = (out / "rv32_bench.md").read_text() + assert failure_reason in markdown + row = next( + line for line in markdown.splitlines() + if line.startswith("| static insns [asm_scan] |") + ) + assert row.rstrip().endswith("| — |") + + # ─────────────────────────────────────────────────────────────────────────── # T7: the provenance audit rejects static counts masquerading as dynamic # ─────────────────────────────────────────────────────────────────────────── @@ -401,10 +503,15 @@ def test_audit_provenance_rejects_static_fallback(): "ops": {"total": 1000, "load": 0, "store": 0, "mul": 0, "add": 0, "madd": 0, "branch": 0}, } + ratio_from_truncated["scratchv"]["output"] = { + "addr": 201326592, "elements": 36, "raw_hex": None, + "q16_16": None, "completion": "budget_exhausted", "partial": True, + } ratio_from_truncated["comparison"] = { "dynamic_instruction_ratio": 0.31, "incomparable_reason": None, } - assert rv32_bench.audit_provenance(ratio_from_truncated) + ratio_violations = rv32_bench.audit_provenance(ratio_from_truncated) + assert any("incomplete/non-simulated" in v for v in ratio_violations) forged_halt = deepcopy(clean) forged_halt["scratchv"]["dynamic"].update( @@ -418,6 +525,59 @@ def test_audit_provenance_rejects_static_fallback(): for v in rv32_bench.audit_provenance(forged_halt) ) + counters_disagree = deepcopy(clean) + counters_disagree["scratchv"]["dynamic"].update( + {"executed": 100, "ops": { + "total": 9999, "load": 0, "store": 0, "mul": 0, + "add": 0, "madd": 0, "branch": 0, + }}, + ) + assert any( + "ops.total" in v + for v in rv32_bench.audit_provenance(counters_disagree) + ) + + budget_overrun = deepcopy(clean) + budget_overrun["scratchv"]["dynamic"].update( + {"completion": "halted", "limit": 50, "executed": 100, "ops": { + "total": 100, "load": 0, "store": 0, "mul": 0, + "add": 0, "madd": 0, "branch": 0, + }}, + ) + assert any( + "budget overrun" in v + for v in rv32_bench.audit_provenance(budget_overrun) + ) + + unmasked_output = deepcopy(clean) + unmasked_output["scratchv"]["dynamic"]["completion"] = "timeout" + assert any( + "output.partial" in v + for v in rv32_bench.audit_provenance(unmasked_output) + ) + + +# ─────────────────────────────────────────────────────────────────────────── +# T7b: output.q16_16 has one stable type (list) regardless of element count +# ─────────────────────────────────────────────────────────────────────────── + +def test_output_q16_16_is_always_a_list(): + class _StubMachine: + available = True + + def __init__(self, words): + self.words = words + + def read_mem_i32(self, addr): + return self.words[addr // 4] + + single = rv32_bench._read_output(_StubMachine([-65536]), 0, 1) + assert isinstance(single["q16_16"], list) + assert single["q16_16"] == [-1.0] + + pair = rv32_bench._read_output(_StubMachine([65536, -131072]), 0, 2) + assert pair["q16_16"] == [1.0, -2.0] + # ─────────────────────────────────────────────────────────────────────────── # T8: schema validation covers every required provenance key @@ -431,8 +591,8 @@ def test_bench_report_schema_required_keys(): joined = "\n".join(empty_errors) for key in ( "schema_version", "model.sha256", "environment", "targets", - "scratchv.compile", "scratchv.dynamic", "llvm.compile", - "llvm.dynamic", "comparison", + "scratchv.compile", "scratchv.dynamic", "scratchv.output", + "llvm.compile", "llvm.dynamic", "comparison", ): assert key in joined, f"{key} missing from {empty_errors}" @@ -456,3 +616,10 @@ def test_bench_report_schema_required_keys(): "comparison.incomparable_reason" in e for e in bench_report.validate_report_schema(bad_ratio) ) + + no_output_partial = deepcopy(report) + del no_output_partial["scratchv"]["output"]["partial"] + assert any( + "scratchv.output.partial" in e + for e in bench_report.validate_report_schema(no_output_partial) + ) From 7887bdd20dc996e5b1160b54e54cd817d793d17e Mon Sep 17 00:00:00 2001 From: Seven Gao <799889633@qq.com> Date: Mon, 14 Sep 2026 22:46:47 +0800 Subject: [PATCH 4/5] docs(topic27): sync design and development docs with implementation - document the instance-level exe shim that stops at halt_addr or the instruction budget, and drop the claim that chunking is the only mechanism available without touching ProfiledMachine - replace the unimplemented estimated_* wall-clock promise with the actual full-mode warning and the documented absence of auto-estimation - fix the run_simulation/parse_labels signatures, the error-path field shape, llvmlite-unavailable-vs-failed semantics, and the output provenance fields (q16_16 list, completion, partial) --- ...00\345\217\221\346\226\207\346\241\243.md" | 32 +++++++++++-------- ...76\350\256\241\346\226\207\346\241\243.md" | 15 +++++---- 2 files changed, 27 insertions(+), 20 deletions(-) diff --git "a/docs/topics/27-RV32\345\205\250\351\207\217Benchmark-\345\274\200\345\217\221\346\226\207\346\241\243.md" "b/docs/topics/27-RV32\345\205\250\351\207\217Benchmark-\345\274\200\345\217\221\346\226\207\346\241\243.md" index 35f2557..7ceda61 100644 --- "a/docs/topics/27-RV32\345\205\250\351\207\217Benchmark-\345\274\200\345\217\221\346\226\207\346\241\243.md" +++ "b/docs/topics/27-RV32\345\205\250\351\207\217Benchmark-\345\274\200\345\217\221\346\226\207\346\241\243.md" @@ -34,7 +34,7 @@ | C5 | 新增 | `load_scratchv_image()`：`load_binary(code_words, 0)` + `load_data(weights, data_offset)` + `load_data(input_blob, 160MiB)` | 权重与输入真实装载 | | C6 | `:160-184` `_prepare_asm_for_tinyfive` | 删除；新增 `parse_labels()`，标签不再被过滤，`.s` 只用于标签映射与静态计数，不用于装载代码 | 修复分支自跳转 | | C7 | `:187-234` `_tinyfive_static_fallback` | 删除；静态计数改由 `static_instruction_mix()` 产出，字段独立为 `static_instruction_mix` | 静态数不冒充动态数 | -| C8 | `:118-157` `run_tinyfive` | 替换为分块执行器：`chunk` 循环 + `m.pc == halt_addr` 停机检测 + `SIGALRM` 超时 + 完整 provenance 输出 | 停机/预算/超时可控 | +| C8 | `:118-157` `run_tinyfive` | 替换为分块执行器：`chunk` 循环 + 实例级 `exe` 停机 shim（`halt_addr`/预算先到者停）+ `SIGALRM` 超时 + 完整 provenance 输出 | 停机/预算/超时可控 | | C9 | `:58-104` `compile_llvm_rv32` | llvmlite 可用时 `llmod.triple="riscv32-unknown-elf"`；新增 `detect_isa_mismatch()`；静态计数只扫 `.text`；失败原因入档 | 统一 RV32 口径与诚实降级 | | C10 | `:241-312` `BenchResult` / `generate_report` | 改为 `build_report()`（schema v2 dict）+ `bench_report.render_*(report)`；删除硬编码模型描述、无依据 ratio 与 “No analytical estimates” 文案 | 诚实报告 | | C11 | `:383-425` `main` | 新增 CLI 参数、退出码、`audit_provenance` 写盘前检查、`--quiet` | 接口规格化 | @@ -55,17 +55,18 @@ sv.tinyfive = run_tinyfive(str(out / "_sv.s"), n_instructions=5000) # :411 ```python def run_simulation( *, - asm_path: str, + asm_path: str, # 登记用；代码装载走 binary_path binary_path: str, + data_offset: int, + workspace_bytes: int, + input_elements: int, + output_elements: int, max_instructions: int = 0, # 0 = 全量 mem_size: int = 268435456, timeout_s: float = 900.0, chunk_instructions: int = 10_000_000, input_seed: int = 42, - input_elements: int, - output_elements: int, - halt_addr: int, -) -> dict: ... +) -> dict: ... # -> {"dynamic": …, "output": …}; halt_addr 由 compute_layout 内部计算 ``` 调用点： @@ -108,7 +109,7 @@ CLI 语义：`--max-instructions 0`（默认）与 `--full` 等价；两者与 ` `.s` 格式：标签独占一行（列 0，`name:`），指令行缩进两格并可能带 `# 注释`（`RISCVEmitter.disassemble()`，`onnx_to_riscv_standalone.py:1291-1305`）。 ```python -def parse_labels(asm_text: str) -> dict[int, str]: +def parse_labels(asm_text: str, expected_code_bytes: int) -> dict[int, str]: pc, labels, seen_tokens = 0, {}, [] for raw in asm_text.splitlines(): line = raw.split("#", 1)[0].rstrip() @@ -143,14 +144,14 @@ def parse_labels(asm_text: str) -> dict[int, str]: | 助记符预检失败 | 子集扫描 `.s` vs TinyFive 支持表 | `completion="not_run"`，exit 4，列出不支持助记符 | | 预算耗尽 | 循环额度判断 | `completion="budget_exhausted"`，`executed==limit`，ratio `null`，exit 0（显式预算属正常） | | 超时 | `SIGALRM` + `_timed_out` 标志 | `completion="timeout"`，保留部分 ops，exit 0（`--fail-on-incomplete` 时 exit 7） | -| `m.last_error` 非空且非超时 | 适配器 `strict=True` 抛错 | `completion="error"`，`dynamic=null`，exit 1 | +| `m.last_error` 非空且非超时 | 适配器 `strict=True` 抛错 | `source="unavailable"` + `completion="error"`，`ops=null`，`output.partial=true`，exit 1 | | 未完成且要求严格 | `--fail-on-incomplete` | exit 7 | 静态兜底已删除：`static_instruction_mix` 只写入 `scratchv.static_instruction_mix`，字段名带 `static_`，永不出现在 `dynamic.ops`。 ### 2.6 LLVM 侧改动（C9） -1. `binding.Target.from_triple("riscv32-unknown-elf")`；`ImportError`/`RuntimeError` → `status="skipped"`，`reason` 记录原始异常。llvmlite 当前环境未安装，该分支即 `skipped`。 +1. `binding.Target.from_triple("riscv32-unknown-elf")`；`ImportError`（llvmlite 未安装）→ `status="skipped"`，`reason` 记录原始异常；成功导入后的目标机/IR/发射阶段异常（如 `RuntimeError`）→ `status="failed"`，`reason` 记录真实原因，`main` 追加 warning 并把它写入 `llvm.dynamic.reason`（不再统一硬编码为 “pipeline not implemented”）。llvmlite 当前环境未安装，该分支即 `skipped`。 2. 成功路径。解析 IR 文本后，用 llvmlite API 覆盖模块头： ```python llmod = binding.parse_assembly(ir_text) @@ -215,6 +216,8 @@ def parse_labels(asm_text: str) -> dict[int, str]: Simulated by tinyfive {environment.tinyfive} | completion={completion} | executed={executed} | limit={limit} | memory={memory_size_bytes} | seed={input_seed} | halt=0x{halt_addr:x} | model sha256={model.sha256} | binary sha256={scratchv.compile.binary_sha256} +output [measured | measured/partial | unavailable]: {output.raw_hex} +| elements={output.elements} | completion={output.completion} ``` 渲染规则： @@ -238,7 +241,7 @@ Simulated by tinyfive {environment.tinyfive} | completion={completion} | execute | `scratchv.compile.status/binary/binary_bytes/binary_sha256/code_bytes/data_offset/data_offset_source/data_bytes/workspace_bytes/static_insns/static_source/elapsed_s` | — | static | 编译与镜像 | | `scratchv.static_instruction_mix.{source,load,store,mul,add,madd,branch,other}` | int | static | 静态助记符分布 | | `scratchv.dynamic.{source,simulator,simulator_version,completion,limit,executed,timeout_s,elapsed_s,memory_size_bytes,input_seed,input_elements,halt_addr,ops{total,load,store,mul,add,madd,branch},x_registers_used,x_usage_total,f_registers_used,per_label,per_label_note,last_error}` | — | measured | 动态执行 | -| `scratchv.output.{addr,elements,raw_hex,q16_16}` | — | measured | 输出读出 | +| `scratchv.output.{addr,elements,raw_hex,q16_16,completion,partial}` | — | measured | 输出读出；`q16_16` 恒为 list（不可用时为 null）；`completion != "halted"` 时 `partial=true`，Markdown/HTML 的 Provenance 段落标 `[measured/partial]` | | `llvm.compile.{status,reason,isa_detected,isa_mismatch,static_insns,static_source,elapsed_s}` | — | static/unavailable | LLVM 侧 | | `llvm.dynamic.{source,simulator,completion,reason,ops}` | — | unavailable | 失败必须 `ops=null` | | `comparison.{dynamic_instruction_ratio,incomparable_reason}` | float\|null | measured 派生 | 比值规则见设计文档 2.1.1 | @@ -265,7 +268,7 @@ Simulated by tinyfive {environment.tinyfive} | completion={completion} | execute ### 4.2 `benchmark.py`（只读使用） -- `estimate_cnn_model(model_spec=None) -> dict`：仅用于 `warnings` 与“预计墙钟”提示；字段进入 `estimated` 分类，前缀 `estimated_`。不得写入 `dynamic`。 +- `estimate_cnn_model(model_spec=None) -> dict`：**未接入 `rv32_bench.py`**（本课题不做自动预估）；仅 `benchmark.py` 自身 CLI 使用。若未来引入，其字段必须进入 `estimated` 分类，前缀 `estimated_`，不得写入 `dynamic`。 - `RV32EmulatorFast` / `run_benchmark`：仅测试用例 1 作为独立功能对照（`load_unified_binary(binary, code_size_base=..., load_addr=0)` + `run(max_instr=...)`）；其计数分类（`Cat_*`）与 TinyFive 不同，只对照 `total/load_count/store_count/branch_total`。若未来作为正式数据源，须以 `simulator="rv32_emulator_fast"` 独立字段呈现，不与 TinyFive 混算。 - `estimate_cnn_instructions` 的 per-MAC 常量（`CONV_INSNS_PER_MAC=8` 等）属解析模型，禁止用于 `measured`。 @@ -399,7 +402,7 @@ make test | # | 风险 | 影响 | 缓解/回退 | |---|------|------|-----------| -| R1 | TinyFive 吞吐低（Python 解释执行），全量 18.5 亿指令可能需数小时 | 全量不可行 | `--max-instructions` 预算并如实标注；先跑 1M 探测得到 MIPS 预估墙钟，写入 `warnings` | +| R1 | TinyFive 吞吐低（Python 解释执行），全量 18.5 亿指令可能需数小时 | 全量不可行 | `--max-instructions` 预算并如实标注；**未实现自动预估**（无 1M 探测/`estimate_cnn_model` 接入），full 模式启动前只写一条“无自动预估、可用预算校准”的 warning；超时/预算中断的结果标 `partial` | | R2 | 256MiB numpy 内存 + 26MB 权重 | 内存压力 | 布局不可压缩（ABI 地址固定）；内存不足只能拒绝并提示，禁止缩容假装成功 | | R3 | `ra` 被生成代码内部 `jal ra, …` 覆盖，`halt_addr` 永不命中 | 无法 `halted` | `completion` 如实标 `budget_exhausted/timeout`；`--fail-on-incomplete` 供 CI | | R4 | 平台无 `SIGALRM` | 超时保护缺失 | 退化为要求 `--max-instructions`；否则拒绝启动 | @@ -443,5 +446,6 @@ make test ### 已知限制 - 对 TinyFive 机器实例的 NumPy 2.x `LW/LH` 兼容 shim 属课题 26 追修（`tinyfive.py` 未改）。 -- 错误路径统一用 `source=unavailable` 表达（而非 `dynamic=null`）。 -- LLVM 侧无 llvmlite 时 `status=skipped`、不打印比值数字。 +- 停机采用实例级 `exe` 重绑 shim（`halt_addr` 与指令预算先到者停），同样未改 `tinyfive.py`；设计文档 2.1.4 已登记该机制。 +- 错误路径统一用 `source=unavailable` 表达（而非 `dynamic=null`），`output.partial=true`。 +- LLVM 侧无 llvmlite 时 `status=skipped`；导入后目标机/IR 失败为 `status=failed` 并带真实 `reason`（main 写 warning）。两种情况都不打印比值数字。 diff --git "a/docs/topics/27-RV32\345\205\250\351\207\217Benchmark-\350\256\276\350\256\241\346\226\207\346\241\243.md" "b/docs/topics/27-RV32\345\205\250\351\207\217Benchmark-\350\256\276\350\256\241\346\226\207\346\241\243.md" index 234bde1..911bf48 100644 --- "a/docs/topics/27-RV32\345\205\250\351\207\217Benchmark-\350\256\276\350\256\241\346\226\207\346\241\243.md" +++ "b/docs/topics/27-RV32\345\205\250\351\207\217Benchmark-\350\256\276\350\256\241\346\226\207\346\241\243.md" @@ -63,7 +63,7 @@ completion ::= "halted" | "budget_exhausted" | "timeout" | "error" | "not_run" 规则： - `--max-instructions 0` 与 `--full` 等价，均为无截断语义；`limit=null`、`executed` 为真实执行数。 -- 分块循环使用 `--chunk-instructions`（默认 10,000,000）作为每块大小；块间检查停机与超时。这是在不修改 `ProfiledMachine` 公共接口的前提下获得“停机检测 + 进度 + 超时”的唯一手段。 +- 分块循环使用 `--chunk-instructions`（默认 10,000,000）作为每块大小；块间检查停机、墙钟与预算。块内精确停机由实例级 `exe` shim（2.1.4）保证，两者都不修改 `ProfiledMachine` 公共接口。 - `budget_exhausted` 时 `executed == limit` 必须为等式不变量；`halted` 时 `limit` 可为 `null`（full）或任意（budgeted 提前停机）。 - 超时只在 `--full` 模式下产生 `timeout`；`budgeted` 模式下先到限额即结束，超时属于异常保护（同样标 `timeout`）。 - 派生指标约束：`comparison.dynamic_instruction_ratio` 仅当两侧 `source=="simulated"` 且 `completion=="halted"` 时计算；任一 `budget_exhausted/timeout` 一律 `null` 并写 `incomparable_reason`。 @@ -109,17 +109,19 @@ data_size ::= len(weight_bytes) = len(.bin) − data_offset #### 2.1.4 停机条件 -- 生成代码结尾为 `_done: ret`（`jalr x0, x1, 0`，`onnx_to_riscv_standalone.py:1553-1556`）。harness 在运行前设 `x1 = halt_addr`，当 PC 到达 `halt_addr` 即判定 `halted`。TinyFive 的 `exe(start, end)` 原生支持按 end 地址停止；`ProfiledMachine` 未暴露该参数，故采用 2.1.1 的分块 `instructions=` 循环 + 每块后检查 `m.pc`。 +- 生成代码结尾为 `_done: ret`（`jalr x0, x1, 0`，`onnx_to_riscv_standalone.py:1553-1556`）。harness 在运行前设 `x1 = halt_addr`，当 PC 到达 `halt_addr` 即判定 `halted`。 +- TinyFive 的 `exe(start, end)` 原生支持按 end 地址停止，但 `ProfiledMachine` 未暴露该参数，且其 `exe` 在给定 `end` 时会忽略 `instructions` 预算。harness 因此只对**机器实例**重绑 `exe`：每步执行前同时检查 `pc == halt_addr` 与指令额度，二者先到者停（`_install_tinyfive_compat`，`rv32_bench.py`）；`scratchv/simulator/tinyfive.py` 与 `ProfiledMachine` 公共接口保持不变。 +- 外层仍以 2.1.1 的分块 `m.run(instructions=chunk, start=pc, strict=True)` 循环驱动，块间做墙钟检查、预算记账与 `pc` 复核；实例级 shim 保证单块内不会越过停机地址空转解码（否则每次解码都会计入 `ops.total`，虚增动态计数）。 - 若生成代码在返回前用 `jal ra, …` 覆盖了 `ra`，或跳转路径异常，PC 永远不会命中 `halt_addr`：`budgeted` 记 `budget_exhausted`，`full` 记 `timeout`。不得谎报 `halted`。 - TinyFive 遇到不支持的指令时打印错误且 PC 不前进（`dec()` 无匹配分支时不调用 `ipc()`），会表现为“卡死”。启动前必须做**助记符白名单预检**：用 ScratchV 的 `_disasm_one` 解析每个 code word，若出现 TinyFive 不支持/无法识别的助记符 → `completion="not_run"`，`exit 4`，禁止开跑。 -- `m.last_error` 非空（适配器捕获到异常）→ `completion="error"`，`dynamic` 整体置 `null`。 +- `m.last_error` 非空（适配器捕获到异常）→ `completion="error"`，该侧 `dynamic.source="unavailable"`、`ops=null`。 #### 2.1.5 超时与预算保护 - `--timeout SECONDS`（默认 900）为单侧仿真墙钟上限。实现：主线程 `signal.setitimer(ITIMER_REAL, remaining)` + `SimulationTimeout` 处理器；`m.run(..., strict=True)` 使适配器 `finally` 仍在异常路径更新 `instr_count`，ops 计数器保留部分值。 - 超时判定不依赖异常类型（适配器会把异常包装成 `RuntimeError`）：置模块级 `_timed_out` 标志，捕获后据此写 `completion="timeout"`，并把 `elapsed_s`、`executed` 落盘。 - 平台不支持 `SIGALRM`（如 Windows）时：`--timeout` 退化为告警，要求用户必须给 `--max-instructions`；否则拒绝启动并提示替代方案。 -- **预算预估仅作告警**：`benchmark.estimate_cnn_model()` 的输出只能进入 `warnings` 与 “预计墙钟” 提示，字段名带 `estimated_` 前缀，绝不填入 `dynamic`。 +- **不做自动预估**：本驱动不运行 1M 探测，也不调用 `benchmark.estimate_cnn_model()`（该解析模型只属于 `benchmark.py` 自身 CLI），因此报告中不存在 `estimated_*` 墙钟数字。full 模式启动前只在 `warnings` 写一条显式提示：无自动预估、可用 `--max-instructions N` 校准、墙钟超时触发时结果如实标 partial。若未来引入任何预估值，字段名必须带 `estimated_` 前缀，且绝不填入 `dynamic`。 ### 2.2 诚实报告规范 @@ -437,7 +439,8 @@ while True: "per_label_note": "tinyfive exe() exposes no per-PC trace", "last_error": null }, - "output": {"addr": 201326592, "elements": 1, "raw_hex": "0x00018000", "q16_16": 1.5} + "output": {"addr": 201326592, "elements": 1, "raw_hex": "0x00018000", + "q16_16": [1.5], "completion": "halted", "partial": false} }, "llvm": { "compile": {"status": "skipped", "reason": "llvmlite not available", @@ -452,7 +455,7 @@ while True: "incomparable_reason": "llvm.dynamic.source=='unavailable'" }, "warnings": [ - "estimated dynamic instructions (analytical): 1.85e9 — used for wall-clock warning only" + "full simulation requested; no automatic instruction-count or wall-clock estimate is available (use --max-instructions N to calibrate); results are marked partial if the wall-clock timeout fires" ], "errors": [] } From e0a70529b25d4125306e89ee60749be115806f42 Mon Sep 17 00:00:00 2001 From: opencode Date: Tue, 15 Sep 2026 00:59:17 +0800 Subject: [PATCH 5/5] feat(topic27): add RV32-bench feature case report and CI regressions --- .github/workflows/ci.yml | 18 + benchmarks/run_topic27_rv32_bench_case.py | 682 +++++++++++++++++++ tests/test_topic27_rv32_bench_case_report.py | 204 ++++++ 3 files changed, 904 insertions(+) create mode 100644 benchmarks/run_topic27_rv32_bench_case.py create mode 100644 tests/test_topic27_rv32_bench_case_report.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 15aae4b..8a8daca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -107,6 +107,13 @@ jobs: run: | python3.12 -m pytest tests/test_pr37_regression.py -v --tb=short + - name: Run topic27 RV32-bench regressions + run: | + python3.12 -m pytest \ + tests/test_rv32_bench.py \ + tests/test_topic27_rv32_bench_case_report.py \ + -v --tb=short + - name: Generate test visualization page if: github.ref == 'refs/heads/main' run: | @@ -218,6 +225,14 @@ jobs: --json benchmark_reports/const_merge_report.json \ --markdown benchmark_reports/const_merge_report.md + # ── 3.1.3 课题27：RV32 bench case 报告（诚实 provenance + 降级） ── + - name: Topic 27 RV32 bench case report + run: | + mkdir -p benchmark_reports + python3.12 benchmarks/run_topic27_rv32_bench_case.py \ + --json benchmark_reports/rv32_bench_case_report.json \ + --markdown benchmark_reports/rv32_bench_case_report.md + # ── 3.2 DSL 用例编译 + 模拟基准 ──────────────────────────────────── - name: DSL case compilation benchmarks run: | @@ -363,6 +378,9 @@ jobs: if [ -f benchmark_reports/const_merge_report.md ]; then cat benchmark_reports/const_merge_report.md >> $GITHUB_STEP_SUMMARY fi + if [ -f benchmark_reports/rv32_bench_case_report.md ]; then + cat benchmark_reports/rv32_bench_case_report.md >> $GITHUB_STEP_SUMMARY + fi echo "" >> $GITHUB_STEP_SUMMARY if [ -f benchmark_reports/github_summary.md ]; then cat benchmark_reports/github_summary.md >> $GITHUB_STEP_SUMMARY diff --git a/benchmarks/run_topic27_rv32_bench_case.py b/benchmarks/run_topic27_rv32_bench_case.py new file mode 100644 index 0000000..c19c5ca --- /dev/null +++ b/benchmarks/run_topic27_rv32_bench_case.py @@ -0,0 +1,682 @@ +#!/usr/bin/env python3 +"""Topic 27 RV32-bench feature case: drive ``rv32_bench`` and audit its report. + +The case runs the real ``rv32_bench.main`` driver on one tiny deterministic +ONNX model (3x3 Conv, 8x8 -> 6x6, fixed seed) and proves the honest-report +contract end to end: + +1. ``rv32_bench.{json,md,html}`` artifacts are written and the JSON passes + ``bench_report.validate_report_schema``; +2. ``completion`` stays inside the documented enumeration and a null + comparison ratio always carries an ``incomparable_reason``; +3. ``audit_provenance`` rejects forged reports, so the honesty gate is not + vacuous; +4. budget truncation (``--max-instructions``) and wall-clock timeout are + labeled honestly, or explicitly reported as ``not_run`` when the + simulator is missing. + +CI has no LLVM toolchain (and may lack TinyFive), so every run uses +``--allow-missing-simulator --skip-llvm`` and degrades to a static-only +report instead of inventing dynamic numbers. This is a deterministic +feature/integration case, not a performance claim: the embedded dynamic +counts are TinyFive emulator instruction counts on a toy model. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +import tempfile +import time +from copy import deepcopy +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from scratchv.standalone import bench_report, rv32_bench + +SCHEMA_VERSION = "topic27-rv32-bench-case/1" +RV32_SCHEMA_VERSION = "rv32-bench/2" +DEFAULT_JSON = Path("benchmark_reports/rv32_bench_case_report.json") +DEFAULT_MARKDOWN = Path("benchmark_reports/rv32_bench_case_report.md") + +ALLOWED_COMPLETIONS = frozenset({ + "halted", "budget_exhausted", "timeout", "error", "not_run", +}) + +CASE_BUILDER = "tiny_conv_8x8_k3" +BUDGET_LIMIT = 1000 +TIMEOUT_PROBE_S = 0.01 +TIMEOUT_PROBE_CHUNK = 1 +RUN_CHUNK = 4096 +RUN_TIMEOUT_S = 120.0 + + +# ═══════════════════════════════════════════════════════════════════════════ +# Case model +# ═══════════════════════════════════════════════════════════════════════════ + +def build_case_model(path: str | Path) -> Path: + """Write the deterministic tiny Conv ONNX model used by the case.""" + import numpy as np + import onnx + from onnx import TensorProto, helper, numpy_helper + + path = Path(path) + rng = np.random.RandomState(0) + inp = helper.make_tensor_value_info( + "input", TensorProto.FLOAT, [1, 1, 8, 8]) + out = helper.make_tensor_value_info( + "output", TensorProto.FLOAT, [1, 1, 6, 6]) + weight = numpy_helper.from_array( + (rng.randn(1, 1, 3, 3).astype(np.float32) * 0.1), "W") + bias = numpy_helper.from_array(np.zeros(1, np.float32), "B") + node = helper.make_node( + "Conv", ["input", "W", "B"], ["output"], + kernel_shape=[3, 3], pads=[0, 0, 0, 0], strides=[1, 1], + ) + graph = helper.make_graph( + [node], "topic27_rv32_bench_case", [inp], [out], [weight, bias]) + model = helper.make_model( + graph, opset_imports=[helper.make_opsetid("", 13)]) + onnx.save(model, str(path)) + return path + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with open(path, "rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +# ═══════════════════════════════════════════════════════════════════════════ +# Driver invocation +# ═══════════════════════════════════════════════════════════════════════════ + +def _run_rv32(model_path: Path, out_dir: Path, extra_args: list[str], *, + timeout_s: float = RUN_TIMEOUT_S, + chunk: int = RUN_CHUNK) -> dict[str, Any]: + """Invoke the real ``rv32_bench.main`` in-process and collect artifacts.""" + out_dir.mkdir(parents=True, exist_ok=True) + argv = [ + str(model_path), + "--quiet", + "--output-dir", str(out_dir), + "--allow-missing-simulator", + "--skip-llvm", + "--timeout", str(timeout_s), + "--chunk-instructions", str(chunk), + ] + [str(arg) for arg in extra_args] + started = time.perf_counter() + status = "ok" + error: str | None = None + exit_code: int | None = None + try: + exit_code = rv32_bench.main(argv) + except Exception as exc: + status = "exception" + error = f"{type(exc).__name__}: {exc}"[:300] + elapsed = time.perf_counter() - started + + json_path = out_dir / "rv32_bench.json" + md_path = out_dir / "rv32_bench.md" + html_path = out_dir / "rv32_bench.html" + rv_report: dict | None = None + if json_path.is_file(): + try: + rv_report = json.loads(json_path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + status = "invalid_json" + error = str(exc)[:300] + + return { + "argv": argv, + "exit_code": exit_code, + "status": status, + "error": error, + "elapsed_s": round(elapsed, 4), + "artifacts": { + "json": str(json_path), + "markdown": str(md_path), + "html": str(html_path), + "json_written": json_path.is_file(), + "markdown_written": md_path.is_file(), + "html_written": html_path.is_file(), + "json_bytes": ( + json_path.stat().st_size if json_path.is_file() else None), + "json_sha256": ( + _sha256(json_path) if json_path.is_file() else None), + }, + "report": rv_report, + } + + +def _probe(model_path: Path, out_dir: Path, extra_args: list[str], + expected: str, *, timeout_s: float = RUN_TIMEOUT_S, + chunk: int = RUN_CHUNK) -> dict[str, Any]: + """Run one truncation probe and summarize its honesty evidence.""" + run = _run_rv32( + model_path, out_dir, extra_args, timeout_s=timeout_s, chunk=chunk) + rv = run.pop("report") + probe: dict[str, Any] = {"expected": expected, **run} + if rv is None: + probe.update({ + "status": "no_report", + "note": run.get("error") or "rv32 report not written", + "completion": None, "source": None, "limit": None, + "executed": None, "ops_total": None, "partial": None, + "ratio": None, "incomparable_reason": None, + "schema_errors": ["rv32 report not written"], + "audit_violations": ["rv32 report not written"], + }) + return probe + + dyn = (rv.get("scratchv") or {}).get("dynamic") or {} + cmp_ = rv.get("comparison") or {} + output = (rv.get("scratchv") or {}).get("output") or {} + probe.update({ + "completion": dyn.get("completion"), + "source": dyn.get("source"), + "reason": dyn.get("reason"), + "limit": dyn.get("limit"), + "executed": dyn.get("executed"), + "ops_total": (dyn.get("ops") or {}).get("total"), + "partial": output.get("partial"), + "ratio": cmp_.get("dynamic_instruction_ratio"), + "incomparable_reason": cmp_.get("incomparable_reason"), + "schema_errors": bench_report.validate_report_schema(rv), + "audit_violations": rv32_bench.audit_provenance(rv), + }) + if dyn.get("source") == "unavailable" and \ + dyn.get("completion") == "not_run": + probe["status"] = "not_run" + probe["note"] = dyn.get("reason") or "simulator unavailable" + elif dyn.get("completion") == expected: + probe["status"] = "confirmed" + probe["note"] = None + else: + probe["status"] = "unexpected" + probe["note"] = ( + f"expected completion={expected!r}, got {dyn.get('completion')!r}") + return probe + + +# ═══════════════════════════════════════════════════════════════════════════ +# Audit probe +# ═══════════════════════════════════════════════════════════════════════════ + +def _tamper(field: str, violations: list[str]) -> dict[str, Any]: + return { + "field": field, + "violations": violations, + "rejected": bool(violations), + } + + +def _audit_probe(rv_report: dict | None) -> dict[str, Any]: + """Forge a copied report and prove ``audit_provenance`` rejects it.""" + if not rv_report: + return { + "tampers": [], + "rejected": False, + "note": "rv32 report not written; audit gate cannot be probed", + } + tampers = [] + + ratio_forgery = deepcopy(rv_report) + ratio_forgery["comparison"] = { + "dynamic_instruction_ratio": 0.31, "incomparable_reason": None, + } + tampers.append(_tamper( + "comparison.dynamic_instruction_ratio=0.31", + rv32_bench.audit_provenance(ratio_forgery), + )) + + schema_forgery = deepcopy(rv_report) + schema_forgery["schema_version"] = "rv32-bench/1" + tampers.append(_tamper( + "schema_version=rv32-bench/1", + rv32_bench.audit_provenance(schema_forgery), + )) + + dyn = (rv_report.get("scratchv") or {}).get("dynamic") or {} + if dyn.get("source") == "simulated" and \ + isinstance(dyn.get("executed"), int): + counters_forgery = deepcopy(rv_report) + counters_forgery["scratchv"]["dynamic"]["executed"] = ( + dyn["executed"] + 999) + tampers.append(_tamper( + "scratchv.dynamic.executed+=999", + rv32_bench.audit_provenance(counters_forgery), + )) + + return { + "tampers": tampers, + "rejected": all(tamper["rejected"] for tamper in tampers), + "note": None, + } + + +# ═══════════════════════════════════════════════════════════════════════════ +# Report assembly and hard checks +# ═══════════════════════════════════════════════════════════════════════════ + +def _simulator_state(rv_report: dict | None): + if not rv_report: + return None + dyn = (rv_report.get("scratchv") or {}).get("dynamic") or {} + source = dyn.get("source") + if source == "simulated": + return True + if source == "unavailable": + return False + return None + + +def _ratio_is_honest(rv_report: dict | None) -> bool: + if not rv_report: + return False + comparison = rv_report.get("comparison") or {} + ratio = comparison.get("dynamic_instruction_ratio") + if ratio is None: + return bool(comparison.get("incomparable_reason")) + if isinstance(ratio, bool) or not isinstance(ratio, (int, float)): + return False + for side in ("scratchv", "llvm"): + dyn = (rv_report.get(side) or {}).get("dynamic") or {} + if dyn.get("source") != "simulated" or \ + dyn.get("completion") != "halted": + return False + return True + + +def _probe_honest(probe: dict | None, expected: str) -> bool: + if not probe: + return False + if probe.get("status") == "confirmed": + consistent = ( + probe.get("completion") == expected + and probe.get("executed") == probe.get("ops_total") + and probe.get("ratio") is None + and bool(probe.get("incomparable_reason")) + and probe.get("schema_errors") == [] + and probe.get("audit_violations") == [] + ) + if expected == "budget_exhausted": + consistent = consistent and ( + probe.get("limit") == probe.get("executed")) + elif expected == "timeout": + consistent = consistent and probe.get("limit") is None + return consistent + if probe.get("status") == "not_run": + return bool(probe.get("reason")) + return False + + +def _hard_checks(report: dict[str, Any]) -> dict[str, bool]: + rv_report = report.get("rv32_report") + artifacts = (report.get("run") or {}).get("artifacts") or {} + return { + "rv32_json_artifact_written": bool(artifacts.get("json_written")), + "rv32_markdown_artifact_written": bool( + artifacts.get("markdown_written")), + "rv32_schema_version_v2": ( + bool(rv_report) + and rv_report.get("schema_version") == RV32_SCHEMA_VERSION + ), + "rv32_schema_valid": report.get("schema_errors") == [], + "provenance_audit_clean": report.get("audit_violations") == [], + "completion_is_legal": ( + report.get("completion") in ALLOWED_COMPLETIONS), + "ratio_is_honest": _ratio_is_honest(rv_report), + "github_summary_rendered": ( + "# RV32 Benchmark Summary" in ( + report.get("github_summary") or "")), + "report_markdown_rendered": ( + "## " in (report.get("report_markdown") or "")), + "audit_gate_rejects_forgery": bool( + (report.get("audit_probe") or {}).get("rejected")), + "budget_probe_honest": _probe_honest( + report.get("budget_probe"), "budget_exhausted"), + "timeout_probe_honest": _probe_honest( + report.get("timeout_probe"), "timeout"), + } + + +def _honesty(rv_report: dict | None, simulator_available) -> str: + if not rv_report: + return ( + "The rv32_bench driver did not produce a report; no dynamic or " + "static number is claimed." + ) + dyn = (rv_report.get("scratchv") or {}).get("dynamic") or {} + parts = [ + "Deterministic feature case executed through the real rv32_bench CLI " + "on a generated tiny Conv model; any dynamic counts are TinyFive " + "emulator instruction counts, not hardware cycles, and this case " + "makes no speedup or performance claim.", + "LLVM compilation is always skipped (--skip-llvm) because CI has no " + "LLVM/clang toolchain; the LLVM side is unavailable by design, so " + "dynamic_instruction_ratio is null with an explicit " + "incomparable_reason.", + ] + if simulator_available is True: + parts.append( + f"TinyFive executed the model " + f"(completion={dyn.get('completion')}); the budget and wall-clock " + "probes confirm that truncated runs are labeled " + "budget_exhausted/timeout and excluded from comparison." + ) + elif simulator_available is False: + parts.append( + "TinyFive is unavailable in this environment, so the dynamic " + "section is source=unavailable with ops=null and the " + "budget/timeout probes are reported as not_run instead of " + "being fabricated." + ) + else: + parts.append( + "Simulator availability could not be determined because the " + "driver failed before simulation; dynamic data is absent and " + "unclaimed." + ) + return " ".join(parts) + + +def evaluate(model_path: str | Path, work_root: str | Path, *, + case_builder: str | None = None) -> dict[str, Any]: + """Run the case (full + probes) and build the auditable report payload.""" + model_path = Path(model_path).resolve() + work_root = Path(work_root) + work_root.mkdir(parents=True, exist_ok=True) + + with tempfile.TemporaryDirectory( + prefix="t27_case_", dir=str(work_root)) as tmp: + root = Path(tmp) + run = _run_rv32(model_path, root / "primary", ["--full"]) + rv_report = run.pop("report") + + completion = None + comparison: dict[str, Any] = { + "dynamic_instruction_ratio": None, "incomparable_reason": None, + } + schema_errors = ["rv32 report not written"] + audit_violations = ["rv32 report not written"] + github_summary = "" + report_markdown = "" + if rv_report is not None: + completion = ( + (rv_report.get("scratchv") or {}).get("dynamic") or {} + ).get("completion") + comparison = dict(rv_report.get("comparison") or {}) + schema_errors = bench_report.validate_report_schema(rv_report) + audit_violations = rv32_bench.audit_provenance(rv_report) + github_summary = bench_report.render_github_summary(rv_report) + report_markdown = bench_report.render_markdown(rv_report) + + simulator_available = _simulator_state(rv_report) + + budget_probe = _probe( + model_path, root / "budget_probe", + ["--max-instructions", str(BUDGET_LIMIT)], "budget_exhausted", + ) + timeout_probe = _probe( + model_path, root / "timeout_probe", [], "timeout", + timeout_s=TIMEOUT_PROBE_S, chunk=TIMEOUT_PROBE_CHUNK, + ) + + report: dict[str, Any] = { + "schema_version": SCHEMA_VERSION, + "topic": "topic27-rv32-bench", + "generated_at": datetime.now(timezone.utc).isoformat(), + "case": { + "model": str(model_path), + "sha256": _sha256(model_path), + "bytes": model_path.stat().st_size, + "builder": case_builder, + "description": ( + "deterministic 3x3 Conv, input [1,1,8,8] -> output [1,1,6,6], " + "seed 0 weights" + ), + }, + "config": { + "flags": ["--allow-missing-simulator", "--skip-llvm"], + "rv32_schema_version": RV32_SCHEMA_VERSION, + "budget_limit": BUDGET_LIMIT, + "timeout_probe_s": TIMEOUT_PROBE_S, + "timeout_probe_chunk": TIMEOUT_PROBE_CHUNK, + "simulator_available": simulator_available, + "llvm_skipped": True, + }, + "completion": completion, + "comparison": comparison, + "schema_errors": schema_errors, + "audit_violations": audit_violations, + "run": run, + "github_summary": github_summary, + "report_markdown": report_markdown, + "budget_probe": budget_probe, + "timeout_probe": timeout_probe, + "audit_probe": _audit_probe(rv_report), + "rv32_report": rv_report, + "honesty": _honesty(rv_report, simulator_available), + } + checks = _hard_checks(report) + report["hard_checks"] = checks + report["hard_failures"] = sorted( + name for name, ok in checks.items() if not ok) + return report + + +# ═══════════════════════════════════════════════════════════════════════════ +# Markdown rendering +# ═══════════════════════════════════════════════════════════════════════════ + +def _cell(value: Any) -> str: + if value is None: + return "—" + if isinstance(value, bool): + return "true" if value else "false" + return str(value) + + +def render_markdown(report: dict[str, Any]) -> str: + """Render the case report as Markdown for stdout and job summaries.""" + rv_report = report.get("rv32_report") or {} + model = rv_report.get("model") or {} + env = rv_report.get("environment") or {} + targets = rv_report.get("targets") or {} + scratchv = rv_report.get("scratchv") or {} + sv_compile = scratchv.get("compile") or {} + sv_dyn = scratchv.get("dynamic") or {} + llvm = rv_report.get("llvm") or {} + ll_compile = llvm.get("compile") or {} + ll_dyn = llvm.get("dynamic") or {} + comparison = rv_report.get("comparison") or {} + case = report.get("case") or {} + run = report.get("run") or {} + ratio = comparison.get("dynamic_instruction_ratio") + tag = "PASS" if not report["hard_failures"] else "FAIL" + + lines = [ + "# Topic 27 RV32-Bench Feature Case", + "", + f"- Schema: `{report.get('schema_version')}` " + f"(embedded report: `{_cell(rv_report.get('schema_version'))}`)", + f"- Case: `{_cell(case.get('model'))}` " + f"(sha256={str(case.get('sha256') or '')[:12]}, " + f"{_cell(case.get('bytes'))} bytes, " + f"builder={_cell(case.get('builder'))})", + f"- Generated: {report.get('generated_at')}", + f"- Hard checks: {tag} " + f"({len(report['hard_checks']) - len(report['hard_failures'])}" + f"/{len(report['hard_checks'])})", + f"- Driver: rv32_bench.main exit={_cell(run.get('exit_code'))}, " + f"status={_cell(run.get('status'))}, " + f"elapsed={_cell(run.get('elapsed_s'))}s", + f"- render_markdown bytes: {len(report.get('report_markdown') or '')}", + "", + "## RV32 report schema", + "", + "| Field | Value |", + "|-------|-------|", + f"| schema_version | {_cell(rv_report.get('schema_version'))} |", + f"| generated_at | {_cell(rv_report.get('generated_at'))} |", + f"| model.path | {_cell(model.get('path'))} |", + f"| model.sha256 | {_cell(model.get('sha256'))} |", + f"| environment.python | {_cell(env.get('python'))} |", + f"| environment.tinyfive | {_cell(env.get('tinyfive'))} |", + f"| environment.llvmlite | {_cell(env.get('llvmlite'))} |", + f"| targets.scratchv.isa | " + f"{_cell((targets.get('scratchv') or {}).get('isa'))} |", + f"| targets.llvm.isa | " + f"{_cell((targets.get('llvm') or {}).get('isa'))} |", + f"| scratchv.compile.status | {_cell(sv_compile.get('status'))} |", + f"| scratchv.compile.static_insns | " + f"{_cell(sv_compile.get('static_insns'))} |", + f"| llvm.compile.status | {_cell(ll_compile.get('status'))} |", + f"| scratchv.dynamic.source | {_cell(sv_dyn.get('source'))} |", + f"| comparison.dynamic_instruction_ratio | {_cell(ratio)} |", + "", + "## Completion / ratio status", + "", + "| Side | source | completion | executed | limit |", + "|------|--------|------------|---------:|------:|", + f"| ScratchV | {_cell(sv_dyn.get('source'))} | " + f"{_cell(sv_dyn.get('completion'))} | " + f"{_cell(sv_dyn.get('executed'))} | {_cell(sv_dyn.get('limit'))} |", + f"| LLVM | {_cell(ll_dyn.get('source'))} | " + f"{_cell(ll_dyn.get('completion'))} | " + f"{_cell(ll_dyn.get('executed'))} | {_cell(ll_dyn.get('limit'))} |", + "", + ] + if ratio is None: + lines.append( + f"- ratio: **null** — " + f"{_cell(comparison.get('incomparable_reason'))}") + else: + lines.append( + f"- ratio: **{ratio:g}** (both sides simulated and halted)") + lines += [ + "", + "## Rendered report summary", + "", + report.get("github_summary") or "_rv32 report not written_", + "", + "## Probes", + "", + "| Probe | Expected | Status | completion | executed | limit | Note |", + "|-------|----------|--------|------------|---------:|------:|------|", + ] + for label, probe in ( + ("budget truncation", report.get("budget_probe")), + ("wall-clock timeout", report.get("timeout_probe")), + ): + probe = probe or {} + lines.append( + f"| {label} | {_cell(probe.get('expected'))} | " + f"{_cell(probe.get('status'))} | " + f"{_cell(probe.get('completion'))} | " + f"{_cell(probe.get('executed'))} | {_cell(probe.get('limit'))} | " + f"{_cell(probe.get('note'))} |" + ) + audit_probe = report.get("audit_probe") or {} + forged = ", ".join( + tamper.get("field", "?") + for tamper in audit_probe.get("tampers") or [] + ) + lines.append( + f"| provenance forgery | rejected | " + f"{'rejected' if audit_probe.get('rejected') else 'NOT rejected'} | " + f"— | — | — | forged: {_cell(forged)} |" + ) + lines += [ + "", + "## Hard checks", + "", + ] + for name, ok in report["hard_checks"].items(): + lines.append(f"- [{'x' if ok else ' '}] {name}") + lines += [ + "", + "## Honesty", + "", + report.get("honesty") or "", + "", + "> Exit codes: 0 = all hard checks pass, 1 = hard-check failure, " + "2 = usage error (missing case model).", + "", + ] + return "\n".join(lines) + + +# ═══════════════════════════════════════════════════════════════════════════ +# CLI +# ═══════════════════════════════════════════════════════════════════════════ + +def _execute(args: argparse.Namespace, work_root: Path) -> int: + work_root.mkdir(parents=True, exist_ok=True) + builder: str | None = None + if args.model is not None: + model_path = args.model.resolve() + else: + args.json.parent.mkdir(parents=True, exist_ok=True) + model_path = args.json.parent / "topic27_rv32_bench_feature.onnx" + try: + build_case_model(model_path) + except Exception as exc: + print( + f"error: case_model_generation_failed: " + f"{type(exc).__name__}: {exc}", + file=sys.stderr, + ) + return 2 + builder = CASE_BUILDER + + report = evaluate(model_path, work_root, case_builder=builder) + markdown = render_markdown(report) + args.json.parent.mkdir(parents=True, exist_ok=True) + args.json.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + args.markdown.parent.mkdir(parents=True, exist_ok=True) + args.markdown.write_text(markdown + "\n", encoding="utf-8") + print(markdown) + if report["hard_failures"]: + print( + "HARD FAILURES: " + ", ".join(report["hard_failures"]), + file=sys.stderr, + ) + return 1 + print(f"reports written: {args.json}, {args.markdown}", file=sys.stderr) + return 0 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--model", type=Path, default=None, + help="Existing ONNX model; default generates the tiny Conv case model", + ) + parser.add_argument("--json", type=Path, default=DEFAULT_JSON) + parser.add_argument("--markdown", type=Path, default=DEFAULT_MARKDOWN) + parser.add_argument( + "--work-dir", type=Path, default=None, + help="Directory for intermediate artifacts (default: temporary dir)", + ) + args = parser.parse_args(argv) + + if args.model is not None and not args.model.is_file(): + print(f"error: case model not found: {args.model}", file=sys.stderr) + return 2 + + if args.work_dir is not None: + return _execute(args, Path(args.work_dir)) + with tempfile.TemporaryDirectory(prefix="topic27_rv32_case_") as tmp: + return _execute(args, Path(tmp)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_topic27_rv32_bench_case_report.py b/tests/test_topic27_rv32_bench_case_report.py new file mode 100644 index 0000000..db669a5 --- /dev/null +++ b/tests/test_topic27_rv32_bench_case_report.py @@ -0,0 +1,204 @@ +"""Tests for the Topic 27 RV32-bench feature case report. + +The report drives the real ``rv32_bench.main`` on a tiny deterministic model +and is the CI artifact that proves the honest-report contract: schema v2, +labeled truncation, null ratio with a reason, and a non-vacuous provenance +audit. Dynamic tests only run the toy case model with explicit budgets. +""" + +from __future__ import annotations + +import json + +import pytest + +from benchmarks import run_topic27_rv32_bench_case as case +from scratchv.standalone import bench_report, rv32_bench + + +class _UnavailableProfiledMachine: + available = False + + def __init__(self, mem_size=0): + self.mem_size = mem_size + + +@pytest.fixture(scope="module") +def case_model(tmp_path_factory): + pytest.importorskip("onnx") + path = ( + tmp_path_factory.mktemp("topic27_case") + / "topic27_rv32_bench_feature.onnx" + ) + case.build_case_model(path) + return path + + +@pytest.fixture(scope="module") +def base_report(case_model, tmp_path_factory): + work = tmp_path_factory.mktemp("topic27_case_run") + return case.evaluate(case_model, work, case_builder=case.CASE_BUILDER) + + +def test_report_passes_all_hard_checks(base_report): + assert base_report["schema_version"] == case.SCHEMA_VERSION + assert base_report["rv32_report"]["schema_version"] == ( + case.RV32_SCHEMA_VERSION) + assert base_report["schema_errors"] == [] + assert base_report["audit_violations"] == [] + assert base_report["hard_failures"] == [] + assert all(base_report["hard_checks"].values()) + assert base_report["honesty"] + + +def test_completion_and_ratio_are_honest(base_report): + assert base_report["completion"] in case.ALLOWED_COMPLETIONS + comparison = base_report["comparison"] + if comparison["dynamic_instruction_ratio"] is None: + assert comparison["incomparable_reason"] + else: + for side in ("scratchv", "llvm"): + dyn = base_report["rv32_report"][side]["dynamic"] + assert dyn["source"] == "simulated" + assert dyn["completion"] == "halted" + + rv_report = base_report["rv32_report"] + assert bench_report.validate_report_schema(rv_report) == [] + dyn = rv_report["scratchv"]["dynamic"] + if base_report["config"]["simulator_available"]: + assert dyn["source"] == "simulated" + assert dyn["ops"]["total"] == dyn["executed"] + else: + assert dyn["source"] == "unavailable" + assert dyn["completion"] == "not_run" + assert dyn["ops"] is None + assert dyn["reason"] + + +def test_main_writes_json_and_markdown(tmp_path, capsys): + json_path = tmp_path / "rv32_bench_case_report.json" + md_path = tmp_path / "rv32_bench_case_report.md" + exit_code = case.main([ + "--json", str(json_path), + "--markdown", str(md_path), + ]) + assert exit_code == 0 + + data = json.loads(json_path.read_text()) + assert data["hard_failures"] == [] + assert data["schema_version"] == case.SCHEMA_VERSION + assert data["rv32_report"]["schema_version"] == case.RV32_SCHEMA_VERSION + assert data["github_summary"].startswith("# RV32 Benchmark Summary") + assert "## " in data["report_markdown"] + assert data["config"]["llvm_skipped"] is True + + markdown = md_path.read_text() + for section in ( + "## RV32 report schema", + "## Completion / ratio status", + "## Rendered report summary", + "## Probes", + "## Hard checks", + "## Honesty", + ): + assert section in markdown, section + assert "Topic 27 RV32-Bench Feature Case" in capsys.readouterr().out + + +def test_main_rejects_missing_model(tmp_path, capsys): + json_path = tmp_path / "missing.json" + exit_code = case.main([ + "--model", str(tmp_path / "nope.onnx"), + "--json", str(json_path), + "--markdown", str(tmp_path / "missing.md"), + ]) + assert exit_code == 2 + assert not json_path.exists() + assert "case model not found" in capsys.readouterr().err + + +def test_audit_probe_rejects_forgery(base_report): + probe = base_report["audit_probe"] + assert probe["rejected"] is True + assert len(probe["tampers"]) >= 2 + assert all(t["rejected"] and t["violations"] for t in probe["tampers"]) + fields = {t["field"] for t in probe["tampers"]} + assert "comparison.dynamic_instruction_ratio=0.31" in fields + assert "schema_version=rv32-bench/1" in fields + + clean = base_report["rv32_report"] + assert rv32_bench.audit_provenance(clean) == [] + forged = json.loads(json.dumps(clean)) + forged["comparison"] = { + "dynamic_instruction_ratio": 0.31, "incomparable_reason": None, + } + violations = rv32_bench.audit_provenance(forged) + assert any("incomplete/non-simulated" in v for v in violations) + + +def test_degraded_mode_is_honest_without_simulator( + case_model, tmp_path, monkeypatch): + monkeypatch.setattr( + rv32_bench, "ProfiledMachine", _UnavailableProfiledMachine) + report = case.evaluate(case_model, tmp_path / "degraded") + + assert report["config"]["simulator_available"] is False + assert report["completion"] == "not_run" + assert report["hard_failures"] == [] + assert report["comparison"]["dynamic_instruction_ratio"] is None + assert "scratchv" in report["comparison"]["incomparable_reason"] + + dyn = report["rv32_report"]["scratchv"]["dynamic"] + assert dyn["source"] == "unavailable" + assert dyn["ops"] is None + assert report["budget_probe"]["status"] == "not_run" + assert report["timeout_probe"]["status"] == "not_run" + assert "TinyFive is unavailable" in report["honesty"] + + markdown = case.render_markdown(report) + assert "not_run" in markdown + assert "[unavailable]" in report["report_markdown"] + + +def test_probes_match_environment(base_report): + budget = base_report["budget_probe"] + timeout = base_report["timeout_probe"] + if base_report["config"]["simulator_available"]: + assert budget["status"] == "confirmed" + assert budget["completion"] == "budget_exhausted" + assert budget["limit"] == budget["executed"] == case.BUDGET_LIMIT + assert budget["ops_total"] == budget["executed"] + assert budget["ratio"] is None and budget["incomparable_reason"] + assert budget["schema_errors"] == [] + assert budget["audit_violations"] == [] + + assert timeout["status"] == "confirmed" + assert timeout["completion"] == "timeout" + assert timeout["partial"] is True + assert timeout["ratio"] is None and timeout["incomparable_reason"] + assert timeout["schema_errors"] == [] + else: + assert budget["status"] == "not_run" and budget["reason"] + assert timeout["status"] == "not_run" and timeout["reason"] + + +def test_hard_check_gate_is_not_vacuous(case_model, tmp_path, monkeypatch): + def silent_driver(_argv): + return rv32_bench.EXIT_OK + + monkeypatch.setattr(rv32_bench, "main", silent_driver) + report = case.evaluate(case_model, tmp_path / "silent") + + assert report["run"]["exit_code"] == 0 + assert report["rv32_report"] is None + for name in ( + "rv32_json_artifact_written", + "rv32_schema_valid", + "provenance_audit_clean", + "completion_is_legal", + "ratio_is_honest", + "audit_gate_rejects_forgery", + "budget_probe_honest", + "timeout_probe_honest", + ): + assert name in report["hard_failures"], report["hard_failures"]