From 27f2bf29dd56aa29b1885f711d1a14057664e153 Mon Sep 17 00:00:00 2001
From: dchaudhari7177 <111210939+dchaudhari7177@users.noreply.github.com>
Date: Wed, 12 Aug 2026 20:42:42 +0530
Subject: [PATCH 1/2] Pin the MMLU/GSM8K extractor invariants exhaustively
(closes #32)
extract_mmlu_answer and extract_gsm8k_answer carry format-tolerance and
precedence logic that had example-based coverage only. This adds ten
invariant tests covering the whole documented input space.
No new dependency. The issue offered hypothesis or hand-rolled parametric
cases; I went with the latter for two reasons:
- The MMLU space is small enough to enumerate *exhaustively* - 10 letters x 6
wrappings x 10 choice-counts - which is strictly stronger than sampling it.
- Adding hypothesis to the dev group meant relocking, and `uv lock` on this
tree regenerates uv.lock into a 2,893-line diff that pulls torch, CUDA and
datasets into the lockfile. Not worth it for a test-only change.
Where a value is genuinely unbounded (a GSM8K number) a seeded Random supplies
boundary values plus a spread, so failures stay reproducible.
Two behaviours worth flagging, both pinned as-is rather than changed:
- The standalone-letter pattern is case-sensitive, so a bare lowercase "a"
extracts nothing, while the explicit "answer: a" form does (it is
IGNORECASE). The docstring lists "lowercase" among the tolerated formats
without that distinction.
- num_choices=0 clamps to 1 rather than rejecting, so it returns 0 for "A".
The range invariant is therefore asserted over num_choices >= 1.
Verified the tests are load-bearing by mutation rather than by passing alone:
making the first standalone letter win, dropping the num_choices filter, and
removing the #### marker preference each fail the corresponding invariant.
---
tests/test_metrics.py | 180 ++++++++++++++++++++++++++++++++++++++++++
1 file changed, 180 insertions(+)
diff --git a/tests/test_metrics.py b/tests/test_metrics.py
index 77ce2e3..dfab9ca 100644
--- a/tests/test_metrics.py
+++ b/tests/test_metrics.py
@@ -9,6 +9,7 @@
import json
import math
+import random
import socket
from pathlib import Path
@@ -355,3 +356,182 @@ def test_no_optional_deps_imported() -> None:
"""
)
subprocess.run([sys.executable, "-c", code], check=True)
+
+
+# --------------------------------------------------------------------------- #
+# SIDE EFFECTS — extractor invariants (exhaustive, deterministic, no new dep)
+# --------------------------------------------------------------------------- #
+#
+# The example-based tests above cover representative formats. These pin the
+# *invariants* from the extractor docstrings across the whole input space rather
+# than at sampled points.
+#
+# The space is small enough to enumerate exhaustively (10 letters x 5 wrappers x
+# 10 choice-counts), which is stronger than sampling it randomly, so no
+# property-based dependency is needed. Where a value is genuinely unbounded (a
+# GSM8K number) a seeded generator supplies the cases, so runs stay
+# reproducible.
+
+_LETTERS = "ABCDEFGHIJ"
+
+#: Wrappings the MMLU extractor documents as equivalent. Filler words are
+#: lowercase on purpose: the standalone pattern only matches *uppercase* single
+#: letters, so a stray capitalised word would itself parse as an answer.
+_MMLU_WRAPPERS = (
+ "{letter}",
+ "({letter})",
+ "{letter}.",
+ "Answer: {letter}",
+ "The answer is {letter}",
+ "hmm, {letter}",
+)
+
+
+@pytest.mark.parametrize("num_choices", range(1, len(_LETTERS) + 1))
+@pytest.mark.parametrize("wrapper", _MMLU_WRAPPERS)
+def test_mmlu_every_wrapping_of_a_valid_letter_extracts_the_same_index(
+ wrapper: str, num_choices: int
+) -> None:
+ """Invariant 1: within num_choices, all documented wrappings agree."""
+ for index in range(num_choices):
+ text = wrapper.format(letter=_LETTERS[index])
+ assert extract_mmlu_answer(text, num_choices=num_choices) == index, text
+
+
+@pytest.mark.parametrize("num_choices", range(1, len(_LETTERS)))
+@pytest.mark.parametrize("wrapper", _MMLU_WRAPPERS)
+def test_mmlu_a_letter_beyond_num_choices_is_never_returned(wrapper: str, num_choices: int) -> None:
+ """Invariant 2: out-of-range letters are ignored, not clamped."""
+ for index in range(num_choices, len(_LETTERS)):
+ text = wrapper.format(letter=_LETTERS[index])
+ assert extract_mmlu_answer(text, num_choices=num_choices) is None, text
+
+
+@pytest.mark.parametrize("num_choices", range(1, len(_LETTERS) + 1))
+def test_mmlu_result_is_always_none_or_a_valid_index(num_choices: int) -> None:
+ """Invariant 3: the return value is None or a usable choice index.
+
+ score_mmlu indexes gold answers with this, so a value outside
+ [0, num_choices) would be a silent mis-score. Checked over a fixed corpus of
+ assorted and adversarial completions.
+ """
+ corpus = [
+ "",
+ "I have no idea",
+ "A B C D E F G H I J",
+ "j i h g f e d c b a",
+ "the answer is Z",
+ "Answer: 3",
+ "(K)",
+ "answer",
+ "answer:",
+ "A" * 50,
+ "The answer is A. No wait, the answer is J.",
+ "item42 costs $3",
+ "\n\t \n",
+ "aAbBcC",
+ "ANSWER IS D",
+ ]
+ rng = random.Random(20260812) # fixed: reproducible corpus
+ alphabet = _LETTERS + _LETTERS.lower() + " .,()\n:0123456789"
+ corpus += ["".join(rng.choice(alphabet) for _ in range(rng.randint(0, 40))) for _ in range(200)]
+
+ for text in corpus:
+ result = extract_mmlu_answer(text, num_choices=num_choices)
+ assert result is None or 0 <= result < num_choices, (num_choices, text, result)
+
+
+@pytest.mark.parametrize("num_choices", range(2, len(_LETTERS) + 1))
+def test_mmlu_last_standalone_letter_wins(num_choices: int) -> None:
+ """Invariant 4: with several standalone letters, the last valid one wins."""
+ for first in range(num_choices):
+ for last in range(num_choices):
+ if first == last:
+ continue
+ text = f"first {_LETTERS[first]} then {_LETTERS[last]}"
+ assert extract_mmlu_answer(text, num_choices=num_choices) == last, text
+
+
+@pytest.mark.parametrize("num_choices", range(2, len(_LETTERS) + 1))
+def test_mmlu_an_explicit_statement_beats_any_standalone_letter(num_choices: int) -> None:
+ """Invariant 5: explicit 'answer is/:' wins over a standalone letter even
+ when the standalone one appears later — precedence is by kind, not position."""
+ for explicit in range(num_choices):
+ for standalone in range(num_choices):
+ if explicit == standalone:
+ continue
+ before = f"{_LETTERS[standalone]} but the answer is {_LETTERS[explicit]}"
+ after = f"the answer is {_LETTERS[explicit]} then {_LETTERS[standalone]}"
+ assert extract_mmlu_answer(before, num_choices=num_choices) == explicit, before
+ assert extract_mmlu_answer(after, num_choices=num_choices) == explicit, after
+
+
+def _gsm8k_numbers() -> list[int]:
+ """Boundary integers plus a seeded spread, so failures are reproducible."""
+ rng = random.Random(20260812)
+ fixed = [0, 1, 7, 9, 10, 42, 100, 999, 1000, 1001, 10_000, 1_000_000]
+ return fixed + [rng.randint(0, 10_000_000) for _ in range(60)]
+
+
+@pytest.mark.parametrize("number", _gsm8k_numbers())
+def test_gsm8k_marker_always_beats_a_trailing_number(number: int) -> None:
+ """Invariant 6: '#### n' is the gold format and outranks any later number."""
+ decoy = number + 1 # guaranteed different
+ text = f"a lot of reasoning\n#### {number}\nand some trailing prose {decoy}"
+ assert extract_gsm8k_answer(text) == str(number)
+
+
+@pytest.mark.parametrize("number", _gsm8k_numbers())
+def test_gsm8k_currency_commas_and_trailing_dot_normalise_alike(number: int) -> None:
+ """Invariant 7: $, thousands commas and a trailing period are cosmetic."""
+ grouped = f"{number:,}"
+ variants = [
+ f"{number}",
+ grouped,
+ f"${number}",
+ f"${grouped}",
+ f"{grouped}.",
+ f"${grouped}.",
+ ]
+ expected = str(number)
+ for variant in variants:
+ assert extract_gsm8k_answer(f"so the total is {variant}") == expected, variant
+ # ...and identically behind the gold marker.
+ assert extract_gsm8k_answer(f"#### {variant}") == expected, variant
+
+
+@pytest.mark.parametrize("number", _gsm8k_numbers())
+def test_gsm8k_takes_the_last_number_without_a_marker(number: int) -> None:
+ """Invariant 8: absent a marker, the final number wins."""
+ text = f"we start with {number + 1} apples and end with {number}"
+ assert extract_gsm8k_answer(text) == str(number)
+
+
+def test_gsm8k_text_without_a_number_is_none() -> None:
+ """Invariant 9: no digits anywhere means no answer."""
+ rng = random.Random(20260812)
+ alphabet = "abcdefghijklmnopqrstuvwxyz ABCDEFGHIJ.,$#\n" # deliberately no digits
+ corpus = ["", "no numbers here", "####", "$", "#### no digits", " \n\t "]
+ corpus += ["".join(rng.choice(alphabet) for _ in range(rng.randint(0, 60))) for _ in range(200)]
+
+ for text in corpus:
+ assert extract_gsm8k_answer(text) is None, text
+
+
+def test_gsm8k_result_is_always_none_or_a_parseable_number() -> None:
+ """Invariant 10: the return value is a canonical numeric string or None.
+
+ score_gsm8k calls float() on it, so anything else would raise there rather
+ than here.
+ """
+ rng = random.Random(20260812)
+ alphabet = "0123456789.,$# abcxyz\n-"
+ corpus = ["", "####", "#### $", "1.2.3", "-", "$,", "1,,2", "....", "-.-", "#### -"]
+ corpus += ["".join(rng.choice(alphabet) for _ in range(rng.randint(0, 40))) for _ in range(300)]
+
+ for text in corpus:
+ result = extract_gsm8k_answer(text)
+ if result is None:
+ continue
+ float(result) # must not raise
+ assert result == result.strip()
From 25835c70af03f3db42301b74449b4e4f2e40a73c Mon Sep 17 00:00:00 2001
From: dchaudhari7177 <111210939+dchaudhari7177@users.noreply.github.com>
Date: Wed, 12 Aug 2026 20:48:26 +0530
Subject: [PATCH 2/2] Stamp reports with a reproducibility/provenance block
(closes #31)
A card stated effect and cliff numbers but not the conditions they were
measured under. Provenance now records n_seeds, model_id, repeng_version,
hardware and wall_clock_s, rendered at the foot of both the markdown and HTML
cards and exposed through --json.
Design notes:
- Used a frozen dataclass rather than the pydantic model the issue suggested.
Core declares `dependencies = []` ("core stays light") and report.py models
every other type as a frozen dataclass, so pydantic here would be the first
core dependency. Easy to switch if you would rather have it.
- n_seeds is the MINIMUM across dose and layer points, not the mean - the
weakest point bounds the claim.
- provenance_from() always recomputes n_seeds from the parsed curves and only
passes the caller's other fields through, so the seed count printed on an
artifact cannot disagree with the CSVs it came from. A test asserts a
caller-supplied n_seeds=999 is overridden.
- Missing fields render as "unknown" rather than failing, and an UNKNOWN seed
count is deliberately not a contract violation - only a known count below
MIN_SEEDS warns.
- build_report() keeps its signature and return type; the new `provenance`
argument is optional, so existing callers are unaffected.
--model was documented as "recorded in output" but was never actually read.
It now feeds model_id as the fallback when no vector is passed; a vector's own
metadata takes precedence over it.
Also adds --hardware and --wall-clock-s for the GPU sweep to supply, and a
stderr warning when a card is built below the seed contract.
---
src/steerbench/cli.py | 40 +++++++++--
src/steerbench/report.py | 147 ++++++++++++++++++++++++++++++++++++++-
tests/test_report.py | 135 +++++++++++++++++++++++++++++++++++
3 files changed, 317 insertions(+), 5 deletions(-)
diff --git a/src/steerbench/cli.py b/src/steerbench/cli.py
index 3c34f46..8b0233d 100644
--- a/src/steerbench/cli.py
+++ b/src/steerbench/cli.py
@@ -26,8 +26,9 @@
_DEFAULT_LAYER_CSV = Path("artifacts/layer_sweep.csv")
-def _summarise_vector(path: Path) -> None:
- """Load a steering vector and print provenance + per-layer L2 norms.
+def _summarise_vector(path: Path) -> tuple[str | None, str | None]:
+ """Load a steering vector, print provenance + per-layer L2 norms, and return
+ its ``(model_id, repeng_version)`` so they can be stamped on the card.
Imported lazily so the CSV → report path works without torch/gguf.
"""
@@ -43,6 +44,7 @@ def _summarise_vector(path: Path) -> None:
print(f" layers : {len(norms)} ({min(norms)}..{max(norms)})")
for layer in sorted(norms):
print(f" L{layer:<3d} ‖dir‖ = {norms[layer]:.4f}")
+ return vec.model_id, vec.repeng_version
def _ensure_side_csv(side_csv: Path | None, out_dir: Path) -> Path:
@@ -95,6 +97,15 @@ def _build_parser() -> argparse.ArgumentParser:
default=None,
help="side-effects CSV (benchmark,unsteered_acc,steered_acc); optional",
)
+ parser.add_argument(
+ "--hardware",
+ help="GPU/CPU the sweep ran on, stamped on the card (e.g. 'A100-40GB')",
+ )
+ parser.add_argument(
+ "--wall-clock-s",
+ type=float,
+ help="sweep wall-clock in seconds, stamped on the card",
+ )
parser.add_argument("--out", type=Path, default=Path("report_out"), help="output directory")
parser.add_argument("--stem", default="report", help="output filename stem")
parser.add_argument(
@@ -120,8 +131,13 @@ def main(argv: list[str] | None = None) -> int:
if args.run:
return _run_modal(args.run, extra)
+ model_id: str | None = args.model
+ repeng_version: str | None = None
if args.vector is not None:
- _summarise_vector(args.vector)
+ # The vector's own metadata is authoritative; --model is the fallback
+ # for a card rendered from CSVs alone.
+ vector_model_id, repeng_version = _summarise_vector(args.vector)
+ model_id = vector_model_id or model_id
for label, csv_path in (("dose-response", args.dose_csv), ("layer-sweep", args.layer_csv)):
if not csv_path.exists():
@@ -133,19 +149,35 @@ def main(argv: list[str] | None = None) -> int:
parser.error(f"side-effects CSV not found: {args.side_csv}")
side_csv = _ensure_side_csv(args.side_csv, args.out)
+ provenance = report.Provenance(
+ model_id=model_id,
+ repeng_version=repeng_version,
+ hardware=args.hardware,
+ wall_clock_s=args.wall_clock_s,
+ )
outputs = report.build_report(
dose_csv=args.dose_csv,
layer_csv=args.layer_csv,
side_csv=side_csv,
out_dir=args.out,
stem=args.stem,
+ provenance=provenance,
)
+ stamped = report.summarise_provenance(args.dose_csv, args.layer_csv, base=provenance)
if args.json:
- print(json.dumps({kind: str(path) for kind, path in outputs.items()}))
+ payload: dict[str, object] = {kind: str(path) for kind, path in outputs.items()}
+ payload["provenance"] = stamped.as_dict()
+ print(json.dumps(payload))
else:
print("[steerbench] wrote:")
for kind, path in outputs.items():
print(f" {kind:9s} {path}")
+ if stamped.below_seed_contract:
+ print(
+ f"[steerbench] warning: {stamped.n_seeds} seed(s) per point, below the "
+ f">= {report.MIN_SEEDS}-seed reproducibility contract",
+ file=sys.stderr,
+ )
return 0
diff --git a/src/steerbench/report.py b/src/steerbench/report.py
index c6333be..684e635 100644
--- a/src/steerbench/report.py
+++ b/src/steerbench/report.py
@@ -51,7 +51,7 @@
import csv
import statistics
from collections import defaultdict
-from dataclasses import dataclass
+from dataclasses import dataclass, field, replace
from pathlib import Path
from typing import TYPE_CHECKING, Literal
@@ -171,6 +171,74 @@ def plateau_span(self) -> tuple[float, float] | None:
return min(xs), max(xs)
+#: Reproducibility contract: a stochastic result is reported as mean ± std over
+#: at least this many seeds. A card built from fewer says so on its face.
+MIN_SEEDS = 3
+
+
+@dataclass(frozen=True)
+class Provenance:
+ """The conditions a card's numbers were measured under.
+
+ Every field is optional: a CPU-only render from bare CSVs knows the seed
+ count but not the hardware, and a card built without a vector knows neither
+ ``model_id`` nor ``repeng_version``. Missing fields render as ``unknown``
+ rather than failing, so provenance never blocks a report.
+
+ ``n_seeds`` is the *minimum* across dose and layer points, not the mean —
+ the weakest point is what bounds the claim.
+ """
+
+ n_seeds: int | None = None
+ model_id: str | None = None
+ repeng_version: str | None = None
+ hardware: str | None = None
+ wall_clock_s: float | None = None
+
+ @property
+ def below_seed_contract(self) -> bool:
+ """True when the sweep has too few seeds for a mean ± std claim.
+
+ An unknown seed count is *not* a violation — it is unknown, and is
+ rendered as such.
+ """
+ return self.n_seeds is not None and self.n_seeds < MIN_SEEDS
+
+ def as_dict(self) -> dict[str, object]:
+ """Machine-readable form, including the derived contract flags."""
+ return {
+ "n_seeds": self.n_seeds,
+ "model_id": self.model_id,
+ "repeng_version": self.repeng_version,
+ "hardware": self.hardware,
+ "wall_clock_s": self.wall_clock_s,
+ "min_seeds": MIN_SEEDS,
+ "below_seed_contract": self.below_seed_contract,
+ }
+
+
+def min_seeds(*curves: list[SweepPoint]) -> int | None:
+ """Smallest ``n_seeds`` across every point of every curve, or None if empty."""
+ counts = [point.n_seeds for curve in curves for point in curve]
+ return min(counts) if counts else None
+
+
+def provenance_from(
+ dose: list[SweepPoint],
+ layer: list[SweepPoint],
+ base: Provenance | None = None,
+) -> Provenance:
+ """``base`` with ``n_seeds`` derived from the parsed curves.
+
+ The seed count is always recomputed from the data rather than trusted from
+ ``base``, so the number printed on the artifact cannot disagree with the CSVs
+ it was rendered from. Everything else (model, hardware, wall-clock) is
+ knowledge only the caller has, and is passed through untouched.
+ """
+ base = base or Provenance()
+ return replace(base, n_seeds=min_seeds(dose, layer))
+
+
@dataclass(frozen=True)
class ReportData:
"""Everything the renderer needs, already parsed and analysed."""
@@ -180,6 +248,7 @@ class ReportData:
layer: list[SweepPoint]
layer_analysis: LayerAnalysis
side_effects: list[SideEffect]
+ provenance: Provenance = field(default_factory=Provenance)
# --------------------------------------------------------------------------- #
@@ -631,6 +700,36 @@ def _layer_trap_lines(data: ReportData) -> list[str]:
]
+def _provenance_rows(prov: Provenance) -> list[tuple[str, str]]:
+ """Label/value pairs for the provenance block; absent values read ``unknown``."""
+
+ def shown(value: object) -> str:
+ return "unknown" if value is None else str(value)
+
+ seeds = shown(prov.n_seeds)
+ if prov.n_seeds is not None:
+ seeds = f"{prov.n_seeds} (contract: ≥ {MIN_SEEDS})"
+ wall_clock = "unknown" if prov.wall_clock_s is None else f"{prov.wall_clock_s:.1f} s"
+ return [
+ ("seeds per point (min)", seeds),
+ ("source model", shown(prov.model_id)),
+ ("repeng version", shown(prov.repeng_version)),
+ ("hardware", shown(prov.hardware)),
+ ("sweep wall-clock", wall_clock),
+ ]
+
+
+def _seed_warning(prov: Provenance) -> str | None:
+ """The visible warning for a card built below the seed contract."""
+ if not prov.below_seed_contract:
+ return None
+ return (
+ f"Measured over {prov.n_seeds} seed(s): below the ≥ {MIN_SEEDS}-seed "
+ f"reproducibility contract, so the ± spreads here are not a reliable "
+ f"error estimate."
+ )
+
+
def render_markdown(data: ReportData, dose_png_name: str, layer_png_name: str) -> str:
"""Assemble the markdown report; images referenced as sidecar PNG files."""
out: list[str] = ["# Steering report card", ""]
@@ -662,6 +761,15 @@ def render_markdown(data: ReportData, dose_png_name: str, layer_png_name: str) -
out += [f"- {line}" for line in _layer_peak_lines(data)]
out += [f"- ⚠️ {line}" for line in _layer_trap_lines(data)]
out.append("")
+
+ # Provenance last: the conditions the numbers above were measured under.
+ out += ["## Reproducibility", ""]
+ out += ["| field | value |", "|---|---|"]
+ out += [f"| {label} | {value} |" for label, value in _provenance_rows(data.provenance)]
+ out.append("")
+ warning = _seed_warning(data.provenance)
+ if warning is not None:
+ out += [f"⚠️ **{warning}**", ""]
return "\n".join(out)
@@ -716,6 +824,14 @@ def render_html(data: ReportData, dose_png: bytes, layer_png: bytes) -> str:
traps = "".join(
f'⚠️ {_html_escape(line)}
' for line in _layer_trap_lines(data)
)
+ prov_rows = "".join(
+ f"| {_html_escape(label)} | {_html_escape(value)} |
"
+ for label, value in _provenance_rows(data.provenance)
+ )
+ seed_warning = _seed_warning(data.provenance)
+ prov_warning = (
+ "" if seed_warning is None else f'⚠️ {_html_escape(seed_warning)}
'
+ )
return f"""
@@ -753,6 +869,14 @@ def render_html(data: ReportData, dose_png: bytes, layer_png: bytes) -> str:
{traps}
+
+Reproducibility
+
+| field | value |
|---|
+{prov_rows}
+
+{prov_warning}
+