[oss] Add read-only jevbench validate-data subcommand

CONTRIBUTING.md

@@ -27,6 +27,21 @@ python -m pytest -q
 
 The suite runs offline in about a second. Please keep it that way.
 
+## Checking task files
+
+```sh
+python -m jevbench.cli validate-data datasets/public/*.jsonl
+```
+
+`validate-data` is read-only. It loads each file with the loader in
+`jevbench/tasks.py`, which runs every task's `validate()`, then checks that ids
+are unique across all files given, that each `question.type` is an allowed kind
+(`noul`, `choice`, `score`), that `expected` has the type its kind needs (an int
+level for `score`, `"no"`/`"yes"` for `noul`, a label string for `choice`) and
+that all items sharing a `group` share one expected answer. It prints a
+per-file summary and one `PROBLEM` line per issue, and exits 1 if there is
+any problem.
+
 ## What not to change in a PR
 
 * Benchmark items (`datasets/`) and published results (`results/`) are

jevbench/cli.py

@@ -7,6 +7,7 @@
       [--price-in-per-m X] [--price-out-per-m X] [--limit N]
   python -m jevbench.cli summarize --tasks <jsonl[,jsonl...]> --results <jsonl> \
       [--ledger PATH] [--public-export PATH] [--include-excluded]
+  python -m jevbench.cli validate-data <jsonl> [<jsonl>...]
 
 Never prints or logs secrets. Raw responses are written into --raw-dir,
 which should live OUTSIDE the repo (defaults to ../private/raw_responses).
@@ -151,6 +152,12 @@ def cmd_summarize(args) -> int:
     return 0
 
 
+def cmd_validate_data(args) -> int:
+    from .validate_data import report
+
+    return report(args.paths, sys.stdout)
+
+
 def main(argv=None) -> int:
     ap = argparse.ArgumentParser(prog="jevbench")
     sub = ap.add_subparsers(dest="cmd", required=True)
@@ -209,6 +216,11 @@ def ds_default(name: str) -> str:
                        help="include non-headline (excluded/unmeasured) items")
     p_sum.set_defaults(fn=cmd_summarize)
 
+    p_val = sub.add_parser("validate-data",
+                           help="read-only checks over task JSONL files")
+    p_val.add_argument("paths", nargs="+", metavar="PATH")
+    p_val.set_defaults(fn=cmd_validate_data)
+
     args = ap.parse_args(argv)
     return args.fn(args)
 

jevbench/validate_data.py

@@ -0,0 +1,104 @@
+"""Read-only checks over JevBench task files (`jevbench validate-data`).
+
+Loads each JSONL file with the canonical loader in jevbench.tasks (which runs
+Task.validate() on every record) and then checks, across all given files:
+
+  * every id is unique
+  * question.type is an allowed kind (jevbench.tasks.QUESTION_TYPES)
+  * the expected answer has the type its kind needs: int level index for
+    score (bool rejected), "no"/"yes" for noul, a label string for choice
+  * group consistency: all items sharing a group share one expected answer
+    (the same check tests/test_protocol.py makes on the public suite)
+
+Never writes anything. Returns a list of problem strings; empty means clean.
+"""
+
+from __future__ import annotations
+
+import json
+from collections import Counter, defaultdict
+
+from .tasks import QUESTION_TYPES, Task
+
+
+def _load(path: str, problems: list) -> list:
+    """Like tasks.load_jsonl, but keeps going past a bad line so every
+    problem in the file is reported, not just the first."""
+    tasks = []
+    try:
+        fh = open(path, "r", encoding="utf-8")
+    except OSError as e:
+        problems.append(f"{path}: cannot open: {e}")
+        return tasks
+    with fh:
+        for lineno, line in enumerate(fh, 1):
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                tasks.append(Task.from_dict(json.loads(line)))
+            except (ValueError, KeyError, TypeError, AttributeError) as e:
+                kind = type(e).__name__
+                problems.append(f"{path}:{lineno}: {kind}: {e}")
+    return tasks
+
+
+def _check_expected(t: Task) -> str | None:
+    kind = t.question.get("type")
+    exp = t.expected
+    if exp is None:
+        return None  # unmeasured ground truth is allowed
+    if kind == "score":
+        if isinstance(exp, bool) or not isinstance(exp, int):
+            return f"score expected must be int, got {type(exp).__name__}"
+    elif kind == "noul":
+        if exp not in ("no", "yes"):
+            return f"noul expected must be 'no' or 'yes', got {exp!r}"
+    elif kind == "choice":
+        if not isinstance(exp, str):
+            return f"choice expected must be a label string, got {type(exp).__name__}"
+    return None
+
+
+def validate_paths(paths: list) -> tuple:
+    """Return (tasks_by_path, problems)."""
+    problems: list = []
+    by_path = {}
+    for p in paths:
+        by_path[p] = _load(p, problems)
+
+    seen: dict = {}
+    groups: dict = defaultdict(set)
+    for p, tasks in by_path.items():
+        for t in tasks:
+            where = f"{p}: {t.id}"
+            if t.id in seen:
+                problems.append(f"{where}: duplicate id (first in {seen[t.id]})")
+            else:
+                seen[t.id] = p
+            if t.question.get("type") not in QUESTION_TYPES:
+                problems.append(f"{where}: kind {t.question.get('type')!r} not in {QUESTION_TYPES}")
+            msg = _check_expected(t)
+            if msg:
+                problems.append(f"{where}: {msg}")
+            if t.group is not None:
+                groups[t.group].add(str(t.expected))
+    for g, answers in sorted(groups.items()):
+        if len(answers) > 1:
+            problems.append(f"group {g!r}: members disagree on expected {sorted(answers)}")
+    return by_path, problems
+
+
+def report(paths: list, out) -> int:
+    by_path, problems = validate_paths(paths)
+    for p, tasks in by_path.items():
+        kinds = Counter(t.question.get("type") for t in tasks)
+        n_groups = len({t.group for t in tasks if t.group is not None})
+        kind_s = ", ".join(f"{k}={v}" for k, v in sorted(kinds.items())) or "-"
+        print(f"{p}: {len(tasks)} tasks ({kind_s}); {n_groups} groups", file=out)
+    for msg in problems:
+        print(f"PROBLEM {msg}", file=out)
+    total = sum(len(t) for t in by_path.values())
+    status = "OK" if not problems else "FAIL"
+    print(f"{status}: {len(paths)} files, {total} tasks, {len(problems)} problems", file=out)
+    return 0 if not problems else 1

tests/test_validate_data.py

@@ -0,0 +1,93 @@
+"""`jevbench validate-data` on tiny invented fixtures.
+
+Every record below is fake (ids start with "fake-", text is placeholder);
+none of them is or paraphrases a benchmark item.
+"""
+
+import io
+import json
+import tempfile
+import unittest
+from contextlib import redirect_stdout
+from pathlib import Path
+
+from jevbench.cli import main
+from jevbench.validate_data import validate_paths
+
+
+def rec(id, qtype="choice", expected="a", labels=None, group=None):
+    if labels is None:
+        labels = {"noul": ["no", "yes"], "score": ["0", "1"]}.get(qtype, ["a", "b"])
+    return {"id": id, "family": "intent", "state": "FAKE STATE",
+            "question": {"type": qtype, "instructions": "FAKE?"},
+            "labels": labels, "expected": expected, "split": "public",
+            "group": group, "provenance": {"source": "test fixture"}}
+
+
+class ValidateData(unittest.TestCase):
+    def setUp(self):
+        self._d = tempfile.TemporaryDirectory()
+        self.dir = Path(self._d.name)
+
+    def tearDown(self):
+        self._d.cleanup()
+
+    def write(self, name, rows):
+        p = self.dir / name
+        p.write_text("".join((r if isinstance(r, str) else json.dumps(r)) + "\n" for r in rows))
+        return str(p)
+
+    def run_cli(self, *paths):
+        buf = io.StringIO()
+        with redirect_stdout(buf):
+            code = main(["validate-data", *paths])
+        return code, buf.getvalue()
+
+    def test_clean_file_passes(self):
+        p = self.write("ok.jsonl", [rec("fake-1", group="g"), rec("fake-2", group="g"),
+                                    rec("fake-3", "noul", "yes"), rec("fake-4", "score", 1),
+                                    rec("fake-5", expected=None)])
+        code, out = self.run_cli(p)
+        self.assertEqual(code, 0, out)
+        self.assertIn("OK: 1 files, 5 tasks, 0 problems", out)
+
+    def test_duplicate_id_across_files(self):
+        a = self.write("a.jsonl", [rec("fake-1")])
+        b = self.write("b.jsonl", [rec("fake-1")])
+        code, out = self.run_cli(a, b)
+        self.assertEqual(code, 1)
+        self.assertIn("duplicate id", out)
+
+    def test_bad_kind_and_bad_line_are_reported_and_loading_continues(self):
+        p = self.write("bad.jsonl", [rec("fake-1", qtype="vibes"), "{not json", rec("fake-2")])
+        by_path, problems = validate_paths([p])
+        self.assertEqual([t.id for t in by_path[p]], ["fake-2"])
+        self.assertEqual(len(problems), 2, problems)
+        self.assertIn(":1:", problems[0]); self.assertIn("bad question type", problems[0])
+        self.assertIn(":2:", problems[1])
+
+    def test_expected_type_must_match_kind(self):
+        # Each passes Task.validate() (expected is in labels) but has the wrong type.
+        rows = [rec("fake-1", "score", True, labels=["0", "True"]),
+                rec("fake-2", "noul", "maybe", labels=["no", "maybe"]),
+                rec("fake-3", "choice", 1, labels=[1, 2])]
+        _, problems = validate_paths([self.write("t.jsonl", rows)])
+        self.assertEqual(len(problems), 3, problems)
+        self.assertIn("score expected must be int", problems[0])
+        self.assertIn("noul expected must be", problems[1])
+        self.assertIn("choice expected must be a label string", problems[2])
+
+    def test_group_members_must_share_expected(self):
+        p = self.write("g.jsonl", [rec("fake-1", group="fake-g"), rec("fake-2", expected="b", group="fake-g")])
+        code, out = self.run_cli(p)
+        self.assertEqual(code, 1)
+        self.assertIn("group 'fake-g': members disagree", out)
+
+    def test_missing_file(self):
+        code, out = self.run_cli(str(self.dir / "nope.jsonl"))
+        self.assertEqual(code, 1)
+        self.assertIn("cannot open", out)
+
+
+if __name__ == "__main__":
+    unittest.main()