From 1c870d021c34b34adbe176a2c60b9a809713de64 Mon Sep 17 00:00:00 2001 From: Note Flow AI Date: Mon, 14 Sep 2026 13:47:49 +0800 Subject: [PATCH] Verify received evaluation evidence without executing candidates --- .github/workflows/ci.yml | 7 ++ CHANGELOG.md | 16 +++++ CITATION.cff | 2 +- README.md | 5 ++ README.zh-CN.md | 5 ++ docs/verification.md | 58 +++++++++++++++++ huggingface/README.md | 7 ++ package-lock.json | 4 +- package.json | 2 +- pyproject.toml | 2 +- scripts/check_installed.py | 74 +++++++++++++++++++++ site/index.html | 8 +++ src/evalarc/__init__.py | 2 +- src/evalarc/cli.py | 36 +++++++++++ src/evalarc/records.py | 24 +++++-- src/evalarc/verify.py | 128 +++++++++++++++++++++++++++++++++++++ tests/test_verify.py | 121 +++++++++++++++++++++++++++++++++++ 17 files changed, 491 insertions(+), 10 deletions(-) create mode 100644 docs/verification.md create mode 100644 scripts/check_installed.py create mode 100644 src/evalarc/verify.py create mode 100644 tests/test_verify.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bb5a124..e0ee10d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,6 +26,13 @@ jobs: - run: ruff format --check . - run: pytest -q - run: python -m build + - name: Verify received evidence from the installed wheel without runtime tools + run: | + python -m venv "$RUNNER_TEMP/installed" + "$RUNNER_TEMP/installed/bin/pip" install --no-deps dist/*.whl + cd "$RUNNER_TEMP" + PYTHONPATH="" "$RUNNER_TEMP/installed/bin/python" \ + "$GITHUB_WORKSPACE/scripts/check_installed.py" --source "$GITHUB_WORKSPACE" docker-audit: runs-on: ubuntu-latest strategy: diff --git a/CHANGELOG.md b/CHANGELOG.md index 711b278..080ca05 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ # Changelog +## 0.7.0 · 2026-09-14 · Research preview + +- Add `evalarc verify` for received evaluation, repetition and comparison + records. Recompute summaries and source identities without candidate + execution, a source checkout, Docker or the original interpreter. +- Return machine-readable results and hashes of every checked JSON input. + Separate consistency success from `--require-resolved` acceptance. +- Bound input sizes and attempt inventory; reject duplicate keys, non-finite + numbers, symlinks and special files. Keep historical evidence readable. +- Document unsupported aggregate formats and the distinction between record + consistency, independent grader execution and producer authentication. + +Task contracts, runtime enforcement and scoring rules are unchanged. This +release verifies existing evidence; it does not add new model trials. + + ## 0.6.0 · 2026-09-14 · Research preview - `init --language python|javascript` supplies starters and references for both diff --git a/CITATION.cff b/CITATION.cff index de2515f..3aec58c 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -4,7 +4,7 @@ title: "EvalArc: Auditable Evaluations for AI Agents" type: software authors: - name: EvalArc contributors -version: 0.6.0 +version: 0.7.0 date-released: 2026-09-14 license: MIT repository-code: "https://github.com/noteflowai/evalarc" diff --git a/README.md b/README.md index 870a694..621d7b6 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,11 @@ ambiguous write outcomes. Python and JavaScript scripted policies use the same host verifier. Browser environments, LLM-provider adapters, and RL training integrations remain planned. No frontier-model benchmark result is claimed. +**Received a report? Verify it without running the candidate.** +`evalarc verify path/to/report --json` checks evaluation, repetition or comparison +evidence and fingerprints every input. Use `--require-resolved` when your handoff +also requires all checks to pass. [Verification and limits](docs/verification.md). + ## Run an audit Clone the source, then install in an isolated Python environment: diff --git a/README.zh-CN.md b/README.zh-CN.md index 0a1d702..d1456ba 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -75,6 +75,11 @@ Docker 评测：参考策略 3/3 轮完全通过，重复写入策略虽然平 当前版本尚未经过前沿模型、真实人类工时或强化学习收益标定。 +**收到报告后，先独立复核。** `evalarc verify 报告路径 --json` 无需执行候选程序， +即可重算单次评测、重复运行和前后对照的汇总，并记录每份输入的指纹。 +需要全部任务通过时加 `--require-resolved`。 +[使用说明与校验范围](docs/verification.md)。 + ## 直接运行 克隆仓库后，在独立 Python 环境中安装： diff --git a/docs/verification.md b/docs/verification.md new file mode 100644 index 0000000..df7f5df --- /dev/null +++ b/docs/verification.md @@ -0,0 +1,58 @@ +# Verify received evaluation evidence + +EvalArc 0.7 adds a read-only handoff check. A customer or CI job can recompute a +report's claims without installing Node, contacting Docker, locating the +original candidate or executing its command. + +```bash +evalarc verify received/evaluation.json --json +evalarc verify received/repetition --json +evalarc verify received/comparison --json +``` + +An evaluation can have any filename. For a repetition, keep `repetition.json` +with `attempts/0001/evaluation.json`, `0002/evaluation.json`, and so on. For a +comparison, keep `comparison.json`, `baseline.json` and `current.json` together. +Pass either the summary file or its directory. A directory must contain exactly +one supported report type. + +The result includes a SHA-256 and byte length for every JSON file actually +checked. These hashes identify the handoff bytes, including historical package +versions. It does not rewrite reports or execute commands embedded in metadata. + +| Evidence | Recomputed checks | +| --- | --- | +| Evaluation v2 | Case/check counts, dimensions, weighted score, validity and resolution | +| Repetition v1 | Exact attempt inventory, input identities, all aggregate counts, scores and variance | +| Comparison v1 | Matching task/runtime/grader/cases, score delta, individual regressions and improvements | + +Malformed or contradictory evidence exits **2**, with `verified: false` in JSON +mode. Default exit **0** means the records are internally consistent, even if +the recorded candidate failed or had an environment error. + +To require full resolution as well: + +```bash +evalarc verify received/repetition --json --require-resolved +``` + +This exits **0** for valid, fully resolved records, **1** for valid but unresolved +records, and **2** for invalid records or a verification error. For a comparison, +full resolution refers to its current evaluation; `has_regressions` is also +reported separately. This flag is not a replacement for a suite's custom gates. + +Suite manifests, custom gate decisions, JUnit, audit summaries and trajectories +are not currently standalone supported inputs. For a suite, verify each +`jobs/` repetition directory; this checks its attempts but does not verify +the suite-level gate decision. The published site additionally checks the +featured suite against its original configuration and JUnit. + +Input files must be regular files without symlinks in the evidence path. +Reads are limited to 64 MiB per JSON and 256 MiB per handoff. Repetitions contain +at most 100 attempts. Duplicate JSON keys and non-finite values are rejected. +Original HTML, logs and source programs are outside the verified inventory. + +This is record consistency, not an independent rerun of the business-state +grader or authentication of the report's producer. A coordinated fabrication +can be internally consistent. Re-run trusted grading on the candidate when +execution evidence is required. diff --git a/huggingface/README.md b/huggingface/README.md index 4ddcb0d..07aced0 100644 --- a/huggingface/README.md +++ b/huggingface/README.md @@ -85,3 +85,10 @@ cases do not establish reliability on unseen tasks or a model success rate. The source and evidence are MIT licensed. `manifest.json` identifies the source commit and SHA-256 of each published file. Publication is performed by the maintainer and does not imply endorsement by Hugging Face. + +### Verify a handoff offline + +With EvalArc 0.7+, run `evalarc verify path/to/report --json` on an evaluation, +repetition or comparison report. It checks the recorded inputs without executing +a candidate; `--require-resolved` separately requires fully resolved results. +[Workflow and limits](https://github.com/noteflowai/evalarc/blob/main/docs/verification.md). diff --git a/package-lock.json b/package-lock.json index 100e1e5..6a24985 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "evalarc-evidence-site", - "version": "0.6.0", + "version": "0.7.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "evalarc-evidence-site", - "version": "0.6.0", + "version": "0.7.0", "devDependencies": { "playwright": "1.63.0" } diff --git a/package.json b/package.json index c92bf69..3db3fa6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "evalarc-evidence-site", - "version": "0.6.0", + "version": "0.7.0", "private": true, "description": "Browser checks for the static EvalArc evidence explorer", "scripts": { diff --git a/pyproject.toml b/pyproject.toml index abad1e5..cff40e3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "evalarc" -version = "0.6.0" +version = "0.7.0" description = "Auditable task environments and evaluations for coding and tool-using agents." readme = "README.md" requires-python = ">=3.11" diff --git a/scripts/check_installed.py b/scripts/check_installed.py new file mode 100644 index 0000000..880b6ad --- /dev/null +++ b/scripts/check_installed.py @@ -0,0 +1,74 @@ +"""Exercise the installed verifier outside the checkout, with no tools on PATH.""" + +import argparse +import hashlib +import json +import os +import shutil +import subprocess +import sys +import tempfile +import tomllib +from importlib.metadata import version +from pathlib import Path + + +def check(source: Path) -> dict: + import evalarc + + source = source.resolve() + module = Path(evalarc.__file__).resolve() + if module.is_relative_to(source) or Path.cwd().is_relative_to(source): + raise ValueError("use a non-editable wheel installation outside the checkout") + expected = tomllib.loads((source / "pyproject.toml").read_text())["project"]["version"] + if version("evalarc") != expected: + raise ValueError("installed version differs from the source") + results = {} + with tempfile.TemporaryDirectory(prefix="evalarc-handoff-") as temporary: + folder = Path(temporary) + environment = {**os.environ, "PATH": "", "PYTHONPATH": "", "PYTHONNOUSERSITE": "1"} + entry = str(Path(sys.executable).with_name("evalarc")) + + def run(path: Path, code: int, *extra: str) -> dict: + process = subprocess.run( + [entry, "verify", str(path), "--json", *extra], + cwd=folder, + env=environment, + capture_output=True, + text=True, + timeout=30, + ) + if process.returncode != code: + raise ValueError(f"unexpected verifier exit: {process.stdout} {process.stderr}") + return json.loads(process.stdout) + + def hashes() -> dict: + return { + p.relative_to(folder).as_posix(): hashlib.sha256(p.read_bytes()).hexdigest() + for p in folder.rglob("*") + if p.is_file() + } + + for name in ("evaluation", "repetition", "repetition-faulty", "comparison"): + target = folder / name + shutil.copytree(source / "examples" / name, target) + before = hashes() + result = run(target, 0) + if not result["verified"] or hashes() != before: + raise ValueError("verification failed or changed evidence") + results[name] = result + run(folder / "repetition", 0, "--require-resolved") + run(folder / "repetition-faulty", 1, "--require-resolved") + changed = folder / "comparison" / "comparison.json" + data = json.loads(changed.read_text()) + data["score_delta"] = 0.123 + changed.write_text(json.dumps(data)) + if run(changed, 2)["verified"]: + raise ValueError("changed summary was accepted") + return {"version": expected, "module": str(module), "path_empty": True, "verified": results} + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source", type=Path, required=True) + print(json.dumps(check(parser.parse_args().source), indent=2)) diff --git a/site/index.html b/site/index.html index a1e24f5..11fe91b 100644 --- a/site/index.html +++ b/site/index.html @@ -149,6 +149,14 @@

Look past
the score.

+
+

HAND OFF EVIDENCE

Received a report?
Check its claims.

Recompute recorded scores, repeated outcomes and regressions without executing the candidate.

Offline verification guide ↗
+
EVALARC 0.7+ / READ-ONLY
evalarc verify received/repetition --json
+
+# Also require valid, fully resolved results:
+evalarc verify received/repetition \
+  --json --require-resolved

Consistency is distinct from task success. No grader rerun or producer authentication is implied.

+

FROM THE BROWSER TO YOUR TERMINAL

Make the grader
earn your trust.

Run the known-good reference and the declared faulty controls. Keep the outcomes, seeds, runtime limits and fingerprints together.

Installation & execution guide ↗
TRUSTED BUNDLED CONTROLS / LINUX / PYTHON 3.11+
git clone https://github.com/noteflowai/evalarc.git
diff --git a/src/evalarc/__init__.py b/src/evalarc/__init__.py
index c1b417c..ad4ab50 100644
--- a/src/evalarc/__init__.py
+++ b/src/evalarc/__init__.py
@@ -1,3 +1,3 @@
 """Auditable evaluations for AI agents."""
 
-__version__ = "0.6.0"
+__version__ = "0.7.0"
diff --git a/src/evalarc/cli.py b/src/evalarc/cli.py
index 176d313..1cec3f1 100644
--- a/src/evalarc/cli.py
+++ b/src/evalarc/cli.py
@@ -23,6 +23,7 @@
 from evalarc.tasks import TASKS
 from evalarc.templates import LANGUAGES, initialize
 from evalarc.trajectory import summarize
+from evalarc.verify import SCOPE, verify
 
 
 def parser() -> argparse.ArgumentParser:
@@ -100,11 +101,46 @@ def parser() -> argparse.ArgumentParser:
     suite.add_argument("--docker-command", default=os.getenv("EVALARC_DOCKER", "docker"))
     suite.add_argument("--progress", action="store_true")
     suite.add_argument("--output", type=Path, default=Path("runs/suite"))
+    verification = commands.add_parser(
+        "verify", help="check saved evaluation, repetition or comparison evidence without execution"
+    )
+    verification.add_argument("evidence", type=Path, help="report JSON or its containing directory")
+    verification.add_argument("--json", action="store_true")
+    verification.add_argument(
+        "--require-resolved",
+        action="store_true",
+        help="also require valid, fully resolved results (current result for comparisons)",
+    )
     return root
 
 
 def main(argv: list[str] | None = None) -> int:
     args = parser().parse_args(argv)
+    if args.command == "verify":
+        try:
+            result = verify(args.evidence)
+            code = 0
+            if args.require_resolved:
+                code = 2 if not result["records_valid"] else (0 if result["fully_resolved"] else 1)
+        except (OSError, ValueError, KeyError, TypeError, IndexError, OverflowError) as error:
+            result = {
+                "schema_version": "evalarc.verification.v1",
+                "verified": False,
+                "error": str(error),
+                "scope": SCOPE,
+            }
+            code = 2
+        if args.json:
+            print(json.dumps(result, indent=2))
+        elif result["verified"]:
+            print(
+                f"Verified {result['kind']}: {len(result['files'])} JSON files | "
+                f"Valid records: {result['records_valid']} | "
+                f"Fully resolved: {result['fully_resolved']}\n{SCOPE}"
+            )
+        else:
+            print(f"Verification failed: {result['error']}", file=sys.stderr)
+        return code
     try:
         if args.command == "suite":
             plan = load_suite(args.config)
diff --git a/src/evalarc/records.py b/src/evalarc/records.py
index 4922ea3..5a2df80 100644
--- a/src/evalarc/records.py
+++ b/src/evalarc/records.py
@@ -4,6 +4,8 @@
 
 import json
 import math
+import os
+import stat
 from pathlib import Path
 
 MAX_REPORT_BYTES = 64 * 1024 * 1024
@@ -30,16 +32,30 @@ def _object(pairs: list[tuple[str, object]]) -> dict:
     return result
 
 
-def read_evaluation(path: Path) -> dict:
-    with path.open("rb") as source:
-        content = source.read(MAX_REPORT_BYTES + 1)
-    if len(content) > MAX_REPORT_BYTES:
+def read_json(path: Path, *, limit: int = MAX_REPORT_BYTES) -> tuple[dict, bytes]:
+    """Read finite, unambiguous JSON from a bounded regular file."""
+    fd = os.open(path, os.O_RDONLY | getattr(os, "O_NONBLOCK", 0) | getattr(os, "O_NOFOLLOW", 0))
+    with os.fdopen(fd, "rb") as source:
+        info = os.fstat(source.fileno())
+        if not stat.S_ISREG(info.st_mode):
+            raise ValueError("report must be a regular file")
+        if info.st_size > limit:
+            raise ValueError("evaluation report exceeds the 64 MiB read limit")
+        content = source.read(limit + 1)
+    if len(content) > limit:
         raise ValueError("evaluation report exceeds the 64 MiB read limit")
     try:
         data = json.loads(content, object_pairs_hook=_object)
         json.dumps(data, allow_nan=False)
+        if not isinstance(data, dict):
+            raise ValueError("report must be an object")
     except (ValueError, UnicodeDecodeError, RecursionError) as error:
         raise ValueError(f"cannot read finite, unambiguous report JSON: {error}") from error
+    return data, content
+
+
+def read_evaluation(path: Path) -> dict:
+    data, _ = read_json(path)
     validate_evaluation(data)
     return data
 
diff --git a/src/evalarc/verify.py b/src/evalarc/verify.py
new file mode 100644
index 0000000..06771b1
--- /dev/null
+++ b/src/evalarc/verify.py
@@ -0,0 +1,128 @@
+"""Verify saved evidence without executing candidates or contacting a runtime."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from pathlib import Path
+
+from evalarc.compare import compare
+from evalarc.records import read_json, validate_evaluation
+from evalarc.repetition import summarize_attempts
+
+MAX_BUNDLE_BYTES = 256 * 1024 * 1024
+REPORTS = ("evaluation.json", "repetition.json", "comparison.json")
+SCOPE = (
+    "Consistency of recorded checks, summaries and identities only. "
+    "No candidate execution, grader rerun, producer authentication or HTML verification."
+)
+
+
+def same_summary(recorded: dict, computed: dict) -> bool:
+    """Ignore generation metadata only; keep booleans distinct from numbers."""
+    ignored = {"created_at", "evalarc_version"}
+    return json.dumps(
+        {key: value for key, value in recorded.items() if key not in ignored},
+        sort_keys=True,
+        allow_nan=False,
+    ) == json.dumps(
+        {key: value for key, value in computed.items() if key not in ignored},
+        sort_keys=True,
+        allow_nan=False,
+    )
+
+
+def verify(path: Path) -> dict:
+    """Verify evaluation, repetition or comparison JSON and its required inputs."""
+    path = path.absolute()
+    if any(item.is_symlink() for item in (path, *path.parents)):
+        raise ValueError("choose an evidence path without symlinks")
+    if path.is_dir():
+        choices = [path / name for name in REPORTS if (path / name).exists()]
+        if len(choices) != 1 or (path / "suite.json").exists():
+            raise ValueError(
+                "choose one evaluation, repetition or comparison report; "
+                "for a suite, verify each jobs/ repetition directory"
+            )
+        path = choices[0]
+    root = path.parent
+    files: dict[str, dict] = {}
+    total = 0
+
+    def read(source: Path) -> dict:
+        nonlocal total
+        if any(item.is_symlink() for item in (source, *source.parents)):
+            raise ValueError("evidence contains a symlink")
+        document, raw = read_json(source, limit=min(64 * 1024 * 1024, MAX_BUNDLE_BYTES - total))
+        total += len(raw)
+        files[source.relative_to(root).as_posix()] = {
+            "sha256": hashlib.sha256(raw).hexdigest(),
+            "bytes": len(raw),
+        }
+        return document
+
+    def evaluation(source: Path) -> dict:
+        document = read(source)
+        validate_evaluation(document)
+        return document
+
+    recorded = read(path)
+    schema = recorded.get("schema_version")
+    regressions = None
+    if schema == "evalarc.evaluation.v2":
+        validate_evaluation(recorded)
+        kind = "evaluation"
+        valid, resolved = recorded["valid"], recorded["resolved"]
+    elif schema == "evalarc.repetition.v1":
+        kind = "repetition"
+        requested, completed = (
+            recorded.get("requested_attempts"),
+            recorded.get("completed_attempts"),
+        )
+        if (
+            type(requested) is not int
+            or type(completed) is not int
+            or not 1 <= completed <= requested <= 100
+        ):
+            raise ValueError("repetition must contain 1–100 completed/requested attempts")
+        directory = root / "attempts"
+        if directory.is_symlink():
+            raise ValueError("attempts directory must not be a symlink")
+        expected = {f"{index:04d}" for index in range(1, completed + 1)}
+        # Bound enumeration too; extra files or directories cannot hide attempts.
+        found = set()
+        for child in directory.iterdir():
+            found.add(child.name)
+            if len(found) > 100:
+                raise ValueError("too many attempt entries")
+        if found != expected:
+            raise ValueError("attempt inventory differs from completed_attempts")
+        reports = [evaluation(directory / name / "evaluation.json") for name in sorted(expected)]
+        computed = summarize_attempts(reports, requested)
+        if not same_summary(recorded, computed):
+            raise ValueError("repetition summary differs from its attempt evaluations")
+        valid, resolved = computed["valid"], computed["all_attempts_resolved"]
+    elif schema == "evalarc.comparison.v1":
+        kind = "comparison"
+        baseline, current = evaluation(root / "baseline.json"), evaluation(root / "current.json")
+        computed = compare(baseline, current)
+        if not same_summary(recorded, computed):
+            raise ValueError("comparison summary differs from its input evaluations")
+        valid, resolved = current["valid"], current["resolved"]
+        regressions = computed["has_regressions"]
+    else:
+        raise ValueError(
+            "unsupported evidence schema; expected evaluation, repetition or comparison"
+        )
+    result = {
+        "schema_version": "evalarc.verification.v1",
+        "verified": True,
+        "kind": kind,
+        "records_valid": valid,
+        "fully_resolved": resolved,
+        "files": files,
+        "scope": SCOPE,
+    }
+    if regressions is not None:
+        result["has_regressions"] = regressions
+    return result
diff --git a/tests/test_verify.py b/tests/test_verify.py
new file mode 100644
index 0000000..e022173
--- /dev/null
+++ b/tests/test_verify.py
@@ -0,0 +1,121 @@
+import hashlib
+import json
+import os
+import shutil
+import subprocess
+from pathlib import Path
+
+import pytest
+
+from evalarc.cli import main
+from evalarc.verify import verify
+
+EXAMPLES = Path(__file__).resolve().parents[1] / "examples"
+
+
+@pytest.mark.parametrize(
+    "name,kind,count,resolved",
+    [
+        ("evaluation", "evaluation", 1, False),
+        ("repetition", "repetition", 4, True),
+        ("repetition-faulty", "repetition", 4, False),
+        ("comparison", "comparison", 3, False),
+    ],
+)
+def test_archived_evidence_verifies_without_runtime_or_writes(
+    tmp_path, monkeypatch, name, kind, count, resolved
+):
+    folder = tmp_path / name
+    shutil.copytree(EXAMPLES / name, folder)
+    before = {p.relative_to(folder): p.read_bytes() for p in folder.rglob("*") if p.is_file()}
+
+    def forbidden(*args, **kwargs):
+        raise AssertionError("verification must not launch any process")
+
+    monkeypatch.setattr(subprocess, "Popen", forbidden)
+    result = verify(folder)
+    assert result["verified"]
+    assert result["kind"] == kind
+    assert result["fully_resolved"] is resolved
+    assert len(result["files"]) == count
+    for name, record in result["files"].items():
+        assert record["sha256"] == hashlib.sha256((folder / name).read_bytes()).hexdigest()
+    assert before == {
+        p.relative_to(folder): p.read_bytes() for p in folder.rglob("*") if p.is_file()
+    }
+
+
+def test_consistency_and_resolution_have_separate_exit_codes(capsys):
+    path = str(EXAMPLES / "repetition-faulty")
+    assert main(["verify", path, "--json"]) == 0
+    result = json.loads(capsys.readouterr().out)
+    assert result["verified"] and not result["fully_resolved"]
+    assert main(["verify", path, "--json", "--require-resolved"]) == 1
+    assert json.loads(capsys.readouterr().out)["records_valid"]
+    assert main(["verify", str(EXAMPLES / "repetition"), "--require-resolved"]) == 0
+
+
+@pytest.mark.parametrize(
+    "name,field", [("repetition", "mean_score"), ("comparison", "score_delta")]
+)
+def test_changed_summary_is_rejected(tmp_path, name, field):
+    shutil.copytree(EXAMPLES / name, tmp_path / "bundle")
+    path = tmp_path / "bundle" / f"{name}.json"
+    data = json.loads(path.read_text())
+    data[field] = 0.123
+    path.write_text(json.dumps(data))
+    with pytest.raises(ValueError, match="summary differs"):
+        verify(path)
+
+
+@pytest.mark.parametrize("change", ["missing", "extra", "symlink", "boolean", "changed-attempt"])
+def test_repetition_requires_exact_attempt_inventory_and_consistent_inputs(tmp_path, change):
+    folder = tmp_path / "bundle"
+    shutil.copytree(EXAMPLES / "repetition", folder)
+    attempt = folder / "attempts" / "0002" / "evaluation.json"
+    if change == "missing":
+        attempt.unlink()
+    elif change == "extra":
+        (folder / "attempts" / "0004").mkdir()
+    elif change == "symlink":
+        attempt.unlink()
+        attempt.symlink_to(EXAMPLES / "repetition" / "attempts" / "0002" / "evaluation.json")
+    elif change == "boolean":
+        path = folder / "repetition.json"
+        data = json.loads(path.read_text())
+        data["mean_score"] = True
+        path.write_text(json.dumps(data))
+    else:
+        data = json.loads(attempt.read_text())
+        data["candidate_sha256"] = "0" * 64
+        attempt.write_text(json.dumps(data))
+    with pytest.raises((ValueError, OSError)):
+        verify(folder)
+
+
+@pytest.mark.parametrize("raw", [b'{"x":1,"x":2}', b'{"x":NaN}', b"\xff", b"[]", b'{"x":1e999}'])
+def test_invalid_json_returns_structured_error(tmp_path, capsys, raw):
+    path = tmp_path / "report.json"
+    path.write_bytes(raw)
+    assert main(["verify", str(path), "--json"]) == 2
+    result = json.loads(capsys.readouterr().out)
+    assert not result["verified"] and result["error"]
+
+
+@pytest.mark.skipif(not hasattr(os, "mkfifo"), reason="POSIX named pipes")
+def test_pipe_is_rejected_without_waiting_for_writer(tmp_path):
+    path = tmp_path / "report.json"
+    os.mkfifo(path)
+    with pytest.raises(ValueError, match="regular file"):
+        verify(path)
+
+
+def test_suite_and_ambiguous_folder_are_explicitly_unsupported(tmp_path):
+    with pytest.raises(ValueError, match="for a suite"):
+        verify(EXAMPLES / "suite")
+    shutil.copytree(EXAMPLES / "repetition", tmp_path / "bundle")
+    shutil.copyfile(
+        EXAMPLES / "evaluation" / "evaluation.json", tmp_path / "bundle/evaluation.json"
+    )
+    with pytest.raises(ValueError, match="choose one"):
+        verify(tmp_path / "bundle")