Add reproducible evaluation artifacts

.gitignore

@@ -1,4 +1,5 @@
+/.venv/
+/runs/
 __pycache__/
 *.pyc
-.venv/
-runs/
+*.egg-info/

README.md

@@ -2,10 +2,9 @@
 
 ARISE is a small experimental framework for testing whether an LLM can revise scientific beliefs when later evidence contradicts earlier conclusions. It tracks claims, provenance, confidence, dependency edges, invalidations, and revision quality across multi-round scenarios.
 
-Current state: the deterministic experiment engine, scenario format, CLI, sample drug-discovery scenario, and metric tests work locally. An LLM adapter is the next step; the included rule-based adapter keeps experiments reproducible without an API key.
+Current state: the deterministic experiment engine, strict provenance validation, canonical scenario fingerprints, repeat-verification CLI, sample drug-discovery scenario, and metric tests work locally. An LLM adapter is the next step; the included rule-based adapter keeps experiments reproducible without an API key.
 
 ```bash
-python -m arise.cli run scenarios/drug_discovery.json --out runs/demo.json
+python -m arise.cli run scenarios/drug_discovery.json --out runs/demo.json --verify-repeat
 python -m unittest discover -s tests -v
 ```
-

arise/cli.py

@@ -13,8 +13,11 @@ def main() -> None:
     run = subparsers.add_parser("run")
     run.add_argument("scenario", type=Path)
     run.add_argument("--out", type=Path)
+    run.add_argument("--verify-repeat", action="store_true", help="run twice and fail if artifacts differ")
     args = parser.parse_args()
     result = run_scenario(args.scenario)
+    if args.verify_repeat and result != run_scenario(args.scenario):
+        raise RuntimeError("repeated run produced a different artifact")
     rendered = json.dumps(result, indent=2)
     if args.out:
         args.out.parent.mkdir(parents=True, exist_ok=True)
@@ -24,4 +27,3 @@ def main() -> None:
 
 if __name__ == "__main__":
     main()
-

arise/runner.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import hashlib
 import json
 from pathlib import Path
 from typing import Any
@@ -11,6 +12,10 @@
 
 def run_scenario(path: Path) -> dict[str, Any]:
     scenario = json.loads(path.read_text())
+    validate_scenario(scenario)
+    fingerprint = hashlib.sha256(
+        json.dumps(scenario, sort_keys=True, separators=(",", ":")).encode()
+    ).hexdigest()
     graph = BeliefGraph()
     judgments: list[RoundJudgment] = []
 
@@ -42,7 +47,60 @@ def run_scenario(path: Path) -> dict[str, Any]:
             tags_correct=sum(1 for raw in round_data.get("evidence", []) if raw["status"] in EpistemicStatus),
             tags_total=len(round_data.get("evidence", [])),
         ))
-    return {"scenario": scenario["name"], "metrics": score(judgments), "graph": graph.snapshot()}
+    return {
+        "protocol_version": 1,
+        "scenario": scenario["name"],
+        "scenario_sha256": fingerprint,
+        "rounds_evaluated": len(judgments),
+        "metrics": score(judgments),
+        "graph": graph.snapshot(),
+    }
+
+
+def validate_scenario(scenario: Any) -> None:
+    if not isinstance(scenario, dict) or not isinstance(scenario.get("name"), str) or not scenario["name"].strip():
+        raise ValueError("scenario requires a non-empty name")
+    if not isinstance(scenario.get("rounds"), list) or not scenario["rounds"]:
+        raise ValueError("scenario requires at least one round")
+    evidence_ids: set[str] = set()
+    claim_ids: set[str] = set()
+    for number, round_data in enumerate(scenario["rounds"], start=1):
+        if not isinstance(round_data, dict):
+            raise ValueError(f"round {number} must be an object")
+        for raw in round_data.get("evidence", []):
+            required = {"id", "statement", "source", "status", "confidence"}
+            if not required <= raw.keys():
+                raise ValueError(f"round {number} evidence is missing required fields")
+            if raw["id"] in evidence_ids:
+                raise ValueError(f"duplicate evidence id: {raw['id']}")
+            EpistemicStatus(raw["status"])
+            if not 0 <= raw["confidence"] <= 1:
+                raise ValueError("evidence confidence must be between 0 and 1")
+            evidence_ids.add(raw["id"])
+        for raw in round_data.get("claims", []):
+            _validate_claim(raw, number, evidence_ids, claim_ids)
+            claim_ids.add(raw["id"])
+        for revision in round_data.get("revisions", []):
+            if revision.get("old_id") not in claim_ids or revision.get("reason") not in evidence_ids:
+                raise ValueError(f"round {number} revision references unknown claim or evidence")
+            replacement = revision.get("replacement", {})
+            _validate_claim(replacement, number, evidence_ids, claim_ids)
+            claim_ids.add(replacement["id"])
+
+
+def _validate_claim(raw: dict[str, Any], round_number: int, evidence_ids: set[str], claim_ids: set[str]) -> None:
+    required = {"id", "statement", "status", "confidence"}
+    if not required <= raw.keys():
+        raise ValueError(f"round {round_number} claim is missing required fields")
+    if raw["id"] in claim_ids:
+        raise ValueError(f"duplicate claim id: {raw['id']}")
+    EpistemicStatus(raw["status"])
+    if not 0 <= raw["confidence"] <= 1:
+        raise ValueError("claim confidence must be between 0 and 1")
+    unknown_evidence = set(raw.get("evidence_ids", [])) - evidence_ids
+    unknown_claims = set(raw.get("depends_on", [])) - claim_ids
+    if unknown_evidence or unknown_claims:
+        raise ValueError(f"round {round_number} claim references unavailable provenance")
 
 
 def _claim(raw: dict[str, Any], round_number: int) -> Claim:
@@ -51,4 +109,3 @@ def _claim(raw: dict[str, Any], round_number: int) -> Claim:
         confidence=raw["confidence"], evidence_ids=raw.get("evidence_ids", []),
         depends_on=raw.get("depends_on", []), created_round=round_number,
     )
-

tests/test_runner.py

@@ -0,0 +1,31 @@
+import json
+import tempfile
+import unittest
+from pathlib import Path
+
+from arise.runner import run_scenario
+
+
+class RunnerTests(unittest.TestCase):
+    def test_artifact_is_reproducible_and_fingerprinted(self) -> None:
+        scenario = Path(__file__).parents[1] / "scenarios" / "drug_discovery.json"
+        first = run_scenario(scenario)
+        second = run_scenario(scenario)
+        self.assertEqual(first, second)
+        self.assertEqual(len(first["scenario_sha256"]), 64)
+        self.assertEqual(first["rounds_evaluated"], 3)
+
+    def test_invalid_forward_provenance_is_rejected(self) -> None:
+        value = {"name": "invalid", "rounds": [{"claims": [{
+            "id": "c1", "statement": "claim", "status": "inferred", "confidence": .5,
+            "evidence_ids": ["later"]
+        }]}]}
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / "scenario.json"
+            path.write_text(json.dumps(value))
+            with self.assertRaisesRegex(ValueError, "unavailable provenance"):
+                run_scenario(path)
+
+
+if __name__ == "__main__":
+    unittest.main()