From 198c9b7d9abc0617639a70d9dc93daec101c7c7e Mon Sep 17 00:00:00 2001 From: MohammadSyed18 <79668964+MohammadSyed18@users.noreply.github.com> Date: Wed, 5 Aug 2026 10:51:07 -0500 Subject: [PATCH] Add reproducible evaluation artifacts --- .gitignore | 5 ++-- README.md | 5 ++-- arise/cli.py | 4 ++- arise/runner.py | 61 ++++++++++++++++++++++++++++++++++++++++++-- tests/test_runner.py | 31 ++++++++++++++++++++++ 5 files changed, 98 insertions(+), 8 deletions(-) create mode 100644 tests/test_runner.py diff --git a/.gitignore b/.gitignore index 2693678..0237381 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ +/.venv/ +/runs/ __pycache__/ *.pyc -.venv/ -runs/ +*.egg-info/ diff --git a/README.md b/README.md index f897363..c8fc98e 100644 --- a/README.md +++ b/README.md @@ -2,10 +2,9 @@ ARISE is a small experimental framework for testing whether an LLM can revise scientific beliefs when later evidence contradicts earlier conclusions. It tracks claims, provenance, confidence, dependency edges, invalidations, and revision quality across multi-round scenarios. -Current state: the deterministic experiment engine, scenario format, CLI, sample drug-discovery scenario, and metric tests work locally. An LLM adapter is the next step; the included rule-based adapter keeps experiments reproducible without an API key. +Current state: the deterministic experiment engine, strict provenance validation, canonical scenario fingerprints, repeat-verification CLI, sample drug-discovery scenario, and metric tests work locally. An LLM adapter is the next step; the included rule-based adapter keeps experiments reproducible without an API key. ```bash -python -m arise.cli run scenarios/drug_discovery.json --out runs/demo.json +python -m arise.cli run scenarios/drug_discovery.json --out runs/demo.json --verify-repeat python -m unittest discover -s tests -v ``` - diff --git a/arise/cli.py b/arise/cli.py index 0e9f6b4..6fde5d1 100644 --- a/arise/cli.py +++ b/arise/cli.py @@ -13,8 +13,11 @@ def main() -> None: run = subparsers.add_parser("run") run.add_argument("scenario", type=Path) run.add_argument("--out", type=Path) + run.add_argument("--verify-repeat", action="store_true", help="run twice and fail if artifacts differ") args = parser.parse_args() result = run_scenario(args.scenario) + if args.verify_repeat and result != run_scenario(args.scenario): + raise RuntimeError("repeated run produced a different artifact") rendered = json.dumps(result, indent=2) if args.out: args.out.parent.mkdir(parents=True, exist_ok=True) @@ -24,4 +27,3 @@ def main() -> None: if __name__ == "__main__": main() - diff --git a/arise/runner.py b/arise/runner.py index fc7e2dc..37f9e1a 100644 --- a/arise/runner.py +++ b/arise/runner.py @@ -1,5 +1,6 @@ from __future__ import annotations +import hashlib import json from pathlib import Path from typing import Any @@ -11,6 +12,10 @@ def run_scenario(path: Path) -> dict[str, Any]: scenario = json.loads(path.read_text()) + validate_scenario(scenario) + fingerprint = hashlib.sha256( + json.dumps(scenario, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() graph = BeliefGraph() judgments: list[RoundJudgment] = [] @@ -42,7 +47,60 @@ def run_scenario(path: Path) -> dict[str, Any]: tags_correct=sum(1 for raw in round_data.get("evidence", []) if raw["status"] in EpistemicStatus), tags_total=len(round_data.get("evidence", [])), )) - return {"scenario": scenario["name"], "metrics": score(judgments), "graph": graph.snapshot()} + return { + "protocol_version": 1, + "scenario": scenario["name"], + "scenario_sha256": fingerprint, + "rounds_evaluated": len(judgments), + "metrics": score(judgments), + "graph": graph.snapshot(), + } + + +def validate_scenario(scenario: Any) -> None: + if not isinstance(scenario, dict) or not isinstance(scenario.get("name"), str) or not scenario["name"].strip(): + raise ValueError("scenario requires a non-empty name") + if not isinstance(scenario.get("rounds"), list) or not scenario["rounds"]: + raise ValueError("scenario requires at least one round") + evidence_ids: set[str] = set() + claim_ids: set[str] = set() + for number, round_data in enumerate(scenario["rounds"], start=1): + if not isinstance(round_data, dict): + raise ValueError(f"round {number} must be an object") + for raw in round_data.get("evidence", []): + required = {"id", "statement", "source", "status", "confidence"} + if not required <= raw.keys(): + raise ValueError(f"round {number} evidence is missing required fields") + if raw["id"] in evidence_ids: + raise ValueError(f"duplicate evidence id: {raw['id']}") + EpistemicStatus(raw["status"]) + if not 0 <= raw["confidence"] <= 1: + raise ValueError("evidence confidence must be between 0 and 1") + evidence_ids.add(raw["id"]) + for raw in round_data.get("claims", []): + _validate_claim(raw, number, evidence_ids, claim_ids) + claim_ids.add(raw["id"]) + for revision in round_data.get("revisions", []): + if revision.get("old_id") not in claim_ids or revision.get("reason") not in evidence_ids: + raise ValueError(f"round {number} revision references unknown claim or evidence") + replacement = revision.get("replacement", {}) + _validate_claim(replacement, number, evidence_ids, claim_ids) + claim_ids.add(replacement["id"]) + + +def _validate_claim(raw: dict[str, Any], round_number: int, evidence_ids: set[str], claim_ids: set[str]) -> None: + required = {"id", "statement", "status", "confidence"} + if not required <= raw.keys(): + raise ValueError(f"round {round_number} claim is missing required fields") + if raw["id"] in claim_ids: + raise ValueError(f"duplicate claim id: {raw['id']}") + EpistemicStatus(raw["status"]) + if not 0 <= raw["confidence"] <= 1: + raise ValueError("claim confidence must be between 0 and 1") + unknown_evidence = set(raw.get("evidence_ids", [])) - evidence_ids + unknown_claims = set(raw.get("depends_on", [])) - claim_ids + if unknown_evidence or unknown_claims: + raise ValueError(f"round {round_number} claim references unavailable provenance") def _claim(raw: dict[str, Any], round_number: int) -> Claim: @@ -51,4 +109,3 @@ def _claim(raw: dict[str, Any], round_number: int) -> Claim: confidence=raw["confidence"], evidence_ids=raw.get("evidence_ids", []), depends_on=raw.get("depends_on", []), created_round=round_number, ) - diff --git a/tests/test_runner.py b/tests/test_runner.py new file mode 100644 index 0000000..ab73720 --- /dev/null +++ b/tests/test_runner.py @@ -0,0 +1,31 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from arise.runner import run_scenario + + +class RunnerTests(unittest.TestCase): + def test_artifact_is_reproducible_and_fingerprinted(self) -> None: + scenario = Path(__file__).parents[1] / "scenarios" / "drug_discovery.json" + first = run_scenario(scenario) + second = run_scenario(scenario) + self.assertEqual(first, second) + self.assertEqual(len(first["scenario_sha256"]), 64) + self.assertEqual(first["rounds_evaluated"], 3) + + def test_invalid_forward_provenance_is_rejected(self) -> None: + value = {"name": "invalid", "rounds": [{"claims": [{ + "id": "c1", "statement": "claim", "status": "inferred", "confidence": .5, + "evidence_ids": ["later"] + }]}]} + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "scenario.json" + path.write_text(json.dumps(value)) + with self.assertRaisesRegex(ValueError, "unavailable provenance"): + run_scenario(path) + + +if __name__ == "__main__": + unittest.main()