From c38a4b5d442d7870bfa475f1d2309a683b40d9d0 Mon Sep 17 00:00:00 2001 From: OpenCode Date: Thu, 9 Jul 2026 13:36:15 +0700 Subject: [PATCH] feat: Phase H H5 simulation-result provenance chain -- traceable audit trail for every result --- Makefile | 12 +- docs/evidence/METRICS_CURRENT.md | 5 +- docs/research/ROADMAP.md | 47 +++++ src/openamp_foundry/cli/commands/reports.py | 74 ++++++++ src/openamp_foundry/cli/main.py | 26 ++- src/openamp_foundry/simulation/__init__.py | 10 ++ src/openamp_foundry/simulation/provenance.py | 91 ++++++++++ tests/simulation/test_provenance.py | 174 +++++++++++++++++++ tests/test_test_count_regression.py | 4 +- 9 files changed, 437 insertions(+), 6 deletions(-) create mode 100644 src/openamp_foundry/simulation/provenance.py create mode 100644 tests/simulation/test_provenance.py diff --git a/Makefile b/Makefile index f7bfe770..c002b2fe 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check +.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3) PYTEST := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest) @@ -735,3 +735,13 @@ adapter-gate-check: --schema-errors "[]" \ --baseline-beaten false @echo "Adapter gate check complete." + +simulation-provenance: + .venv/bin/openamp-foundry simulation-provenance \ + --run-id test-run-001 \ + --module-id membrane_proxy \ + --module-version 0.1.0 \ + --timestamp-utc 2026-07-09T12:00:00Z \ + --input-sequence AKLWKR \ + --scores-json '{"binding_energy": 0.75}' + @echo "Provenance record complete." diff --git a/docs/evidence/METRICS_CURRENT.md b/docs/evidence/METRICS_CURRENT.md index 38e782e4..35a43558 100644 --- a/docs/evidence/METRICS_CURRENT.md +++ b/docs/evidence/METRICS_CURRENT.md @@ -5,7 +5,8 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees > with this file, this file wins. Updated whenever benchmark/benchmark config changes. > -> **Last updated:** 2026-07-09 (Phase H H4 — fail-closed adapter integration tests — v0.5.92) +> **Last updated:** 2026-07-09 (Phase H H5 — simulation-result provenance chain — v0.5.93) +> **New in v0.5.93:** Simulation-result provenance chain (H5) — `SimulationProvenanceRecord` dataclass with run_id, module_id, module_version, timestamp_utc, input_hash (SHA-256 of input sequence), result_hash (SHA-256 of sorted scores dict), calibration_set, notes, and dry_lab_only. `make_provenance_record()` computes input_hash and result_hash deterministically (sort_keys=True for result_hash). `validate_provenance_record()` checks run_id, module_id, module_version non-empty; ISO 8601 timestamp contains 'T'; hashes are 64-char hex; dry_lab_only is True. `provenance_summary()` aggregates total, unique modules, run_ids, and dry_lab_only flag. CLI (`openamp-foundry simulation-provenance`) with `--run-id`, `--module-id`, `--module-version`, `--timestamp-utc`, `--input-sequence`, `--scores-json`, `--calibration-set`, `--format text|json`. `make simulation-provenance` target with demo invocation. 19 tests. **3181 total.** Every simulation result carries a traceable provenance chain so results can be audited, reproduced, or invalidated later without relying on memory. > **New in v0.5.92:** Fail-closed adapter integration tests (H4) — `FAIL_CLOSED_REASONS` dict keys (6 entries: timeout, connection_refused, invalid_response, schema_violation, module_unavailable, baseline_not_beaten). `AdapterGateResult` dataclass with module_id, passed, failure_reason, failure_detail, dry_lab_only. `evaluate_adapter_gate()` fail-closed: returns passed=False on ANY failure signal with priority ordering (timeout > connection_refused > invalid_response > schema_violation > module_unavailable > baseline_not_beaten). `run_adapter_gate_batch()` aggregates multiple adapter calls with total/passed/failed/any_failed/results/dry_lab_only. CLI (`openamp-foundry adapter-gate-check`) with `--module-id`, `--timeout`, `--connection-refused`, `--schema-errors`, `--module-unavailable`, `--baseline-beaten`, `--format text|json`. `make adapter-gate-check` target. 21 tests. **3162 total.** Avoids hidden external failures — when the adapter to an external simulation service is down or misbehaves, the pipeline must fail loudly rather than silently passing garbage through. > **New in v0.5.91:** Per-module cheapest-baseline declaration (H3) — `BaselineDeclaration` dataclass with module_id, module_name, baseline_description, baseline_type, evidence_level_ceiling, and notes. `BASELINE_DECLARATIONS` list (4 entries: membrane_proxy, structure_proxy, dummy_membrane_proxy, external_adapter_placeholder). `get_baseline_declaration()` and `list_baseline_declarations()` for lookup. `check_baseline_requirement()` caps effective_evidence_level to ceiling when baseline not beaten; returns dict with module_id, baseline_beaten, claimed/effective evidence levels, capped flag, message, and dry_lab_only. `validate_baseline_declarations()` checks module_id non-empty, baseline_description non-empty, baseline_type in {heuristic, random, constant, length}, evidence_level_ceiling 1-6, no duplicate module_ids. CLI (`openamp-foundry simulation-baseline-check`) with `--module-id`, `--claimed-level`, `--baseline-beaten`, `--format text|json`. `make simulation-baseline-check` target. 16 tests. **3141 total.** Forces honest enemy comparison — every simulation module must declare the simplest baseline it must beat. > **New in v0.5.90:** Simulation-result schema and validator (H2) — `schemas/simulation_result.schema.json` (Draft 2020-12) validates SimulationResult outputs with uncertainty 0.0–1.0 range, required fields (module, version, scope, scores, uncertainty, calibration_set, validated_against, notes), and optional `dry_lab_context` const "dry-lab-only". `validate_simulation_result()` checks module, version, scope, scores, uncertainty, validated_against, notes. Strict mode adds: module must not be "dummy" or contain "stub", uncertainty < 1.0, validated_against non-empty. `validate_simulation_result_batch()` aggregates results with counts and `any_invalid` flag. CLI (`openamp-foundry validate-simulation-result`) with `--results-json`, `--strict`, `--out-json`. `make validate-simulation-result-schema` target. 19 tests. **3125 total.** Dry-lab only. @@ -83,7 +84,7 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **New in v0.5.31:** Added dipeptide-order features for sequence-order awareness. `dipeptide_order_score` achieves AUROC 0.7861 on AMP-vs-scrambled discrimination — the strongest order-dependent feature in the pipeline. Only 7/31 features survive scrambling (amphipathicity/helix-wheel + dipeptide). All composition features are purely position-independent (exactly 0.5000 AUROC on scrambled test). > **New in v0.5.30:** Easy baseline benchmark added — charge density alone (AUROC 0.8166) outperforms the full pipeline ensemble (0.7792) on AMP-vs-Swiss-Prot-decoy discrimination. Honest finding documented: expected because pipeline optimizes for safety, not raw discrimination. > **New in v0.5.29:** Expanded benchmark to 500 AMPs + 500 composition-matched decoys (n=1000). AUROC 0.7792 (CI₉₅: 0.7505–0.8065) confirms signal generalizes. Cluster-aware CI: 0.746–0.8102. Representative AUROC: 0.778. Standard benchmark (n=191) retained for backward comparison. -> **Pipeline version:** v0.5.92 (H4 — fail-closed adapter integration tests) +> **Pipeline version:** v0.5.93 (H5 — simulation-result provenance chain) > **Branch:** main --- diff --git a/docs/research/ROADMAP.md b/docs/research/ROADMAP.md index 30b840fc..8ac0e3b8 100644 --- a/docs/research/ROADMAP.md +++ b/docs/research/ROADMAP.md @@ -1,5 +1,52 @@ # Roadmap +## v0.5.93 — Loop 93: Phase H H5 — Simulation-Result Provenance Chain ✓ (2026-07-09) + +`SimulationProvenanceRecord` dataclass with run_id, module_id, module_version, +timestamp_utc, input_hash (SHA-256 of input sequence), result_hash (SHA-256 of +sorted scores dict), calibration_set, notes, and dry_lab_only. +`make_provenance_record()` computes hashes deterministically (sort_keys=True for +result_hash). `validate_provenance_record()` checks non-empty fields, ISO 8601 +timestamp, 64-char hex hashes, and dry_lab_only=True. `provenance_summary()` +aggregates total, unique modules, run_ids, and dry_lab_only flag. Every simulation +result carries a traceable provenance chain so results can be audited, reproduced, +or invalidated later without relying on memory. + +Changes: +- `src/openamp_foundry/simulation/provenance.py` (H5) — Core module with + `SimulationProvenanceRecord` dataclass (9 fields), `make_provenance_record()` + computing SHA-256 input_hash and sorted result_hash, `validate_provenance_record()` + with 7 integrity checks, `provenance_summary()` with aggregation. +- `src/openamp_foundry/simulation/__init__.py` — Exports `SimulationProvenanceRecord`, + `make_provenance_record`, `validate_provenance_record`, `provenance_summary`. +- `src/openamp_foundry/cli/main.py` — Registered `simulation-provenance` subcommand + with `--run-id`, `--module-id`, `--module-version`, `--timestamp-utc`, + `--input-sequence`, `--scores-json`, `--calibration-set`, `--format` flags. + Added import and dispatch. +- `src/openamp_foundry/cli/commands/reports.py` — Added `_run_simulation_provenance()` + CLI handler with JSON parsing, validation, text and JSON output, exit code 3 on + validation error. +- `Makefile` — Added `simulation-provenance` target with demo invocation using + `test-run-001`. Added to `.PHONY`. +- `tests/simulation/test_provenance.py` — 19 tests covering: record creation, + deterministic hashes, dry_lab_only always True, all validation checks + (empty fields, bad timestamp, wrong-length hash, dry_lab_only=False), + provenance_summary with 0 and N records. +- `docs/evidence/METRICS_CURRENT.md` — v0.5.93 H5 changelog. Test count: 3181. +- `tests/test_test_count_regression.py` — baseline updated to 3181. + +Honest boundaries: +- SHA-256 hashes prove content integrity, not biological activity. +- The provenance record attests that a simulation ran, not that the result + is biologically meaningful. +- `input_hash` covers the input sequence only; other input parameters + (e.g., engine settings) are not hashed. +- `result_hash` uses JSON serialisation with sort_keys=True; any future + change to the serialisation format will change the hash for identical data. +- Timestamps are self-reported by the caller and not independently verified. +- Provenance records are dry-lab only and must not be presented as + biological proof. + ## v0.5.92 — Loop 92: Phase H H4 — Fail-Closed Adapter Integration Tests ✓ (2026-07-09) `FAIL_CLOSED_REASONS` dict (6 keys) enumerates known adapter failure reasons. diff --git a/src/openamp_foundry/cli/commands/reports.py b/src/openamp_foundry/cli/commands/reports.py index aad83db8..c60f8201 100644 --- a/src/openamp_foundry/cli/commands/reports.py +++ b/src/openamp_foundry/cli/commands/reports.py @@ -1495,3 +1495,77 @@ def _run_adapter_gate_check(args: argparse.Namespace) -> int: print("not biological proof.") return 0 if gate_result.passed else 3 + + +def _run_simulation_provenance(args: argparse.Namespace) -> int: + """Generate and display a simulation-result provenance record.""" + import json + + from openamp_foundry.simulation.provenance import ( + make_provenance_record, + validate_provenance_record, + ) + + run_id = args.run_id + module_id = args.module_id + module_version = args.module_version + timestamp_utc = args.timestamp_utc + input_sequence = args.input_sequence + output_format = getattr(args, "format", "text") + + try: + scores = json.loads(args.scores_json) + except (json.JSONDecodeError, TypeError) as e: + print(json.dumps({ + "status": "error", + "error": f"Invalid JSON for --scores-json: {e}", + })) + return 2 + + if not isinstance(scores, dict): + print(json.dumps({ + "status": "error", + "error": "--scores-json must be a JSON object (dict)", + })) + return 2 + + calibration_set = getattr(args, "calibration_set", None) + + record = make_provenance_record( + run_id=run_id, + module_id=module_id, + module_version=module_version, + timestamp_utc=timestamp_utc, + input_sequence=input_sequence, + result_scores=scores, + calibration_set=calibration_set, + ) + + validation_errors = validate_provenance_record(record) + if validation_errors: + print(json.dumps({ + "status": "validation_error", + "errors": validation_errors, + })) + return 3 + + if output_format == "json": + print(json.dumps(record.to_dict(), indent=2)) + else: + print(f"Simulation Provenance Record") + print(f" Run ID: {record.run_id}") + print(f" Module: {record.module_id} v{record.module_version}") + print(f" Timestamp (UTC): {record.timestamp_utc}") + print(f" Input hash: {record.input_hash}") + print(f" Result hash: {record.result_hash}") + if record.calibration_set: + print(f" Calibration set: {record.calibration_set}") + if record.notes: + for note in record.notes: + print(f" Note: {note}") + print(f" Dry-lab only: {record.dry_lab_only}") + print() + print("Dry-lab only. Provenance records are computational audit trails,") + print("not biological proof.") + + return 0 diff --git a/src/openamp_foundry/cli/main.py b/src/openamp_foundry/cli/main.py index 64fc0c87..f92a9b09 100644 --- a/src/openamp_foundry/cli/main.py +++ b/src/openamp_foundry/cli/main.py @@ -5,7 +5,7 @@ from openamp_foundry.cli.commands.selection import _run_pilot_panel, _run_pilot_confident, _run_diversity_check, _run_select_batch, _run_batch_rationale from openamp_foundry.cli.commands.external import _run_external_predict, _run_external_consensus from openamp_foundry.cli.commands.qc import _run_synthesis_order, _run_presynth_qc -from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter, _run_synthetic_result_policy_check, _run_calibration_decision_checklist, _run_calibration_rollback_plan, _run_simulation_registry, _run_validate_simulation_result, _run_simulation_baseline_check, _run_adapter_gate_check +from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter, _run_synthetic_result_policy_check, _run_calibration_decision_checklist, _run_calibration_rollback_plan, _run_simulation_registry, _run_validate_simulation_result, _run_simulation_baseline_check, _run_adapter_gate_check, _run_simulation_provenance from openamp_foundry.cli.commands.gates import _run_gate_check import argparse @@ -1469,6 +1469,27 @@ def build_parser() -> argparse.ArgumentParser: help="Output format (default: text).", ) + # ── Simulation-result provenance (H5) ──────────────────────────────── + sp = sub.add_parser( + "simulation-provenance", + help=( + "Generate a simulation-result provenance record with SHA-256 " + "input and result hashes. Dry-lab only." + ), + ) + sp.add_argument("--run-id", type=str, required=True, help="Unique run ID (e.g. UUID).") + sp.add_argument("--module-id", type=str, required=True, help="Module ID (e.g. membrane_proxy).") + sp.add_argument("--module-version", type=str, required=True, help="Module version (e.g. 0.1.0).") + sp.add_argument("--timestamp-utc", type=str, required=True, help="ISO 8601 UTC timestamp.") + sp.add_argument("--input-sequence", type=str, required=True, help="Input peptide sequence.") + sp.add_argument("--scores-json", type=str, required=True, help="JSON object of str→float scores.") + sp.add_argument("--calibration-set", type=str, default=None, help="Optional calibration set identifier.") + sp.add_argument( + "--format", type=str, default="text", + choices=["text", "json"], + help="Output format (default: text).", + ) + return parser @@ -1636,6 +1657,9 @@ def main(argv: list[str] | None = None) -> int: if args.command == "adapter-gate-check": return _run_adapter_gate_check(args) + if args.command == "simulation-provenance": + return _run_simulation_provenance(args) + parser.error("unknown command") return 2 diff --git a/src/openamp_foundry/simulation/__init__.py b/src/openamp_foundry/simulation/__init__.py index 57da86b5..7d59d5d9 100644 --- a/src/openamp_foundry/simulation/__init__.py +++ b/src/openamp_foundry/simulation/__init__.py @@ -6,6 +6,12 @@ evaluate_adapter_gate, run_adapter_gate_batch, ) +from .provenance import ( + SimulationProvenanceRecord, + make_provenance_record, + provenance_summary, + validate_provenance_record, +) from .baseline_registry import ( BASELINE_DECLARATIONS, BaselineDeclaration, @@ -57,6 +63,10 @@ "get_active_modules", "registry_summary", "validate_registry", + "SimulationProvenanceRecord", + "make_provenance_record", + "validate_provenance_record", + "provenance_summary", "validate_simulation_result", "validate_simulation_result_batch", "BaselineDeclaration", diff --git a/src/openamp_foundry/simulation/provenance.py b/src/openamp_foundry/simulation/provenance.py new file mode 100644 index 00000000..9c896820 --- /dev/null +++ b/src/openamp_foundry/simulation/provenance.py @@ -0,0 +1,91 @@ +"""Simulation-result provenance chain — traceable audit trail for every result. + +Every simulation result must carry a traceable provenance record (which module +produced it, with what version, run_id, and when) so that results can be +audited, reproduced, or invalidated later without relying on memory. +""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass, field, asdict +from typing import Any + + +@dataclass +class SimulationProvenanceRecord: + run_id: str + module_id: str + module_version: str + timestamp_utc: str + input_hash: str + result_hash: str + calibration_set: str | None = None + notes: list[str] = field(default_factory=list) + dry_lab_only: bool = True + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +def make_provenance_record( + run_id: str, + module_id: str, + module_version: str, + timestamp_utc: str, + input_sequence: str, + result_scores: dict[str, float], + calibration_set: str | None = None, + notes: list[str] | None = None, +) -> SimulationProvenanceRecord: + input_hash = hashlib.sha256(input_sequence.encode()).hexdigest() + scores_json = json.dumps(result_scores, sort_keys=True).encode() + result_hash = hashlib.sha256(scores_json).hexdigest() + return SimulationProvenanceRecord( + run_id=run_id, + module_id=module_id, + module_version=module_version, + timestamp_utc=timestamp_utc, + input_hash=input_hash, + result_hash=result_hash, + calibration_set=calibration_set, + notes=notes or [], + ) + + +def validate_provenance_record(record: SimulationProvenanceRecord) -> list[str]: + errors: list[str] = [] + if not record.run_id: + errors.append("run_id must be non-empty") + if not record.module_id: + errors.append("module_id must be non-empty") + if not record.module_version: + errors.append("module_version must be non-empty") + if not record.timestamp_utc or "T" not in record.timestamp_utc: + errors.append("timestamp_utc must be an ISO 8601 string containing 'T'") + if len(record.input_hash) != 64 or not _is_hex(record.input_hash): + errors.append("input_hash must be a 64-character hex string") + if len(record.result_hash) != 64 or not _is_hex(record.result_hash): + errors.append("result_hash must be a 64-character hex string") + if record.dry_lab_only is not True: + errors.append("dry_lab_only must be True") + return errors + + +def _is_hex(s: str) -> bool: + try: + int(s, 16) + return True + except ValueError: + return False + + +def provenance_summary(records: list[SimulationProvenanceRecord]) -> dict: + modules = sorted({r.module_id for r in records}) + return { + "total": len(records), + "modules": modules, + "runs": [r.run_id for r in records], + "dry_lab_only": True, + } diff --git a/tests/simulation/test_provenance.py b/tests/simulation/test_provenance.py new file mode 100644 index 00000000..1b003f54 --- /dev/null +++ b/tests/simulation/test_provenance.py @@ -0,0 +1,174 @@ +"""Tests for simulation-result provenance chain.""" +from __future__ import annotations + +import pytest + +from openamp_foundry.simulation.provenance import ( + SimulationProvenanceRecord, + make_provenance_record, + provenance_summary, + validate_provenance_record, +) + + +def _make_valid_record(**overrides) -> SimulationProvenanceRecord: + record = SimulationProvenanceRecord( + run_id="test-run-001", + module_id="membrane_proxy", + module_version="0.1.0", + timestamp_utc="2026-07-09T12:00:00Z", + input_hash="a" * 64, + result_hash="b" * 64, + ) + for k, v in overrides.items(): + setattr(record, k, v) + return record + + +class TestMakeProvenanceRecord: + def test_returns_simulation_provenance_record(self): + record = make_provenance_record( + run_id="test-run-001", + module_id="membrane_proxy", + module_version="0.1.0", + timestamp_utc="2026-07-09T12:00:00Z", + input_sequence="GIGKFLHSAKKFGKAFVGEIMNS", + result_scores={"binding_energy": 0.75, "stability": 0.42}, + ) + assert isinstance(record, SimulationProvenanceRecord) + assert record.run_id == "test-run-001" + assert record.module_id == "membrane_proxy" + + def test_input_hash_is_64_char_hex(self): + record = make_provenance_record( + run_id="r1", + module_id="m", + module_version="1", + timestamp_utc="2026-07-09T12:00:00Z", + input_sequence="AKLWKR", + result_scores={"x": 0.5}, + ) + assert len(record.input_hash) == 64 + int(record.input_hash, 16) + + def test_result_hash_is_64_char_hex(self): + record = make_provenance_record( + run_id="r1", + module_id="m", + module_version="1", + timestamp_utc="2026-07-09T12:00:00Z", + input_sequence="AKLWKR", + result_scores={"x": 0.5}, + ) + assert len(record.result_hash) == 64 + int(record.result_hash, 16) + + def test_same_input_same_input_hash(self): + scores = {"binding_energy": 0.75} + r1 = make_provenance_record("r1", "m", "1", "2026-07-09T12:00:00Z", "AKLWKR", scores) + r2 = make_provenance_record("r2", "m", "1", "2026-07-09T12:00:00Z", "AKLWKR", scores) + assert r1.input_hash == r2.input_hash + + def test_different_input_different_input_hash(self): + scores = {"binding_energy": 0.75} + r1 = make_provenance_record("r1", "m", "1", "2026-07-09T12:00:00Z", "AKLWKR", scores) + r2 = make_provenance_record("r2", "m", "1", "2026-07-09T12:00:00Z", "GIGKFL", scores) + assert r1.input_hash != r2.input_hash + + def test_same_scores_same_result_hash(self): + r1 = make_provenance_record("r1", "m", "1", "2026-07-09T12:00:00Z", "AKLWKR", {"a": 0.5, "b": 0.3}) + r2 = make_provenance_record("r2", "m", "1", "2026-07-09T12:00:00Z", "AKLWKR", {"b": 0.3, "a": 0.5}) + assert r1.result_hash == r2.result_hash + + def test_dry_lab_only_always_true(self): + record = make_provenance_record( + run_id="r1", + module_id="m", + module_version="1", + timestamp_utc="2026-07-09T12:00:00Z", + input_sequence="AKLWKR", + result_scores={"x": 0.5}, + ) + assert record.dry_lab_only is True + + def test_calibration_set_preserved(self): + record = make_provenance_record( + run_id="r1", + module_id="m", + module_version="1", + timestamp_utc="2026-07-09T12:00:00Z", + input_sequence="AKLWKR", + result_scores={"x": 0.5}, + calibration_set="demo_2026", + ) + assert record.calibration_set == "demo_2026" + + def test_notes_stored(self): + record = make_provenance_record( + run_id="r1", + module_id="m", + module_version="1", + timestamp_utc="2026-07-09T12:00:00Z", + input_sequence="AKLWKR", + result_scores={"x": 0.5}, + notes=["Test note."], + ) + assert record.notes == ["Test note."] + + +class TestValidateProvenanceRecord: + def test_valid_record_returns_empty_list(self): + record = _make_valid_record() + errors = validate_provenance_record(record) + assert errors == [] + + def test_empty_run_id(self): + record = _make_valid_record(run_id="") + errors = validate_provenance_record(record) + assert "run_id must be non-empty" in errors + + def test_empty_module_id(self): + record = _make_valid_record(module_id="") + errors = validate_provenance_record(record) + assert "module_id must be non-empty" in errors + + def test_empty_module_version(self): + record = _make_valid_record(module_version="") + errors = validate_provenance_record(record) + assert "module_version must be non-empty" in errors + + def test_missing_t_in_timestamp(self): + record = _make_valid_record(timestamp_utc="2026-07-09 12:00:00") + errors = validate_provenance_record(record) + assert any("T" in e for e in errors) + + def test_wrong_length_hash(self): + record = _make_valid_record(input_hash="abc123") + errors = validate_provenance_record(record) + assert "input_hash must be a 64-character hex string" in errors + + def test_dry_lab_only_not_true(self): + record = _make_valid_record(dry_lab_only=False) + errors = validate_provenance_record(record) + assert "dry_lab_only must be True" in errors + + +class TestProvenanceSummary: + def test_zero_records(self): + summary = provenance_summary([]) + assert summary["total"] == 0 + assert summary["modules"] == [] + assert summary["runs"] == [] + + def test_n_records_correct_total_and_modules(self): + r1 = _make_valid_record(run_id="r1", module_id="membrane_proxy") + r2 = _make_valid_record(run_id="r2", module_id="structure_proxy") + r3 = _make_valid_record(run_id="r3", module_id="membrane_proxy") + summary = provenance_summary([r1, r2, r3]) + assert summary["total"] == 3 + assert summary["modules"] == ["membrane_proxy", "structure_proxy"] + assert summary["runs"] == ["r1", "r2", "r3"] + + def test_dry_lab_only_true(self): + summary = provenance_summary([]) + assert summary["dry_lab_only"] is True diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index 3c92d683..64c951c6 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -2,9 +2,9 @@ import subprocess import sys -# Baseline: 3162 tests as of Loop 92 (2026-07-09) -- Phase H H4 (fail-closed adapter integration tests) +# Baseline: 3181 tests as of Loop 93 (2026-07-09) -- Phase H H5 (simulation-result provenance chain) # Raise threshold if legitimate tests are removed; lower threshold is okay. -BASELINE = 3162 +BASELINE = 3181 TOLERANCE = 0.05 # allow 5% fluctuation