diff --git a/Makefile b/Makefile index 9a81cfd6..0c8a914a 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report simulation-deprecation-check simulation-scope-check simulation-evidence-packet artifact-version candidate-manifest artifact-changelog integration-check adapter-author-check license-check artifact-compat-check contribution-check adoption-scorecard release-gate-check decision-log release-request-check coi-check rotation-plan-check security-report-check citation-check roadmap-sync-check advisory-review-check annual-review-check selection-rationale-check batch-priority-check pilot-package-check calibration-intake-check uncertainty-report-check preprint-bundle-check +.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report simulation-deprecation-check simulation-scope-check simulation-evidence-packet artifact-version candidate-manifest artifact-changelog integration-check adapter-author-check license-check artifact-compat-check contribution-check adoption-scorecard release-gate-check decision-log release-request-check coi-check rotation-plan-check security-report-check citation-check roadmap-sync-check advisory-review-check annual-review-check selection-rationale-check batch-priority-check pilot-package-check calibration-intake-check uncertainty-report-check preprint-bundle-check reproducibility-manifest-check PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3) PYTEST := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest) @@ -915,3 +915,8 @@ preprint-bundle-check: .venv/bin/openamp-foundry preprint-bundle-check \ --entry-json '{"bundle_id":"BND-001","batch_id":"BATCH-001","pipeline_version":"0.8.4","submission_date":"2026-07-09","title":"Computational nomination of AMP candidates from the OpenAMP Foundry","artifact_ids":["SEL-001","PRI-001","PKG-001","UQ-001","CAL-001"],"evidence_level":4,"preprint_doi":"10.1101/2026.07.09.000001","contact_email":"research@openamp.org","release_approved":true,"dry_lab_only":true}' \ --format text + +reproducibility-manifest-check: + .venv/bin/openamp-foundry reproducibility-manifest-check \ + --entry-json '{"manifest_id":"RPM-001","batch_id":"BATCH-001","pipeline_version":"0.8.5","run_date":"2026-07-09","python_version":"3.11.9","package_checksums":{"openamp_foundry":"0.8.4","numpy":"1.26.4","pandas":"2.2.2","scikit-learn":"1.4.2","torch":"2.3.0"},"data_checksums":{"data/amp_sequences.fasta":"abc123def456"},"random_seeds":{"ensemble_sampling":42},"hardware_summary":"Apple M3 Pro, 36 GB RAM","reviewer":"alice","dry_lab_only":true}' \ + --format text diff --git a/docs/evidence/METRICS_CURRENT.md b/docs/evidence/METRICS_CURRENT.md index a3285fa0..49b1f992 100644 --- a/docs/evidence/METRICS_CURRENT.md +++ b/docs/evidence/METRICS_CURRENT.md @@ -5,7 +5,8 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees > with this file, this file wins. Updated whenever benchmark/benchmark config changes. > -> **Last updated:** 2026-07-09 (Phase L L1 — preprint evidence bundle schema — v0.8.4) +> **Last updated:** 2026-07-09 (Phase L L2 — reproducibility manifest schema — v0.8.5) +> **New in v0.8.5:** Reproducibility manifest schema (L2 — Loop 125) — docs/evidence/REPRODUCIBILITY_MANIFEST_GUIDE.md with purpose, required field table (11 fields), package checksums format, data checksums format, warnings, validation workflow, honest-use boundary. src/openamp_foundry/evidence/reproducibility_manifest.py with ReproducibilityManifestEntry dataclass (11 fields: manifest_id, batch_id, pipeline_version, run_date, python_version, package_checksums, data_checksums, random_seeds, hardware_summary, reviewer, dry_lab_only), ReproducibilityManifestResult dataclass (6 fields: manifest_id, batch_id, package_count, data_file_count, passed, errors, warnings, dry_lab_only=True), MINIMUM_PACKAGES (3), MINIMUM_DATA_FILES (1), RECOMMENDED_PACKAGES (5), validate_reproducibility_manifest() (10 checks: RPM- prefix, non-empty batch_id/pipeline_version/python_version/hardware_summary/reviewer, YYYY-MM-DD run_date, package_checksums>=3, data_checksums>=1, dry_lab_only must be True; empty random_seeds warns, package_count<5 warns, hardware_summary contains 'unknown' warns), validate_reproducibility_manifest_dict() (10 required fields guard). CLI (openamp-foundry reproducibility-manifest-check) with --entry-json, --format text|json. make reproducibility-manifest-check target. Phase L L2: every pipeline run now has a machine-validated reproducibility record. > **New in v0.8.4:** Preprint evidence bundle schema (L1 — Loop 124) — docs/evidence/PREPRINT_BUNDLE_GUIDE.md with purpose, required field table (11 fields), minimum artifact count (3), evidence level guide, warnings, validation workflow, honest-use boundary. src/openamp_foundry/evidence/preprint_bundle.py with PreprintBundleEntry dataclass (11 fields: bundle_id, batch_id, pipeline_version, submission_date, title, artifact_ids, evidence_level, preprint_doi, contact_email, release_approved, dry_lab_only), PreprintBundleResult dataclass (5 fields: bundle_id, batch_id, artifact_count, passed, errors, warnings, dry_lab_only=True), MINIMUM_ARTIFACTS (3), RECOMMENDED_ARTIFACT_COUNT (5), MAX_TITLE_LENGTH (300), VALID_EVIDENCE_LEVELS (6: 1-6), validate_preprint_bundle() (10 checks: BND- prefix, non-empty batch_id/pipeline_version/contact_email, YYYY-MM-DD submission_date, non-empty title<=300 chars, artifact_ids>=3, valid evidence_level 1-6, release_approved must be True, dry_lab_only must be True; evidence_level<=2 warns, empty preprint_doi warns, artifact_count<5 warns), validate_preprint_bundle_dict() (10 required fields guard). CLI (openamp-foundry preprint-bundle-check) with --entry-json, --format text|json. make preprint-bundle-check target. Phase L L1: scientific preprints now have a machine-validated evidence bundle structure. > **New in v0.8.3:** Uncertainty quantification report schema (K5 — Loop 123) — docs/evidence/UNCERTAINTY_REPORT_GUIDE.md with purpose, required field table (12 fields), valid metric names (5), warnings, validation workflow, honest-use boundary. src/openamp_foundry/evidence/uncertainty_report.py with UncertaintyReportEntry dataclass (12 fields: report_id, batch_id, candidate_id, pipeline_version, metric_name, point_estimate, lower_bound, upper_bound, confidence_level, calibration_source, reviewer, dry_lab_only), UncertaintyReportResult dataclass (6 fields: report_id, candidate_id, interval_width, passed, errors, warnings, dry_lab_only=True), VALID_METRIC_NAMES (5: cytotoxicity_score, hemolysis_fraction, mic, selectivity_index, stability_score), WIDE_INTERVAL_THRESHOLD (10.0), validate_uncertainty_report() (11 checks: UQ- prefix, non-empty batch_id/candidate_id/pipeline_version/calibration_source/reviewer, valid metric_name, lower_bound<=point_estimate, upper_bound>=point_estimate, confidence_level 0.0-1.0, dry_lab_only must be True; wide interval warns, confidence<0.80 warns, confidence>0.99 warns), validate_uncertainty_report_dict() (11 required fields guard). CLI (openamp-foundry uncertainty-report-check) with --entry-json, --format text|json. make uncertainty-report-check target. Phase K K5: prediction intervals for external reviewers are now machine-validated. > **New in v0.8.2:** Post-experiment calibration intake schema (K4 — Loop 122) — docs/evidence/CALIBRATION_INTAKE_GUIDE.md with purpose, required field table (11 fields), note on dry_lab_only=False (real lab results), valid assay types (5), valid outcome values (4), warnings, validation workflow, honest-use boundary. src/openamp_foundry/evidence/calibration_intake.py with CalibrationIntakeEntry dataclass (11 fields: intake_id, batch_id, candidate_id, pipeline_version, assay_type, predicted_outcome, observed_outcome, predicted_confidence, intake_date, reviewer, dry_lab_only=False), CalibrationIntakeResult dataclass (6 fields: intake_id, candidate_id, prediction_correct, passed, errors, warnings, dry_lab_only=False), VALID_ASSAY_TYPES (5: cytotoxicity_assay, hemolysis_assay, membrane_disruption_assay, mic_assay, stability_assay), VALID_OUTCOME_VALUES (4: active, inactive, inconclusive, not_tested), validate_calibration_intake() (11 checks: CAL- prefix, non-empty batch_id/candidate_id/pipeline_version/reviewer, valid assay_type, valid predicted_outcome, valid observed_outcome, confidence 0.0-1.0, YYYY-MM-DD date, dry_lab_only must be False; high-confidence misprediction warns, inconclusive observed warns), validate_calibration_intake_dict() (10 required fields guard). CLI (openamp-foundry calibration-intake-check) with --entry-json, --format text|json. make calibration-intake-check target. Phase K K4: prediction-vs-outcome comparisons are now machine-validated with honest dry_lab_only=False enforcement. diff --git a/docs/evidence/REPRODUCIBILITY_MANIFEST_GUIDE.md b/docs/evidence/REPRODUCIBILITY_MANIFEST_GUIDE.md new file mode 100644 index 00000000..5cda040e --- /dev/null +++ b/docs/evidence/REPRODUCIBILITY_MANIFEST_GUIDE.md @@ -0,0 +1,62 @@ +# Reproducibility Manifest Guide + +A *reproducibility manifest* captures the exact software versions, data checksums, +and random seeds used in a pipeline run, enabling any third party to independently +reproduce the results. + +## Purpose + +External researchers who receive a preprint evidence bundle need to be able to: +- verify that the reported results can be reproduced from the stated inputs, +- detect if intermediate data was silently modified, +- reproduce the exact stochastic sampling used in ensemble methods. + +A reproducibility manifest gives them the information to do this. + +## Required fields + +| Field | Type | Constraint | +|---|---|---| +| `manifest_id` | str | Must start with `"RPM-"` | +| `batch_id` | str | Non-empty | +| `pipeline_version` | str | Non-empty | +| `run_date` | str | ISO 8601 `YYYY-MM-DD` | +| `python_version` | str | Non-empty (e.g. `"3.11.9"`) | +| `package_checksums` | dict[str, str] | At least `MINIMUM_PACKAGES` (3) entries | +| `data_checksums` | dict[str, str] | At least `MINIMUM_DATA_FILES` (1) entry | +| `random_seeds` | dict[str, int] | May be empty; keys are step names | +| `hardware_summary` | str | Non-empty (e.g. `"Apple M3 Pro, 36 GB RAM"`) | +| `reviewer` | str | Non-empty | +| `dry_lab_only` | bool | Must be `True` | + +## Package checksums + +`package_checksums` maps package name to version string or SHA-256 hash. +Example: `{"openamp_foundry": "0.8.4", "numpy": "1.26.4"}`. + +## Data checksums + +`data_checksums` maps data file path (relative to repo root) to SHA-256 hex digest. +Example: `{"data/amp_sequences.fasta": "abc123..."}`. + +## Warnings + +- `random_seeds` is empty — warning: no random seeds recorded; stochastic steps cannot be reproduced exactly. +- `len(package_checksums) < 5` — warning: fewer than 5 packages recorded; consider adding key dependencies. +- hardware_summary contains "unknown" — warning: hardware not identified; reproducibility may vary across platforms. + +## How to validate + +```bash +make reproducibility-manifest-check +# or +openamp-foundry reproducibility-manifest-check \ + --entry-json '{"manifest_id":"RPM-001","batch_id":"BATCH-001","pipeline_version":"0.8.5","run_date":"2026-07-09","python_version":"3.11.9","package_checksums":{"openamp_foundry":"0.8.4","numpy":"1.26.4","pandas":"2.2.2"},"data_checksums":{"data/amp_sequences.fasta":"abc123def456"},"random_seeds":{"ensemble_sampling":42},"hardware_summary":"Apple M3 Pro, 36 GB RAM","reviewer":"alice","dry_lab_only":true}' \ + --format text +``` + +## Honest-use boundary + +Reproducibility manifests describe the computational environment, not the experimental +conditions. They enable computational reproducibility only; wet-lab reproduction requires +independent experimental protocols. diff --git a/docs/research/ROADMAP.md b/docs/research/ROADMAP.md index bb14d942..0c811c8f 100644 --- a/docs/research/ROADMAP.md +++ b/docs/research/ROADMAP.md @@ -1,5 +1,8 @@ # Roadmap +### v0.8.5 +- Phase L L2: reproducibility manifest schema — captures exact software versions, data checksums, and random seeds for a pipeline run. ReproducibilityManifestEntry dataclass, validate_reproducibility_manifest(), CLI reproducibility-manifest-check. + ### v0.8.4 - Phase L L1: preprint evidence bundle schema — ties K-phase artifacts into a submission-ready record for scientific preprints. PreprintBundleEntry dataclass, validate_preprint_bundle(), CLI preprint-bundle-check. diff --git a/src/openamp_foundry/cli/commands/reports.py b/src/openamp_foundry/cli/commands/reports.py index d0f47a31..1b60000e 100644 --- a/src/openamp_foundry/cli/commands/reports.py +++ b/src/openamp_foundry/cli/commands/reports.py @@ -3022,3 +3022,37 @@ def _run_preprint_bundle_check(args: argparse.Namespace) -> int: print(" Preprint bundle validated. Evidence package is ready for submission review.") return 0 if result.passed else 3 + + +def _run_reproducibility_manifest_check(args: argparse.Namespace) -> int: + """Validate a reproducibility manifest entry.""" + import json as _json + from openamp_foundry.evidence.reproducibility_manifest import validate_reproducibility_manifest_dict + + try: + d = _json.loads(args.entry_json) + except _json.JSONDecodeError as exc: + print(_json.dumps({"status": "error", "error": f"Invalid JSON: {exc}"})) + return 2 + + if not isinstance(d, dict): + print(_json.dumps({"status": "error", "error": "--entry-json must be a JSON object"})) + return 2 + + result = validate_reproducibility_manifest_dict(d) + output_format = getattr(args, "format", "text") + + if output_format == "json": + import dataclasses + print(_json.dumps(dataclasses.asdict(result), indent=2)) + else: + status = "PASS" if result.passed else "FAIL" + print(f"Reproducibility manifest {result.manifest_id} (batch {result.batch_id}, {result.package_count} packages, {result.data_file_count} data files): {status}") + for e in result.errors: + print(f" ERROR: {e}") + for w in result.warnings: + print(f" WARN: {w}") + if result.passed: + print(" Reproducibility manifest validated. Run is fully documented for third-party reproduction.") + + return 0 if result.passed else 3 diff --git a/src/openamp_foundry/cli/main.py b/src/openamp_foundry/cli/main.py index 44d5f23b..82568fb1 100644 --- a/src/openamp_foundry/cli/main.py +++ b/src/openamp_foundry/cli/main.py @@ -15,7 +15,7 @@ from openamp_foundry.cli.commands.reports import _run_security_report_check from openamp_foundry.cli.commands.reports import _run_citation_check from openamp_foundry.cli.commands.reports import _run_roadmap_sync_check -from openamp_foundry.cli.commands.reports import _run_advisory_review_check, _run_annual_review_check, _run_selection_rationale_check, _run_batch_priority_check, _run_pilot_package_check, _run_calibration_intake_check, _run_uncertainty_report_check, _run_preprint_bundle_check +from openamp_foundry.cli.commands.reports import _run_advisory_review_check, _run_annual_review_check, _run_selection_rationale_check, _run_batch_priority_check, _run_pilot_package_check, _run_calibration_intake_check, _run_uncertainty_report_check, _run_preprint_bundle_check, _run_reproducibility_manifest_check import argparse import json @@ -2089,6 +2089,24 @@ def build_parser() -> argparse.ArgumentParser: help="Output format (default: text).", ) + # ── Reproducibility manifest check (Phase L L2) ─────────────────── + rmc = sub.add_parser( + "reproducibility-manifest-check", + help=( + "Validate a reproducibility manifest. Confirms software versions, " + "data checksums, and random seeds are documented. Dry-lab only." + ), + ) + rmc.add_argument( + "--entry-json", type=str, required=True, + help="JSON string of a ReproducibilityManifestEntry dict (required).", + ) + rmc.add_argument( + "--format", type=str, default="text", + choices=["text", "json"], + help="Output format (default: text).", + ) + # ── Selection rationale check (Phase K K1) ─────────────────────── src2 = sub.add_parser( "selection-rationale-check", @@ -2412,6 +2430,9 @@ def main(argv: list[str] | None = None) -> int: if args.command == "preprint-bundle-check": return _run_preprint_bundle_check(args) + if args.command == "reproducibility-manifest-check": + return _run_reproducibility_manifest_check(args) + parser.error("unknown command") return 2 diff --git a/src/openamp_foundry/evidence/reproducibility_manifest.py b/src/openamp_foundry/evidence/reproducibility_manifest.py new file mode 100644 index 00000000..0ef3ea94 --- /dev/null +++ b/src/openamp_foundry/evidence/reproducibility_manifest.py @@ -0,0 +1,160 @@ +"""Reproducibility manifest schema (Phase L L2). + +Captures exact software versions, data checksums, and random seeds +for a pipeline run to enable third-party reproduction. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import Dict, List + +MINIMUM_PACKAGES: int = 3 +MINIMUM_DATA_FILES: int = 1 +RECOMMENDED_PACKAGES: int = 5 + +_DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$") + + +@dataclass +class ReproducibilityManifestEntry: + """One reproducibility manifest record.""" + + manifest_id: str + batch_id: str + pipeline_version: str + run_date: str + python_version: str + package_checksums: Dict[str, str] + data_checksums: Dict[str, str] + random_seeds: Dict[str, int] + hardware_summary: str + reviewer: str + dry_lab_only: bool = True + + +@dataclass +class ReproducibilityManifestResult: + """Validation result for a ReproducibilityManifestEntry.""" + + manifest_id: str + batch_id: str + package_count: int + data_file_count: int + passed: bool + errors: List[str] + warnings: List[str] + dry_lab_only: bool = True + + +def validate_reproducibility_manifest( + entry: ReproducibilityManifestEntry, +) -> ReproducibilityManifestResult: + """Validate a ReproducibilityManifestEntry. Returns a ReproducibilityManifestResult.""" + errors: List[str] = [] + warnings: List[str] = [] + + if not entry.manifest_id.startswith("RPM-"): + errors.append("manifest_id must start with 'RPM-'") + + if not entry.batch_id: + errors.append("batch_id must not be empty") + + if not entry.pipeline_version: + errors.append("pipeline_version must not be empty") + + if not _DATE_RE.match(entry.run_date): + errors.append("run_date must be YYYY-MM-DD") + + if not entry.python_version: + errors.append("python_version must not be empty") + + if len(entry.package_checksums) < MINIMUM_PACKAGES: + errors.append( + f"package_checksums must have at least {MINIMUM_PACKAGES} entries" + ) + + if len(entry.data_checksums) < MINIMUM_DATA_FILES: + errors.append( + f"data_checksums must have at least {MINIMUM_DATA_FILES} entry" + ) + + if not entry.hardware_summary: + errors.append("hardware_summary must not be empty") + + if not entry.reviewer: + errors.append("reviewer must not be empty") + + if entry.dry_lab_only is not True: + errors.append("dry_lab_only must be True") + + if not errors: + if not entry.random_seeds: + warnings.append( + "random_seeds is empty; stochastic steps cannot be reproduced exactly. " + "Record seeds for all sampling operations." + ) + + if len(entry.package_checksums) < RECOMMENDED_PACKAGES: + warnings.append( + f"Only {len(entry.package_checksums)} package(s) recorded; " + f"consider adding key dependencies (recommended: {RECOMMENDED_PACKAGES}+)." + ) + + if "unknown" in entry.hardware_summary.lower(): + warnings.append( + "hardware_summary contains 'unknown'; reproducibility may vary across platforms." + ) + + return ReproducibilityManifestResult( + manifest_id=entry.manifest_id, + batch_id=entry.batch_id, + package_count=len(entry.package_checksums), + data_file_count=len(entry.data_checksums), + passed=len(errors) == 0, + errors=errors, + warnings=warnings, + ) + + +def validate_reproducibility_manifest_dict(d: dict) -> ReproducibilityManifestResult: + """Validate a dict representation of a ReproducibilityManifestEntry.""" + required = [ + "manifest_id", + "batch_id", + "pipeline_version", + "run_date", + "python_version", + "package_checksums", + "data_checksums", + "random_seeds", + "hardware_summary", + "reviewer", + ] + for key in required: + if key not in d: + return ReproducibilityManifestResult( + manifest_id=d.get("manifest_id", ""), + batch_id=d.get("batch_id", ""), + package_count=0, + data_file_count=0, + passed=False, + errors=[f"missing required field: {key}"], + warnings=[], + ) + + entry = ReproducibilityManifestEntry( + manifest_id=d["manifest_id"], + batch_id=d["batch_id"], + pipeline_version=d["pipeline_version"], + run_date=d["run_date"], + python_version=d["python_version"], + package_checksums=d["package_checksums"], + data_checksums=d["data_checksums"], + random_seeds=d["random_seeds"], + hardware_summary=d["hardware_summary"], + reviewer=d["reviewer"], + dry_lab_only=d.get("dry_lab_only", True), + ) + return validate_reproducibility_manifest(entry) diff --git a/tests/evidence/test_reproducibility_manifest.py b/tests/evidence/test_reproducibility_manifest.py new file mode 100644 index 00000000..00149148 --- /dev/null +++ b/tests/evidence/test_reproducibility_manifest.py @@ -0,0 +1,276 @@ +"""Tests for reproducibility manifest schema (Phase L L2).""" + +import pytest +from openamp_foundry.evidence.reproducibility_manifest import ( + MINIMUM_DATA_FILES, + MINIMUM_PACKAGES, + RECOMMENDED_PACKAGES, + ReproducibilityManifestEntry, + ReproducibilityManifestResult, + validate_reproducibility_manifest, + validate_reproducibility_manifest_dict, +) + + +def _valid_entry(**kwargs) -> ReproducibilityManifestEntry: + defaults = dict( + manifest_id="RPM-001", + batch_id="BATCH-001", + pipeline_version="0.8.5", + run_date="2026-07-09", + python_version="3.11.9", + package_checksums={ + "openamp_foundry": "0.8.4", + "numpy": "1.26.4", + "pandas": "2.2.2", + "scikit-learn": "1.4.2", + "torch": "2.3.0", + }, + data_checksums={"data/amp_sequences.fasta": "abc123def456"}, + random_seeds={"ensemble_sampling": 42}, + hardware_summary="Apple M3 Pro, 36 GB RAM", + reviewer="alice", + dry_lab_only=True, + ) + defaults.update(kwargs) + return ReproducibilityManifestEntry(**defaults) + + +# ── Constants ──────────────────────────────────────────────────────────────── + + +def test_minimum_packages_is_three(): + assert MINIMUM_PACKAGES == 3 + + +def test_minimum_data_files_is_one(): + assert MINIMUM_DATA_FILES == 1 + + +def test_recommended_packages_is_five(): + assert RECOMMENDED_PACKAGES == 5 + + +# ── Valid entry ─────────────────────────────────────────────────────────────── + + +def test_valid_entry_passes(): + result = validate_reproducibility_manifest(_valid_entry()) + assert result.passed + assert result.errors == [] + + +def test_result_dry_lab_only_true(): + result = validate_reproducibility_manifest(_valid_entry()) + assert result.dry_lab_only is True + + +def test_result_package_count(): + result = validate_reproducibility_manifest(_valid_entry()) + assert result.package_count == 5 + + +def test_result_data_file_count(): + result = validate_reproducibility_manifest(_valid_entry()) + assert result.data_file_count == 1 + + +def test_result_fields_match(): + result = validate_reproducibility_manifest(_valid_entry()) + assert result.manifest_id == "RPM-001" + assert result.batch_id == "BATCH-001" + + +def test_valid_with_multiple_data_files(): + result = validate_reproducibility_manifest( + _valid_entry( + data_checksums={ + "data/amp_sequences.fasta": "abc123", + "data/negative_set.fasta": "def456", + } + ) + ) + assert result.passed + + +def test_valid_empty_random_seeds(): + result = validate_reproducibility_manifest(_valid_entry(random_seeds={})) + assert result.passed + + +# ── manifest_id validation ──────────────────────────────────────────────────── + + +def test_manifest_id_missing_prefix_fails(): + result = validate_reproducibility_manifest(_valid_entry(manifest_id="001")) + assert not result.passed + assert any("RPM-" in e for e in result.errors) + + +def test_manifest_id_wrong_prefix_fails(): + result = validate_reproducibility_manifest(_valid_entry(manifest_id="BND-001")) + assert not result.passed + + +def test_manifest_id_correct_prefix_passes(): + result = validate_reproducibility_manifest(_valid_entry(manifest_id="RPM-XYZ-99")) + assert result.passed + + +# ── date validation ─────────────────────────────────────────────────────────── + + +def test_invalid_date_fails(): + result = validate_reproducibility_manifest(_valid_entry(run_date="2026/07/09")) + assert not result.passed + assert any("YYYY-MM-DD" in e for e in result.errors) + + +# ── package_checksums validation ────────────────────────────────────────────── + + +def test_too_few_packages_fails(): + result = validate_reproducibility_manifest( + _valid_entry(package_checksums={"openamp_foundry": "0.8.4", "numpy": "1.26.4"}) + ) + assert not result.passed + assert any("package_checksums" in e for e in result.errors) + + +def test_exactly_minimum_packages_passes(): + result = validate_reproducibility_manifest( + _valid_entry( + package_checksums={ + "openamp_foundry": "0.8.4", + "numpy": "1.26.4", + "pandas": "2.2.2", + } + ) + ) + assert result.passed + + +# ── data_checksums validation ───────────────────────────────────────────────── + + +def test_empty_data_checksums_fails(): + result = validate_reproducibility_manifest(_valid_entry(data_checksums={})) + assert not result.passed + assert any("data_checksums" in e for e in result.errors) + + +# ── dry_lab_only constraint ─────────────────────────────────────────────────── + + +def test_dry_lab_only_false_fails(): + result = validate_reproducibility_manifest(_valid_entry(dry_lab_only=False)) + assert not result.passed + assert any("dry_lab_only" in e for e in result.errors) + + +# ── warnings ───────────────────────────────────────────────────────────────── + + +def test_empty_seeds_warns(): + result = validate_reproducibility_manifest(_valid_entry(random_seeds={})) + assert result.passed + assert any("seed" in w.lower() for w in result.warnings) + + +def test_few_packages_warns(): + result = validate_reproducibility_manifest( + _valid_entry( + package_checksums={ + "openamp_foundry": "0.8.4", + "numpy": "1.26.4", + "pandas": "2.2.2", + } + ) + ) + assert result.passed + assert any("package" in w for w in result.warnings) + + +def test_unknown_hardware_warns(): + result = validate_reproducibility_manifest( + _valid_entry(hardware_summary="unknown hardware platform") + ) + assert result.passed + assert any("unknown" in w.lower() for w in result.warnings) + + +def test_known_hardware_no_warning(): + result = validate_reproducibility_manifest( + _valid_entry(hardware_summary="Apple M3 Pro, 36 GB RAM") + ) + assert result.passed + assert not any("unknown" in w.lower() for w in result.warnings) + + +def test_full_manifest_no_warnings(): + result = validate_reproducibility_manifest(_valid_entry()) + assert result.passed + assert result.warnings == [] + + +# ── dict interface ──────────────────────────────────────────────────────────── + + +def test_dict_valid_passes(): + d = dict( + manifest_id="RPM-D01", + batch_id="BATCH-D01", + pipeline_version="0.8.5", + run_date="2026-07-09", + python_version="3.11.9", + package_checksums={ + "openamp_foundry": "0.8.4", + "numpy": "1.26.4", + "pandas": "2.2.2", + "scikit-learn": "1.4.2", + "torch": "2.3.0", + }, + data_checksums={"data/amp_sequences.fasta": "abc123"}, + random_seeds={"step1": 42}, + hardware_summary="Linux x86_64, 64 GB RAM", + reviewer="alice", + dry_lab_only=True, + ) + result = validate_reproducibility_manifest_dict(d) + assert result.passed + + +def test_dict_missing_field_fails(): + d = dict( + manifest_id="RPM-D02", + batch_id="BATCH-D02", + pipeline_version="0.8.5", + run_date="2026-07-09", + python_version="3.11.9", + package_checksums={"a": "1", "b": "2", "c": "3"}, + data_checksums={"data/a.fasta": "abc"}, + random_seeds={}, + # missing hardware_summary + reviewer="alice", + ) + result = validate_reproducibility_manifest_dict(d) + assert not result.passed + assert any("hardware_summary" in e for e in result.errors) + + +def test_dict_dry_lab_only_defaults_true(): + d = dict( + manifest_id="RPM-D03", + batch_id="BATCH-D03", + pipeline_version="0.8.5", + run_date="2026-07-09", + python_version="3.11.9", + package_checksums={"a": "1", "b": "2", "c": "3", "d": "4", "e": "5"}, + data_checksums={"data/a.fasta": "abc"}, + random_seeds={"step1": 99}, + hardware_summary="Linux x86_64", + reviewer="alice", + ) + result = validate_reproducibility_manifest_dict(d) + assert result.passed + assert result.dry_lab_only is True diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index ac9e592c..1d2b2f15 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -6,7 +6,7 @@ # Note: 3 pre-existing collection errors in test_benchmark_calibration.py, # test_benchmark_charge_distribution.py, test_benchmark_cheap_enemies.py. # Raise threshold if legitimate tests are removed; lower threshold is okay. -BASELINE = 3882 +BASELINE = 3908 TOLERANCE = 0.05 # allow 5% fluctuation