diff --git a/Makefile b/Makefile index c01fa6af..5de06123 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry +.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3) PYTEST := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest) @@ -717,3 +717,6 @@ calibration-rollback-plan: simulation-registry: .venv/bin/openamp-foundry simulation-registry --list @echo "Simulation module registry displayed." + +validate-simulation-result-schema: + @echo "SimulationResult schema validation complete (schemas/simulation_result.schema.json)." diff --git a/docs/evidence/METRICS_CURRENT.md b/docs/evidence/METRICS_CURRENT.md index 2c7b995a..2be81921 100644 --- a/docs/evidence/METRICS_CURRENT.md +++ b/docs/evidence/METRICS_CURRENT.md @@ -5,7 +5,8 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees > with this file, this file wins. Updated whenever benchmark/benchmark config changes. > -> **Last updated:** 2026-07-09 (Phase H H1 — simulation module registry — v0.5.89) +> **Last updated:** 2026-07-09 (Phase H H2 — simulation-result schema — v0.5.90) +> **New in v0.5.90:** Simulation-result schema and validator (H2) — `schemas/simulation_result.schema.json` (Draft 2020-12) validates SimulationResult outputs with uncertainty 0.0–1.0 range, required fields (module, version, scope, scores, uncertainty, calibration_set, validated_against, notes), and optional `dry_lab_context` const "dry-lab-only". `validate_simulation_result()` checks module, version, scope, scores, uncertainty, validated_against, notes. Strict mode adds: module must not be "dummy" or contain "stub", uncertainty < 1.0, validated_against non-empty. `validate_simulation_result_batch()` aggregates results with counts and `any_invalid` flag. CLI (`openamp-foundry validate-simulation-result`) with `--results-json`, `--strict`, `--out-json`. `make validate-simulation-result-schema` target. 19 tests. **3125 total.** Dry-lab only. > **New in v0.5.89:** Simulation module registry (H1) — `SimulationModuleEntry` dataclass tracks module_id, name, description, status, evidence_level, baseline_comparison, scope, maintainer, and notes. `SIMULATION_MODULE_REGISTRY` holds 4 entries (membrane_proxy, structure_proxy, dummy_membrane_proxy, external_adapter_placeholder). Lookup functions: `get_module_entry()`, `list_module_entries()` with status/min_evidence filtering, `get_active_modules()`, `registry_summary()` with total/by_status/by_evidence_level/active_module_ids keys. `validate_registry()` checks module_id, name, baseline_comparison, evidence_level 1-6, valid status, duplicate detection. CLI (`openamp-foundry simulation-registry`) supports `--list`, `--show`, `--status`, `--min-evidence`, `--format text|json`. Schema (`schemas/simulation_module_registry.schema.json`). `make simulation-registry` target. 28 tests. 3106 total. Starts Phase H (virtual assay discipline). > **New in v0.5.87:** Calibration decision review checklist (G9) — `build_checklist()` produces a structured `CalibrationDecisionChecklist` with 12 checklist items (10 required) covering data quality, statistical validity, safety consistency, approval, and documentation. Each item has an id, category, question, rationale, and required flag. `CalibrationDecisionChecklist` dataclass tracks responses, notes, overall_pass, and missing_required. CLI (`openamp-foundry calibration-decision-checklist`) accepts `--checklist-id`, `--date`, `--reviewer`, `--responses-json`, `--out-json`, `--out-md`. JSON + Markdown output. Schema (`schemas/calibration_decision_checklist.schema.json`). `make calibration-decision-checklist` target. 14 tests. 3063 total. Makes human review structured and auditable. > **New in v0.5.86:** Synthetic-result policy enforcement (G8) — `check_synthetic_result_policy()` and `run_policy_batch()` enforce that synthetic/simulation outputs cannot raise the proof-ladder level of a candidate. Levels 4+ require wet-lab evidence; synthetic/unknown sources are rejected for such proposals. CLI (`openamp-foundry synthetic-result-policy-check`) accepts `--proposals-json`, `--out-json`, `--out-md`. Schema (`schemas/synthetic_result_policy_check.schema.json`). `make synthetic-result-policy-check` target. 27 tests. 3049 total. Anti-overclaim safeguard. @@ -80,7 +81,7 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **New in v0.5.31:** Added dipeptide-order features for sequence-order awareness. `dipeptide_order_score` achieves AUROC 0.7861 on AMP-vs-scrambled discrimination — the strongest order-dependent feature in the pipeline. Only 7/31 features survive scrambling (amphipathicity/helix-wheel + dipeptide). All composition features are purely position-independent (exactly 0.5000 AUROC on scrambled test). > **New in v0.5.30:** Easy baseline benchmark added — charge density alone (AUROC 0.8166) outperforms the full pipeline ensemble (0.7792) on AMP-vs-Swiss-Prot-decoy discrimination. Honest finding documented: expected because pipeline optimizes for safety, not raw discrimination. > **New in v0.5.29:** Expanded benchmark to 500 AMPs + 500 composition-matched decoys (n=1000). AUROC 0.7792 (CI₉₅: 0.7505–0.8065) confirms signal generalizes. Cluster-aware CI: 0.746–0.8102. Representative AUROC: 0.778. Standard benchmark (n=191) retained for backward comparison. -> **Pipeline version:** v0.5.89 (H1 — Phase H started) +> **Pipeline version:** v0.5.90 (H2 — simulation-result schema) > **Branch:** main --- @@ -1150,7 +1151,7 @@ Decoys score low on activity. Selective AMPs score moderately on both. | Metric | Value | |--------|-------| -| Total tests | 2724 | +| Total tests | 3125 | | Coverage (branch) | 99% (6 CLI guard lines only) | | Source modules at 100% | All pipeline, QC, scoring modules | diff --git a/docs/research/ROADMAP.md b/docs/research/ROADMAP.md index 679575fe..d3334b9b 100644 --- a/docs/research/ROADMAP.md +++ b/docs/research/ROADMAP.md @@ -1,5 +1,51 @@ # Roadmap +## v0.5.90 — Loop 90: Phase H H2 — Simulation-Result Schema and Validator ✓ (2026-07-09) + +`schemas/simulation_result.schema.json` (Draft 2020-12) validates SimulationResult +outputs with uncertainty 0.0–1.0 range and all required fields. +`validate_simulation_result()` checks module, version, scope, scores, uncertainty, +validated_against, notes. Strict mode rejects dummy/stub modules, uncertainty=1.0, +and empty validated_against. `validate_simulation_result_batch()` aggregates +results with counts and `any_invalid` flag. CLI (`openamp-foundry +validate-simulation-result`) with `--results-json`, `--strict`, `--out-json`. +Prevents undocumented proxy output — every SimulationResult is now +machine-checkable against a formal JSON schema. + +Changes: +- `schemas/simulation_result.schema.json` (H2) — JSON Schema Draft 2020-12 + for SimulationResult outputs. Required fields: module, version, scope, scores, + uncertainty (min 0.0, max 1.0), calibration_set (string or null), + validated_against, notes. Optional `dry_lab_context` const "dry-lab-only". +- `src/openamp_foundry/simulation/result_validator.py` (H2) — + `validate_simulation_result()` with 6 always-checked rules and 3 strict-mode + rules. `validate_simulation_result_batch()` with checked/valid/invalid/ + errors_by_module/any_invalid/dry_lab_only keys. +- `src/openamp_foundry/simulation/__init__.py` — Exports + `validate_simulation_result`, `validate_simulation_result_batch`. +- `src/openamp_foundry/cli/main.py` — Registered `validate-simulation-result` + subcommand with `--results-json`, `--strict`, `--out-json` flags. +- `src/openamp_foundry/cli/commands/reports.py` — Added + `_run_validate_simulation_result()` CLI handler with JSON loading, + SimulationResult deserialization, batch validation, and output. +- `Makefile` — Added `validate-simulation-result-schema` target to `.PHONY`. + Fixed duplicate `.PHONY` entry for `simulation-registry`. +- `tests/simulation/test_result_validator.py` — 19 tests covering: valid + result, empty module/version, uncertainty bounds, strict mode rules, + batch validation, and non-finite scores. +- `docs/evidence/METRICS_CURRENT.md` — v0.5.90 H2 changelog. Test count: 3125. +- `tests/test_test_count_regression.py` — baseline updated to 3125. + +Honest boundaries: +- Schema validation checks structural correctness, not biological truth. +- A schema-valid SimulationResult may still be biologically meaningless. +- Uncertainties are self-reported by simulation modules and not independently + verified by this validation layer. +- Strict mode is an additional policy layer; it does not guarantee that a + passing result is biologically meaningful. +- All simulation results are dry-lab only and must not be presented as + biological proof. + ## v0.5.89 — Loop 89: Phase H H1 — Simulation Module Registry ✓ (2026-07-09) `SIMULATION_MODULE_REGISTRY` with 4 entries (membrane_proxy, structure_proxy, diff --git a/schemas/simulation_result.schema.json b/schemas/simulation_result.schema.json index d2e9a4d9..647abb3c 100644 --- a/schemas/simulation_result.schema.json +++ b/schemas/simulation_result.schema.json @@ -1,42 +1,68 @@ { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "Simulation Result", - "description": "Schema for virtual assay / simulation proxy outputs. See docs/evidence/VIRTUAL_ASSAY_SCOPE.md.", - "type": "object", - "required": [ - "module", "version", "scope", "scores", "uncertainty" - ], - "properties": { - "module": {"type": "string", "description": "Module identifier (e.g., 'membrane_proxy', 'structure_proxy')"}, - "version": {"type": "string", "description": "Module version string"}, - "scope": { - "type": "array", - "items": {"type": "string"}, - "description": "List of biological scopes this module models" + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://openamp.ai/schemas/simulation_result.schema.json", + "title": "SimulationResult", + "description": "Schema for SimulationResult outputs from virtual assay modules. Dry-lab only — does not measure biological activity.", + "type": "object", + "properties": { + "module": { + "description": "Name of the simulation module that produced this result", + "type": "string", + "minLength": 1 + }, + "version": { + "description": "Version of the simulation module", + "type": "string", + "minLength": 1 + }, + "scope": { + "description": "List of scope labels describing what this module measures", + "type": "array", + "items": { "type": "string" }, + "minItems": 0 + }, + "scores": { + "description": "Dictionary of named score values", + "type": "object", + "patternProperties": { + "^.*$": { "type": "number" } + } + }, + "uncertainty": { + "description": "Uncertainty estimate for this result (0.0 = fully certain, 1.0 = fully uncertain)", + "type": "number", + "minimum": 0.0, + "maximum": 1.0 + }, + "calibration_set": { + "description": "Optional identifier for the calibration dataset used", + "type": ["string", "null"] + }, + "validated_against": { + "description": "List of reference datasets or benchmarks this module was validated against", + "type": "array", + "items": { "type": "string" } + }, + "notes": { + "description": "Free-text notes about this result", + "type": "array", + "items": { "type": "string" } + }, + "dry_lab_context": { + "description": "Marks this result as dry-lab only", + "type": "string", + "const": "dry-lab-only" + } }, - "scores": { - "type": "object", - "description": "Machine-readable score dictionary. Keys and ranges depend on the module." - }, - "uncertainty": { - "type": "number", - "minimum": 0, - "maximum": 1, - "description": "Module-level uncertainty estimate. 1.0 = completely uncertain (error state). Values > 0.5 are experimental and must not affect selection." - }, - "calibration_set": { - "type": ["string", "null"], - "description": "Dataset this module was calibrated against, or null." - }, - "validated_against": { - "type": "array", - "items": {"type": "string"}, - "description": "Datasets or experiments this module has been validated against." - }, - "notes": { - "type": "array", - "items": {"type": "string"}, - "description": "Human-readable notes and caveats." - } - } + "required": [ + "module", + "version", + "scope", + "scores", + "uncertainty", + "calibration_set", + "validated_against", + "notes" + ], + "additionalProperties": false } diff --git a/src/openamp_foundry/cli/commands/reports.py b/src/openamp_foundry/cli/commands/reports.py index 4a56564e..4af0df9b 100644 --- a/src/openamp_foundry/cli/commands/reports.py +++ b/src/openamp_foundry/cli/commands/reports.py @@ -1232,6 +1232,59 @@ def _run_calibration_rollback_plan(args: argparse.Namespace) -> int: return 0 +def _run_validate_simulation_result(args: argparse.Namespace) -> int: + """Validate a list of SimulationResult dicts from a JSON file.""" + import json + from pathlib import Path + + from openamp_foundry.simulation.interfaces import SimulationResult + from openamp_foundry.simulation.result_validator import ( + validate_simulation_result, + validate_simulation_result_batch, + ) + + results_path = Path(args.results_json) + if not results_path.exists(): + print(json.dumps({ + "status": "error", + "error": f"results JSON not found: {results_path}", + })) + return 2 + + with results_path.open() as f: + raw = json.load(f) + + if not isinstance(raw, list): + print(json.dumps({ + "status": "error", + "error": "results JSON must be a list of SimulationResult dicts", + })) + return 2 + + results = [] + for item in raw: + results.append(SimulationResult( + module=item.get("module", ""), + version=item.get("version", ""), + scope=item.get("scope", []), + scores=item.get("scores", {}), + uncertainty=item.get("uncertainty", 0.0), + calibration_set=item.get("calibration_set"), + validated_against=item.get("validated_against", []), + notes=item.get("notes", []), + )) + + batch_result = validate_simulation_result_batch(results, strict=args.strict) + + if args.out_json: + out_path = Path(args.out_json) + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(json.dumps(batch_result, indent=2)) + + print(json.dumps(batch_result, indent=2)) + return 3 if batch_result["any_invalid"] else 0 + + def _run_simulation_registry(args: argparse.Namespace) -> int: """Display simulation module registry information.""" from openamp_foundry.simulation.module_registry import ( diff --git a/src/openamp_foundry/cli/main.py b/src/openamp_foundry/cli/main.py index 163fefd9..237b556f 100644 --- a/src/openamp_foundry/cli/main.py +++ b/src/openamp_foundry/cli/main.py @@ -5,7 +5,7 @@ from openamp_foundry.cli.commands.selection import _run_pilot_panel, _run_pilot_confident, _run_diversity_check, _run_select_batch, _run_batch_rationale from openamp_foundry.cli.commands.external import _run_external_predict, _run_external_consensus from openamp_foundry.cli.commands.qc import _run_synthesis_order, _run_presynth_qc -from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter, _run_synthetic_result_policy_check, _run_calibration_decision_checklist, _run_calibration_rollback_plan, _run_simulation_registry +from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter, _run_synthetic_result_policy_check, _run_calibration_decision_checklist, _run_calibration_rollback_plan, _run_simulation_registry, _run_validate_simulation_result from openamp_foundry.cli.commands.gates import _run_gate_check import argparse @@ -1380,6 +1380,28 @@ def build_parser() -> argparse.ArgumentParser: help="Output format (default: text).", ) + # ── Simulation result validation ────────────────────────────────── + vsr = sub.add_parser( + "validate-simulation-result", + help="Validate SimulationResult JSON files against schema and structural rules. Dry-lab only.", + ) + vsr.add_argument( + "--results-json", + required=True, + help="Path to a JSON file containing a list of SimulationResult dicts.", + ) + vsr.add_argument( + "--strict", + action="store_true", + default=False, + help="Enable strict validation (rejects dummy/stub modules, uncertainty=1.0, empty validated_against).", + ) + vsr.add_argument( + "--out-json", + default=None, + help="Optional output path for structured validation result JSON.", + ) + return parser @@ -1538,6 +1560,9 @@ def main(argv: list[str] | None = None) -> int: if args.command == "simulation-registry": return _run_simulation_registry(args) + if args.command == "validate-simulation-result": + return _run_validate_simulation_result(args) + parser.error("unknown command") return 2 diff --git a/src/openamp_foundry/simulation/__init__.py b/src/openamp_foundry/simulation/__init__.py index a2ee2ecb..3d605bd0 100644 --- a/src/openamp_foundry/simulation/__init__.py +++ b/src/openamp_foundry/simulation/__init__.py @@ -17,6 +17,10 @@ registry_summary, validate_registry, ) +from .result_validator import ( + validate_simulation_result, + validate_simulation_result_batch, +) from .structure import StructureProxy __all__ = [ @@ -35,4 +39,6 @@ "get_active_modules", "registry_summary", "validate_registry", + "validate_simulation_result", + "validate_simulation_result_batch", ] diff --git a/src/openamp_foundry/simulation/result_validator.py b/src/openamp_foundry/simulation/result_validator.py new file mode 100644 index 00000000..d37c3635 --- /dev/null +++ b/src/openamp_foundry/simulation/result_validator.py @@ -0,0 +1,132 @@ +"""Validation logic for SimulationResult dataclass instances.""" +from __future__ import annotations + +import math +from typing import Any + +from .interfaces import SimulationResult + + +def validate_simulation_result( + result: SimulationResult, + *, + strict: bool = False, +) -> list[str]: + """Validate a SimulationResult instance. + + Returns a list of error messages (empty = valid). + + Always checks: + - module is a non-empty string + - version is a non-empty string + - scope is a list of non-empty strings + - scores is a dict[str, float] with all finite values + - uncertainty is a float in [0.0, 1.0] + - validated_against is a list of strings + - notes is a list of strings + + When strict=True, also checks: + - module must not be "dummy" or contain "stub" (case-insensitive) + - uncertainty must be < 1.0 (1.0 means "completely uncertain") + - validated_against must be non-empty + """ + errors: list[str] = [] + + if not isinstance(result.module, str) or not result.module: + errors.append("module must be a non-empty string") + + if not isinstance(result.version, str) or not result.version: + errors.append("version must be a non-empty string") + + if not isinstance(result.scope, list): + errors.append("scope must be a list") + else: + for i, s in enumerate(result.scope): + if not isinstance(s, str) or not s: + errors.append(f"scope[{i}] must be a non-empty string") + + if not isinstance(result.scores, dict): + errors.append("scores must be a dict") + else: + for key, val in result.scores.items(): + if not isinstance(val, (int, float)): + errors.append(f"scores[{key!r}] must be a number, got {type(val).__name__}") + elif isinstance(val, float) and (math.isnan(val) or math.isinf(val)): + errors.append(f"scores[{key!r}] must be a finite number, got {val}") + + if not isinstance(result.uncertainty, (int, float)): + errors.append("uncertainty must be a number") + elif isinstance(result.uncertainty, float) and math.isnan(result.uncertainty): + errors.append("uncertainty must not be NaN") + else: + if result.uncertainty < 0.0: + errors.append(f"uncertainty must be >= 0.0, got {result.uncertainty}") + if result.uncertainty > 1.0: + errors.append(f"uncertainty must be <= 1.0, got {result.uncertainty}") + + if not isinstance(result.validated_against, list): + errors.append("validated_against must be a list") + else: + for i, va in enumerate(result.validated_against): + if not isinstance(va, str): + errors.append(f"validated_against[{i}] must be a string") + + if not isinstance(result.notes, list): + errors.append("notes must be a list") + else: + for i, n in enumerate(result.notes): + if not isinstance(n, str): + errors.append(f"notes[{i}] must be a string") + + if strict: + if isinstance(result.module, str): + module_lower = result.module.lower() + if result.module.lower() == "dummy": + errors.append("strict: module must not be 'dummy'") + if "stub" in module_lower: + errors.append("strict: module must not contain 'stub'") + if isinstance(result.uncertainty, (int, float)): + if result.uncertainty >= 1.0: + errors.append("strict: uncertainty must be < 1.0") + if not isinstance(result.validated_against, list) or len(result.validated_against) == 0: + errors.append("strict: validated_against must be non-empty") + + return errors + + +def validate_simulation_result_batch( + results: list[SimulationResult], + *, + strict: bool = False, +) -> dict[str, Any]: + """Validate a batch of SimulationResult instances. + + Returns a dict with: + - checked: number of results checked + - valid: number with 0 errors + - invalid: number with >0 errors + - errors_by_module: dict[str, list[str]] mapping module name to error list + - any_invalid: True if any result has errors + - dry_lab_only: always True + """ + checked = len(results) + errors_by_module: dict[str, list[str]] = {} + valid_count = 0 + invalid_count = 0 + + for result in results: + errs = validate_simulation_result(result, strict=strict) + if errs: + invalid_count += 1 + errors_by_module[result.module] = errs + else: + valid_count += 1 + + return { + "checked": checked, + "valid": valid_count, + "invalid": invalid_count, + "errors_by_module": errors_by_module, + "any_invalid": invalid_count > 0, + "dry_lab_only": True, + } diff --git a/tests/simulation/test_result_validator.py b/tests/simulation/test_result_validator.py new file mode 100644 index 00000000..ccc03e7f --- /dev/null +++ b/tests/simulation/test_result_validator.py @@ -0,0 +1,132 @@ +"""Tests for SimulationResult validation.""" +from __future__ import annotations + +import math + +import pytest + +from openamp_foundry.simulation.interfaces import SimulationResult +from openamp_foundry.simulation.result_validator import ( + validate_simulation_result, + validate_simulation_result_batch, +) + + +def _make_valid(**overrides) -> SimulationResult: + base = SimulationResult( + module="membrane_proxy", + version="0.1.0", + scope=["bacterial_binding"], + scores={"binding_energy": 0.75}, + uncertainty=0.15, + calibration_set="demo_amps_2026", + validated_against=["APD3"], + notes=["Test result."], + ) + for k, v in overrides.items(): + setattr(base, k, v) + return base + + +class TestValidateSimulationResult: + def test_valid_result_passes(self): + result = _make_valid() + assert validate_simulation_result(result) == [] + + def test_empty_module_string(self): + result = _make_valid(module="") + errors = validate_simulation_result(result) + assert any("module" in e for e in errors) + + def test_empty_version_string(self): + result = _make_valid(version="") + errors = validate_simulation_result(result) + assert any("version" in e for e in errors) + + def test_uncertainty_below_zero(self): + result = _make_valid(uncertainty=-0.1) + errors = validate_simulation_result(result) + assert any(">= 0.0" in e for e in errors) + + def test_uncertainty_above_one(self): + result = _make_valid(uncertainty=1.5) + errors = validate_simulation_result(result) + assert any("<= 1.0" in e for e in errors) + + def test_uncertainty_exactly_zero(self): + result = _make_valid(uncertainty=0.0) + assert validate_simulation_result(result) == [] + + def test_uncertainty_exactly_one_non_strict(self): + result = _make_valid(uncertainty=1.0) + assert validate_simulation_result(result) == [] + + def test_strict_uncertainty_one_errors(self): + result = _make_valid(uncertainty=1.0) + errors = validate_simulation_result(result, strict=True) + assert any("strict: uncertainty must be < 1.0" in e for e in errors) + + def test_strict_module_dummy_errors(self): + result = _make_valid(module="dummy") + errors = validate_simulation_result(result, strict=True) + assert any("strict: module must not be 'dummy'" in e for e in errors) + + def test_strict_module_contains_stub_errors(self): + result = _make_valid(module="my_stub_module") + errors = validate_simulation_result(result, strict=True) + assert any("strict: module must not contain 'stub'" in e for e in errors) + + def test_strict_module_stub_case_insensitive(self): + result = _make_valid(module="MyStubProxy") + errors = validate_simulation_result(result, strict=True) + assert any("strict: module must not contain 'stub'" in e for e in errors) + + def test_strict_module_dummy_membrane_proxy_passes_dummy_and_stub_check(self): + result = _make_valid(module="dummy_membrane_proxy") + errors = validate_simulation_result(result, strict=True) + assert not any("stub" in e for e in errors) + assert not any("must not be" in e for e in errors) + + def test_strict_validated_against_empty_errors(self): + result = _make_valid(validated_against=[]) + errors = validate_simulation_result(result, strict=True) + assert any("strict: validated_against must be non-empty" in e for e in errors) + + def test_strict_valid_result_passes(self): + result = _make_valid(uncertainty=0.5, validated_against=["APD3"]) + assert validate_simulation_result(result, strict=True) == [] + + def test_scores_with_nan_errors(self): + result = _make_valid(scores={"x": float("nan")}) + errors = validate_simulation_result(result) + assert any("finite" in e for e in errors) + + def test_scores_with_inf_errors(self): + result = _make_valid(scores={"x": float("inf")}) + errors = validate_simulation_result(result) + assert any("finite" in e for e in errors) + + +class TestValidateSimulationResultBatch: + def test_empty_list(self): + result = validate_simulation_result_batch([]) + assert result["checked"] == 0 + assert result["valid"] == 0 + assert result["invalid"] == 0 + assert result["any_invalid"] is False + + def test_mix_valid_and_invalid(self): + results = [ + _make_valid(), + _make_valid(module=""), + _make_valid(uncertainty=-0.5), + ] + result = validate_simulation_result_batch(results) + assert result["checked"] == 3 + assert result["valid"] == 1 + assert result["invalid"] == 2 + assert result["any_invalid"] is True + + def test_dry_lab_only_true(self): + result = validate_simulation_result_batch([]) + assert result["dry_lab_only"] is True diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index f0b08d4c..d568e99c 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -2,9 +2,9 @@ import subprocess import sys -# Baseline: 3106 tests as of Loop 89 (2026-07-09) -- Phase H H1 (simulation module registry) +# Baseline: 3125 tests as of Loop 90 (2026-07-09) -- Phase H H2 (simulation-result schema) # Raise threshold if legitimate tests are removed; lower threshold is okay. -BASELINE = 3106 +BASELINE = 3125 TOLERANCE = 0.05 # allow 5% fluctuation