From aad2fb8cf9c19c6789bd8ba884b78695139b4092 Mon Sep 17 00:00:00 2001 From: OpenCode Date: Thu, 9 Jul 2026 13:10:22 +0700 Subject: [PATCH 1/2] feat: Phase H H1 simulation module registry -- tracks status and evidence level of virtual assay modules --- Makefile | 6 +- docs/evidence/METRICS_CURRENT.md | 6 +- docs/research/ROADMAP.md | 61 +++++ .../simulation_module_registry.schema.json | 39 +++ src/openamp_foundry/cli/commands/reports.py | 108 ++++++++ src/openamp_foundry/cli/main.py | 36 ++- src/openamp_foundry/simulation/__init__.py | 16 ++ .../simulation/module_registry.py | 147 +++++++++++ tests/simulation/test_module_registry.py | 230 ++++++++++++++++++ tests/test_test_count_regression.py | 4 +- 10 files changed, 646 insertions(+), 7 deletions(-) create mode 100644 schemas/simulation_module_registry.schema.json create mode 100644 src/openamp_foundry/simulation/module_registry.py create mode 100644 tests/simulation/test_module_registry.py diff --git a/Makefile b/Makefile index e3b7014e..c551215f 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan +.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry simulation-registry PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3) PYTEST := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest) @@ -713,3 +713,7 @@ calibration-rollback-plan: --out-json /tmp/rollback_plan.json \ --out-md /tmp/rollback_plan.md @echo "Rollback plan generated. See /tmp/rollback_plan.json" + +simulation-registry: + .venv/bin/openamp-foundry simulation-registry --list + @echo "Simulation module registry displayed." diff --git a/docs/evidence/METRICS_CURRENT.md b/docs/evidence/METRICS_CURRENT.md index cc81fd47..2c7b995a 100644 --- a/docs/evidence/METRICS_CURRENT.md +++ b/docs/evidence/METRICS_CURRENT.md @@ -5,8 +5,8 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees > with this file, this file wins. Updated whenever benchmark/benchmark config changes. > -> **Last updated:** 2026-07-09 (Phase G G10 — recalibration rollback plan — v0.5.88) -> **New in v0.5.88:** Recalibration rollback plan (G10) — `build_rollback_plan()` produces a structured `RollbackPlan` with 5 rollback triggers (RT-01 through RT-05) covering AUROC regression, calibration metric below threshold, safety filter bypass, cohort distribution shift, and unexplained weight delta. Each trigger has id, name, description, threshold, and severity. `RollbackStep` dataclass with step_number, action, responsible, detail, and dry_lab_only. `RollbackPlan` dataclass with plan_id, version, triggered_by, steps, notes, dry_lab_only, and to_dict(). 6 default rollback steps: halt, document, restore, verify, root-cause, and log. CLI (`openamp-foundry calibration-rollback-plan`) accepts `--plan-id`, `--version`, `--triggered-by`, `--notes`, `--out-json`, `--out-md`. JSON + Markdown output. Schema (`schemas/calibration_rollback_plan.schema.json`). `make calibration-rollback-plan` target. 15 tests. 3078 total. This completes Phase G (G1-G10 — calibration and active-learning rigor). +> **Last updated:** 2026-07-09 (Phase H H1 — simulation module registry — v0.5.89) +> **New in v0.5.89:** Simulation module registry (H1) — `SimulationModuleEntry` dataclass tracks module_id, name, description, status, evidence_level, baseline_comparison, scope, maintainer, and notes. `SIMULATION_MODULE_REGISTRY` holds 4 entries (membrane_proxy, structure_proxy, dummy_membrane_proxy, external_adapter_placeholder). Lookup functions: `get_module_entry()`, `list_module_entries()` with status/min_evidence filtering, `get_active_modules()`, `registry_summary()` with total/by_status/by_evidence_level/active_module_ids keys. `validate_registry()` checks module_id, name, baseline_comparison, evidence_level 1-6, valid status, duplicate detection. CLI (`openamp-foundry simulation-registry`) supports `--list`, `--show`, `--status`, `--min-evidence`, `--format text|json`. Schema (`schemas/simulation_module_registry.schema.json`). `make simulation-registry` target. 28 tests. 3106 total. Starts Phase H (virtual assay discipline). > **New in v0.5.87:** Calibration decision review checklist (G9) — `build_checklist()` produces a structured `CalibrationDecisionChecklist` with 12 checklist items (10 required) covering data quality, statistical validity, safety consistency, approval, and documentation. Each item has an id, category, question, rationale, and required flag. `CalibrationDecisionChecklist` dataclass tracks responses, notes, overall_pass, and missing_required. CLI (`openamp-foundry calibration-decision-checklist`) accepts `--checklist-id`, `--date`, `--reviewer`, `--responses-json`, `--out-json`, `--out-md`. JSON + Markdown output. Schema (`schemas/calibration_decision_checklist.schema.json`). `make calibration-decision-checklist` target. 14 tests. 3063 total. Makes human review structured and auditable. > **New in v0.5.86:** Synthetic-result policy enforcement (G8) — `check_synthetic_result_policy()` and `run_policy_batch()` enforce that synthetic/simulation outputs cannot raise the proof-ladder level of a candidate. Levels 4+ require wet-lab evidence; synthetic/unknown sources are rejected for such proposals. CLI (`openamp-foundry synthetic-result-policy-check`) accepts `--proposals-json`, `--out-json`, `--out-md`. Schema (`schemas/synthetic_result_policy_check.schema.json`). `make synthetic-result-policy-check` target. 27 tests. 3049 total. Anti-overclaim safeguard. > **New in v0.5.85:** Result-quality flag propagation (G7) — `assess_result_quality()` and `filter_results_for_calibration()` propagate result-quality flags into the calibration engine. Candidates with contamination or assay interference are excluded; candidates with 2+ quality flags are excluded; candidates with 1 flag are included with caution; clean candidates are fully included. `QUALITY_FLAGS` dictionary (8 flags). `ResultQualityReport` dataclass. CLI (`openamp-foundry result-quality-filter`) accepts `--results-json`, `--out-json`, `--out-md`. JSON + Markdown output. Schema (`schemas/result_quality_report.schema.json`). `make result-quality-filter` target. 27 tests. 3022 total. @@ -80,7 +80,7 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **New in v0.5.31:** Added dipeptide-order features for sequence-order awareness. `dipeptide_order_score` achieves AUROC 0.7861 on AMP-vs-scrambled discrimination — the strongest order-dependent feature in the pipeline. Only 7/31 features survive scrambling (amphipathicity/helix-wheel + dipeptide). All composition features are purely position-independent (exactly 0.5000 AUROC on scrambled test). > **New in v0.5.30:** Easy baseline benchmark added — charge density alone (AUROC 0.8166) outperforms the full pipeline ensemble (0.7792) on AMP-vs-Swiss-Prot-decoy discrimination. Honest finding documented: expected because pipeline optimizes for safety, not raw discrimination. > **New in v0.5.29:** Expanded benchmark to 500 AMPs + 500 composition-matched decoys (n=1000). AUROC 0.7792 (CI₉₅: 0.7505–0.8065) confirms signal generalizes. Cluster-aware CI: 0.746–0.8102. Representative AUROC: 0.778. Standard benchmark (n=191) retained for backward comparison. -> **Pipeline version:** v0.5.88 (G10 — Phase G complete) +> **Pipeline version:** v0.5.89 (H1 — Phase H started) > **Branch:** main --- diff --git a/docs/research/ROADMAP.md b/docs/research/ROADMAP.md index 990dd7b8..679575fe 100644 --- a/docs/research/ROADMAP.md +++ b/docs/research/ROADMAP.md @@ -1,5 +1,66 @@ # Roadmap +## v0.5.89 — Loop 89: Phase H H1 — Simulation Module Registry ✓ (2026-07-09) + +`SIMULATION_MODULE_REGISTRY` with 4 entries (membrane_proxy, structure_proxy, +dummy_membrane_proxy, external_adapter_placeholder). `SimulationModuleEntry` +dataclass tracks module_id, name, description, status, evidence_level, +baseline_comparison, scope, maintainer, and notes. Lookup functions: +`get_module_entry()`, `list_module_entries()` with status/min_evidence +filtering, `get_active_modules()`, `registry_summary()` with +total/by_status/by_evidence_level/active_module_ids keys. +`validate_registry()` checks module_id, name, baseline_comparison, +evidence_level 1-6, valid status, duplicate detection. +CLI (`openamp-foundry simulation-registry`) with `--list`, `--show`, +`--status`, `--min-evidence`, `--format text|json`. +Schema (`schemas/simulation_module_registry.schema.json`). +`make simulation-registry` target. 28 tests. 3106 total. +Starts Phase H (virtual assay discipline). + +Changes: +- `src/openamp_foundry/simulation/module_registry.py` (H1) — Core module with + `SimulationModuleStatus` literal type, `SimulationModuleEntry` dataclass + (9 fields), `SIMULATION_MODULE_REGISTRY` list (4 entries), + `get_module_entry()`, `list_module_entries()`, `get_active_modules()`, + `registry_summary()` with aggregation, `validate_registry()` with 6 checks. + Imports `PROOF_LADDER_LEVELS` from `evidence/synthetic_result_policy.py`. +- `schemas/simulation_module_registry.schema.json` (H1) — JSON Schema Draft + 2020-12 for the registry_summary() output. Validates total, by_status + (all 4 status keys), by_evidence_level, active_module_ids. +- `src/openamp_foundry/simulation/__init__.py` — Exports all module registry + symbols. +- `src/openamp_foundry/cli/main.py` — Registered `simulation-registry` + subcommand with `--list`, `--show`, `--status`, `--min-evidence`, + `--format` flags. +- `src/openamp_foundry/cli/commands/reports.py` — Added + `_run_simulation_registry()` CLI handler with text and JSON output, + per-module details, status/evidence filtering, and validation. +- `Makefile` — Added `simulation-registry` target with `--list` default. + Added to `.PHONY`. +- `tests/simulation/test_module_registry.py` — 28 tests covering: registry + size ≥ 4, all entries pass validation, required field presence, evidence + level range 1-6, valid status values, get_module_entry known/unknown, + list_module_entries no filter/status filter/min_evidence filter, + get_active_modules returns only active, registry_summary keys and totals, + validate_registry detection of empty fields, invalid level, invalid + status, duplicate ids, PROOF_LADDER_LEVELS completeness. +- `docs/evidence/METRICS_CURRENT.md` — v0.5.89 H1 changelog. Test count: 3106. +- `tests/test_test_count_regression.py` — baseline updated to 3106. + +Honest boundaries: +- The registry lists module status and evidence level for informational + purposes only. It does not measure biological activity, safety, or + real-world performance. +- Registry validation checks structural correctness, not scientific validity. + A valid entry may still produce biologically meaningless results. +- The PROOF_LADDER_LEVELS mapping is a claim-level taxonomy. An evidence_level + of 2 ("virtual-assay support") means the module supports computational + exploration — it does not constitute biological proof. +- The registry is dry-lab only. All module entries carry dry-lab caveats + regardless of their status or evidence_level. +- "active" status means the module is available for use, not that it has + been biologically validated. + ## v0.5.88 — Loop 88: Phase G G10 — Recalibration Rollback Plan ✓ (2026-07-09) `build_rollback_plan()` produces a structured `RollbackPlan` with 5 rollback diff --git a/schemas/simulation_module_registry.schema.json b/schemas/simulation_module_registry.schema.json new file mode 100644 index 00000000..1e587a5b --- /dev/null +++ b/schemas/simulation_module_registry.schema.json @@ -0,0 +1,39 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://openamp.ai/schemas/simulation_module_registry.schema.json", + "title": "Simulation Module Registry Summary", + "description": "Schema for the registry_summary() output. Dry-lab only — tracks module status and evidence level, not biological activity.", + "type": "object", + "properties": { + "total": { + "description": "Total number of registered modules", + "type": "integer", + "minimum": 0 + }, + "by_status": { + "description": "Count of modules per status level", + "type": "object", + "properties": { + "active": { "type": "integer", "minimum": 0 }, + "experimental": { "type": "integer", "minimum": 0 }, + "deprecated": { "type": "integer", "minimum": 0 }, + "unavailable": { "type": "integer", "minimum": 0 } + }, + "required": ["active", "experimental", "deprecated", "unavailable"] + }, + "by_evidence_level": { + "description": "Count of modules per proof-ladder evidence level (1-6)", + "type": "object", + "patternProperties": { + "^[1-6]$": { "type": "integer", "minimum": 0 } + } + }, + "active_module_ids": { + "description": "List of module IDs with status=active", + "type": "array", + "items": { "type": "string" } + } + }, + "required": ["total", "by_status", "by_evidence_level", "active_module_ids"], + "additionalProperties": false +} diff --git a/src/openamp_foundry/cli/commands/reports.py b/src/openamp_foundry/cli/commands/reports.py index 1e0f2bfc..4a56564e 100644 --- a/src/openamp_foundry/cli/commands/reports.py +++ b/src/openamp_foundry/cli/commands/reports.py @@ -1230,3 +1230,111 @@ def _run_calibration_rollback_plan(args: argparse.Namespace) -> int: "out_md": args.out_md, }, indent=2)) return 0 + + +def _run_simulation_registry(args: argparse.Namespace) -> int: + """Display simulation module registry information.""" + from openamp_foundry.simulation.module_registry import ( + SIMULATION_MODULE_REGISTRY, + get_module_entry, + list_module_entries, + registry_summary, + VALID_STATUSES, + ) + + PROOF_LADDER_LABELS = { + 1: "computational nomination", + 2: "virtual-assay support", + 3: "in-silico ensemble agreement", + 4: "ex-vivo preliminary", + 5: "in-vivo preliminary", + 6: "clinical evidence", + } + + show = getattr(args, "show", None) + status_filter = getattr(args, "status", None) + min_evidence = getattr(args, "min_evidence", None) + output_format = getattr(args, "format", "text") + + if show: + entry = get_module_entry(show) + if entry is None: + print(json.dumps({"status": "error", "error": f"Unknown module_id: '{show}'"})) + return 3 + if output_format == "json": + print(json.dumps({ + "module_id": entry.module_id, + "name": entry.name, + "description": entry.description, + "status": entry.status, + "evidence_level": entry.evidence_level, + "evidence_label": PROOF_LADDER_LABELS.get(entry.evidence_level, "unknown"), + "baseline_comparison": entry.baseline_comparison, + "scope": entry.scope, + "maintainer": entry.maintainer, + "notes": entry.notes, + }, indent=2)) + else: + label = PROOF_LADDER_LABELS.get(entry.evidence_level, "unknown") + print(f"Module: {entry.module_id}") + print(f" Name: {entry.name}") + print(f" Description: {entry.description}") + print(f" Status: {entry.status}") + print(f" Evidence Level: {entry.evidence_level} ({label})") + print(f" Baseline: {entry.baseline_comparison}") + print(f" Scope: {', '.join(entry.scope)}") + print(f" Maintainer: {entry.maintainer}") + if entry.notes: + print(f" Notes: {entry.notes}") + return 0 + + if status_filter: + if status_filter not in VALID_STATUSES: + print(json.dumps({ + "status": "error", + "error": f"Invalid status '{status_filter}'. Valid: {', '.join(sorted(VALID_STATUSES))}" + })) + return 3 + entries = list_module_entries(status=status_filter) + elif min_evidence is not None: + entries = list_module_entries(min_evidence_level=min_evidence) + else: + entries = list(SIMULATION_MODULE_REGISTRY) + + summary = registry_summary() + + if output_format == "json": + print(json.dumps({ + "modules": [ + { + "module_id": e.module_id, + "name": e.name, + "status": e.status, + "evidence_level": e.evidence_level, + "evidence_label": PROOF_LADDER_LABELS.get(e.evidence_level, "unknown"), + "baseline_comparison": e.baseline_comparison, + "scope": e.scope, + "maintainer": e.maintainer, + "notes": e.notes, + } + for e in entries + ], + "summary": summary, + }, indent=2)) + else: + print("Simulation Module Registry") + print("=" * 60) + print(f"Total modules: {summary['total']}") + print(f"By status: {dict(sorted(summary['by_status'].items()))}") + print(f"By evidence: {dict(sorted(summary['by_evidence_level'].items()))}") + print(f"Active IDs: {summary['active_module_ids']}") + print() + for entry in entries: + label = PROOF_LADDER_LABELS.get(entry.evidence_level, "unknown") + print(f" [{entry.status:13}] {entry.module_id:35s} " + f"L{entry.evidence_level} {label}") + print() + print("Dry-lab only. Registry status is informational — it does not") + print("measure biological activity, safety, or real-world performance.") + + return 0 diff --git a/src/openamp_foundry/cli/main.py b/src/openamp_foundry/cli/main.py index 4a709ba6..163fefd9 100644 --- a/src/openamp_foundry/cli/main.py +++ b/src/openamp_foundry/cli/main.py @@ -5,7 +5,7 @@ from openamp_foundry.cli.commands.selection import _run_pilot_panel, _run_pilot_confident, _run_diversity_check, _run_select_batch, _run_batch_rationale from openamp_foundry.cli.commands.external import _run_external_predict, _run_external_consensus from openamp_foundry.cli.commands.qc import _run_synthesis_order, _run_presynth_qc -from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter, _run_synthetic_result_policy_check, _run_calibration_decision_checklist, _run_calibration_rollback_plan +from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter, _run_synthetic_result_policy_check, _run_calibration_decision_checklist, _run_calibration_rollback_plan, _run_simulation_registry from openamp_foundry.cli.commands.gates import _run_gate_check import argparse @@ -1349,6 +1349,37 @@ def build_parser() -> argparse.ArgumentParser: cdc.add_argument("--out-json", default=None, help="Optional output path for the checklist JSON.") cdc.add_argument("--out-md", default=None, help="Optional output path for the checklist Markdown.") + # ── Simulation module registry ─────────────────────────────────── + simreg = sub.add_parser( + "simulation-registry", + help="Display the simulation module registry (status and evidence level of virtual assay modules). Dry-lab only.", + ) + simreg.add_argument( + "--list", action="store_true", default=True, + dest="list_modules", + help="List all registered modules (default).", + ) + simreg.add_argument( + "--show", type=str, default=None, + metavar="MODULE_ID", + help="Show details for a specific module ID.", + ) + simreg.add_argument( + "--status", type=str, default=None, + choices=["active", "experimental", "deprecated", "unavailable"], + help="Filter by status: active, experimental, deprecated, unavailable.", + ) + simreg.add_argument( + "--min-evidence", type=int, default=None, + dest="min_evidence", + help="Minimum evidence level (1-6) to filter by.", + ) + simreg.add_argument( + "--format", type=str, default="text", + choices=["text", "json"], + help="Output format (default: text).", + ) + return parser @@ -1504,6 +1535,9 @@ def main(argv: list[str] | None = None) -> int: if args.command == "calibration-rollback-plan": return _run_calibration_rollback_plan(args) + if args.command == "simulation-registry": + return _run_simulation_registry(args) + parser.error("unknown command") return 2 diff --git a/src/openamp_foundry/simulation/__init__.py b/src/openamp_foundry/simulation/__init__.py index 14592b34..a2ee2ecb 100644 --- a/src/openamp_foundry/simulation/__init__.py +++ b/src/openamp_foundry/simulation/__init__.py @@ -8,6 +8,15 @@ VirtualAssayProxy, ) from .membrane import MembraneProxy +from .module_registry import ( + SIMULATION_MODULE_REGISTRY, + SimulationModuleEntry, + get_active_modules, + get_module_entry, + list_module_entries, + registry_summary, + validate_registry, +) from .structure import StructureProxy __all__ = [ @@ -19,4 +28,11 @@ "StructureProxy", "SimulationGateVerdict", "evaluate_simulation_gate", + "SimulationModuleEntry", + "SIMULATION_MODULE_REGISTRY", + "get_module_entry", + "list_module_entries", + "get_active_modules", + "registry_summary", + "validate_registry", ] diff --git a/src/openamp_foundry/simulation/module_registry.py b/src/openamp_foundry/simulation/module_registry.py new file mode 100644 index 00000000..5a4e1d58 --- /dev/null +++ b/src/openamp_foundry/simulation/module_registry.py @@ -0,0 +1,147 @@ +"""Simulation module registry — tracks status and evidence level of each +virtual assay module, preventing undocumented use of simulation outputs. + +Dry-lab only. This is a registry of computational modules, not biological proof. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Literal + +from openamp_foundry.evidence.synthetic_result_policy import PROOF_LADDER_LEVELS + +SimulationModuleStatus = Literal["active", "experimental", "deprecated", "unavailable"] + +VALID_STATUSES: set[str] = {"active", "experimental", "deprecated", "unavailable"} + + +@dataclass +class SimulationModuleEntry: + module_id: str + name: str + description: str + status: str + evidence_level: int + baseline_comparison: str + scope: list[str] = field(default_factory=list) + maintainer: str = "pipeline" + notes: str = "" + + +SIMULATION_MODULE_REGISTRY: list[SimulationModuleEntry] = [ + SimulationModuleEntry( + module_id="membrane_proxy", + name="Membrane Proxy", + description="Wimley-White interfacial/octanol scale coarse-grained bacterial and mammalian binding energy scores", + status="active", + evidence_level=2, + baseline_comparison="charge density (net charge at pH 7.4)", + scope=["bacterial_membrane_binding"], + maintainer="pipeline", + notes="", + ), + SimulationModuleEntry( + module_id="structure_proxy", + name="Structure Proxy", + description="Chou-Fasman 3-state secondary structure prediction (helix/sheet/coil) with non-helical flag", + status="experimental", + evidence_level=2, + baseline_comparison="length alone (no structural signal)", + scope=["helical_structure_prediction"], + maintainer="pipeline", + notes="", + ), + SimulationModuleEntry( + module_id="dummy_membrane_proxy", + name="Dummy Membrane Proxy", + description="Stub membrane proxy for testing only", + status="deprecated", + evidence_level=1, + baseline_comparison="charge density (net charge at pH 7.4)", + scope=["bacterial_membrane_binding"], + maintainer="pipeline", + notes="stub for testing only; do not use for scoring", + ), + SimulationModuleEntry( + module_id="external_adapter_placeholder", + name="External Adapter Placeholder", + description="Placeholder for future third-party simulation adapters", + status="unavailable", + evidence_level=1, + baseline_comparison="TBD", + scope=["external_simulation"], + maintainer="pipeline", + notes="placeholder for future third-party adapters", + ), +] + + +def get_module_entry(module_id: str) -> SimulationModuleEntry | None: + for entry in SIMULATION_MODULE_REGISTRY: + if entry.module_id == module_id: + return entry + return None + + +def list_module_entries( + status: str | None = None, + min_evidence_level: int | None = None, +) -> list[SimulationModuleEntry]: + results = list(SIMULATION_MODULE_REGISTRY) + if status is not None: + results = [e for e in results if e.status == status] + if min_evidence_level is not None: + results = [e for e in results if e.evidence_level >= min_evidence_level] + return results + + +def get_active_modules() -> list[SimulationModuleEntry]: + return list_module_entries(status="active") + + +def registry_summary() -> dict: + by_status: dict[str, int] = {} + by_evidence_level: dict[str, int] = {} + active_ids: list[str] = [] + + for entry in SIMULATION_MODULE_REGISTRY: + by_status[entry.status] = by_status.get(entry.status, 0) + 1 + key = str(entry.evidence_level) + by_evidence_level[key] = by_evidence_level.get(key, 0) + 1 + if entry.status == "active": + active_ids.append(entry.module_id) + + return { + "total": len(SIMULATION_MODULE_REGISTRY), + "by_status": dict(sorted(by_status.items())), + "by_evidence_level": dict(sorted(by_evidence_level.items())), + "active_module_ids": active_ids, + } + + +def validate_registry() -> list[str]: + errors: list[str] = [] + seen_ids: set[str] = set() + + for i, entry in enumerate(SIMULATION_MODULE_REGISTRY): + if not entry.module_id: + errors.append(f"Entry {i}: module_id is empty") + if not entry.name: + errors.append(f"Entry {i} ({entry.module_id}): name is empty") + if not entry.baseline_comparison: + errors.append(f"Entry {i} ({entry.module_id}): baseline_comparison is empty") + if entry.evidence_level not in PROOF_LADDER_LEVELS: + errors.append( + f"Entry {i} ({entry.module_id}): evidence_level {entry.evidence_level} " + f"must be 1..6" + ) + if entry.status not in VALID_STATUSES: + errors.append( + f"Entry {i} ({entry.module_id}): unknown status '{entry.status}'" + ) + if entry.module_id in seen_ids: + errors.append(f"Duplicate module_id: '{entry.module_id}'") + seen_ids.add(entry.module_id) + + return errors diff --git a/tests/simulation/test_module_registry.py b/tests/simulation/test_module_registry.py new file mode 100644 index 00000000..5d100b99 --- /dev/null +++ b/tests/simulation/test_module_registry.py @@ -0,0 +1,230 @@ +"""Tests for the simulation module registry.""" + +import pytest +from openamp_foundry.simulation.module_registry import ( + SIMULATION_MODULE_REGISTRY, + SimulationModuleEntry, + SimulationModuleStatus, + VALID_STATUSES, + get_module_entry, + get_active_modules, + list_module_entries, + registry_summary, + validate_registry, +) +from openamp_foundry.evidence.synthetic_result_policy import PROOF_LADDER_LEVELS + + +class TestRegistryContents: + def test_has_at_least_four_entries(self): + assert len(SIMULATION_MODULE_REGISTRY) >= 4 + + def test_all_entries_pass_validation(self): + errors = validate_registry() + assert errors == [] + + def test_all_entries_have_required_fields(self): + for entry in SIMULATION_MODULE_REGISTRY: + assert entry.module_id, "module_id must be non-empty" + assert entry.name, "name must be non-empty" + assert entry.description, "description must be non-empty" + assert entry.baseline_comparison, "baseline_comparison must be non-empty" + + def test_all_evidence_levels_in_range(self): + for entry in SIMULATION_MODULE_REGISTRY: + assert 1 <= entry.evidence_level <= 6, ( + f"evidence_level {entry.evidence_level} must be 1..6" + ) + + def test_all_statuses_valid(self): + for entry in SIMULATION_MODULE_REGISTRY: + assert entry.status in VALID_STATUSES, ( + f"unknown status '{entry.status}' for {entry.module_id}" + ) + + +class TestGetModuleEntry: + def test_known_id_returns_entry(self): + entry = get_module_entry("membrane_proxy") + assert entry is not None + assert entry.module_id == "membrane_proxy" + assert entry.name == "Membrane Proxy" + + def test_unknown_id_returns_none(self): + assert get_module_entry("nonexistent_module") is None + + def test_empty_string_returns_none(self): + assert get_module_entry("") is None + + +class TestListModuleEntries: + def test_no_filter_returns_all(self): + all_entries = list_module_entries() + assert len(all_entries) == len(SIMULATION_MODULE_REGISTRY) + + def test_status_filter_active(self): + active = list_module_entries(status="active") + assert all(e.status == "active" for e in active) + + def test_status_filter_deprecated(self): + deprecated = list_module_entries(status="deprecated") + assert all(e.status == "deprecated" for e in deprecated) + + def test_min_evidence_level_filter(self): + high_evidence = list_module_entries(min_evidence_level=2) + assert all(e.evidence_level >= 2 for e in high_evidence) + + def test_min_evidence_level_1_includes_all(self): + all_entries = list_module_entries(min_evidence_level=1) + assert len(all_entries) == len(SIMULATION_MODULE_REGISTRY) + + +class TestGetActiveModules: + def test_returns_only_active(self): + active = get_active_modules() + assert all(e.status == "active" for e in active) + + def test_returns_list(self): + assert isinstance(get_active_modules(), list) + + +class TestRegistrySummary: + def test_has_required_keys(self): + summary = registry_summary() + assert "total" in summary + assert "by_status" in summary + assert "by_evidence_level" in summary + assert "active_module_ids" in summary + + def test_total_matches_registry_length(self): + summary = registry_summary() + assert summary["total"] == len(SIMULATION_MODULE_REGISTRY) + + def test_by_status_sum_matches_total(self): + summary = registry_summary() + assert sum(summary["by_status"].values()) == summary["total"] + + def test_by_evidence_level_sum_matches_total(self): + summary = registry_summary() + assert sum(summary["by_evidence_level"].values()) == summary["total"] + + def test_active_module_ids_are_all_active(self): + summary = registry_summary() + for mid in summary["active_module_ids"]: + entry = get_module_entry(mid) + assert entry is not None + assert entry.status == "active" + + +class TestValidateRegistry: + def test_default_registry_has_no_errors(self): + errors = validate_registry() + assert errors == [] + + def test_detects_empty_module_id(self, monkeypatch): + bad_entry = SimulationModuleEntry( + module_id="", + name="Bad Module", + description="test", + status="active", + evidence_level=1, + baseline_comparison="charge density", + scope=[], + ) + monkeypatch.setattr( + "openamp_foundry.simulation.module_registry.SIMULATION_MODULE_REGISTRY", + [bad_entry], + ) + errors = validate_registry() + assert any("module_id is empty" in e for e in errors) + + def test_detects_empty_name(self, monkeypatch): + bad_entry = SimulationModuleEntry( + module_id="bad_mod", + name="", + description="test", + status="active", + evidence_level=1, + baseline_comparison="charge density", + scope=[], + ) + monkeypatch.setattr( + "openamp_foundry.simulation.module_registry.SIMULATION_MODULE_REGISTRY", + [bad_entry], + ) + errors = validate_registry() + assert any("name is empty" in e for e in errors) + + def test_detects_empty_baseline(self, monkeypatch): + bad_entry = SimulationModuleEntry( + module_id="bad_mod", + name="Bad Module", + description="test", + status="active", + evidence_level=1, + baseline_comparison="", + scope=[], + ) + monkeypatch.setattr( + "openamp_foundry.simulation.module_registry.SIMULATION_MODULE_REGISTRY", + [bad_entry], + ) + errors = validate_registry() + assert any("baseline_comparison is empty" in e for e in errors) + + def test_detects_invalid_evidence_level(self, monkeypatch): + bad_entry = SimulationModuleEntry( + module_id="bad_mod", + name="Bad Module", + description="test", + status="active", + evidence_level=0, + baseline_comparison="charge density", + scope=[], + ) + monkeypatch.setattr( + "openamp_foundry.simulation.module_registry.SIMULATION_MODULE_REGISTRY", + [bad_entry], + ) + errors = validate_registry() + assert any("evidence_level" in e for e in errors) + + def test_detects_invalid_status(self, monkeypatch): + bad_entry = SimulationModuleEntry( + module_id="bad_mod", + name="Bad Module", + description="test", + status="super_active", + evidence_level=1, + baseline_comparison="charge density", + scope=[], + ) + monkeypatch.setattr( + "openamp_foundry.simulation.module_registry.SIMULATION_MODULE_REGISTRY", + [bad_entry], + ) + errors = validate_registry() + assert any("unknown status" in e for e in errors) + + def test_detects_duplicate_module_ids(self, monkeypatch): + dup_entry = SimulationModuleEntry( + module_id="duplicate", + name="Dup1", + description="test", + status="active", + evidence_level=1, + baseline_comparison="charge density", + scope=[], + ) + monkeypatch.setattr( + "openamp_foundry.simulation.module_registry.SIMULATION_MODULE_REGISTRY", + [dup_entry, dup_entry], + ) + errors = validate_registry() + assert any("Duplicate module_id" in e for e in errors) + + +class TestProofLadderLevels: + def test_proof_ladder_levels_are_imported(self): + assert isinstance(PROOF_LADDER_LEVELS, dict) + assert set(PROOF_LADDER_LEVELS.keys()) == {1, 2, 3, 4, 5, 6} diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index 3ece7aad..f0b08d4c 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -2,9 +2,9 @@ import subprocess import sys -# Baseline: 3078 tests as of Loop 88 (2026-07-09) -- Phase G G10 (recalibration rollback plan, Phase G complete) +# Baseline: 3106 tests as of Loop 89 (2026-07-09) -- Phase H H1 (simulation module registry) # Raise threshold if legitimate tests are removed; lower threshold is okay. -BASELINE = 3078 +BASELINE = 3106 TOLERANCE = 0.05 # allow 5% fluctuation From 33601a1931d590823c07ab724dfe17beeb80b67b Mon Sep 17 00:00:00 2001 From: OpenCode Date: Thu, 9 Jul 2026 13:11:18 +0700 Subject: [PATCH 2/2] feat: Phase H H1 simulation module registry -- tracks status and evidence level of virtual assay modules --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c551215f..c01fa6af 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry simulation-registry +.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3) PYTEST := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest)