diff --git a/Makefile b/Makefile index 5de06123..0efb3b9d 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema +.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3) PYTEST := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest) @@ -720,3 +720,10 @@ simulation-registry: validate-simulation-result-schema: @echo "SimulationResult schema validation complete (schemas/simulation_result.schema.json)." + +simulation-baseline-check: + .venv/bin/openamp-foundry simulation-baseline-check \ + --module-id membrane_proxy \ + --claimed-level 2 \ + --baseline-beaten false + @echo "Baseline check complete." diff --git a/docs/evidence/METRICS_CURRENT.md b/docs/evidence/METRICS_CURRENT.md index 2be81921..4f61d853 100644 --- a/docs/evidence/METRICS_CURRENT.md +++ b/docs/evidence/METRICS_CURRENT.md @@ -5,7 +5,8 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees > with this file, this file wins. Updated whenever benchmark/benchmark config changes. > -> **Last updated:** 2026-07-09 (Phase H H2 — simulation-result schema — v0.5.90) +> **Last updated:** 2026-07-09 (Phase H H3 — per-module cheapest-baseline declaration — v0.5.91) +> **New in v0.5.91:** Per-module cheapest-baseline declaration (H3) — `BaselineDeclaration` dataclass with module_id, module_name, baseline_description, baseline_type, evidence_level_ceiling, and notes. `BASELINE_DECLARATIONS` list (4 entries: membrane_proxy, structure_proxy, dummy_membrane_proxy, external_adapter_placeholder). `get_baseline_declaration()` and `list_baseline_declarations()` for lookup. `check_baseline_requirement()` caps effective_evidence_level to ceiling when baseline not beaten; returns dict with module_id, baseline_beaten, claimed/effective evidence levels, capped flag, message, and dry_lab_only. `validate_baseline_declarations()` checks module_id non-empty, baseline_description non-empty, baseline_type in {heuristic, random, constant, length}, evidence_level_ceiling 1-6, no duplicate module_ids. CLI (`openamp-foundry simulation-baseline-check`) with `--module-id`, `--claimed-level`, `--baseline-beaten`, `--format text|json`. `make simulation-baseline-check` target. 16 tests. **3141 total.** Forces honest enemy comparison — every simulation module must declare the simplest baseline it must beat. > **New in v0.5.90:** Simulation-result schema and validator (H2) — `schemas/simulation_result.schema.json` (Draft 2020-12) validates SimulationResult outputs with uncertainty 0.0–1.0 range, required fields (module, version, scope, scores, uncertainty, calibration_set, validated_against, notes), and optional `dry_lab_context` const "dry-lab-only". `validate_simulation_result()` checks module, version, scope, scores, uncertainty, validated_against, notes. Strict mode adds: module must not be "dummy" or contain "stub", uncertainty < 1.0, validated_against non-empty. `validate_simulation_result_batch()` aggregates results with counts and `any_invalid` flag. CLI (`openamp-foundry validate-simulation-result`) with `--results-json`, `--strict`, `--out-json`. `make validate-simulation-result-schema` target. 19 tests. **3125 total.** Dry-lab only. > **New in v0.5.89:** Simulation module registry (H1) — `SimulationModuleEntry` dataclass tracks module_id, name, description, status, evidence_level, baseline_comparison, scope, maintainer, and notes. `SIMULATION_MODULE_REGISTRY` holds 4 entries (membrane_proxy, structure_proxy, dummy_membrane_proxy, external_adapter_placeholder). Lookup functions: `get_module_entry()`, `list_module_entries()` with status/min_evidence filtering, `get_active_modules()`, `registry_summary()` with total/by_status/by_evidence_level/active_module_ids keys. `validate_registry()` checks module_id, name, baseline_comparison, evidence_level 1-6, valid status, duplicate detection. CLI (`openamp-foundry simulation-registry`) supports `--list`, `--show`, `--status`, `--min-evidence`, `--format text|json`. Schema (`schemas/simulation_module_registry.schema.json`). `make simulation-registry` target. 28 tests. 3106 total. Starts Phase H (virtual assay discipline). > **New in v0.5.87:** Calibration decision review checklist (G9) — `build_checklist()` produces a structured `CalibrationDecisionChecklist` with 12 checklist items (10 required) covering data quality, statistical validity, safety consistency, approval, and documentation. Each item has an id, category, question, rationale, and required flag. `CalibrationDecisionChecklist` dataclass tracks responses, notes, overall_pass, and missing_required. CLI (`openamp-foundry calibration-decision-checklist`) accepts `--checklist-id`, `--date`, `--reviewer`, `--responses-json`, `--out-json`, `--out-md`. JSON + Markdown output. Schema (`schemas/calibration_decision_checklist.schema.json`). `make calibration-decision-checklist` target. 14 tests. 3063 total. Makes human review structured and auditable. @@ -81,7 +82,7 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **New in v0.5.31:** Added dipeptide-order features for sequence-order awareness. `dipeptide_order_score` achieves AUROC 0.7861 on AMP-vs-scrambled discrimination — the strongest order-dependent feature in the pipeline. Only 7/31 features survive scrambling (amphipathicity/helix-wheel + dipeptide). All composition features are purely position-independent (exactly 0.5000 AUROC on scrambled test). > **New in v0.5.30:** Easy baseline benchmark added — charge density alone (AUROC 0.8166) outperforms the full pipeline ensemble (0.7792) on AMP-vs-Swiss-Prot-decoy discrimination. Honest finding documented: expected because pipeline optimizes for safety, not raw discrimination. > **New in v0.5.29:** Expanded benchmark to 500 AMPs + 500 composition-matched decoys (n=1000). AUROC 0.7792 (CI₉₅: 0.7505–0.8065) confirms signal generalizes. Cluster-aware CI: 0.746–0.8102. Representative AUROC: 0.778. Standard benchmark (n=191) retained for backward comparison. -> **Pipeline version:** v0.5.90 (H2 — simulation-result schema) +> **Pipeline version:** v0.5.91 (H3 — per-module cheapest-baseline declaration) > **Branch:** main --- diff --git a/docs/research/ROADMAP.md b/docs/research/ROADMAP.md index d3334b9b..428def1a 100644 --- a/docs/research/ROADMAP.md +++ b/docs/research/ROADMAP.md @@ -1,5 +1,57 @@ # Roadmap +## v0.5.91 — Loop 91: Phase H H3 — Per-Module Cheapest-Baseline Declaration ✓ (2026-07-09) + +`BaselineDeclaration` dataclass with module_id, module_name, baseline_description, +baseline_type, evidence_level_ceiling, and notes. `BASELINE_DECLARATIONS` list +(4 entries: membrane_proxy, structure_proxy, dummy_membrane_proxy, +external_adapter_placeholder). `get_baseline_declaration()` and +`list_baseline_declarations()` for lookup. `check_baseline_requirement()` caps +effective_evidence_level to ceiling when baseline not beaten. +`validate_baseline_declarations()` checks module_id, baseline_description, +baseline_type, evidence_level_ceiling, and duplicate detection. Forces honest +enemy comparison — every simulation module must declare the simplest baseline +it must beat, making it easy to detect "simulation theater." + +Changes: +- `src/openamp_foundry/simulation/baseline_registry.py` (H3) — Core module with + `BaselineDeclaration` dataclass (6 fields), `BASELINE_DECLARATIONS` list + (4 entries), `get_baseline_declaration()`, `list_baseline_declarations()`, + `check_baseline_requirement()` with ceiling logic, + `validate_baseline_declarations()` with 5 checks. +- `src/openamp_foundry/simulation/__init__.py` — Exports all baseline registry + symbols. +- `src/openamp_foundry/cli/main.py` — Registered `simulation-baseline-check` + subcommand with `--module-id`, `--claimed-level`, `--baseline-beaten`, + `--format` flags. Added import and dispatch. +- `src/openamp_foundry/cli/commands/reports.py` — Added + `_run_simulation_baseline_check()` CLI handler with text and JSON output, + error handling for unknown module IDs, exit code 3 if capped. +- `Makefile` — Added `simulation-baseline-check` target with default + `membrane_proxy` check. Added to `.PHONY`. +- `tests/simulation/test_baseline_registry.py` — 16 tests covering: at least + 4 entries, validation passes, get/list lookup, required field presence, + baseline type validity, check logic (beaten/not beaten/capped/uncapped), + dry_lab_only always True, unknown module handling, duplicate detection, + invalid baseline type detection. +- `docs/evidence/METRICS_CURRENT.md` — v0.5.91 H3 changelog. Test count: 3141. +- `tests/test_test_count_regression.py` — baseline updated to 3141. + +Honest boundaries: +- Baseline declarations are self-reported by module maintainers and must be + manually kept in sync with the module registry. +- `check_baseline_requirement()` caps evidence levels based on declared + ceilings, not on independent biological validation. +- A module that beats its cheapest baseline may still produce biologically + meaningless results — beating a heuristic baseline is a necessary condition + for evidence, not a sufficient one. +- Baseline types are categorical labels; a "heuristic" baseline and a "length" + baseline are not directly comparable in difficulty. +- The evidence_level_ceiling is a policy rule, not a biological guarantee. + Higher ceilings should be justified by actual benchmark performance. +- All baseline comparisons are dry-lab only and must not be presented as + biological proof. + ## v0.5.90 — Loop 90: Phase H H2 — Simulation-Result Schema and Validator ✓ (2026-07-09) `schemas/simulation_result.schema.json` (Draft 2020-12) validates SimulationResult diff --git a/src/openamp_foundry/cli/commands/reports.py b/src/openamp_foundry/cli/commands/reports.py index 4af0df9b..25ad58b5 100644 --- a/src/openamp_foundry/cli/commands/reports.py +++ b/src/openamp_foundry/cli/commands/reports.py @@ -1391,3 +1391,45 @@ def _run_simulation_registry(args: argparse.Namespace) -> int: print("measure biological activity, safety, or real-world performance.") return 0 + + +def _run_simulation_baseline_check(args: argparse.Namespace) -> int: + """Check whether a simulation module beats its cheapest declared baseline.""" + from openamp_foundry.simulation.baseline_registry import ( + check_baseline_requirement, + get_baseline_declaration, + ) + + module_id = args.module_id + claimed_level = args.claimed_level + baseline_beaten = args.baseline_beaten.lower() == "true" + output_format = getattr(args, "format", "text") + + entry = get_baseline_declaration(module_id) + if entry is None: + print(json.dumps({ + "status": "error", + "error": f"Unknown module_id '{module_id}'. Use --list to see available modules.", + })) + return 3 + + result = check_baseline_requirement(module_id, claimed_level, baseline_beaten) + + if output_format == "json": + print(json.dumps(result, indent=2)) + else: + print(f"Baseline Check: {module_id}") + print(f" Module: {entry.module_name}") + print(f" Baseline: {entry.baseline_description}") + print(f" Baseline type: {entry.baseline_type}") + print(f" Evidence level ceiling: {entry.evidence_level_ceiling}") + print(f" Baseline beaten: {baseline_beaten}") + print(f" Claimed evidence level: {claimed_level}") + print(f" Effective evidence: {result['effective_evidence_level']}") + print(f" Capped: {result['capped']}") + print(f" Message: {result['message']}") + print() + print("Dry-lab only. Baseline comparisons are computational benchmarks,") + print("not biological proof.") + + return 3 if result["capped"] else 0 diff --git a/src/openamp_foundry/cli/main.py b/src/openamp_foundry/cli/main.py index 237b556f..53edd404 100644 --- a/src/openamp_foundry/cli/main.py +++ b/src/openamp_foundry/cli/main.py @@ -5,7 +5,7 @@ from openamp_foundry.cli.commands.selection import _run_pilot_panel, _run_pilot_confident, _run_diversity_check, _run_select_batch, _run_batch_rationale from openamp_foundry.cli.commands.external import _run_external_predict, _run_external_consensus from openamp_foundry.cli.commands.qc import _run_synthesis_order, _run_presynth_qc -from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter, _run_synthetic_result_policy_check, _run_calibration_decision_checklist, _run_calibration_rollback_plan, _run_simulation_registry, _run_validate_simulation_result +from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter, _run_synthetic_result_policy_check, _run_calibration_decision_checklist, _run_calibration_rollback_plan, _run_simulation_registry, _run_validate_simulation_result, _run_simulation_baseline_check from openamp_foundry.cli.commands.gates import _run_gate_check import argparse @@ -1402,6 +1402,34 @@ def build_parser() -> argparse.ArgumentParser: help="Optional output path for structured validation result JSON.", ) + # ── Simulation baseline check (H3) ──────────────────────────────── + sbc = sub.add_parser( + "simulation-baseline-check", + help=( + "Check whether a simulation module beats its cheapest declared baseline. " + "Caps evidence level if baseline not beaten. Dry-lab only." + ), + ) + sbc.add_argument( + "--module-id", type=str, required=True, + help="Module ID to check (e.g. membrane_proxy).", + ) + sbc.add_argument( + "--claimed-level", type=int, required=True, + choices=range(1, 7), + help="Claimed evidence level (1-6).", + ) + sbc.add_argument( + "--baseline-beaten", type=str, required=True, + choices=["true", "false"], + help="Whether the baseline has been beaten (true or false).", + ) + sbc.add_argument( + "--format", type=str, default="text", + choices=["text", "json"], + help="Output format (default: text).", + ) + return parser @@ -1563,6 +1591,9 @@ def main(argv: list[str] | None = None) -> int: if args.command == "validate-simulation-result": return _run_validate_simulation_result(args) + if args.command == "simulation-baseline-check": + return _run_simulation_baseline_check(args) + parser.error("unknown command") return 2 diff --git a/src/openamp_foundry/simulation/__init__.py b/src/openamp_foundry/simulation/__init__.py index 3d605bd0..a1ec5245 100644 --- a/src/openamp_foundry/simulation/__init__.py +++ b/src/openamp_foundry/simulation/__init__.py @@ -1,5 +1,13 @@ """Virtual assay simulation layer for OpenAMP Foundry.""" +from .baseline_registry import ( + BASELINE_DECLARATIONS, + BaselineDeclaration, + check_baseline_requirement, + get_baseline_declaration, + list_baseline_declarations, + validate_baseline_declarations, +) from .gate import SimulationGateVerdict, evaluate_simulation_gate from .interfaces import ( EmulatorBaseline, @@ -41,4 +49,10 @@ "validate_registry", "validate_simulation_result", "validate_simulation_result_batch", + "BaselineDeclaration", + "BASELINE_DECLARATIONS", + "get_baseline_declaration", + "list_baseline_declarations", + "check_baseline_requirement", + "validate_baseline_declarations", ] diff --git a/src/openamp_foundry/simulation/baseline_registry.py b/src/openamp_foundry/simulation/baseline_registry.py new file mode 100644 index 00000000..32b64203 --- /dev/null +++ b/src/openamp_foundry/simulation/baseline_registry.py @@ -0,0 +1,162 @@ +"""Per-module cheapest-baseline declarations for virtual assay modules. + +Every simulation module must declare the simplest baseline it must beat, +making it easy to detect simulation theater (modules that don't beat trivial baselines). +Dry-lab only. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +VALID_BASELINE_TYPES: set[str] = {"heuristic", "random", "constant", "length"} + + +@dataclass +class BaselineDeclaration: + module_id: str + module_name: str + baseline_description: str + baseline_type: str + evidence_level_ceiling: int + notes: str = "" + + +BASELINE_DECLARATIONS: list[BaselineDeclaration] = [ + BaselineDeclaration( + module_id="membrane_proxy", + module_name="Membrane Proxy", + baseline_description="charge density (net charge at pH 7.4)", + baseline_type="heuristic", + evidence_level_ceiling=1, + notes="Must beat charge density alone on AMP-vs-decoy discrimination before contributing to level-2 evidence.", + ), + BaselineDeclaration( + module_id="structure_proxy", + module_name="Structure Proxy", + baseline_description="length alone (no structural signal)", + baseline_type="length", + evidence_level_ceiling=1, + notes="Must beat sequence length alone on helical structure prediction before contributing to level-2 evidence.", + ), + BaselineDeclaration( + module_id="dummy_membrane_proxy", + module_name="Dummy Membrane Proxy", + baseline_description="charge density (net charge at pH 7.4)", + baseline_type="heuristic", + evidence_level_ceiling=1, + notes="Stub for testing only. Trivially fails the baseline check; should not be used for scoring.", + ), + BaselineDeclaration( + module_id="external_adapter_placeholder", + module_name="External Adapter Placeholder", + baseline_description="constant 0.5 (no signal)", + baseline_type="constant", + evidence_level_ceiling=1, + notes="Placeholder. Any external adapter must declare and beat its own baseline before contributing evidence.", + ), +] + + +def get_baseline_declaration(module_id: str) -> BaselineDeclaration | None: + for decl in BASELINE_DECLARATIONS: + if decl.module_id == module_id: + return decl + return None + + +def list_baseline_declarations() -> list[BaselineDeclaration]: + return list(BASELINE_DECLARATIONS) + + +def check_baseline_requirement( + module_id: str, + claimed_evidence_level: int, + baseline_beaten: bool, +) -> dict[str, Any]: + """Check whether a module can claim a given evidence level. + + If the module has not beaten its declared baseline, the effective evidence + level is capped at its evidence_level_ceiling. + + Returns a dict with keys: + module_id, baseline_beaten, claimed_evidence_level, + effective_evidence_level, capped, message, dry_lab_only + """ + decl = get_baseline_declaration(module_id) + if decl is None: + return { + "module_id": module_id, + "baseline_beaten": baseline_beaten, + "claimed_evidence_level": claimed_evidence_level, + "effective_evidence_level": claimed_evidence_level, + "capped": False, + "message": f"No baseline declaration found for module '{module_id}'. Proceeding uncapped.", + "dry_lab_only": True, + } + + if baseline_beaten or claimed_evidence_level <= decl.evidence_level_ceiling: + effective = claimed_evidence_level + capped = False + if baseline_beaten: + msg = ( + f"Module '{module_id}' has beaten its baseline " + f"({decl.baseline_description}). Evidence level {claimed_evidence_level} is permitted." + ) + else: + msg = ( + f"Module '{module_id}' has not beaten its baseline " + f"({decl.baseline_description}), but claimed level {claimed_evidence_level} " + f"is within the allowed ceiling ({decl.evidence_level_ceiling}). " + f"Caution: baseline not beaten." + ) + else: + effective = decl.evidence_level_ceiling + capped = True + msg = ( + f"Module '{module_id}' has NOT beaten its declared baseline " + f"({decl.baseline_description}). " + f"Evidence level capped from {claimed_evidence_level} to {effective} " + f"(ceiling for unbeaten baseline)." + ) + + return { + "module_id": module_id, + "baseline_beaten": baseline_beaten, + "claimed_evidence_level": claimed_evidence_level, + "effective_evidence_level": effective, + "capped": capped, + "message": msg, + "dry_lab_only": True, + } + + +def validate_baseline_declarations() -> list[str]: + """Validate all entries in BASELINE_DECLARATIONS. + + Returns a list of error strings (empty = valid). + """ + errors: list[str] = [] + seen_ids: set[str] = set() + + for i, decl in enumerate(BASELINE_DECLARATIONS): + if not decl.module_id: + errors.append(f"Entry {i}: module_id is empty") + if not decl.baseline_description: + errors.append(f"Entry {i} ({decl.module_id}): baseline_description is empty") + if decl.baseline_type not in VALID_BASELINE_TYPES: + errors.append( + f"Entry {i} ({decl.module_id}): invalid baseline_type '{decl.baseline_type}'. " + f"Must be one of: {sorted(VALID_BASELINE_TYPES)}" + ) + if not (1 <= decl.evidence_level_ceiling <= 6): + errors.append( + f"Entry {i} ({decl.module_id}): evidence_level_ceiling " + f"{decl.evidence_level_ceiling} must be 1..6" + ) + if decl.module_id in seen_ids: + errors.append(f"Duplicate module_id: '{decl.module_id}'") + seen_ids.add(decl.module_id) + + return errors diff --git a/tests/simulation/test_baseline_registry.py b/tests/simulation/test_baseline_registry.py new file mode 100644 index 00000000..0ed693bc --- /dev/null +++ b/tests/simulation/test_baseline_registry.py @@ -0,0 +1,119 @@ +"""Tests for simulation baseline registry (Phase H H3).""" + +from __future__ import annotations + +from openamp_foundry.simulation.baseline_registry import ( + BASELINE_DECLARATIONS, + BaselineDeclaration, + VALID_BASELINE_TYPES, + check_baseline_requirement, + get_baseline_declaration, + list_baseline_declarations, + validate_baseline_declarations, +) + + +class TestBaselineDeclarations: + def test_has_at_least_4_entries(self): + assert len(BASELINE_DECLARATIONS) >= 4 + + def test_all_entries_pass_validation(self): + errors = validate_baseline_declarations() + assert errors == [], f"Validation errors: {errors}" + + def test_get_baseline_declaration_known_id(self): + entry = get_baseline_declaration("membrane_proxy") + assert entry is not None + assert entry.module_id == "membrane_proxy" + assert entry.baseline_description == "charge density (net charge at pH 7.4)" + + def test_get_baseline_declaration_unknown_id(self): + entry = get_baseline_declaration("nonexistent_module") + assert entry is None + + def test_list_baseline_declarations_returns_all(self): + entries = list_baseline_declarations() + assert len(entries) == len(BASELINE_DECLARATIONS) + + def test_each_entry_has_non_empty_module_id(self): + for entry in BASELINE_DECLARATIONS: + assert entry.module_id, f"Empty module_id in {entry}" + + def test_each_entry_has_non_empty_baseline_description(self): + for entry in BASELINE_DECLARATIONS: + assert entry.baseline_description, f"Empty baseline_description in {entry.module_id}" + + def test_baseline_type_in_valid_set(self): + for entry in BASELINE_DECLARATIONS: + assert entry.baseline_type in VALID_BASELINE_TYPES, ( + f"Invalid baseline_type '{entry.baseline_type}' in {entry.module_id}" + ) + + +class TestCheckBaselineRequirement: + def test_baseline_beaten_no_cap(self): + result = check_baseline_requirement("membrane_proxy", 2, baseline_beaten=True) + assert result["capped"] is False + assert result["effective_evidence_level"] == 2 + assert result["claimed_evidence_level"] == 2 + assert result["baseline_beaten"] is True + + def test_baseline_not_beaten_claimed_above_ceiling_capped(self): + result = check_baseline_requirement("membrane_proxy", 5, baseline_beaten=False) + assert result["capped"] is True + assert result["effective_evidence_level"] == 1 + assert result["claimed_evidence_level"] == 5 + + def test_baseline_not_beaten_claimed_at_or_below_ceiling_no_cap(self): + result = check_baseline_requirement("membrane_proxy", 1, baseline_beaten=False) + assert result["capped"] is False + assert result["effective_evidence_level"] == 1 + assert result["claimed_evidence_level"] == 1 + + def test_dry_lab_only_always_true(self): + result = check_baseline_requirement("membrane_proxy", 2, baseline_beaten=False) + assert result["dry_lab_only"] is True + + def test_unknown_module_id_returns_no_cap(self): + result = check_baseline_requirement("unknown_module", 3, baseline_beaten=False) + assert result["capped"] is False + assert result["effective_evidence_level"] == 3 + assert "No baseline declaration found" in result["message"] + + +class TestValidateBaselineDeclarations: + def test_returns_empty_for_valid_registry(self): + errors = validate_baseline_declarations() + assert errors == [] + + def test_detects_duplicate_module_id(self, monkeypatch): + dup = BaselineDeclaration( + module_id="membrane_proxy", + module_name="Dup", + baseline_description="test", + baseline_type="constant", + evidence_level_ceiling=1, + ) + monkeypatch.setattr( + "openamp_foundry.simulation.baseline_registry.BASELINE_DECLARATIONS", + BASELINE_DECLARATIONS + [dup], + ) + errors = validate_baseline_declarations() + dup_errors = [e for e in errors if "Duplicate" in e and "membrane_proxy" in e] + assert len(dup_errors) >= 1 + + def test_detects_invalid_baseline_type(self, monkeypatch): + bad = BaselineDeclaration( + module_id="test_bad_type", + module_name="Bad", + baseline_description="test", + baseline_type="invalid_type", + evidence_level_ceiling=1, + ) + monkeypatch.setattr( + "openamp_foundry.simulation.baseline_registry.BASELINE_DECLARATIONS", + [bad], + ) + errors = validate_baseline_declarations() + type_errors = [e for e in errors if "invalid_type" in e] + assert len(type_errors) >= 1 diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index d568e99c..fbda6fa9 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -2,9 +2,9 @@ import subprocess import sys -# Baseline: 3125 tests as of Loop 90 (2026-07-09) -- Phase H H2 (simulation-result schema) +# Baseline: 3141 tests as of Loop 91 (2026-07-09) -- Phase H H3 (per-module cheapest-baseline declaration) # Raise threshold if legitimate tests are removed; lower threshold is okay. -BASELINE = 3125 +BASELINE = 3141 TOLERANCE = 0.05 # allow 5% fluctuation