Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report
.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report simulation-deprecation-check

PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3)
PYTEST := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest)
Expand Down Expand Up @@ -756,3 +756,8 @@ simulation-ci-report:
.venv/bin/openamp-foundry simulation-ci-report \
--results-json '[{"module":"membrane_proxy","version":"0.1.0","scope":["bacterial_binding"],"scores":{"binding_energy":0.75},"uncertainty":0.1,"calibration_set":null,"validated_against":["APD3"],"notes":[]},{"module":"structure_proxy","version":"0.1.0","scope":["bacterial_binding"],"scores":{"binding_energy":0.80},"uncertainty":0.15,"calibration_set":null,"validated_against":["APD3"],"notes":[]}]'
@echo "CI report complete."

simulation-deprecation-check:
.venv/bin/openamp-foundry simulation-deprecation-check \
--module-ids membrane_proxy,dummy_membrane_proxy,external_adapter_placeholder
@echo "Deprecation check complete."
7 changes: 4 additions & 3 deletions docs/evidence/METRICS_CURRENT.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak
> **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees
> with this file, this file wins. Updated whenever benchmark/benchmark config changes.
>
> **Last updated:** 2026-07-09 (Phase H H7 — simulation-result confidence interval reporter — v0.5.95)
> **Last updated:** 2026-07-09 (Phase H H8 — simulation-module deprecation enforcer — v0.5.96)
> **New in v0.5.96:** Simulation-module deprecation enforcer (H8) — `DeprecationCheckResult` dataclass (5 fields: module_id, status, is_blocked, block_reason, dry_lab_only). `BLOCKED_STATUSES = {"deprecated", "unavailable"}`. `check_module_deprecation()` looks up module in registry: not-found returns is_blocked=True status="unknown"; deprecated/unavailable returns is_blocked=True with reason; active/experimental returns is_blocked=False. `enforce_deprecation()` filters list[SimulationResult] to only non-blocked modules, returns total_input/passed/blocked/blocked_modules/passed_results/checks/dry_lab_only. `run_deprecation_check_batch()` bulk-checks module_ids with total/blocked/allowed/any_blocked/results/dry_lab_only. CLI (`openamp-foundry simulation-deprecation-check`) with `--module-ids`, `--format text|json`. `make simulation-deprecation-check` target. 21 tests. **3238 total.** Deprecated simulation modules must not be used in production scoring — the enforcer prevents stale or unreliable modules from tainting evidence packets.
> **New in v0.5.95:** Simulation-result confidence interval reporter (H7) — `ci_reporter.py` with `ScoreCI` dataclass (9 fields: module_id, score_key, point_estimate, uncertainty, ci_lower, ci_upper, ci_width, overlaps_with, dry_lab_only). `compute_score_ci()` builds CIs from SimulationResult.scores[score_key] ± uncertainty, returns None if score_key missing. `compare_cis()` pairwise checks overlap condition (a_lo <= b_hi and b_lo <= a_hi), returns new list with overlaps_with populated (no in-place mutation). `ci_report()` produces full report with n_results, cis list, any_overlap flag, and dry_lab_only=True. CLI (`openamp-foundry simulation-ci-report`) with `--results-json`, `--score-key`, `--format text|json`. `make simulation-ci-report` target. 16 tests. **3217 total.** Raw scores without uncertainty ranges make it impossible to judge whether two candidates are distinguishable. The CI reporter makes uncertainty explicit and auditable.
> **New in v0.5.94:** Simulation-ensemble agreement checker (H6) — `ensemble_checker.py` with `EnsembleAgreementResult` dataclass (9 fields: sequence, modules_checked, agreement_level, agreement_description, mean_score, score_range, scores_by_module, threshold, dry_lab_only). `AGREEMENT_LEVELS` dict (5 levels: strong, moderate, weak, conflict, insufficient). `check_ensemble_agreement()` extracts score_key from each SimulationResult's scores dict, computes score_range, classifies into strong (≥3 modules within threshold), moderate (2 modules within threshold), weak (1 module), conflict (beyond threshold), or insufficient (no results). `run_ensemble_check_batch()` aggregates multiple calls with counts and any_conflict flag. CLI (`openamp-foundry simulation-ensemble-check`) with `--sequence`, `--results-json`, `--score-key`, `--threshold`, `--format text|json`. `make simulation-ensemble-check` target. 20 tests. **3201 total.** When multiple simulation modules independently agree on a candidate, that agreement is stronger evidence than a single module alone. The ensemble checker makes this agreement explicit and auditable.
> **New in v0.5.93:** Simulation-result provenance chain (H5) — `SimulationProvenanceRecord` dataclass with run_id, module_id, module_version, timestamp_utc, input_hash (SHA-256 of input sequence), result_hash (SHA-256 of sorted scores dict), calibration_set, notes, and dry_lab_only. `make_provenance_record()` computes input_hash and result_hash deterministically (sort_keys=True for result_hash). `validate_provenance_record()` checks run_id, module_id, module_version non-empty; ISO 8601 timestamp contains 'T'; hashes are 64-char hex; dry_lab_only is True. `provenance_summary()` aggregates total, unique modules, run_ids, and dry_lab_only flag. CLI (`openamp-foundry simulation-provenance`) with `--run-id`, `--module-id`, `--module-version`, `--timestamp-utc`, `--input-sequence`, `--scores-json`, `--calibration-set`, `--format text|json`. `make simulation-provenance` target with demo invocation. 19 tests. **3181 total.** Every simulation result carries a traceable provenance chain so results can be audited, reproduced, or invalidated later without relying on memory.
Expand Down Expand Up @@ -86,7 +87,7 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak
> **New in v0.5.31:** Added dipeptide-order features for sequence-order awareness. `dipeptide_order_score` achieves AUROC 0.7861 on AMP-vs-scrambled discrimination — the strongest order-dependent feature in the pipeline. Only 7/31 features survive scrambling (amphipathicity/helix-wheel + dipeptide). All composition features are purely position-independent (exactly 0.5000 AUROC on scrambled test).
> **New in v0.5.30:** Easy baseline benchmark added — charge density alone (AUROC 0.8166) outperforms the full pipeline ensemble (0.7792) on AMP-vs-Swiss-Prot-decoy discrimination. Honest finding documented: expected because pipeline optimizes for safety, not raw discrimination.
> **New in v0.5.29:** Expanded benchmark to 500 AMPs + 500 composition-matched decoys (n=1000). AUROC 0.7792 (CI₉₅: 0.7505–0.8065) confirms signal generalizes. Cluster-aware CI: 0.746–0.8102. Representative AUROC: 0.778. Standard benchmark (n=191) retained for backward comparison.
> **Pipeline version:** v0.5.95 (H7 — simulation-result confidence interval reporter)
> **Pipeline version:** v0.5.96 (H8 — simulation-module deprecation enforcer)
> **Branch:** main

---
Expand Down Expand Up @@ -1156,7 +1157,7 @@ Decoys score low on activity. Selective AMPs score moderately on both.

| Metric | Value |
|--------|-------|
| Total tests | 3201 |
| Total tests | 3238 |
| Coverage (branch) | 99% (6 CLI guard lines only) |
| Source modules at 100% | All pipeline, QC, scoring modules |

Expand Down
46 changes: 46 additions & 0 deletions docs/research/ROADMAP.md
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,52 @@ Honest boundaries:
not invalidate results; it highlights them for human review.
- All CI reports are dry-lab only and must not be presented as biological proof.

## v0.5.96 — Loop 96: Phase H H8 — Simulation-Module Deprecation Enforcer ✓ (2026-07-09)

`DeprecationCheckResult` dataclass (5 fields: module_id, status, is_blocked,
block_reason, dry_lab_only). `BLOCKED_STATUSES = {"deprecated", "unavailable"}`.
`check_module_deprecation()` looks up module in registry: not-found returns
is_blocked=True status="unknown"; deprecated/unavailable returns is_blocked=True
with reason; active/experimental returns is_blocked=False.
`enforce_deprecation()` filters list[SimulationResult] to only non-blocked
modules, returns dict with total_input/passed/blocked/blocked_modules/
passed_results/checks/dry_lab_only. `run_deprecation_check_batch()` bulk-checks
module_ids with total/blocked/allowed/any_blocked/results/dry_lab_only.
CLI (`openamp-foundry simulation-deprecation-check`) with `--module-ids`,
`--format text|json`. `make simulation-deprecation-check` target with demo
invocation. Deprecated simulation modules must not be used in production
scoring — the enforcer prevents stale or unreliable modules from tainting
evidence packets.

Changes:
- `src/openamp_foundry/simulation/deprecation_enforcer.py` (H8) — Core
module with `DeprecationCheckResult` dataclass, `check_module_deprecation()`,
`enforce_deprecation()`, `run_deprecation_check_batch()`.
- `src/openamp_foundry/simulation/__init__.py` — Exports `BLOCKED_STATUSES`,
`DeprecationCheckResult`, `check_module_deprecation`, `enforce_deprecation`,
`run_deprecation_check_batch`.
- `src/openamp_foundry/cli/main.py` — Registered `simulation-deprecation-check`
subcommand with `--module-ids`, `--format` flags. Added import and dispatch.
- `src/openamp_foundry/cli/commands/reports.py` — Added
`_run_simulation_deprecation_check()` CLI handler.
- `Makefile` — Added `simulation-deprecation-check` target. Added to `.PHONY`.
- `tests/simulation/test_deprecation_enforcer.py` — 21 tests covering: active
module not blocked, deprecated/unavailable blocked, unknown blocked with
status="unknown", dry_lab_only always True, enforce_deprecation filters
results, passed count, blocked_modules sorted/deduplicated, dry_lab_only flag,
all-allowed blocked=0, batch total/any_blocked/not blocked.
- `docs/evidence/METRICS_CURRENT.md` — v0.5.96 H8 changelog. Test count: 3238.
- `tests/test_test_count_regression.py` — baseline updated to 3238.

Honest boundaries:
- Deprecation checks are based on registry status, not biological validity.
A module may be "active" in the registry but still produce unreliable scores.
- Blocking removes results from evidence packets but does not invalidate them
as research data — blocked results may still be useful for internal analysis.
- The enforcer does not evaluate whether a module's underlying science is sound;
it enforces the pipeline's declared module lifecycle policy.
- All outputs are dry-lab only and must not be presented as biological proof.

## v0.5.94 — Loop 94: Phase H H6 — Simulation-Ensemble Agreement Checker ✓ (2026-07-09)

`EnsembleAgreementResult` dataclass (9 fields: sequence, modules_checked,
Expand Down
37 changes: 37 additions & 0 deletions src/openamp_foundry/cli/commands/reports.py
Original file line number Diff line number Diff line change
Expand Up @@ -1704,3 +1704,40 @@ def _run_simulation_provenance(args: argparse.Namespace) -> int:
print("not biological proof.")

return 0


def _run_simulation_deprecation_check(args: argparse.Namespace) -> int:
"""Check whether simulation modules are deprecated or unavailable."""
import json

from openamp_foundry.simulation.deprecation_enforcer import (
run_deprecation_check_batch,
)

module_ids = [m.strip() for m in args.module_ids.split(",") if m.strip()]
if not module_ids:
print(json.dumps({"status": "error", "error": "No module IDs provided."}))
return 2

result = run_deprecation_check_batch(module_ids)
output_format = getattr(args, "format", "text")

if output_format == "json":
print(json.dumps(result, indent=2))
else:
print(f"Simulation Deprecation Check")
print(f" Total modules: {result['total']}")
print(f" Blocked: {result['blocked']}")
print(f" Allowed: {result['allowed']}")
print(f" Any blocked: {result['any_blocked']}")
print()
for check in result["results"]:
status = "BLOCKED" if check["is_blocked"] else "ALLOWED"
print(f" [{status:7}] {check['module_id']:35s} "
f"status={check['status']:15s} "
f"{check['block_reason']}")
print()
print("Dry-lab only. Deprecation checks are computational safeguards,")
print("not biological proof.")

return 3 if result["any_blocked"] else 0
24 changes: 23 additions & 1 deletion src/openamp_foundry/cli/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
from openamp_foundry.cli.commands.selection import _run_pilot_panel, _run_pilot_confident, _run_diversity_check, _run_select_batch, _run_batch_rationale
from openamp_foundry.cli.commands.external import _run_external_predict, _run_external_consensus
from openamp_foundry.cli.commands.qc import _run_synthesis_order, _run_presynth_qc
from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter, _run_synthetic_result_policy_check, _run_calibration_decision_checklist, _run_calibration_rollback_plan, _run_simulation_registry, _run_validate_simulation_result, _run_simulation_baseline_check, _run_adapter_gate_check, _run_simulation_provenance, _run_simulation_ensemble_check, _run_simulation_ci_report
from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter, _run_synthetic_result_policy_check, _run_calibration_decision_checklist, _run_calibration_rollback_plan, _run_simulation_registry, _run_validate_simulation_result, _run_simulation_baseline_check, _run_adapter_gate_check, _run_simulation_provenance, _run_simulation_ensemble_check, _run_simulation_ci_report, _run_simulation_deprecation_check
from openamp_foundry.cli.commands.gates import _run_gate_check

import argparse
Expand Down Expand Up @@ -1519,6 +1519,25 @@ def build_parser() -> argparse.ArgumentParser:
help="Output format (default: text).",
)

# ── Simulation deprecation check (H8) ────────────────────────────
sdc = sub.add_parser(
"simulation-deprecation-check",
help=(
"Check whether simulation modules are deprecated or unavailable. "
"Blocks deprecated modules from contributing to evidence packets. "
"Dry-lab only."
),
)
sdc.add_argument(
"--module-ids", type=str, required=True,
help="Comma-separated module IDs (e.g. membrane_proxy,dummy_membrane_proxy).",
)
sdc.add_argument(
"--format", type=str, default="text",
choices=["text", "json"],
help="Output format (default: text).",
)

# ── Simulation-result confidence interval report (H7) ────────────
sicr = sub.add_parser(
"simulation-ci-report",
Expand Down Expand Up @@ -1718,6 +1737,9 @@ def main(argv: list[str] | None = None) -> int:
if args.command == "simulation-ci-report":
return _run_simulation_ci_report(args)

if args.command == "simulation-deprecation-check":
return _run_simulation_deprecation_check(args)

parser.error("unknown command")
return 2

Expand Down
12 changes: 12 additions & 0 deletions src/openamp_foundry/simulation/__init__.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,12 @@
"""Virtual assay simulation layer for OpenAMP Foundry."""

from .deprecation_enforcer import (
BLOCKED_STATUSES,
DeprecationCheckResult,
check_module_deprecation,
enforce_deprecation,
run_deprecation_check_batch,
)
from .adapter_gate import (
AdapterGateResult,
FAIL_CLOSED_REASONS,
Expand Down Expand Up @@ -56,6 +63,11 @@
from .structure import StructureProxy

__all__ = [
"BLOCKED_STATUSES",
"DeprecationCheckResult",
"check_module_deprecation",
"enforce_deprecation",
"run_deprecation_check_batch",
"AdapterGateResult",
"FAIL_CLOSED_REASONS",
"evaluate_adapter_gate",
Expand Down
Loading
Loading