Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan
.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry

PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3)
PYTEST := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest)
Expand Down Expand Up @@ -713,3 +713,7 @@ calibration-rollback-plan:
--out-json /tmp/rollback_plan.json \
--out-md /tmp/rollback_plan.md
@echo "Rollback plan generated. See /tmp/rollback_plan.json"

simulation-registry:
.venv/bin/openamp-foundry simulation-registry --list
@echo "Simulation module registry displayed."
6 changes: 3 additions & 3 deletions docs/evidence/METRICS_CURRENT.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak
> **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees
> with this file, this file wins. Updated whenever benchmark/benchmark config changes.
>
> **Last updated:** 2026-07-09 (Phase G G10 — recalibration rollback plan — v0.5.88)
> **New in v0.5.88:** Recalibration rollback plan (G10) — `build_rollback_plan()` produces a structured `RollbackPlan` with 5 rollback triggers (RT-01 through RT-05) covering AUROC regression, calibration metric below threshold, safety filter bypass, cohort distribution shift, and unexplained weight delta. Each trigger has id, name, description, threshold, and severity. `RollbackStep` dataclass with step_number, action, responsible, detail, and dry_lab_only. `RollbackPlan` dataclass with plan_id, version, triggered_by, steps, notes, dry_lab_only, and to_dict(). 6 default rollback steps: halt, document, restore, verify, root-cause, and log. CLI (`openamp-foundry calibration-rollback-plan`) accepts `--plan-id`, `--version`, `--triggered-by`, `--notes`, `--out-json`, `--out-md`. JSON + Markdown output. Schema (`schemas/calibration_rollback_plan.schema.json`). `make calibration-rollback-plan` target. 15 tests. 3078 total. This completes Phase G (G1-G10 — calibration and active-learning rigor).
> **Last updated:** 2026-07-09 (Phase H H1 — simulation module registry — v0.5.89)
> **New in v0.5.89:** Simulation module registry (H1) — `SimulationModuleEntry` dataclass tracks module_id, name, description, status, evidence_level, baseline_comparison, scope, maintainer, and notes. `SIMULATION_MODULE_REGISTRY` holds 4 entries (membrane_proxy, structure_proxy, dummy_membrane_proxy, external_adapter_placeholder). Lookup functions: `get_module_entry()`, `list_module_entries()` with status/min_evidence filtering, `get_active_modules()`, `registry_summary()` with total/by_status/by_evidence_level/active_module_ids keys. `validate_registry()` checks module_id, name, baseline_comparison, evidence_level 1-6, valid status, duplicate detection. CLI (`openamp-foundry simulation-registry`) supports `--list`, `--show`, `--status`, `--min-evidence`, `--format text|json`. Schema (`schemas/simulation_module_registry.schema.json`). `make simulation-registry` target. 28 tests. 3106 total. Starts Phase H (virtual assay discipline).
> **New in v0.5.87:** Calibration decision review checklist (G9) — `build_checklist()` produces a structured `CalibrationDecisionChecklist` with 12 checklist items (10 required) covering data quality, statistical validity, safety consistency, approval, and documentation. Each item has an id, category, question, rationale, and required flag. `CalibrationDecisionChecklist` dataclass tracks responses, notes, overall_pass, and missing_required. CLI (`openamp-foundry calibration-decision-checklist`) accepts `--checklist-id`, `--date`, `--reviewer`, `--responses-json`, `--out-json`, `--out-md`. JSON + Markdown output. Schema (`schemas/calibration_decision_checklist.schema.json`). `make calibration-decision-checklist` target. 14 tests. 3063 total. Makes human review structured and auditable.
> **New in v0.5.86:** Synthetic-result policy enforcement (G8) — `check_synthetic_result_policy()` and `run_policy_batch()` enforce that synthetic/simulation outputs cannot raise the proof-ladder level of a candidate. Levels 4+ require wet-lab evidence; synthetic/unknown sources are rejected for such proposals. CLI (`openamp-foundry synthetic-result-policy-check`) accepts `--proposals-json`, `--out-json`, `--out-md`. Schema (`schemas/synthetic_result_policy_check.schema.json`). `make synthetic-result-policy-check` target. 27 tests. 3049 total. Anti-overclaim safeguard.
> **New in v0.5.85:** Result-quality flag propagation (G7) — `assess_result_quality()` and `filter_results_for_calibration()` propagate result-quality flags into the calibration engine. Candidates with contamination or assay interference are excluded; candidates with 2+ quality flags are excluded; candidates with 1 flag are included with caution; clean candidates are fully included. `QUALITY_FLAGS` dictionary (8 flags). `ResultQualityReport` dataclass. CLI (`openamp-foundry result-quality-filter`) accepts `--results-json`, `--out-json`, `--out-md`. JSON + Markdown output. Schema (`schemas/result_quality_report.schema.json`). `make result-quality-filter` target. 27 tests. 3022 total.
Expand Down Expand Up @@ -80,7 +80,7 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak
> **New in v0.5.31:** Added dipeptide-order features for sequence-order awareness. `dipeptide_order_score` achieves AUROC 0.7861 on AMP-vs-scrambled discrimination — the strongest order-dependent feature in the pipeline. Only 7/31 features survive scrambling (amphipathicity/helix-wheel + dipeptide). All composition features are purely position-independent (exactly 0.5000 AUROC on scrambled test).
> **New in v0.5.30:** Easy baseline benchmark added — charge density alone (AUROC 0.8166) outperforms the full pipeline ensemble (0.7792) on AMP-vs-Swiss-Prot-decoy discrimination. Honest finding documented: expected because pipeline optimizes for safety, not raw discrimination.
> **New in v0.5.29:** Expanded benchmark to 500 AMPs + 500 composition-matched decoys (n=1000). AUROC 0.7792 (CI₉₅: 0.7505–0.8065) confirms signal generalizes. Cluster-aware CI: 0.746–0.8102. Representative AUROC: 0.778. Standard benchmark (n=191) retained for backward comparison.
> **Pipeline version:** v0.5.88 (G10 — Phase G complete)
> **Pipeline version:** v0.5.89 (H1 — Phase H started)
> **Branch:** main

---
Expand Down
61 changes: 61 additions & 0 deletions docs/research/ROADMAP.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,66 @@
# Roadmap

## v0.5.89 — Loop 89: Phase H H1 — Simulation Module Registry ✓ (2026-07-09)

`SIMULATION_MODULE_REGISTRY` with 4 entries (membrane_proxy, structure_proxy,
dummy_membrane_proxy, external_adapter_placeholder). `SimulationModuleEntry`
dataclass tracks module_id, name, description, status, evidence_level,
baseline_comparison, scope, maintainer, and notes. Lookup functions:
`get_module_entry()`, `list_module_entries()` with status/min_evidence
filtering, `get_active_modules()`, `registry_summary()` with
total/by_status/by_evidence_level/active_module_ids keys.
`validate_registry()` checks module_id, name, baseline_comparison,
evidence_level 1-6, valid status, duplicate detection.
CLI (`openamp-foundry simulation-registry`) with `--list`, `--show`,
`--status`, `--min-evidence`, `--format text|json`.
Schema (`schemas/simulation_module_registry.schema.json`).
`make simulation-registry` target. 28 tests. 3106 total.
Starts Phase H (virtual assay discipline).

Changes:
- `src/openamp_foundry/simulation/module_registry.py` (H1) — Core module with
`SimulationModuleStatus` literal type, `SimulationModuleEntry` dataclass
(9 fields), `SIMULATION_MODULE_REGISTRY` list (4 entries),
`get_module_entry()`, `list_module_entries()`, `get_active_modules()`,
`registry_summary()` with aggregation, `validate_registry()` with 6 checks.
Imports `PROOF_LADDER_LEVELS` from `evidence/synthetic_result_policy.py`.
- `schemas/simulation_module_registry.schema.json` (H1) — JSON Schema Draft
2020-12 for the registry_summary() output. Validates total, by_status
(all 4 status keys), by_evidence_level, active_module_ids.
- `src/openamp_foundry/simulation/__init__.py` — Exports all module registry
symbols.
- `src/openamp_foundry/cli/main.py` — Registered `simulation-registry`
subcommand with `--list`, `--show`, `--status`, `--min-evidence`,
`--format` flags.
- `src/openamp_foundry/cli/commands/reports.py` — Added
`_run_simulation_registry()` CLI handler with text and JSON output,
per-module details, status/evidence filtering, and validation.
- `Makefile` — Added `simulation-registry` target with `--list` default.
Added to `.PHONY`.
- `tests/simulation/test_module_registry.py` — 28 tests covering: registry
size ≥ 4, all entries pass validation, required field presence, evidence
level range 1-6, valid status values, get_module_entry known/unknown,
list_module_entries no filter/status filter/min_evidence filter,
get_active_modules returns only active, registry_summary keys and totals,
validate_registry detection of empty fields, invalid level, invalid
status, duplicate ids, PROOF_LADDER_LEVELS completeness.
- `docs/evidence/METRICS_CURRENT.md` — v0.5.89 H1 changelog. Test count: 3106.
- `tests/test_test_count_regression.py` — baseline updated to 3106.

Honest boundaries:
- The registry lists module status and evidence level for informational
purposes only. It does not measure biological activity, safety, or
real-world performance.
- Registry validation checks structural correctness, not scientific validity.
A valid entry may still produce biologically meaningless results.
- The PROOF_LADDER_LEVELS mapping is a claim-level taxonomy. An evidence_level
of 2 ("virtual-assay support") means the module supports computational
exploration — it does not constitute biological proof.
- The registry is dry-lab only. All module entries carry dry-lab caveats
regardless of their status or evidence_level.
- "active" status means the module is available for use, not that it has
been biologically validated.

## v0.5.88 — Loop 88: Phase G G10 — Recalibration Rollback Plan ✓ (2026-07-09)

`build_rollback_plan()` produces a structured `RollbackPlan` with 5 rollback
Expand Down
39 changes: 39 additions & 0 deletions schemas/simulation_module_registry.schema.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://openamp.ai/schemas/simulation_module_registry.schema.json",
"title": "Simulation Module Registry Summary",
"description": "Schema for the registry_summary() output. Dry-lab only — tracks module status and evidence level, not biological activity.",
"type": "object",
"properties": {
"total": {
"description": "Total number of registered modules",
"type": "integer",
"minimum": 0
},
"by_status": {
"description": "Count of modules per status level",
"type": "object",
"properties": {
"active": { "type": "integer", "minimum": 0 },
"experimental": { "type": "integer", "minimum": 0 },
"deprecated": { "type": "integer", "minimum": 0 },
"unavailable": { "type": "integer", "minimum": 0 }
},
"required": ["active", "experimental", "deprecated", "unavailable"]
},
"by_evidence_level": {
"description": "Count of modules per proof-ladder evidence level (1-6)",
"type": "object",
"patternProperties": {
"^[1-6]$": { "type": "integer", "minimum": 0 }
}
},
"active_module_ids": {
"description": "List of module IDs with status=active",
"type": "array",
"items": { "type": "string" }
}
},
"required": ["total", "by_status", "by_evidence_level", "active_module_ids"],
"additionalProperties": false
}
108 changes: 108 additions & 0 deletions src/openamp_foundry/cli/commands/reports.py
Original file line number Diff line number Diff line change
Expand Up @@ -1230,3 +1230,111 @@ def _run_calibration_rollback_plan(args: argparse.Namespace) -> int:
"out_md": args.out_md,
}, indent=2))
return 0


def _run_simulation_registry(args: argparse.Namespace) -> int:
"""Display simulation module registry information."""
from openamp_foundry.simulation.module_registry import (
SIMULATION_MODULE_REGISTRY,
get_module_entry,
list_module_entries,
registry_summary,
VALID_STATUSES,
)

PROOF_LADDER_LABELS = {
1: "computational nomination",
2: "virtual-assay support",
3: "in-silico ensemble agreement",
4: "ex-vivo preliminary",
5: "in-vivo preliminary",
6: "clinical evidence",
}

show = getattr(args, "show", None)
status_filter = getattr(args, "status", None)
min_evidence = getattr(args, "min_evidence", None)
output_format = getattr(args, "format", "text")

if show:
entry = get_module_entry(show)
if entry is None:
print(json.dumps({"status": "error", "error": f"Unknown module_id: '{show}'"}))
return 3
if output_format == "json":
print(json.dumps({
"module_id": entry.module_id,
"name": entry.name,
"description": entry.description,
"status": entry.status,
"evidence_level": entry.evidence_level,
"evidence_label": PROOF_LADDER_LABELS.get(entry.evidence_level, "unknown"),
"baseline_comparison": entry.baseline_comparison,
"scope": entry.scope,
"maintainer": entry.maintainer,
"notes": entry.notes,
}, indent=2))
else:
label = PROOF_LADDER_LABELS.get(entry.evidence_level, "unknown")
print(f"Module: {entry.module_id}")
print(f" Name: {entry.name}")
print(f" Description: {entry.description}")
print(f" Status: {entry.status}")
print(f" Evidence Level: {entry.evidence_level} ({label})")
print(f" Baseline: {entry.baseline_comparison}")
print(f" Scope: {', '.join(entry.scope)}")
print(f" Maintainer: {entry.maintainer}")
if entry.notes:
print(f" Notes: {entry.notes}")
return 0

if status_filter:
if status_filter not in VALID_STATUSES:
print(json.dumps({
"status": "error",
"error": f"Invalid status '{status_filter}'. Valid: {', '.join(sorted(VALID_STATUSES))}"
}))
return 3
entries = list_module_entries(status=status_filter)
elif min_evidence is not None:
entries = list_module_entries(min_evidence_level=min_evidence)
else:
entries = list(SIMULATION_MODULE_REGISTRY)

summary = registry_summary()

if output_format == "json":
print(json.dumps({
"modules": [
{
"module_id": e.module_id,
"name": e.name,
"status": e.status,
"evidence_level": e.evidence_level,
"evidence_label": PROOF_LADDER_LABELS.get(e.evidence_level, "unknown"),
"baseline_comparison": e.baseline_comparison,
"scope": e.scope,
"maintainer": e.maintainer,
"notes": e.notes,
}
for e in entries
],
"summary": summary,
}, indent=2))
else:
print("Simulation Module Registry")
print("=" * 60)
print(f"Total modules: {summary['total']}")
print(f"By status: {dict(sorted(summary['by_status'].items()))}")
print(f"By evidence: {dict(sorted(summary['by_evidence_level'].items()))}")
print(f"Active IDs: {summary['active_module_ids']}")
print()
for entry in entries:
label = PROOF_LADDER_LABELS.get(entry.evidence_level, "unknown")
print(f" [{entry.status:13}] {entry.module_id:35s} "
f"L{entry.evidence_level} {label}")
print()
print("Dry-lab only. Registry status is informational — it does not")
print("measure biological activity, safety, or real-world performance.")

return 0
Loading