diff --git a/Makefile b/Makefile index bd001908..5bad16ce 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report simulation-deprecation-check simulation-scope-check simulation-evidence-packet artifact-version +.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report simulation-deprecation-check simulation-scope-check simulation-evidence-packet artifact-version candidate-manifest PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3) PYTEST := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest) @@ -780,3 +780,15 @@ simulation-evidence-packet: artifact-version: .venv/bin/openamp-foundry artifact-version --list @echo "Artifact version list complete." + +candidate-manifest: + .venv/bin/openamp-foundry candidate-manifest \ + --candidate-id AMP-001 \ + --sequence AKLWKR \ + --evidence-level 2 \ + --scopes bacterial_binding \ + --scores-json '{"binding_energy": 0.75}' \ + --uncertainty 0.1 \ + --source-modules membrane_proxy \ + --validate + @echo "Candidate manifest complete." diff --git a/docs/evidence/METRICS_CURRENT.md b/docs/evidence/METRICS_CURRENT.md index f1efb716..593a323c 100644 --- a/docs/evidence/METRICS_CURRENT.md +++ b/docs/evidence/METRICS_CURRENT.md @@ -5,7 +5,8 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees > with this file, this file wins. Updated whenever benchmark/benchmark config changes. > -> **Last updated:** 2026-07-09 (Phase I I1 — artifact versioning policy — v0.5.99) +> **Last updated:** 2026-07-09 (Phase I I2 — candidate manifest schema — v0.6.0) +> **New in v0.6.0:** Candidate manifest schema (I2 — Loop 100) — `schemas/candidate_manifest.schema.json` (Draft 2020-12, 14 required fields: candidate_id, sequence, evidence_level, scopes, scores, uncertainty, source_modules, calibration_set, safety_flags, provenance_run_id, dry_lab_only, version, created_at, notes). `$schema`, `$id`, `title`, `description`, `additionalProperties: false`. `src/openamp_foundry/manifests/` module with `CandidateManifest` dataclass (14 fields), `make_candidate_manifest()`, `validate_candidate_manifest()` (8 checks: non-empty candidate_id, non-empty sequence, evidence_level 1-6, non-empty scopes, uncertainty 0.0-1.0, non-empty source_modules, dry_lab_only must be True, version MAJOR.MINOR.PATCH), `manifest_to_dict()`, `manifest_summary()` (total, by_evidence_level, with_safety_flags, dry_lab_only). CLI (`openamp-foundry candidate-manifest`) with `--candidate-id`, `--sequence`, `--evidence-level`, `--scopes`, `--scores-json`, `--uncertainty`, `--source-modules`, `--validate`, `--format text|json`. `make candidate-manifest` target. 19 tests. **3310 total.** A candidate manifest is the core interoperable artifact — it describes a dry-lab candidate (sequence, scores, evidence level, scopes, safety flags, provenance) in a machine-readable format that external tools can consume without the full OpenAMP stack. > **New in v0.5.99:** Artifact versioning policy (I1 — starts Phase I) — `docs/engineering/ARTIFACT_VERSIONING_POLICY.md` with structured versioning policy covering scope, SemVer format, breaking/non-breaking change definitions, deprecation timeline (at least one minor version warning), schema `$id` policy, changelog requirement, and three stability tiers (stable/experimental/internal). `src/openamp_foundry/versioning/` module with `ArtifactVersionInfo` dataclass (7 fields), `STABILITY_TIERS` dict, `VERSIONED_ARTIFACTS` list (6 entries covering candidate, lab_result, run_manifest, external_review_packet, safety_release_decision, simulation_result at v1.0.0), `get_artifact_version()`, `list_versioned_artifacts()`, `validate_version_format()`, `artifact_version_summary()`. CLI (`openamp-foundry artifact-version`) with `--list`, `--show`, `--tier`, `--format text|json`. `make artifact-version` target. 19 tests. **3291 total.** Starts Phase I (interoperability) — external users now have stability guarantees for schemas, evidence certificates, and candidate manifests. > **New in v0.5.97:** Simulation-scope coverage checker (H9) — `ScopeCoverageResult` dataclass (8 fields: module_id, requested_scopes, module_scopes, covered, uncovered, coverage_fraction, is_fully_covered, dry_lab_only). `check_scope_coverage()` looks up module in registry, computes covered (intersection) and uncovered (requested scopes not in module scopes), coverage_fraction = len(covered)/len(requested_scopes) if requested else 1.0. `check_result_scope()` uses conservative intersection of registry scopes and result.scope as effective module_scopes. `scope_coverage_report()` returns full dict. CLI (`openamp-foundry simulation-scope-check`) with `--module-id`, `--requested-scopes`, `--format text|json`. `make simulation-scope-check` target. 17 tests. **3255 total.** A simulation module may cover only some biological scopes. If a candidate is evaluated for a scope the module doesn't cover, that result must be flagged as out-of-scope rather than silently trusted. > **New in v0.5.96:** Simulation-module deprecation enforcer (H8) — `DeprecationCheckResult` dataclass (5 fields: module_id, status, is_blocked, block_reason, dry_lab_only). `BLOCKED_STATUSES = {"deprecated", "unavailable"}`. `check_module_deprecation()` looks up module in registry: not-found returns is_blocked=True status="unknown"; deprecated/unavailable returns is_blocked=True with reason; active/experimental returns is_blocked=False. `enforce_deprecation()` filters list[SimulationResult] to only non-blocked modules, returns total_input/passed/blocked/blocked_modules/passed_results/checks/dry_lab_only. `run_deprecation_check_batch()` bulk-checks module_ids with total/blocked/allowed/any_blocked/results/dry_lab_only. CLI (`openamp-foundry simulation-deprecation-check`) with `--module-ids`, `--format text|json`. `make simulation-deprecation-check` target. 21 tests. **3238 total.** Deprecated simulation modules must not be used in production scoring — the enforcer prevents stale or unreliable modules from tainting evidence packets. @@ -89,7 +90,7 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **New in v0.5.31:** Added dipeptide-order features for sequence-order awareness. `dipeptide_order_score` achieves AUROC 0.7861 on AMP-vs-scrambled discrimination — the strongest order-dependent feature in the pipeline. Only 7/31 features survive scrambling (amphipathicity/helix-wheel + dipeptide). All composition features are purely position-independent (exactly 0.5000 AUROC on scrambled test). > **New in v0.5.30:** Easy baseline benchmark added — charge density alone (AUROC 0.8166) outperforms the full pipeline ensemble (0.7792) on AMP-vs-Swiss-Prot-decoy discrimination. Honest finding documented: expected because pipeline optimizes for safety, not raw discrimination. > **New in v0.5.29:** Expanded benchmark to 500 AMPs + 500 composition-matched decoys (n=1000). AUROC 0.7792 (CI₉₅: 0.7505–0.8065) confirms signal generalizes. Cluster-aware CI: 0.746–0.8102. Representative AUROC: 0.778. Standard benchmark (n=191) retained for backward comparison. -> **Pipeline version:** v0.5.99 (I1 — artifact versioning policy) +> **Pipeline version:** v0.6.0 (I2 — candidate manifest schema) > **Branch:** main --- diff --git a/docs/research/ROADMAP.md b/docs/research/ROADMAP.md index 37b6f548..22226d98 100644 --- a/docs/research/ROADMAP.md +++ b/docs/research/ROADMAP.md @@ -1,5 +1,72 @@ # Roadmap +## v0.6.0 — Loop 100: Phase I I2 — Candidate Manifest Schema ✓ (2026-07-09) + +`schemas/candidate_manifest.schema.json` (Draft 2020-12, 14 required fields: +candidate_id, sequence, evidence_level, scopes, scores, uncertainty, +source_modules, calibration_set, safety_flags, provenance_run_id, dry_lab_only, +version, created_at, notes). `$schema`, `$id`, `title`, `description`, +`additionalProperties: false`. + +`src/openamp_foundry/manifests/` module with `CandidateManifest` dataclass +(14 fields), `make_candidate_manifest()`, `validate_candidate_manifest()` +(8 checks: non-empty candidate_id, non-empty sequence, evidence_level 1-6, +non-empty scopes, uncertainty 0.0-1.0, non-empty source_modules, +dry_lab_only must be True, version MAJOR.MINOR.PATCH), `manifest_to_dict()`, +`manifest_summary()` (total, by_evidence_level, with_safety_flags, dry_lab_only). + +CLI (`openamp-foundry candidate-manifest`) with `--candidate-id`, `--sequence`, +`--evidence-level`, `--scopes`, `--scores-json`, `--uncertainty`, +`--source-modules`, `--validate`, `--format text|json`. +`make candidate-manifest` target. + +19 tests. **3310 total.** A candidate manifest is the core interoperable +artifact — it describes a dry-lab candidate (sequence, scores, evidence level, +scopes, safety flags, provenance) in a machine-readable format that external +tools can consume without the full OpenAMP stack. This is Loop 100 — the +100th PR in the NEXT_100_PR_MAP series. + +Changes: +- `schemas/candidate_manifest.schema.json` (I2) — Draft 2020-12 schema with + 14 required fields, `$id`, `title`, `description`, `additionalProperties: false`. +- `src/openamp_foundry/manifests/__init__.py` (I2) — Package init, exports + `CandidateManifest`, `make_candidate_manifest`, `validate_candidate_manifest`, + `manifest_to_dict`, `manifest_summary`. +- `src/openamp_foundry/manifests/candidate_manifest.py` (I2) — Core module + with `CandidateManifest` dataclass (14 fields), `make_candidate_manifest()`, + `validate_candidate_manifest()`, `manifest_to_dict()`, `manifest_summary()`. +- `src/openamp_foundry/cli/main.py` (I2) — Registered `candidate-manifest` + subcommand with `--candidate-id`, `--sequence`, `--evidence-level`, + `--scopes`, `--scores-json`, `--uncertainty`, `--source-modules`, + `--validate`, `--format` flags. Added import and dispatch. +- `src/openamp_foundry/cli/commands/reports.py` (I2) — Added + `_run_candidate_manifest()` CLI handler with JSON and text output, + manifest creation, optional validation, and error handling. +- `Makefile` (I2) — Added `candidate-manifest` target with demo invocation + using AMP-001. Added to `.PHONY`. +- `tests/manifests/__init__.py` (I2) — Empty package init. +- `tests/manifests/test_candidate_manifest.py` (I2) — 19 tests covering: + make_candidate_manifest returns CandidateManifest, dry_lab_only always True, + valid manifest passes validation, empty candidate_id fails, empty sequence + fails, evidence_level 0 or 7 fails, empty scopes fails, uncertainty out of + range fails, empty source_modules fails, dry_lab_only=False fails, + invalid version fails, manifest_to_dict returns all fields, manifest_summary + total correct, with_safety_flags correct, dry_lab_only=True, by_evidence_level + correct, schema file exists. +- `docs/evidence/METRICS_CURRENT.md` — v0.6.0 I2 changelog. Test count: 3310. +- `tests/test_test_count_regression.py` — baseline updated to 3310. + +Honest boundaries: +- Candidate manifests describe dry-lab candidates only — they are computational + artifacts, not biological proof. +- Scores and uncertainty are pipeline outputs, not measured biological properties. +- The manifest schema is versioned (currently 1.0.0) and should be updated when + fields are added or changed. +- Validation checks structural correctness only — it does not verify biological + plausibility, safety, or efficacy. +- `dry_lab_only: true` is a const field — all manifests are inherently dry-lab + and must never be presented as validated wet-lab candidates. + ## v0.5.95 — Loop 95: Phase H H7 — Simulation-Result Confidence Interval Reporter ✓ (2026-07-09) `ScoreCI` dataclass (9 fields: module_id, score_key, point_estimate, uncertainty, diff --git a/schemas/candidate_manifest.schema.json b/schemas/candidate_manifest.schema.json new file mode 100644 index 00000000..0517b968 --- /dev/null +++ b/schemas/candidate_manifest.schema.json @@ -0,0 +1,100 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://openamp-foundry.org/schemas/candidate_manifest.schema.json", + "title": "CandidateManifest", + "description": "Schema for a dry-lab candidate manifest — a machine-readable artifact describing a candidate AMP sequence, its scores, evidence level, scopes, safety flags, and provenance. Dry-lab only — does not measure biological activity.", + "type": "object", + "properties": { + "candidate_id": { + "description": "Unique identifier for the candidate", + "type": "string", + "minLength": 1 + }, + "sequence": { + "description": "Amino acid sequence of the candidate AMP", + "type": "string", + "minLength": 1 + }, + "evidence_level": { + "description": "Proof-ladder evidence level (1=computational nomination, 2=virtual-assay support, 3=in-silico ensemble agreement, 4=ex-vivo preliminary, 5=in-vivo preliminary, 6=clinical evidence)", + "type": "integer", + "minimum": 1, + "maximum": 6 + }, + "scopes": { + "description": "List of biological scopes this candidate is assessed for (e.g. bacterial_binding, anti_fungal)", + "type": "array", + "items": { "type": "string" }, + "minItems": 1 + }, + "scores": { + "description": "Dictionary of named score values from pipeline modules", + "type": "object", + "patternProperties": { + "^.*$": { "type": "number" } + } + }, + "uncertainty": { + "description": "Overall uncertainty estimate (0.0 = fully certain, 1.0 = fully uncertain)", + "type": "number", + "minimum": 0.0, + "maximum": 1.0 + }, + "source_modules": { + "description": "List of pipeline modules or scorers that produced this manifest", + "type": "array", + "items": { "type": "string" }, + "minItems": 1 + }, + "calibration_set": { + "description": "Optional identifier for the calibration dataset used", + "type": ["string", "null"] + }, + "safety_flags": { + "description": "List of safety flags (empty array = no flags raised)", + "type": "array", + "items": { "type": "string" } + }, + "provenance_run_id": { + "description": "Optional run ID linking this manifest to a specific pipeline execution", + "type": ["string", "null"] + }, + "dry_lab_only": { + "description": "Must be true — this is a dry-lab computational artifact", + "type": "boolean", + "const": true + }, + "version": { + "description": "Manifest schema version (MAJOR.MINOR.PATCH)", + "type": "string", + "pattern": "^\\d+\\.\\d+\\.\\d+$" + }, + "created_at": { + "description": "ISO 8601 UTC timestamp of manifest creation", + "type": "string", + "format": "date-time" + }, + "notes": { + "description": "Free-text notes about this candidate", + "type": "array", + "items": { "type": "string" } + } + }, + "required": [ + "candidate_id", + "sequence", + "evidence_level", + "scopes", + "scores", + "uncertainty", + "source_modules", + "calibration_set", + "safety_flags", + "provenance_run_id", + "dry_lab_only", + "version", + "created_at", + "notes" + ], + "additionalProperties": false +} diff --git a/src/openamp_foundry/cli/commands/reports.py b/src/openamp_foundry/cli/commands/reports.py index 0d5281f1..ab86daaf 100644 --- a/src/openamp_foundry/cli/commands/reports.py +++ b/src/openamp_foundry/cli/commands/reports.py @@ -1936,3 +1936,81 @@ def _run_artifact_version(args: argparse.Namespace) -> int: print("See docs/engineering/ARTIFACT_VERSIONING_POLICY.md for policy details.") return 0 + + +def _run_candidate_manifest(args: argparse.Namespace) -> int: + """Build and optionally validate a candidate manifest from CLI arguments.""" + from openamp_foundry.manifests.candidate_manifest import ( + make_candidate_manifest, + manifest_to_dict, + validate_candidate_manifest, + ) + + try: + scores = json.loads(args.scores_json) + except (json.JSONDecodeError, TypeError) as e: + print(json.dumps({"status": "error", "error": f"Invalid --scores-json: {e}"})) + return 2 + + if not isinstance(scores, dict): + print(json.dumps({"status": "error", "error": "--scores-json must be a JSON object"})) + return 2 + + scopes = [s.strip() for s in args.scopes.split(",") if s.strip()] + source_modules = [s.strip() for s in args.source_modules.split(",") if s.strip()] + + try: + evidence_level = int(args.evidence_level) + except (TypeError, ValueError): + print(json.dumps({"status": "error", "error": "--evidence-level must be an integer"})) + return 2 + + manifest = make_candidate_manifest( + candidate_id=args.candidate_id, + sequence=args.sequence, + evidence_level=evidence_level, + scopes=scopes, + scores=scores, + uncertainty=args.uncertainty, + source_modules=source_modules, + ) + + validation_errors: list[str] = [] + if getattr(args, "validate", False): + validation_errors = validate_candidate_manifest(manifest) + + output_format = getattr(args, "format", "text") + if output_format == "json": + result = manifest_to_dict(manifest) + if validation_errors: + result["validation_errors"] = validation_errors + result["valid"] = len(validation_errors) == 0 + print(json.dumps(result, indent=2)) + else: + print(f"Candidate Manifest: {manifest.candidate_id}") + print(f" Sequence: {manifest.sequence}") + print(f" Evidence level: {manifest.evidence_level}") + print(f" Scopes: {', '.join(manifest.scopes)}") + print(f" Scores: {manifest.scores}") + print(f" Uncertainty: {manifest.uncertainty}") + print(f" Source modules: {', '.join(manifest.source_modules)}") + print(f" Calibration set: {manifest.calibration_set}") + print(f" Safety flags: {', '.join(manifest.safety_flags) if manifest.safety_flags else '(none)'}") + print(f" Provenance run: {manifest.provenance_run_id}") + print(f" Dry-lab only: {manifest.dry_lab_only}") + print(f" Version: {manifest.version}") + print(f" Created at: {manifest.created_at}") + print(f" Notes: {len(manifest.notes)}") + if validation_errors: + print(f" Validation: {len(validation_errors)} error(s)") + for err in validation_errors: + print(f" - {err}") + else: + print(f" Validation: passed") + print() + print("Dry-lab only. Candidate manifests are computational artifacts,") + print("not biological proof.") + + if validation_errors: + return 3 + return 0 diff --git a/src/openamp_foundry/cli/main.py b/src/openamp_foundry/cli/main.py index 56f195f5..b764d4c9 100644 --- a/src/openamp_foundry/cli/main.py +++ b/src/openamp_foundry/cli/main.py @@ -5,7 +5,7 @@ from openamp_foundry.cli.commands.selection import _run_pilot_panel, _run_pilot_confident, _run_diversity_check, _run_select_batch, _run_batch_rationale from openamp_foundry.cli.commands.external import _run_external_predict, _run_external_consensus from openamp_foundry.cli.commands.qc import _run_synthesis_order, _run_presynth_qc -from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter, _run_synthetic_result_policy_check, _run_calibration_decision_checklist, _run_calibration_rollback_plan, _run_simulation_registry, _run_validate_simulation_result, _run_simulation_baseline_check, _run_adapter_gate_check, _run_simulation_provenance, _run_simulation_ensemble_check, _run_simulation_ci_report, _run_simulation_deprecation_check, _run_simulation_scope_check, _run_simulation_evidence_packet, _run_artifact_version +from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter, _run_synthetic_result_policy_check, _run_calibration_decision_checklist, _run_calibration_rollback_plan, _run_simulation_registry, _run_validate_simulation_result, _run_simulation_baseline_check, _run_adapter_gate_check, _run_simulation_provenance, _run_simulation_ensemble_check, _run_simulation_ci_report, _run_simulation_deprecation_check, _run_simulation_scope_check, _run_simulation_evidence_packet, _run_artifact_version, _run_candidate_manifest from openamp_foundry.cli.commands.gates import _run_gate_check import argparse @@ -1625,6 +1625,24 @@ def build_parser() -> argparse.ArgumentParser: help="Output format (default: text).", ) + # ── Candidate manifest (Phase I I2) ────────────────────────────── + cm = sub.add_parser( + "candidate-manifest", + help=( + "Build and optionally validate a candidate manifest — the core " + "interoperable artifact describing a dry-lab candidate. Dry-lab only." + ), + ) + cm.add_argument("--candidate-id", type=str, required=True, help="Candidate identifier (required).") + cm.add_argument("--sequence", type=str, required=True, help="Amino acid sequence (required).") + cm.add_argument("--evidence-level", type=str, required=True, help="Proof-ladder level 1-6 (required).") + cm.add_argument("--scopes", type=str, required=True, help="Comma-separated scopes (required).") + cm.add_argument("--scores-json", type=str, required=True, help="JSON object of score name→value (required).") + cm.add_argument("--uncertainty", type=float, required=True, help="Uncertainty 0.0-1.0 (required).") + cm.add_argument("--source-modules", type=str, required=True, help="Comma-separated source modules (required).") + cm.add_argument("--validate", action="store_true", default=False, help="Run validate_candidate_manifest.") + cm.add_argument("--format", type=str, default="text", choices=["text", "json"], help="Output format (default: text).") + # ── Simulation-result confidence interval report (H7) ──────────── sicr = sub.add_parser( "simulation-ci-report", @@ -1836,6 +1854,9 @@ def main(argv: list[str] | None = None) -> int: if args.command == "artifact-version": return _run_artifact_version(args) + if args.command == "candidate-manifest": + return _run_candidate_manifest(args) + parser.error("unknown command") return 2 diff --git a/src/openamp_foundry/manifests/__init__.py b/src/openamp_foundry/manifests/__init__.py new file mode 100644 index 00000000..c11ab7b4 --- /dev/null +++ b/src/openamp_foundry/manifests/__init__.py @@ -0,0 +1,15 @@ +from openamp_foundry.manifests.candidate_manifest import ( + CandidateManifest, + make_candidate_manifest, + validate_candidate_manifest, + manifest_to_dict, + manifest_summary, +) + +__all__ = [ + "CandidateManifest", + "make_candidate_manifest", + "validate_candidate_manifest", + "manifest_to_dict", + "manifest_summary", +] diff --git a/src/openamp_foundry/manifests/candidate_manifest.py b/src/openamp_foundry/manifests/candidate_manifest.py new file mode 100644 index 00000000..d39a6d82 --- /dev/null +++ b/src/openamp_foundry/manifests/candidate_manifest.py @@ -0,0 +1,110 @@ +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from datetime import datetime, timezone + + +@dataclass +class CandidateManifest: + candidate_id: str + sequence: str + evidence_level: int + scopes: list[str] + scores: dict[str, float] + uncertainty: float + source_modules: list[str] + calibration_set: str | None + safety_flags: list[str] + provenance_run_id: str | None + dry_lab_only: bool = True + version: str = "1.0.0" + created_at: str = "" + notes: list[str] = field(default_factory=list) + + +def make_candidate_manifest( + candidate_id: str, + sequence: str, + evidence_level: int, + scopes: list[str], + scores: dict[str, float], + uncertainty: float, + source_modules: list[str], + *, + calibration_set: str | None = None, + safety_flags: list[str] | None = None, + provenance_run_id: str | None = None, + created_at: str = "", + notes: list[str] | None = None, +) -> CandidateManifest: + return CandidateManifest( + candidate_id=candidate_id, + sequence=sequence, + evidence_level=evidence_level, + scopes=scopes, + scores=scores, + uncertainty=uncertainty, + source_modules=source_modules, + calibration_set=calibration_set, + safety_flags=safety_flags or [], + provenance_run_id=provenance_run_id, + created_at=created_at or datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + notes=notes or [], + ) + + +def validate_candidate_manifest(manifest: CandidateManifest) -> list[str]: + errors: list[str] = [] + if not manifest.candidate_id: + errors.append("candidate_id must be non-empty") + if not manifest.sequence: + errors.append("sequence must be non-empty") + if manifest.evidence_level < 1 or manifest.evidence_level > 6: + errors.append(f"evidence_level must be 1-6, got {manifest.evidence_level}") + if not manifest.scopes: + errors.append("scopes must be a non-empty list") + if manifest.uncertainty < 0.0 or manifest.uncertainty > 1.0: + errors.append(f"uncertainty must be in [0.0, 1.0], got {manifest.uncertainty}") + if not manifest.source_modules: + errors.append("source_modules must be a non-empty list") + if not manifest.dry_lab_only: + errors.append("dry_lab_only must be True") + if not re.match(r"^\d+\.\d+\.\d+$", manifest.version): + errors.append(f"version must match MAJOR.MINOR.PATCH, got {manifest.version!r}") + return errors + + +def manifest_to_dict(manifest: CandidateManifest) -> dict: + return { + "candidate_id": manifest.candidate_id, + "sequence": manifest.sequence, + "evidence_level": manifest.evidence_level, + "scopes": list(manifest.scopes), + "scores": dict(manifest.scores), + "uncertainty": manifest.uncertainty, + "source_modules": list(manifest.source_modules), + "calibration_set": manifest.calibration_set, + "safety_flags": list(manifest.safety_flags), + "provenance_run_id": manifest.provenance_run_id, + "dry_lab_only": manifest.dry_lab_only, + "version": manifest.version, + "created_at": manifest.created_at, + "notes": list(manifest.notes), + } + + +def manifest_summary(manifests: list[CandidateManifest]) -> dict: + by_evidence: dict[str, int] = {} + with_safety_flags = 0 + for m in manifests: + level_key = str(m.evidence_level) + by_evidence[level_key] = by_evidence.get(level_key, 0) + 1 + if m.safety_flags: + with_safety_flags += 1 + return { + "total": len(manifests), + "by_evidence_level": by_evidence, + "with_safety_flags": with_safety_flags, + "dry_lab_only": True, + } diff --git a/tests/manifests/__init__.py b/tests/manifests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/manifests/test_candidate_manifest.py b/tests/manifests/test_candidate_manifest.py new file mode 100644 index 00000000..1db12500 --- /dev/null +++ b/tests/manifests/test_candidate_manifest.py @@ -0,0 +1,184 @@ +"""Tests for candidate manifest module.""" +from pathlib import Path + +import pytest + +from openamp_foundry.manifests.candidate_manifest import ( + CandidateManifest, + make_candidate_manifest, + validate_candidate_manifest, + manifest_to_dict, + manifest_summary, +) + + +def _valid_manifest() -> CandidateManifest: + return make_candidate_manifest( + candidate_id="AMP-001", + sequence="AKLWKR", + evidence_level=2, + scopes=["bacterial_binding"], + scores={"binding_energy": 0.75}, + uncertainty=0.1, + source_modules=["membrane_proxy"], + ) + + +class TestMakeCandidateManifest: + def test_returns_candidate_manifest(self): + m = make_candidate_manifest( + candidate_id="AMP-001", + sequence="AKLWKR", + evidence_level=2, + scopes=["bacterial_binding"], + scores={"binding_energy": 0.75}, + uncertainty=0.1, + source_modules=["membrane_proxy"], + ) + assert isinstance(m, CandidateManifest) + + def test_dry_lab_only_always_true(self): + m = _valid_manifest() + assert m.dry_lab_only is True + + m2 = make_candidate_manifest( + candidate_id="AMP-002", + sequence="GIGKFLHSAKK", + evidence_level=1, + scopes=["anti_fungal"], + scores={"charge": 3.0}, + uncertainty=0.2, + source_modules=["charge_scorer"], + ) + assert m2.dry_lab_only is True + + +class TestValidateCandidateManifest: + def test_valid_manifest_returns_empty_list(self): + m = _valid_manifest() + errors = validate_candidate_manifest(m) + assert errors == [] + + def test_empty_candidate_id(self): + m = _valid_manifest() + m.candidate_id = "" + errors = validate_candidate_manifest(m) + assert any("candidate_id" in e for e in errors) + + def test_empty_sequence(self): + m = _valid_manifest() + m.sequence = "" + errors = validate_candidate_manifest(m) + assert any("sequence" in e for e in errors) + + def test_evidence_level_below_range(self): + m = _valid_manifest() + m.evidence_level = 0 + errors = validate_candidate_manifest(m) + assert any("evidence_level" in e for e in errors) + + def test_evidence_level_above_range(self): + m = _valid_manifest() + m.evidence_level = 7 + errors = validate_candidate_manifest(m) + assert any("evidence_level" in e for e in errors) + + def test_empty_scopes(self): + m = _valid_manifest() + m.scopes = [] + errors = validate_candidate_manifest(m) + assert any("scopes" in e for e in errors) + + def test_uncertainty_below_range(self): + m = _valid_manifest() + m.uncertainty = -0.1 + errors = validate_candidate_manifest(m) + assert any("uncertainty" in e for e in errors) + + def test_uncertainty_above_range(self): + m = _valid_manifest() + m.uncertainty = 1.1 + errors = validate_candidate_manifest(m) + assert any("uncertainty" in e for e in errors) + + def test_empty_source_modules(self): + m = _valid_manifest() + m.source_modules = [] + errors = validate_candidate_manifest(m) + assert any("source_modules" in e for e in errors) + + def test_dry_lab_only_false(self): + m = _valid_manifest() + m.dry_lab_only = False + errors = validate_candidate_manifest(m) + assert any("dry_lab_only" in e for e in errors) + + def test_invalid_version_format(self): + m = _valid_manifest() + m.version = "1.0" + errors = validate_candidate_manifest(m) + assert any("version" in e for e in errors) + + +class TestManifestToDict: + def test_returns_dict_with_all_fields(self): + m = _valid_manifest() + d = manifest_to_dict(m) + assert isinstance(d, dict) + assert d["candidate_id"] == "AMP-001" + assert d["sequence"] == "AKLWKR" + assert d["evidence_level"] == 2 + assert d["scopes"] == ["bacterial_binding"] + assert d["scores"] == {"binding_energy": 0.75} + assert d["uncertainty"] == 0.1 + assert d["source_modules"] == ["membrane_proxy"] + assert d["calibration_set"] is None + assert d["safety_flags"] == [] + assert d["provenance_run_id"] is None + assert d["dry_lab_only"] is True + assert "version" in d + assert "created_at" in d + assert d["notes"] == [] + + +class TestManifestSummary: + def test_total_correct(self): + m1 = _valid_manifest() + m2 = _valid_manifest() + m2.candidate_id = "AMP-002" + m2.evidence_level = 3 + summary = manifest_summary([m1, m2]) + assert summary["total"] == 2 + + def test_with_safety_flags_correct(self): + m1 = _valid_manifest() + m2 = _valid_manifest() + m2.candidate_id = "AMP-002" + m2.safety_flags = ["hemolytic_risk"] + summary = manifest_summary([m1, m2]) + assert summary["with_safety_flags"] == 1 + + def test_dry_lab_only_true(self): + m = _valid_manifest() + summary = manifest_summary([m]) + assert summary["dry_lab_only"] is True + + def test_by_evidence_level(self): + m1 = _valid_manifest() + m2 = _valid_manifest() + m2.candidate_id = "AMP-002" + m2.evidence_level = 3 + m3 = _valid_manifest() + m3.candidate_id = "AMP-003" + m3.evidence_level = 2 + summary = manifest_summary([m1, m2, m3]) + assert summary["by_evidence_level"]["2"] == 2 + assert summary["by_evidence_level"]["3"] == 1 + + +class TestSchemaFileExists: + def test_schema_file_exists(self): + schema_path = Path("schemas/candidate_manifest.schema.json") + assert schema_path.exists(), ( + f"Schema file not found at {schema_path}" + ) diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index af7e4c30..b8b4fc9e 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -2,9 +2,9 @@ import subprocess import sys -# Baseline: 3291 tests as of Loop 99 (2026-07-09) -- Phase I I1 (artifact versioning policy) +# Baseline: 3310 tests as of Loop 100 (2026-07-09) -- Phase I I2 (candidate manifest schema) # Raise threshold if legitimate tests are removed; lower threshold is okay. -BASELINE = 3291 +BASELINE = 3310 TOLERANCE = 0.05 # allow 5% fluctuation