Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report simulation-deprecation-check simulation-scope-check simulation-evidence-packet artifact-version candidate-manifest artifact-changelog integration-check adapter-author-check license-check artifact-compat-check contribution-check adoption-scorecard release-gate-check decision-log release-request-check coi-check rotation-plan-check security-report-check citation-check roadmap-sync-check advisory-review-check annual-review-check selection-rationale-check batch-priority-check pilot-package-check calibration-intake-check uncertainty-report-check
.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report simulation-deprecation-check simulation-scope-check simulation-evidence-packet artifact-version candidate-manifest artifact-changelog integration-check adapter-author-check license-check artifact-compat-check contribution-check adoption-scorecard release-gate-check decision-log release-request-check coi-check rotation-plan-check security-report-check citation-check roadmap-sync-check advisory-review-check annual-review-check selection-rationale-check batch-priority-check pilot-package-check calibration-intake-check uncertainty-report-check preprint-bundle-check

PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3)
PYTEST := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest)
Expand Down Expand Up @@ -910,3 +910,8 @@ uncertainty-report-check:
.venv/bin/openamp-foundry uncertainty-report-check \
--entry-json '{"report_id":"UQ-001","batch_id":"BATCH-001","candidate_id":"AMP-001","pipeline_version":"0.8.3","metric_name":"mic","point_estimate":4.0,"lower_bound":2.0,"upper_bound":8.0,"confidence_level":0.90,"calibration_source":"historical_holdout_v2","reviewer":"alice","dry_lab_only":true}' \
--format text

preprint-bundle-check:
.venv/bin/openamp-foundry preprint-bundle-check \
--entry-json '{"bundle_id":"BND-001","batch_id":"BATCH-001","pipeline_version":"0.8.4","submission_date":"2026-07-09","title":"Computational nomination of AMP candidates from the OpenAMP Foundry","artifact_ids":["SEL-001","PRI-001","PKG-001","UQ-001","CAL-001"],"evidence_level":4,"preprint_doi":"10.1101/2026.07.09.000001","contact_email":"research@openamp.org","release_approved":true,"dry_lab_only":true}' \
--format text
3 changes: 2 additions & 1 deletion docs/evidence/METRICS_CURRENT.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak
> **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees
> with this file, this file wins. Updated whenever benchmark/benchmark config changes.
>
> **Last updated:** 2026-07-09 (Phase K K5 — uncertainty quantification report schema — v0.8.3)
> **Last updated:** 2026-07-09 (Phase L L1 — preprint evidence bundle schema — v0.8.4)
> **New in v0.8.4:** Preprint evidence bundle schema (L1 — Loop 124) — docs/evidence/PREPRINT_BUNDLE_GUIDE.md with purpose, required field table (11 fields), minimum artifact count (3), evidence level guide, warnings, validation workflow, honest-use boundary. src/openamp_foundry/evidence/preprint_bundle.py with PreprintBundleEntry dataclass (11 fields: bundle_id, batch_id, pipeline_version, submission_date, title, artifact_ids, evidence_level, preprint_doi, contact_email, release_approved, dry_lab_only), PreprintBundleResult dataclass (5 fields: bundle_id, batch_id, artifact_count, passed, errors, warnings, dry_lab_only=True), MINIMUM_ARTIFACTS (3), RECOMMENDED_ARTIFACT_COUNT (5), MAX_TITLE_LENGTH (300), VALID_EVIDENCE_LEVELS (6: 1-6), validate_preprint_bundle() (10 checks: BND- prefix, non-empty batch_id/pipeline_version/contact_email, YYYY-MM-DD submission_date, non-empty title<=300 chars, artifact_ids>=3, valid evidence_level 1-6, release_approved must be True, dry_lab_only must be True; evidence_level<=2 warns, empty preprint_doi warns, artifact_count<5 warns), validate_preprint_bundle_dict() (10 required fields guard). CLI (openamp-foundry preprint-bundle-check) with --entry-json, --format text|json. make preprint-bundle-check target. Phase L L1: scientific preprints now have a machine-validated evidence bundle structure.
> **New in v0.8.3:** Uncertainty quantification report schema (K5 — Loop 123) — docs/evidence/UNCERTAINTY_REPORT_GUIDE.md with purpose, required field table (12 fields), valid metric names (5), warnings, validation workflow, honest-use boundary. src/openamp_foundry/evidence/uncertainty_report.py with UncertaintyReportEntry dataclass (12 fields: report_id, batch_id, candidate_id, pipeline_version, metric_name, point_estimate, lower_bound, upper_bound, confidence_level, calibration_source, reviewer, dry_lab_only), UncertaintyReportResult dataclass (6 fields: report_id, candidate_id, interval_width, passed, errors, warnings, dry_lab_only=True), VALID_METRIC_NAMES (5: cytotoxicity_score, hemolysis_fraction, mic, selectivity_index, stability_score), WIDE_INTERVAL_THRESHOLD (10.0), validate_uncertainty_report() (11 checks: UQ- prefix, non-empty batch_id/candidate_id/pipeline_version/calibration_source/reviewer, valid metric_name, lower_bound<=point_estimate, upper_bound>=point_estimate, confidence_level 0.0-1.0, dry_lab_only must be True; wide interval warns, confidence<0.80 warns, confidence>0.99 warns), validate_uncertainty_report_dict() (11 required fields guard). CLI (openamp-foundry uncertainty-report-check) with --entry-json, --format text|json. make uncertainty-report-check target. Phase K K5: prediction intervals for external reviewers are now machine-validated.
> **New in v0.8.2:** Post-experiment calibration intake schema (K4 — Loop 122) — docs/evidence/CALIBRATION_INTAKE_GUIDE.md with purpose, required field table (11 fields), note on dry_lab_only=False (real lab results), valid assay types (5), valid outcome values (4), warnings, validation workflow, honest-use boundary. src/openamp_foundry/evidence/calibration_intake.py with CalibrationIntakeEntry dataclass (11 fields: intake_id, batch_id, candidate_id, pipeline_version, assay_type, predicted_outcome, observed_outcome, predicted_confidence, intake_date, reviewer, dry_lab_only=False), CalibrationIntakeResult dataclass (6 fields: intake_id, candidate_id, prediction_correct, passed, errors, warnings, dry_lab_only=False), VALID_ASSAY_TYPES (5: cytotoxicity_assay, hemolysis_assay, membrane_disruption_assay, mic_assay, stability_assay), VALID_OUTCOME_VALUES (4: active, inactive, inconclusive, not_tested), validate_calibration_intake() (11 checks: CAL- prefix, non-empty batch_id/candidate_id/pipeline_version/reviewer, valid assay_type, valid predicted_outcome, valid observed_outcome, confidence 0.0-1.0, YYYY-MM-DD date, dry_lab_only must be False; high-confidence misprediction warns, inconclusive observed warns), validate_calibration_intake_dict() (10 required fields guard). CLI (openamp-foundry calibration-intake-check) with --entry-json, --format text|json. make calibration-intake-check target. Phase K K4: prediction-vs-outcome comparisons are now machine-validated with honest dry_lab_only=False enforcement.
> **New in v0.8.1:** Pilot package completeness checker (K3 — Loop 121) — docs/evidence/PILOT_PACKAGE_GUIDE.md with purpose, required field table (11 fields), mandatory artifact types table (3 types), valid artifact types, warnings, validation workflow, honest-use boundary. src/openamp_foundry/evidence/pilot_package.py with PilotPackageEntry dataclass (11 fields: package_id, batch_id, submission_date, pipeline_version, included_artifacts, missing_artifacts, reviewer, approver, completeness_score, ready_to_submit, dry_lab_only), PilotPackageResult dataclass (5 fields, dry_lab_only=True), MINIMUM_REQUIRED_ARTIFACTS (3), READINESS_SCORE_THRESHOLD (0.80), MANDATORY_ARTIFACT_TYPES (3: batch_priority, evidence_certificate, selection_rationale), VALID_ARTIFACT_TYPES (8 types), validate_pilot_package() (11 checks: PKG- prefix, non-empty batch_id/pipeline_version/reviewer/approver, YYYY-MM-DD submission_date, included_artifacts>=3, mandatory types covered, completeness_score 0.0-1.0, ready_to_submit must be False when score<0.80, dry_lab_only must be True; missing_artifacts non-empty warns, score<0.90 warns, same reviewer/approver warns), validate_pilot_package_dict() (10 required fields guard). CLI (openamp-foundry pilot-package-check) with --entry-json, --format text|json. make pilot-package-check target. Phase K K3: every pilot submission is now machine-validated for completeness.
Expand Down
71 changes: 71 additions & 0 deletions docs/evidence/PREPRINT_BUNDLE_GUIDE.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
# Preprint Evidence Bundle Guide

A *preprint evidence bundle* ties together all Phase K artifacts for a specific batch
into a single structured record that can be submitted alongside a scientific preprint.
This guide defines what a bundle must contain, how to validate it, and the honesty
constraints that apply.

## Purpose

Before submitting a preprint describing a computationally-nominated antimicrobial peptide
family, the evidence bundle ensures that:
- all K-phase artifacts (selection rationale, batch priority, pilot package, uncertainty reports)
are referenced,
- the claimed evidence level is consistent with actual artifacts,
- the bundle passes structural review before public release.

## Required fields

| Field | Type | Constraint |
|---|---|---|
| `bundle_id` | str | Must start with `"BND-"` |
| `batch_id` | str | Non-empty |
| `pipeline_version` | str | Non-empty |
| `submission_date` | str | ISO 8601 `YYYY-MM-DD` |
| `title` | str | Non-empty, ≤ 300 characters |
| `artifact_ids` | list[str] | ≥ `MINIMUM_ARTIFACTS` (3) entries |
| `evidence_level` | int | 1 – 6 (from PROOF_LADDER.md) |
| `preprint_doi` | str | May be empty (DOI assigned after submission) |
| `contact_email` | str | Non-empty |
| `release_approved` | bool | Must be `True` for a valid bundle |
| `dry_lab_only` | bool | Must be `True` |

## Minimum artifact types

At least 3 artifact IDs must be provided. The bundle does not enforce specific artifact
types — it enforces count — because artifact IDs are opaque references tracked in the
pilot package system.

## Evidence level

Use the same 1–6 scale as PROOF_LADDER.md:
1. Sequence only
2. Physicochemical
3. Computational model
4. Ensemble
5. Virtual assay
6. Cross-validated virtual assay

A preprint with evidence level 1 or 2 warrants a warning.

## Warnings

- `evidence_level <= 2` — warning: low evidence level; reviewers should apply extra scrutiny.
- `preprint_doi` empty — warning: DOI not yet assigned; update before final release.
- `len(artifact_ids) < 5` — warning: fewer than 5 artifacts; consider adding uncertainty reports and calibration records.

## How to validate

```bash
make preprint-bundle-check
# or
openamp-foundry preprint-bundle-check \
--entry-json '{"bundle_id":"BND-001","batch_id":"BATCH-001","pipeline_version":"0.8.4","submission_date":"2026-07-09","title":"Computational nomination of AMP candidates from the OpenAMP Foundry","artifact_ids":["SEL-001","PRI-001","PKG-001","UQ-001","CAL-001"],"evidence_level":4,"preprint_doi":"10.1101/2026.07.09.000001","contact_email":"research@openamp.org","release_approved":true,"dry_lab_only":true}' \
--format text
```

## Honest-use boundary

Preprint evidence bundles describe computational nominations, not experimental validations.
The bundle does not imply experimental success. Wet-lab validation results must be reported
separately and must not be attributed to the pipeline.
12 changes: 12 additions & 0 deletions docs/research/NEXT_100_PR_MAP.md
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,18 @@ Build the evidence and accountability artifacts needed for credible external val
| K4 | Add post-experiment calibration intake schema. | Structured result-to-prediction comparison. | B/D |
| K5 | Add uncertainty quantification report schema (complete — Loop 123). | Honest prediction intervals for reviewers. | B |

## Phase L — Scientific Communication

Bridge the gap between evidence packages and published science.

| PR | Task | Why it matters | Review class |
|---:|---|---|---|
| L1 | Add preprint evidence bundle schema (complete — Loop 124). | Ties K-phase artifacts into a submission-ready package. | B |
| L2 | Add reproducibility manifest schema. | Captures exact versions, checksums, seeds for full reproduction. | B |
| L3 | Add candidate summary card schema. | Publication-ready per-candidate structured summary. | B |
| L4 | Add multi-candidate comparison schema. | Structured comparison table for ≥2 candidates. | B |
| L5 | Add dataset release package checker. | Validates open dataset releases meet data governance. | B/D |


## Prioritization rule

Expand Down
3 changes: 3 additions & 0 deletions docs/research/ROADMAP.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
# Roadmap

### v0.8.4
- Phase L L1: preprint evidence bundle schema — ties K-phase artifacts into a submission-ready record for scientific preprints. PreprintBundleEntry dataclass, validate_preprint_bundle(), CLI preprint-bundle-check.

### v0.8.3
- Phase K K5: uncertainty quantification report schema — validates prediction intervals, confidence levels, and calibration source for dry-lab candidate recommendations. UncertaintyReportEntry dataclass, validate_uncertainty_report(), CLI uncertainty-report-check.

Expand Down
34 changes: 34 additions & 0 deletions src/openamp_foundry/cli/commands/reports.py
Original file line number Diff line number Diff line change
Expand Up @@ -2988,3 +2988,37 @@ def _run_uncertainty_report_check(args: argparse.Namespace) -> int:
print(" Uncertainty report validated. Prediction intervals are documented.")

return 0 if result.passed else 3


def _run_preprint_bundle_check(args: argparse.Namespace) -> int:
"""Validate a preprint evidence bundle entry."""
import json as _json
from openamp_foundry.evidence.preprint_bundle import validate_preprint_bundle_dict

try:
d = _json.loads(args.entry_json)
except _json.JSONDecodeError as exc:
print(_json.dumps({"status": "error", "error": f"Invalid JSON: {exc}"}))
return 2

if not isinstance(d, dict):
print(_json.dumps({"status": "error", "error": "--entry-json must be a JSON object"}))
return 2

result = validate_preprint_bundle_dict(d)
output_format = getattr(args, "format", "text")

if output_format == "json":
import dataclasses
print(_json.dumps(dataclasses.asdict(result), indent=2))
else:
status = "PASS" if result.passed else "FAIL"
print(f"Preprint bundle {result.bundle_id} (batch {result.batch_id}, {result.artifact_count} artifacts): {status}")
for e in result.errors:
print(f" ERROR: {e}")
for w in result.warnings:
print(f" WARN: {w}")
if result.passed:
print(" Preprint bundle validated. Evidence package is ready for submission review.")

return 0 if result.passed else 3
23 changes: 22 additions & 1 deletion src/openamp_foundry/cli/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
from openamp_foundry.cli.commands.reports import _run_security_report_check
from openamp_foundry.cli.commands.reports import _run_citation_check
from openamp_foundry.cli.commands.reports import _run_roadmap_sync_check
from openamp_foundry.cli.commands.reports import _run_advisory_review_check, _run_annual_review_check, _run_selection_rationale_check, _run_batch_priority_check, _run_pilot_package_check, _run_calibration_intake_check, _run_uncertainty_report_check
from openamp_foundry.cli.commands.reports import _run_advisory_review_check, _run_annual_review_check, _run_selection_rationale_check, _run_batch_priority_check, _run_pilot_package_check, _run_calibration_intake_check, _run_uncertainty_report_check, _run_preprint_bundle_check

import argparse
import json
Expand Down Expand Up @@ -2071,6 +2071,24 @@ def build_parser() -> argparse.ArgumentParser:
help="Output format (default: text).",
)

# ── Preprint bundle check (Phase L L1) ───────────────────────────
pbc = sub.add_parser(
"preprint-bundle-check",
help=(
"Validate a preprint evidence bundle. Confirms all K-phase artifacts "
"are referenced and release is approved. Dry-lab only."
),
)
pbc.add_argument(
"--entry-json", type=str, required=True,
help="JSON string of a PreprintBundleEntry dict (required).",
)
pbc.add_argument(
"--format", type=str, default="text",
choices=["text", "json"],
help="Output format (default: text).",
)

# ── Selection rationale check (Phase K K1) ───────────────────────
src2 = sub.add_parser(
"selection-rationale-check",
Expand Down Expand Up @@ -2391,6 +2409,9 @@ def main(argv: list[str] | None = None) -> int:
if args.command == "uncertainty-report-check":
return _run_uncertainty_report_check(args)

if args.command == "preprint-bundle-check":
return _run_preprint_bundle_check(args)

parser.error("unknown command")
return 2

Expand Down
Loading
Loading