Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report simulation-deprecation-check simulation-scope-check simulation-evidence-packet artifact-version candidate-manifest artifact-changelog integration-check adapter-author-check license-check artifact-compat-check contribution-check adoption-scorecard release-gate-check decision-log release-request-check coi-check rotation-plan-check security-report-check citation-check roadmap-sync-check advisory-review-check annual-review-check selection-rationale-check batch-priority-check pilot-package-check calibration-intake-check uncertainty-report-check preprint-bundle-check reproducibility-manifest-check
.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report simulation-deprecation-check simulation-scope-check simulation-evidence-packet artifact-version candidate-manifest artifact-changelog integration-check adapter-author-check license-check artifact-compat-check contribution-check adoption-scorecard release-gate-check decision-log release-request-check coi-check rotation-plan-check security-report-check citation-check roadmap-sync-check advisory-review-check annual-review-check selection-rationale-check batch-priority-check pilot-package-check calibration-intake-check uncertainty-report-check preprint-bundle-check reproducibility-manifest-check candidate-summary-card-check

PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3)
PYTEST := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest)
Expand Down Expand Up @@ -920,3 +920,8 @@ reproducibility-manifest-check:
.venv/bin/openamp-foundry reproducibility-manifest-check \
--entry-json '{"manifest_id":"RPM-001","batch_id":"BATCH-001","pipeline_version":"0.8.5","run_date":"2026-07-09","python_version":"3.11.9","package_checksums":{"openamp_foundry":"0.8.4","numpy":"1.26.4","pandas":"2.2.2","scikit-learn":"1.4.2","torch":"2.3.0"},"data_checksums":{"data/amp_sequences.fasta":"abc123def456"},"random_seeds":{"ensemble_sampling":42},"hardware_summary":"Apple M3 Pro, 36 GB RAM","reviewer":"alice","dry_lab_only":true}' \
--format text

candidate-summary-card-check:
.venv/bin/openamp-foundry candidate-summary-card-check \
--entry-json '{"card_id":"CRD-001","candidate_id":"AMP-001","batch_id":"BATCH-001","pipeline_version":"0.8.6","sequence":"KWKLFKKIEK","sequence_length":10,"evidence_level":4,"predicted_activity":"high_activity","safety_flags":[],"selection_rationale_id":"SEL-001","reviewer":"alice","dry_lab_only":true}' \
--format text
61 changes: 61 additions & 0 deletions docs/evidence/CANDIDATE_SUMMARY_CARD_GUIDE.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
# Candidate Summary Card Guide

A *candidate summary card* is a publication-ready, per-candidate structured record
that consolidates the key computational properties, evidence level, safety flags,
and selection rationale for a single candidate into a reviewable snapshot.

## Purpose

Summary cards allow:
- external reviewers to quickly assess individual candidates without reading the full evidence package,
- authors to generate structured supplementary tables for preprints, and
- downstream tooling to render consistent candidate reports.

## Required fields

| Field | Type | Constraint |
|---|---|---|
| `card_id` | str | Must start with `"CRD-"` |
| `candidate_id` | str | Non-empty |
| `batch_id` | str | Non-empty |
| `pipeline_version` | str | Non-empty |
| `sequence` | str | Non-empty, only standard amino acid letters (A-Z, no B/J/O/U/X/Z) |
| `sequence_length` | int | Must equal `len(sequence)` |
| `evidence_level` | int | 1 – 6 |
| `predicted_activity` | str | One of `VALID_ACTIVITY_LABELS` |
| `safety_flags` | list[str] | May be empty; each entry is a named safety concern |
| `selection_rationale_id` | str | Must start with `"SEL-"` |
| `reviewer` | str | Non-empty |
| `dry_lab_only` | bool | Must be `True` |

## Valid activity labels

`high_activity`, `moderate_activity`, `low_activity`, `inactive`, `uncertain`

## Valid amino acid letters

Standard one-letter codes: A C D E F G H I K L M N P Q R S T V W Y
(B, J, O, U, X, Z are excluded)

## Warnings

- `evidence_level <= 2` — warning: low evidence; apply extra scrutiny.
- `len(safety_flags) > 0` — warning: safety concerns are flagged; review before synthesis.
- `predicted_activity == "uncertain"` — warning: activity is uncertain; prioritize candidates with clearer predictions.
- `sequence_length > 50` — warning: long peptide; synthesis cost may be high.

## How to validate

```bash
make candidate-summary-card-check
# or
openamp-foundry candidate-summary-card-check \
--entry-json '{"card_id":"CRD-001","candidate_id":"AMP-001","batch_id":"BATCH-001","pipeline_version":"0.8.6","sequence":"KWKLFKKIEKVGQNIRDGIVKAGPAVADPQGRPVPVPVDPVDPVD","sequence_length":45,"evidence_level":4,"predicted_activity":"high_activity","safety_flags":[],"selection_rationale_id":"SEL-001","reviewer":"alice","dry_lab_only":true}' \
--format text
```

## Honest-use boundary

Candidate summary cards are dry-lab outputs. Predicted activity and safety flags are
computational estimates, not experimental results. They must not be presented as
experimental measurements or used to make clinical or safety decisions.
3 changes: 2 additions & 1 deletion docs/evidence/METRICS_CURRENT.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak
> **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees
> with this file, this file wins. Updated whenever benchmark/benchmark config changes.
>
> **Last updated:** 2026-07-09 (Phase L L2 — reproducibility manifest schema — v0.8.5)
> **Last updated:** 2026-07-09 (Phase L L3 — candidate summary card schema — v0.8.6)
> **New in v0.8.6:** Candidate summary card schema (L3 — Loop 126) — docs/evidence/CANDIDATE_SUMMARY_CARD_GUIDE.md with purpose, required field table (12 fields), valid activity labels (5), valid amino acid set (20 standard), warnings, validation workflow, honest-use boundary. src/openamp_foundry/evidence/candidate_summary_card.py with CandidateSummaryCardEntry dataclass (12 fields: card_id, candidate_id, batch_id, pipeline_version, sequence, sequence_length, evidence_level, predicted_activity, safety_flags, selection_rationale_id, reviewer, dry_lab_only), CandidateSummaryCardResult dataclass (5 fields: card_id, candidate_id, sequence_length, passed, errors, warnings, dry_lab_only=True), VALID_ACTIVITY_LABELS (5: high_activity, inactive, low_activity, moderate_activity, uncertain), VALID_AMINO_ACIDS (20 standard one-letter codes), LONG_PEPTIDE_THRESHOLD (50), VALID_EVIDENCE_LEVELS (6: 1-6), validate_candidate_summary_card() (11 checks: CRD- prefix, non-empty candidate_id/batch_id/pipeline_version/reviewer, non-empty sequence with valid amino acids, sequence_length==len(sequence), valid evidence_level, valid predicted_activity, SEL- prefix on selection_rationale_id, dry_lab_only must be True; evidence_level<=2 warns, safety_flags non-empty warns, uncertain activity warns, length>50 warns), validate_candidate_summary_card_dict() (11 required fields guard). CLI (openamp-foundry candidate-summary-card-check) with --entry-json, --format text|json. make candidate-summary-card-check target. Phase L L3: every candidate now has a machine-validated publication-ready summary card.
> **New in v0.8.5:** Reproducibility manifest schema (L2 — Loop 125) — docs/evidence/REPRODUCIBILITY_MANIFEST_GUIDE.md with purpose, required field table (11 fields), package checksums format, data checksums format, warnings, validation workflow, honest-use boundary. src/openamp_foundry/evidence/reproducibility_manifest.py with ReproducibilityManifestEntry dataclass (11 fields: manifest_id, batch_id, pipeline_version, run_date, python_version, package_checksums, data_checksums, random_seeds, hardware_summary, reviewer, dry_lab_only), ReproducibilityManifestResult dataclass (6 fields: manifest_id, batch_id, package_count, data_file_count, passed, errors, warnings, dry_lab_only=True), MINIMUM_PACKAGES (3), MINIMUM_DATA_FILES (1), RECOMMENDED_PACKAGES (5), validate_reproducibility_manifest() (10 checks: RPM- prefix, non-empty batch_id/pipeline_version/python_version/hardware_summary/reviewer, YYYY-MM-DD run_date, package_checksums>=3, data_checksums>=1, dry_lab_only must be True; empty random_seeds warns, package_count<5 warns, hardware_summary contains 'unknown' warns), validate_reproducibility_manifest_dict() (10 required fields guard). CLI (openamp-foundry reproducibility-manifest-check) with --entry-json, --format text|json. make reproducibility-manifest-check target. Phase L L2: every pipeline run now has a machine-validated reproducibility record.
> **New in v0.8.4:** Preprint evidence bundle schema (L1 — Loop 124) — docs/evidence/PREPRINT_BUNDLE_GUIDE.md with purpose, required field table (11 fields), minimum artifact count (3), evidence level guide, warnings, validation workflow, honest-use boundary. src/openamp_foundry/evidence/preprint_bundle.py with PreprintBundleEntry dataclass (11 fields: bundle_id, batch_id, pipeline_version, submission_date, title, artifact_ids, evidence_level, preprint_doi, contact_email, release_approved, dry_lab_only), PreprintBundleResult dataclass (5 fields: bundle_id, batch_id, artifact_count, passed, errors, warnings, dry_lab_only=True), MINIMUM_ARTIFACTS (3), RECOMMENDED_ARTIFACT_COUNT (5), MAX_TITLE_LENGTH (300), VALID_EVIDENCE_LEVELS (6: 1-6), validate_preprint_bundle() (10 checks: BND- prefix, non-empty batch_id/pipeline_version/contact_email, YYYY-MM-DD submission_date, non-empty title<=300 chars, artifact_ids>=3, valid evidence_level 1-6, release_approved must be True, dry_lab_only must be True; evidence_level<=2 warns, empty preprint_doi warns, artifact_count<5 warns), validate_preprint_bundle_dict() (10 required fields guard). CLI (openamp-foundry preprint-bundle-check) with --entry-json, --format text|json. make preprint-bundle-check target. Phase L L1: scientific preprints now have a machine-validated evidence bundle structure.
> **New in v0.8.3:** Uncertainty quantification report schema (K5 — Loop 123) — docs/evidence/UNCERTAINTY_REPORT_GUIDE.md with purpose, required field table (12 fields), valid metric names (5), warnings, validation workflow, honest-use boundary. src/openamp_foundry/evidence/uncertainty_report.py with UncertaintyReportEntry dataclass (12 fields: report_id, batch_id, candidate_id, pipeline_version, metric_name, point_estimate, lower_bound, upper_bound, confidence_level, calibration_source, reviewer, dry_lab_only), UncertaintyReportResult dataclass (6 fields: report_id, candidate_id, interval_width, passed, errors, warnings, dry_lab_only=True), VALID_METRIC_NAMES (5: cytotoxicity_score, hemolysis_fraction, mic, selectivity_index, stability_score), WIDE_INTERVAL_THRESHOLD (10.0), validate_uncertainty_report() (11 checks: UQ- prefix, non-empty batch_id/candidate_id/pipeline_version/calibration_source/reviewer, valid metric_name, lower_bound<=point_estimate, upper_bound>=point_estimate, confidence_level 0.0-1.0, dry_lab_only must be True; wide interval warns, confidence<0.80 warns, confidence>0.99 warns), validate_uncertainty_report_dict() (11 required fields guard). CLI (openamp-foundry uncertainty-report-check) with --entry-json, --format text|json. make uncertainty-report-check target. Phase K K5: prediction intervals for external reviewers are now machine-validated.
Expand Down
3 changes: 3 additions & 0 deletions docs/research/ROADMAP.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
# Roadmap

### v0.8.6
- Phase L L3: candidate summary card schema — validates publication-ready per-candidate structured summaries with sequence, evidence level, activity prediction, and safety flags. CandidateSummaryCardEntry dataclass, validate_candidate_summary_card(), CLI candidate-summary-card-check.

### v0.8.5
- Phase L L2: reproducibility manifest schema — captures exact software versions, data checksums, and random seeds for a pipeline run. ReproducibilityManifestEntry dataclass, validate_reproducibility_manifest(), CLI reproducibility-manifest-check.

Expand Down
34 changes: 34 additions & 0 deletions src/openamp_foundry/cli/commands/reports.py
Original file line number Diff line number Diff line change
Expand Up @@ -3056,3 +3056,37 @@ def _run_reproducibility_manifest_check(args: argparse.Namespace) -> int:
print(" Reproducibility manifest validated. Run is fully documented for third-party reproduction.")

return 0 if result.passed else 3


def _run_candidate_summary_card_check(args: argparse.Namespace) -> int:
"""Validate a candidate summary card entry."""
import json as _json
from openamp_foundry.evidence.candidate_summary_card import validate_candidate_summary_card_dict

try:
d = _json.loads(args.entry_json)
except _json.JSONDecodeError as exc:
print(_json.dumps({"status": "error", "error": f"Invalid JSON: {exc}"}))
return 2

if not isinstance(d, dict):
print(_json.dumps({"status": "error", "error": "--entry-json must be a JSON object"}))
return 2

result = validate_candidate_summary_card_dict(d)
output_format = getattr(args, "format", "text")

if output_format == "json":
import dataclasses
print(_json.dumps(dataclasses.asdict(result), indent=2))
else:
status = "PASS" if result.passed else "FAIL"
print(f"Candidate summary card {result.card_id} (candidate {result.candidate_id}, length {result.sequence_length}): {status}")
for e in result.errors:
print(f" ERROR: {e}")
for w in result.warnings:
print(f" WARN: {w}")
if result.passed:
print(" Candidate summary card validated. Ready for reviewer packet.")

return 0 if result.passed else 3
23 changes: 22 additions & 1 deletion src/openamp_foundry/cli/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
from openamp_foundry.cli.commands.reports import _run_security_report_check
from openamp_foundry.cli.commands.reports import _run_citation_check
from openamp_foundry.cli.commands.reports import _run_roadmap_sync_check
from openamp_foundry.cli.commands.reports import _run_advisory_review_check, _run_annual_review_check, _run_selection_rationale_check, _run_batch_priority_check, _run_pilot_package_check, _run_calibration_intake_check, _run_uncertainty_report_check, _run_preprint_bundle_check, _run_reproducibility_manifest_check
from openamp_foundry.cli.commands.reports import _run_advisory_review_check, _run_annual_review_check, _run_selection_rationale_check, _run_batch_priority_check, _run_pilot_package_check, _run_calibration_intake_check, _run_uncertainty_report_check, _run_preprint_bundle_check, _run_reproducibility_manifest_check, _run_candidate_summary_card_check

import argparse
import json
Expand Down Expand Up @@ -2107,6 +2107,24 @@ def build_parser() -> argparse.ArgumentParser:
help="Output format (default: text).",
)

# ── Candidate summary card check (Phase L L3) ─────────────────────
cscc = sub.add_parser(
"candidate-summary-card-check",
help=(
"Validate a candidate summary card. Confirms sequence, evidence level, "
"activity label, and safety flags. Dry-lab only."
),
)
cscc.add_argument(
"--entry-json", type=str, required=True,
help="JSON string of a CandidateSummaryCardEntry dict (required).",
)
cscc.add_argument(
"--format", type=str, default="text",
choices=["text", "json"],
help="Output format (default: text).",
)

# ── Selection rationale check (Phase K K1) ───────────────────────
src2 = sub.add_parser(
"selection-rationale-check",
Expand Down Expand Up @@ -2433,6 +2451,9 @@ def main(argv: list[str] | None = None) -> int:
if args.command == "reproducibility-manifest-check":
return _run_reproducibility_manifest_check(args)

if args.command == "candidate-summary-card-check":
return _run_candidate_summary_card_check(args)

parser.error("unknown command")
return 2

Expand Down
Loading
Loading