Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report simulation-deprecation-check simulation-scope-check simulation-evidence-packet artifact-version candidate-manifest artifact-changelog integration-check adapter-author-check license-check artifact-compat-check contribution-check adoption-scorecard release-gate-check decision-log release-request-check coi-check rotation-plan-check security-report-check citation-check roadmap-sync-check advisory-review-check annual-review-check selection-rationale-check batch-priority-check pilot-package-check calibration-intake-check uncertainty-report-check preprint-bundle-check reproducibility-manifest-check candidate-summary-card-check multi-candidate-comparison-check dataset-release-check
.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report simulation-deprecation-check simulation-scope-check simulation-evidence-packet artifact-version candidate-manifest artifact-changelog integration-check adapter-author-check license-check artifact-compat-check contribution-check adoption-scorecard release-gate-check decision-log release-request-check coi-check rotation-plan-check security-report-check citation-check roadmap-sync-check advisory-review-check annual-review-check selection-rationale-check batch-priority-check pilot-package-check calibration-intake-check uncertainty-report-check preprint-bundle-check reproducibility-manifest-check candidate-summary-card-check multi-candidate-comparison-check pipeline-decision-audit-check dataset-release-check

PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3)
PYTEST := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest)
Expand Down Expand Up @@ -931,6 +931,11 @@ multi-candidate-comparison-check:
--entry-json '{"comparison_id":"CMP-001","batch_id":"BATCH-001","pipeline_version":"0.8.7","comparison_date":"2026-07-09","candidate_ids":["AMP-001","AMP-002","AMP-003"],"comparison_criteria":["predicted_mic","hemolysis_fraction","selectivity_index"],"top_candidate_id":"AMP-001","top_candidate_rationale":"Highest selectivity index with lowest hemolysis fraction.","evidence_level":4,"reviewer":"alice","dry_lab_only":true}' \
--format text

pipeline-decision-audit-check:
.venv/bin/openamp-foundry pipeline-decision-audit-check \
--entry-json '{"audit_id":"AUD-001","batch_id":"BATCH-001","pipeline_version":"0.8.9","decision_date":"2026-07-09","decision_type":"filter_applied","decision_description":"Applied hemolysis fraction filter at threshold 0.15.","rationale":"Threshold pre-specified in BENCHMARK_GOVERNANCE.md based on Wave 0 data.","alternatives_considered":["threshold 0.10","threshold 0.20"],"affected_candidate_count":47,"evidence_level":4,"pre_specified":true,"reviewer":"alice","dry_lab_only":true}' \
--format text

dataset-release-check:
.venv/bin/openamp-foundry dataset-release-check \
--entry-json '{"release_id":"DSR-001","dataset_name":"OpenAMP Wave 0.5 Candidates","dataset_version":"1.0.0","release_date":"2026-07-09","license_identifier":"CC-BY-4.0","data_sources":["UniProt AMP database","APD3","in-house pipeline"],"contains_sequences":true,"contains_activity_data":false,"dual_use_assessed":true,"usage_policy_url":"https://openamp.example.org/data-policy","contact_email":"data@openamp.example.org","release_approved":true,"dry_lab_only":true}' \
Expand Down
3 changes: 2 additions & 1 deletion docs/evidence/METRICS_CURRENT.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak
> **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees
> with this file, this file wins. Updated whenever benchmark/benchmark config changes.
>
> **Last updated:** 2026-07-09 (Phase L L5 — dataset release package checker — v0.8.8)
> **Last updated:** 2026-07-09 (Phase M M1 — pipeline decision audit entry schema — v0.8.9)
> **New in v0.8.9:** Pipeline decision audit entry schema (M1 — Loop 128) — docs/evidence/PIPELINE_DECISION_AUDIT_GUIDE.md with purpose, required field table (13 fields), valid decision types (7), warnings, validation workflow, honest-use boundary. src/openamp_foundry/evidence/pipeline_decision_audit.py with PipelineDecisionAuditEntry dataclass (13 fields: audit_id, batch_id, pipeline_version, decision_date, decision_type, decision_description, rationale, alternatives_considered, affected_candidate_count, evidence_level, pre_specified, reviewer, dry_lab_only), PipelineDecisionAuditResult dataclass (5 fields: audit_id, batch_id, decision_type, passed, errors, warnings, dry_lab_only=True), VALID_DECISION_TYPES (7: benchmark_updated, calibration_adjusted, candidate_ranked, candidate_rejected, filter_applied, safety_flag_applied, threshold_chosen), VALID_EVIDENCE_LEVELS (6: 1-6), MAX_DESCRIPTION_LENGTH (500), MAX_RATIONALE_LENGTH (1000), validate_pipeline_decision_audit() (12 checks: AUD- prefix, non-empty batch_id/pipeline_version/reviewer, YYYY-MM-DD decision_date, valid decision_type, non-empty description<=500 chars, non-empty rationale<=1000 chars, affected_candidate_count>=0, valid evidence_level, dry_lab_only must be True; post-hoc warns, empty alternatives warns, low evidence warns, zero affected warns), validate_pipeline_decision_audit_dict() (12 required fields guard). CLI (openamp-foundry pipeline-decision-audit-check) with --entry-json, --format text|json. make pipeline-decision-audit-check target. Phase M M1: every pipeline decision is now machine-validated and traceable for external audit.
> **New in v0.8.8:** Dataset release package checker (L5 — Loop 127) — docs/evidence/DATASET_RELEASE_GUIDE.md with purpose, required field table (13 fields), valid license identifiers (5), warnings, validation workflow, honest-use boundary. src/openamp_foundry/evidence/dataset_release.py with DatasetReleaseEntry dataclass (13 fields: release_id, dataset_name, dataset_version, release_date, license_identifier, data_sources, contains_sequences, contains_activity_data, dual_use_assessed, usage_policy_url, contact_email, release_approved, dry_lab_only), DatasetReleaseResult dataclass (4 fields: release_id, dataset_name, passed, errors, warnings, dry_lab_only=True), VALID_LICENSE_IDENTIFIERS (5: Apache-2.0, CC-BY-4.0, CC-BY-NC-4.0, CC0-1.0, MIT), MINIMUM_DATA_SOURCES (1), validate_dataset_release() (11 checks: DSR- prefix, non-empty dataset_name/dataset_version/usage_policy_url/contact_email, YYYY-MM-DD release_date, valid license_identifier, data_sources>=1, dual_use_assessed must be True, release_approved must be True, dry_lab_only must be True; CC-BY-NC-4.0 warns, single source warns), validate_dataset_release_dict() (12 required fields guard). CLI (openamp-foundry dataset-release-check) with --entry-json, --format text|json. make dataset-release-check target. Phase L L5 completes Phase L: open dataset releases now have machine-validated data governance checks.
> **New in v0.8.7:** Multi-candidate comparison schema (L4 — Loop 126b) — docs/evidence/MULTI_CANDIDATE_COMPARISON_GUIDE.md with purpose, required field table (11 fields), minimum requirements (2 candidates, 2 criteria), warnings, validation workflow, honest-use boundary. src/openamp_foundry/evidence/multi_candidate_comparison.py with MultiCandidateComparisonEntry dataclass (11 fields: comparison_id, batch_id, pipeline_version, comparison_date, candidate_ids, comparison_criteria, top_candidate_id, top_candidate_rationale, evidence_level, reviewer, dry_lab_only), MultiCandidateComparisonResult dataclass (5 fields: comparison_id, batch_id, candidate_count, passed, errors, warnings, dry_lab_only=True), MINIMUM_CANDIDATES (2), MINIMUM_CRITERIA (2), RECOMMENDED_CRITERIA (3), MAX_RATIONALE_LENGTH (500), LARGE_CANDIDATE_SET_THRESHOLD (10), VALID_EVIDENCE_LEVELS (6: 1-6), validate_multi_candidate_comparison() (11 checks: CMP- prefix, non-empty batch_id/pipeline_version/reviewer, YYYY-MM-DD comparison_date, candidate_ids>=2, comparison_criteria>=2, top_candidate_id in candidate_ids, non-empty rationale<=500 chars, valid evidence_level, dry_lab_only must be True; evidence_level<=2 warns, candidate_count>10 warns, criteria_count<3 warns), validate_multi_candidate_comparison_dict() (10 required fields guard). CLI (openamp-foundry multi-candidate-comparison-check) with --entry-json, --format text|json. make multi-candidate-comparison-check target. Phase L L4: side-by-side candidate comparisons are now machine-validated for publication-ready supplementary tables.
> **New in v0.8.6:** Candidate summary card schema (L3 — Loop 126) — docs/evidence/CANDIDATE_SUMMARY_CARD_GUIDE.md with purpose, required field table (12 fields), valid activity labels (5), valid amino acid set (20 standard), warnings, validation workflow, honest-use boundary. src/openamp_foundry/evidence/candidate_summary_card.py with CandidateSummaryCardEntry dataclass (12 fields: card_id, candidate_id, batch_id, pipeline_version, sequence, sequence_length, evidence_level, predicted_activity, safety_flags, selection_rationale_id, reviewer, dry_lab_only), CandidateSummaryCardResult dataclass (5 fields: card_id, candidate_id, sequence_length, passed, errors, warnings, dry_lab_only=True), VALID_ACTIVITY_LABELS (5: high_activity, inactive, low_activity, moderate_activity, uncertain), VALID_AMINO_ACIDS (20 standard one-letter codes), LONG_PEPTIDE_THRESHOLD (50), VALID_EVIDENCE_LEVELS (6: 1-6), validate_candidate_summary_card() (11 checks: CRD- prefix, non-empty candidate_id/batch_id/pipeline_version/reviewer, non-empty sequence with valid amino acids, sequence_length==len(sequence), valid evidence_level, valid predicted_activity, SEL- prefix on selection_rationale_id, dry_lab_only must be True; evidence_level<=2 warns, safety_flags non-empty warns, uncertain activity warns, length>50 warns), validate_candidate_summary_card_dict() (11 required fields guard). CLI (openamp-foundry candidate-summary-card-check) with --entry-json, --format text|json. make candidate-summary-card-check target. Phase L L3: every candidate now has a machine-validated publication-ready summary card.
Expand Down
59 changes: 59 additions & 0 deletions docs/evidence/PIPELINE_DECISION_AUDIT_GUIDE.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
# Pipeline Decision Audit Entry Guide

A *pipeline decision audit entry* records each significant decision made during candidate
ranking, filtering, or selection — with the rationale, evidence level, and alternatives
considered. This guide defines the schema, validation rules, and honest-use constraints.

## Purpose

External auditors reviewing a preprint or evidence package need to:
- understand which filters were applied and why,
- verify that threshold choices were pre-specified rather than post-hoc,
- confirm that rejected candidates were documented, not silently dropped,
- trace each decision back to a specific pipeline version and reviewer.

## Required fields

| Field | Type | Constraint |
|---|---|---|
| `audit_id` | str | Must start with `"AUD-"` |
| `batch_id` | str | Non-empty |
| `pipeline_version` | str | Non-empty |
| `decision_date` | str | ISO 8601 `YYYY-MM-DD` |
| `decision_type` | str | One of `VALID_DECISION_TYPES` |
| `decision_description` | str | Non-empty, ≤ 500 characters |
| `rationale` | str | Non-empty, ≤ 1000 characters |
| `alternatives_considered` | list[str] | May be empty |
| `affected_candidate_count` | int | ≥ 0 |
| `evidence_level` | int | 1 – 6 |
| `pre_specified` | bool | Required field |
| `reviewer` | str | Non-empty |
| `dry_lab_only` | bool | Must be `True` |

## Valid decision types

`filter_applied`, `threshold_chosen`, `candidate_ranked`, `candidate_rejected`,
`benchmark_updated`, `safety_flag_applied`, `calibration_adjusted`

## Warnings

- `pre_specified` is `False` — warning: post-hoc decision; document justification carefully.
- `alternatives_considered` is empty — warning: no alternatives documented; consider recording what else was considered.
- `evidence_level <= 2` — warning: low evidence level for this decision.
- `affected_candidate_count == 0` — warning: decision affected no candidates; verify this is intentional.

## How to validate

```bash
make pipeline-decision-audit-check
# or
openamp-foundry pipeline-decision-audit-check \
--entry-json '{"audit_id":"AUD-001","batch_id":"BATCH-001","pipeline_version":"0.8.9","decision_date":"2026-07-09","decision_type":"filter_applied","decision_description":"Applied hemolysis fraction filter at threshold 0.15.","rationale":"Threshold pre-specified in BENCHMARK_GOVERNANCE.md based on Wave 0 data.","alternatives_considered":["threshold 0.10","threshold 0.20"],"affected_candidate_count":47,"evidence_level":4,"pre_specified":true,"reviewer":"alice","dry_lab_only":true}' \
--format text
```

## Honest-use boundary

Pipeline decision audit entries are dry-lab records. They document computational choices,
not experimental outcomes. A well-audited decision does not imply the selected candidates
are biologically active.
12 changes: 12 additions & 0 deletions docs/research/NEXT_100_PR_MAP.md
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,18 @@ Bridge the gap between evidence packages and published science.
| L5 | Add dataset release package checker (complete — Loop 127). | Validates open dataset releases meet data governance. | B/D |


## Phase M — Audit Trail Infrastructure

Make every pipeline decision traceable, independently verifiable, and resistant to post-hoc rationalization.

| PR | Task | Why it matters | Review class |
|---:|---|---|---|
| M1 | Add pipeline decision audit entry schema (complete — Loop 128). | Records filter/threshold/rank decisions with rationale for external audit. | B |
| M2 | Add claim-to-evidence mapper schema. | Maps each scientific claim to the artifact that supports it. | B |
| M3 | Add score decomposition report schema. | Documents how composite scores are computed from components. | B |
| M4 | Add reviewer briefing package checker. | One-stop summary that external reviewers need before auditing. | B |
| M5 | Add audit chain completeness checker. | Validates the evidence chain has no gaps from sequences to submission. | B/D |

## Prioritization rule

When choosing between tasks, prefer the one that:
Expand Down
3 changes: 3 additions & 0 deletions docs/research/ROADMAP.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
# Roadmap

### v0.8.9
- Phase M M1: pipeline decision audit entry schema — records each filter/threshold/rank decision with rationale and alternatives considered for external audit. PipelineDecisionAuditEntry dataclass, validate_pipeline_decision_audit(), CLI pipeline-decision-audit-check.

### v0.8.8
- Phase L L5: dataset release package checker — validates that open dataset releases meet data governance requirements (license, provenance, dual-use assessment, release approval). DatasetReleaseEntry dataclass, validate_dataset_release(), CLI dataset-release-check. Completes Phase L.

Expand Down
34 changes: 34 additions & 0 deletions src/openamp_foundry/cli/commands/reports.py
Original file line number Diff line number Diff line change
Expand Up @@ -3158,3 +3158,37 @@ def _run_dataset_release_check(args: argparse.Namespace) -> int:
print(" Dataset release validated. Package meets data governance requirements.")

return 0 if result.passed else 3


def _run_pipeline_decision_audit_check(args: argparse.Namespace) -> int:
"""Validate a pipeline decision audit entry."""
import json as _json
from openamp_foundry.evidence.pipeline_decision_audit import validate_pipeline_decision_audit_dict

try:
d = _json.loads(args.entry_json)
except _json.JSONDecodeError as exc:
print(_json.dumps({"status": "error", "error": f"Invalid JSON: {exc}"}))
return 2

if not isinstance(d, dict):
print(_json.dumps({"status": "error", "error": "--entry-json must be a JSON object"}))
return 2

result = validate_pipeline_decision_audit_dict(d)
output_format = getattr(args, "format", "text")

if output_format == "json":
import dataclasses
print(_json.dumps(dataclasses.asdict(result), indent=2))
else:
status = "PASS" if result.passed else "FAIL"
print(f"Pipeline decision audit {result.audit_id} (batch {result.batch_id}, type {result.decision_type}): {status}")
for e in result.errors:
print(f" ERROR: {e}")
for w in result.warnings:
print(f" WARN: {w}")
if result.passed:
print(" Pipeline decision audit validated. Decision is traceable and reviewable.")

return 0 if result.passed else 3
Loading
Loading