diff --git a/Makefile b/Makefile index 0d709541..db56de2c 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report simulation-deprecation-check simulation-scope-check simulation-evidence-packet artifact-version candidate-manifest artifact-changelog integration-check adapter-author-check license-check artifact-compat-check contribution-check adoption-scorecard release-gate-check decision-log release-request-check coi-check rotation-plan-check security-report-check citation-check roadmap-sync-check advisory-review-check annual-review-check selection-rationale-check batch-priority-check pilot-package-check calibration-intake-check uncertainty-report-check preprint-bundle-check reproducibility-manifest-check candidate-summary-card-check multi-candidate-comparison-check dataset-release-check +.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report simulation-deprecation-check simulation-scope-check simulation-evidence-packet artifact-version candidate-manifest artifact-changelog integration-check adapter-author-check license-check artifact-compat-check contribution-check adoption-scorecard release-gate-check decision-log release-request-check coi-check rotation-plan-check security-report-check citation-check roadmap-sync-check advisory-review-check annual-review-check selection-rationale-check batch-priority-check pilot-package-check calibration-intake-check uncertainty-report-check preprint-bundle-check reproducibility-manifest-check candidate-summary-card-check multi-candidate-comparison-check pipeline-decision-audit-check dataset-release-check PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3) PYTEST := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest) @@ -931,6 +931,11 @@ multi-candidate-comparison-check: --entry-json '{"comparison_id":"CMP-001","batch_id":"BATCH-001","pipeline_version":"0.8.7","comparison_date":"2026-07-09","candidate_ids":["AMP-001","AMP-002","AMP-003"],"comparison_criteria":["predicted_mic","hemolysis_fraction","selectivity_index"],"top_candidate_id":"AMP-001","top_candidate_rationale":"Highest selectivity index with lowest hemolysis fraction.","evidence_level":4,"reviewer":"alice","dry_lab_only":true}' \ --format text +pipeline-decision-audit-check: + .venv/bin/openamp-foundry pipeline-decision-audit-check \ + --entry-json '{"audit_id":"AUD-001","batch_id":"BATCH-001","pipeline_version":"0.8.9","decision_date":"2026-07-09","decision_type":"filter_applied","decision_description":"Applied hemolysis fraction filter at threshold 0.15.","rationale":"Threshold pre-specified in BENCHMARK_GOVERNANCE.md based on Wave 0 data.","alternatives_considered":["threshold 0.10","threshold 0.20"],"affected_candidate_count":47,"evidence_level":4,"pre_specified":true,"reviewer":"alice","dry_lab_only":true}' \ + --format text + dataset-release-check: .venv/bin/openamp-foundry dataset-release-check \ --entry-json '{"release_id":"DSR-001","dataset_name":"OpenAMP Wave 0.5 Candidates","dataset_version":"1.0.0","release_date":"2026-07-09","license_identifier":"CC-BY-4.0","data_sources":["UniProt AMP database","APD3","in-house pipeline"],"contains_sequences":true,"contains_activity_data":false,"dual_use_assessed":true,"usage_policy_url":"https://openamp.example.org/data-policy","contact_email":"data@openamp.example.org","release_approved":true,"dry_lab_only":true}' \ diff --git a/docs/evidence/METRICS_CURRENT.md b/docs/evidence/METRICS_CURRENT.md index 183f7b90..dd6f8542 100644 --- a/docs/evidence/METRICS_CURRENT.md +++ b/docs/evidence/METRICS_CURRENT.md @@ -5,7 +5,8 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees > with this file, this file wins. Updated whenever benchmark/benchmark config changes. > -> **Last updated:** 2026-07-09 (Phase L L5 — dataset release package checker — v0.8.8) +> **Last updated:** 2026-07-09 (Phase M M1 — pipeline decision audit entry schema — v0.8.9) +> **New in v0.8.9:** Pipeline decision audit entry schema (M1 — Loop 128) — docs/evidence/PIPELINE_DECISION_AUDIT_GUIDE.md with purpose, required field table (13 fields), valid decision types (7), warnings, validation workflow, honest-use boundary. src/openamp_foundry/evidence/pipeline_decision_audit.py with PipelineDecisionAuditEntry dataclass (13 fields: audit_id, batch_id, pipeline_version, decision_date, decision_type, decision_description, rationale, alternatives_considered, affected_candidate_count, evidence_level, pre_specified, reviewer, dry_lab_only), PipelineDecisionAuditResult dataclass (5 fields: audit_id, batch_id, decision_type, passed, errors, warnings, dry_lab_only=True), VALID_DECISION_TYPES (7: benchmark_updated, calibration_adjusted, candidate_ranked, candidate_rejected, filter_applied, safety_flag_applied, threshold_chosen), VALID_EVIDENCE_LEVELS (6: 1-6), MAX_DESCRIPTION_LENGTH (500), MAX_RATIONALE_LENGTH (1000), validate_pipeline_decision_audit() (12 checks: AUD- prefix, non-empty batch_id/pipeline_version/reviewer, YYYY-MM-DD decision_date, valid decision_type, non-empty description<=500 chars, non-empty rationale<=1000 chars, affected_candidate_count>=0, valid evidence_level, dry_lab_only must be True; post-hoc warns, empty alternatives warns, low evidence warns, zero affected warns), validate_pipeline_decision_audit_dict() (12 required fields guard). CLI (openamp-foundry pipeline-decision-audit-check) with --entry-json, --format text|json. make pipeline-decision-audit-check target. Phase M M1: every pipeline decision is now machine-validated and traceable for external audit. > **New in v0.8.8:** Dataset release package checker (L5 — Loop 127) — docs/evidence/DATASET_RELEASE_GUIDE.md with purpose, required field table (13 fields), valid license identifiers (5), warnings, validation workflow, honest-use boundary. src/openamp_foundry/evidence/dataset_release.py with DatasetReleaseEntry dataclass (13 fields: release_id, dataset_name, dataset_version, release_date, license_identifier, data_sources, contains_sequences, contains_activity_data, dual_use_assessed, usage_policy_url, contact_email, release_approved, dry_lab_only), DatasetReleaseResult dataclass (4 fields: release_id, dataset_name, passed, errors, warnings, dry_lab_only=True), VALID_LICENSE_IDENTIFIERS (5: Apache-2.0, CC-BY-4.0, CC-BY-NC-4.0, CC0-1.0, MIT), MINIMUM_DATA_SOURCES (1), validate_dataset_release() (11 checks: DSR- prefix, non-empty dataset_name/dataset_version/usage_policy_url/contact_email, YYYY-MM-DD release_date, valid license_identifier, data_sources>=1, dual_use_assessed must be True, release_approved must be True, dry_lab_only must be True; CC-BY-NC-4.0 warns, single source warns), validate_dataset_release_dict() (12 required fields guard). CLI (openamp-foundry dataset-release-check) with --entry-json, --format text|json. make dataset-release-check target. Phase L L5 completes Phase L: open dataset releases now have machine-validated data governance checks. > **New in v0.8.7:** Multi-candidate comparison schema (L4 — Loop 126b) — docs/evidence/MULTI_CANDIDATE_COMPARISON_GUIDE.md with purpose, required field table (11 fields), minimum requirements (2 candidates, 2 criteria), warnings, validation workflow, honest-use boundary. src/openamp_foundry/evidence/multi_candidate_comparison.py with MultiCandidateComparisonEntry dataclass (11 fields: comparison_id, batch_id, pipeline_version, comparison_date, candidate_ids, comparison_criteria, top_candidate_id, top_candidate_rationale, evidence_level, reviewer, dry_lab_only), MultiCandidateComparisonResult dataclass (5 fields: comparison_id, batch_id, candidate_count, passed, errors, warnings, dry_lab_only=True), MINIMUM_CANDIDATES (2), MINIMUM_CRITERIA (2), RECOMMENDED_CRITERIA (3), MAX_RATIONALE_LENGTH (500), LARGE_CANDIDATE_SET_THRESHOLD (10), VALID_EVIDENCE_LEVELS (6: 1-6), validate_multi_candidate_comparison() (11 checks: CMP- prefix, non-empty batch_id/pipeline_version/reviewer, YYYY-MM-DD comparison_date, candidate_ids>=2, comparison_criteria>=2, top_candidate_id in candidate_ids, non-empty rationale<=500 chars, valid evidence_level, dry_lab_only must be True; evidence_level<=2 warns, candidate_count>10 warns, criteria_count<3 warns), validate_multi_candidate_comparison_dict() (10 required fields guard). CLI (openamp-foundry multi-candidate-comparison-check) with --entry-json, --format text|json. make multi-candidate-comparison-check target. Phase L L4: side-by-side candidate comparisons are now machine-validated for publication-ready supplementary tables. > **New in v0.8.6:** Candidate summary card schema (L3 — Loop 126) — docs/evidence/CANDIDATE_SUMMARY_CARD_GUIDE.md with purpose, required field table (12 fields), valid activity labels (5), valid amino acid set (20 standard), warnings, validation workflow, honest-use boundary. src/openamp_foundry/evidence/candidate_summary_card.py with CandidateSummaryCardEntry dataclass (12 fields: card_id, candidate_id, batch_id, pipeline_version, sequence, sequence_length, evidence_level, predicted_activity, safety_flags, selection_rationale_id, reviewer, dry_lab_only), CandidateSummaryCardResult dataclass (5 fields: card_id, candidate_id, sequence_length, passed, errors, warnings, dry_lab_only=True), VALID_ACTIVITY_LABELS (5: high_activity, inactive, low_activity, moderate_activity, uncertain), VALID_AMINO_ACIDS (20 standard one-letter codes), LONG_PEPTIDE_THRESHOLD (50), VALID_EVIDENCE_LEVELS (6: 1-6), validate_candidate_summary_card() (11 checks: CRD- prefix, non-empty candidate_id/batch_id/pipeline_version/reviewer, non-empty sequence with valid amino acids, sequence_length==len(sequence), valid evidence_level, valid predicted_activity, SEL- prefix on selection_rationale_id, dry_lab_only must be True; evidence_level<=2 warns, safety_flags non-empty warns, uncertain activity warns, length>50 warns), validate_candidate_summary_card_dict() (11 required fields guard). CLI (openamp-foundry candidate-summary-card-check) with --entry-json, --format text|json. make candidate-summary-card-check target. Phase L L3: every candidate now has a machine-validated publication-ready summary card. diff --git a/docs/evidence/PIPELINE_DECISION_AUDIT_GUIDE.md b/docs/evidence/PIPELINE_DECISION_AUDIT_GUIDE.md new file mode 100644 index 00000000..e9265048 --- /dev/null +++ b/docs/evidence/PIPELINE_DECISION_AUDIT_GUIDE.md @@ -0,0 +1,59 @@ +# Pipeline Decision Audit Entry Guide + +A *pipeline decision audit entry* records each significant decision made during candidate +ranking, filtering, or selection — with the rationale, evidence level, and alternatives +considered. This guide defines the schema, validation rules, and honest-use constraints. + +## Purpose + +External auditors reviewing a preprint or evidence package need to: +- understand which filters were applied and why, +- verify that threshold choices were pre-specified rather than post-hoc, +- confirm that rejected candidates were documented, not silently dropped, +- trace each decision back to a specific pipeline version and reviewer. + +## Required fields + +| Field | Type | Constraint | +|---|---|---| +| `audit_id` | str | Must start with `"AUD-"` | +| `batch_id` | str | Non-empty | +| `pipeline_version` | str | Non-empty | +| `decision_date` | str | ISO 8601 `YYYY-MM-DD` | +| `decision_type` | str | One of `VALID_DECISION_TYPES` | +| `decision_description` | str | Non-empty, ≤ 500 characters | +| `rationale` | str | Non-empty, ≤ 1000 characters | +| `alternatives_considered` | list[str] | May be empty | +| `affected_candidate_count` | int | ≥ 0 | +| `evidence_level` | int | 1 – 6 | +| `pre_specified` | bool | Required field | +| `reviewer` | str | Non-empty | +| `dry_lab_only` | bool | Must be `True` | + +## Valid decision types + +`filter_applied`, `threshold_chosen`, `candidate_ranked`, `candidate_rejected`, +`benchmark_updated`, `safety_flag_applied`, `calibration_adjusted` + +## Warnings + +- `pre_specified` is `False` — warning: post-hoc decision; document justification carefully. +- `alternatives_considered` is empty — warning: no alternatives documented; consider recording what else was considered. +- `evidence_level <= 2` — warning: low evidence level for this decision. +- `affected_candidate_count == 0` — warning: decision affected no candidates; verify this is intentional. + +## How to validate + +```bash +make pipeline-decision-audit-check +# or +openamp-foundry pipeline-decision-audit-check \ + --entry-json '{"audit_id":"AUD-001","batch_id":"BATCH-001","pipeline_version":"0.8.9","decision_date":"2026-07-09","decision_type":"filter_applied","decision_description":"Applied hemolysis fraction filter at threshold 0.15.","rationale":"Threshold pre-specified in BENCHMARK_GOVERNANCE.md based on Wave 0 data.","alternatives_considered":["threshold 0.10","threshold 0.20"],"affected_candidate_count":47,"evidence_level":4,"pre_specified":true,"reviewer":"alice","dry_lab_only":true}' \ + --format text +``` + +## Honest-use boundary + +Pipeline decision audit entries are dry-lab records. They document computational choices, +not experimental outcomes. A well-audited decision does not imply the selected candidates +are biologically active. diff --git a/docs/research/NEXT_100_PR_MAP.md b/docs/research/NEXT_100_PR_MAP.md index d206f622..91f81afa 100644 --- a/docs/research/NEXT_100_PR_MAP.md +++ b/docs/research/NEXT_100_PR_MAP.md @@ -213,6 +213,18 @@ Bridge the gap between evidence packages and published science. | L5 | Add dataset release package checker (complete — Loop 127). | Validates open dataset releases meet data governance. | B/D | +## Phase M — Audit Trail Infrastructure + +Make every pipeline decision traceable, independently verifiable, and resistant to post-hoc rationalization. + +| PR | Task | Why it matters | Review class | +|---:|---|---|---| +| M1 | Add pipeline decision audit entry schema (complete — Loop 128). | Records filter/threshold/rank decisions with rationale for external audit. | B | +| M2 | Add claim-to-evidence mapper schema. | Maps each scientific claim to the artifact that supports it. | B | +| M3 | Add score decomposition report schema. | Documents how composite scores are computed from components. | B | +| M4 | Add reviewer briefing package checker. | One-stop summary that external reviewers need before auditing. | B | +| M5 | Add audit chain completeness checker. | Validates the evidence chain has no gaps from sequences to submission. | B/D | + ## Prioritization rule When choosing between tasks, prefer the one that: diff --git a/docs/research/ROADMAP.md b/docs/research/ROADMAP.md index d0ddb4e1..3c1a10dc 100644 --- a/docs/research/ROADMAP.md +++ b/docs/research/ROADMAP.md @@ -1,5 +1,8 @@ # Roadmap +### v0.8.9 +- Phase M M1: pipeline decision audit entry schema — records each filter/threshold/rank decision with rationale and alternatives considered for external audit. PipelineDecisionAuditEntry dataclass, validate_pipeline_decision_audit(), CLI pipeline-decision-audit-check. + ### v0.8.8 - Phase L L5: dataset release package checker — validates that open dataset releases meet data governance requirements (license, provenance, dual-use assessment, release approval). DatasetReleaseEntry dataclass, validate_dataset_release(), CLI dataset-release-check. Completes Phase L. diff --git a/src/openamp_foundry/cli/commands/reports.py b/src/openamp_foundry/cli/commands/reports.py index e77a29ec..bf9b8adf 100644 --- a/src/openamp_foundry/cli/commands/reports.py +++ b/src/openamp_foundry/cli/commands/reports.py @@ -3158,3 +3158,37 @@ def _run_dataset_release_check(args: argparse.Namespace) -> int: print(" Dataset release validated. Package meets data governance requirements.") return 0 if result.passed else 3 + + +def _run_pipeline_decision_audit_check(args: argparse.Namespace) -> int: + """Validate a pipeline decision audit entry.""" + import json as _json + from openamp_foundry.evidence.pipeline_decision_audit import validate_pipeline_decision_audit_dict + + try: + d = _json.loads(args.entry_json) + except _json.JSONDecodeError as exc: + print(_json.dumps({"status": "error", "error": f"Invalid JSON: {exc}"})) + return 2 + + if not isinstance(d, dict): + print(_json.dumps({"status": "error", "error": "--entry-json must be a JSON object"})) + return 2 + + result = validate_pipeline_decision_audit_dict(d) + output_format = getattr(args, "format", "text") + + if output_format == "json": + import dataclasses + print(_json.dumps(dataclasses.asdict(result), indent=2)) + else: + status = "PASS" if result.passed else "FAIL" + print(f"Pipeline decision audit {result.audit_id} (batch {result.batch_id}, type {result.decision_type}): {status}") + for e in result.errors: + print(f" ERROR: {e}") + for w in result.warnings: + print(f" WARN: {w}") + if result.passed: + print(" Pipeline decision audit validated. Decision is traceable and reviewable.") + + return 0 if result.passed else 3 diff --git a/src/openamp_foundry/cli/main.py b/src/openamp_foundry/cli/main.py index 1b0d6491..689878b5 100644 --- a/src/openamp_foundry/cli/main.py +++ b/src/openamp_foundry/cli/main.py @@ -15,7 +15,7 @@ from openamp_foundry.cli.commands.reports import _run_security_report_check from openamp_foundry.cli.commands.reports import _run_citation_check from openamp_foundry.cli.commands.reports import _run_roadmap_sync_check -from openamp_foundry.cli.commands.reports import _run_advisory_review_check, _run_annual_review_check, _run_selection_rationale_check, _run_batch_priority_check, _run_pilot_package_check, _run_calibration_intake_check, _run_uncertainty_report_check, _run_preprint_bundle_check, _run_reproducibility_manifest_check, _run_candidate_summary_card_check, _run_multi_candidate_comparison_check, _run_dataset_release_check +from openamp_foundry.cli.commands.reports import _run_advisory_review_check, _run_annual_review_check, _run_selection_rationale_check, _run_batch_priority_check, _run_pilot_package_check, _run_calibration_intake_check, _run_uncertainty_report_check, _run_preprint_bundle_check, _run_reproducibility_manifest_check, _run_candidate_summary_card_check, _run_multi_candidate_comparison_check, _run_dataset_release_check, _run_pipeline_decision_audit_check import argparse import json @@ -2161,6 +2161,24 @@ def build_parser() -> argparse.ArgumentParser: help="Output format (default: text).", ) + # ── Pipeline decision audit check (Phase M M1) ──────────────────── + pdac = sub.add_parser( + "pipeline-decision-audit-check", + help=( + "Validate a pipeline decision audit entry. Records filter/threshold/rank " + "decisions with rationale for external audit. Dry-lab only." + ), + ) + pdac.add_argument( + "--entry-json", type=str, required=True, + help="JSON string of a PipelineDecisionAuditEntry dict (required).", + ) + pdac.add_argument( + "--format", type=str, default="text", + choices=["text", "json"], + help="Output format (default: text).", + ) + # ── Selection rationale check (Phase K K1) ─────────────────────── src2 = sub.add_parser( "selection-rationale-check", @@ -2496,6 +2514,9 @@ def main(argv: list[str] | None = None) -> int: if args.command == "dataset-release-check": return _run_dataset_release_check(args) + if args.command == "pipeline-decision-audit-check": + return _run_pipeline_decision_audit_check(args) + parser.error("unknown command") return 2 diff --git a/src/openamp_foundry/evidence/pipeline_decision_audit.py b/src/openamp_foundry/evidence/pipeline_decision_audit.py new file mode 100644 index 00000000..f646c4df --- /dev/null +++ b/src/openamp_foundry/evidence/pipeline_decision_audit.py @@ -0,0 +1,195 @@ +"""Pipeline decision audit entry schema (Phase M M1). + +Records each significant decision made during candidate ranking, +filtering, or selection to support external audit of the pipeline. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import List + +VALID_DECISION_TYPES: set[str] = { + "benchmark_updated", + "calibration_adjusted", + "candidate_ranked", + "candidate_rejected", + "filter_applied", + "safety_flag_applied", + "threshold_chosen", +} + +VALID_EVIDENCE_LEVELS: set[int] = {1, 2, 3, 4, 5, 6} +MAX_DESCRIPTION_LENGTH: int = 500 +MAX_RATIONALE_LENGTH: int = 1000 + +_DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$") + + +@dataclass +class PipelineDecisionAuditEntry: + """One pipeline decision audit record.""" + + audit_id: str + batch_id: str + pipeline_version: str + decision_date: str + decision_type: str + decision_description: str + rationale: str + alternatives_considered: List[str] + affected_candidate_count: int + evidence_level: int + pre_specified: bool + reviewer: str + dry_lab_only: bool = True + + +@dataclass +class PipelineDecisionAuditResult: + """Validation result for a PipelineDecisionAuditEntry.""" + + audit_id: str + batch_id: str + decision_type: str + passed: bool + errors: List[str] + warnings: List[str] + dry_lab_only: bool = True + + +def validate_pipeline_decision_audit( + entry: PipelineDecisionAuditEntry, +) -> PipelineDecisionAuditResult: + """Validate a PipelineDecisionAuditEntry. Returns a PipelineDecisionAuditResult.""" + errors: List[str] = [] + warnings: List[str] = [] + + if not entry.audit_id.startswith("AUD-"): + errors.append("audit_id must start with 'AUD-'") + + if not entry.batch_id: + errors.append("batch_id must not be empty") + + if not entry.pipeline_version: + errors.append("pipeline_version must not be empty") + + if not _DATE_RE.match(entry.decision_date): + errors.append("decision_date must be YYYY-MM-DD") + + if entry.decision_type not in VALID_DECISION_TYPES: + errors.append( + f"decision_type '{entry.decision_type}' is not valid; " + f"must be one of {sorted(VALID_DECISION_TYPES)}" + ) + + if not entry.decision_description: + errors.append("decision_description must not be empty") + elif len(entry.decision_description) > MAX_DESCRIPTION_LENGTH: + errors.append( + f"decision_description exceeds {MAX_DESCRIPTION_LENGTH} characters " + f"({len(entry.decision_description)} chars)" + ) + + if not entry.rationale: + errors.append("rationale must not be empty") + elif len(entry.rationale) > MAX_RATIONALE_LENGTH: + errors.append( + f"rationale exceeds {MAX_RATIONALE_LENGTH} characters " + f"({len(entry.rationale)} chars)" + ) + + if entry.affected_candidate_count < 0: + errors.append("affected_candidate_count must be >= 0") + + if entry.evidence_level not in VALID_EVIDENCE_LEVELS: + errors.append( + f"evidence_level {entry.evidence_level} is not valid; " + f"must be one of {sorted(VALID_EVIDENCE_LEVELS)}" + ) + + if not entry.reviewer: + errors.append("reviewer must not be empty") + + if entry.dry_lab_only is not True: + errors.append("dry_lab_only must be True") + + if not errors: + if not entry.pre_specified: + warnings.append( + "pre_specified is False; this is a post-hoc decision. " + "Document the justification carefully to avoid selection bias." + ) + + if not entry.alternatives_considered: + warnings.append( + "alternatives_considered is empty; consider recording what " + "alternative thresholds or approaches were evaluated." + ) + + if entry.evidence_level <= 2: + warnings.append( + f"evidence_level {entry.evidence_level} is low (≤2); " + "this decision rests on limited computational evidence." + ) + + if entry.affected_candidate_count == 0: + warnings.append( + "affected_candidate_count is 0; verify this decision " + "actually affected no candidates." + ) + + return PipelineDecisionAuditResult( + audit_id=entry.audit_id, + batch_id=entry.batch_id, + decision_type=entry.decision_type, + passed=len(errors) == 0, + errors=errors, + warnings=warnings, + ) + + +def validate_pipeline_decision_audit_dict(d: dict) -> PipelineDecisionAuditResult: + """Validate a dict representation of a PipelineDecisionAuditEntry.""" + required = [ + "audit_id", + "batch_id", + "pipeline_version", + "decision_date", + "decision_type", + "decision_description", + "rationale", + "alternatives_considered", + "affected_candidate_count", + "evidence_level", + "pre_specified", + "reviewer", + ] + for key in required: + if key not in d: + return PipelineDecisionAuditResult( + audit_id=d.get("audit_id", ""), + batch_id=d.get("batch_id", ""), + decision_type=d.get("decision_type", ""), + passed=False, + errors=[f"missing required field: {key}"], + warnings=[], + ) + + entry = PipelineDecisionAuditEntry( + audit_id=d["audit_id"], + batch_id=d["batch_id"], + pipeline_version=d["pipeline_version"], + decision_date=d["decision_date"], + decision_type=d["decision_type"], + decision_description=d["decision_description"], + rationale=d["rationale"], + alternatives_considered=d["alternatives_considered"], + affected_candidate_count=d["affected_candidate_count"], + evidence_level=d["evidence_level"], + pre_specified=d["pre_specified"], + reviewer=d["reviewer"], + dry_lab_only=d.get("dry_lab_only", True), + ) + return validate_pipeline_decision_audit(entry) diff --git a/tests/evidence/test_pipeline_decision_audit.py b/tests/evidence/test_pipeline_decision_audit.py new file mode 100644 index 00000000..f11a0bf5 --- /dev/null +++ b/tests/evidence/test_pipeline_decision_audit.py @@ -0,0 +1,284 @@ +"""Tests for pipeline decision audit entry schema (Phase M M1).""" + +from openamp_foundry.evidence.pipeline_decision_audit import ( + MAX_DESCRIPTION_LENGTH, + MAX_RATIONALE_LENGTH, + VALID_DECISION_TYPES, + VALID_EVIDENCE_LEVELS, + PipelineDecisionAuditEntry, + validate_pipeline_decision_audit, + validate_pipeline_decision_audit_dict, +) + + +def _valid_entry(**kwargs) -> PipelineDecisionAuditEntry: + defaults = dict( + audit_id="AUD-001", + batch_id="BATCH-001", + pipeline_version="0.8.9", + decision_date="2026-07-09", + decision_type="filter_applied", + decision_description="Applied hemolysis fraction filter at threshold 0.15.", + rationale="Threshold pre-specified in BENCHMARK_GOVERNANCE.md based on Wave 0 data.", + alternatives_considered=["threshold 0.10", "threshold 0.20"], + affected_candidate_count=47, + evidence_level=4, + pre_specified=True, + reviewer="alice", + dry_lab_only=True, + ) + defaults.update(kwargs) + return PipelineDecisionAuditEntry(**defaults) + + +# ── Constants ──────────────────────────────────────────────────────────────── + + +def test_valid_decision_types_count(): + assert len(VALID_DECISION_TYPES) == 7 + + +def test_filter_applied_in_decision_types(): + assert "filter_applied" in VALID_DECISION_TYPES + + +def test_safety_flag_applied_in_decision_types(): + assert "safety_flag_applied" in VALID_DECISION_TYPES + + +def test_max_description_length(): + assert MAX_DESCRIPTION_LENGTH == 500 + + +def test_max_rationale_length(): + assert MAX_RATIONALE_LENGTH == 1000 + + +# ── Valid entry ─────────────────────────────────────────────────────────────── + + +def test_valid_entry_passes(): + result = validate_pipeline_decision_audit(_valid_entry()) + assert result.passed + assert result.errors == [] + + +def test_result_dry_lab_only_true(): + result = validate_pipeline_decision_audit(_valid_entry()) + assert result.dry_lab_only is True + + +def test_result_fields_match(): + result = validate_pipeline_decision_audit(_valid_entry()) + assert result.audit_id == "AUD-001" + assert result.batch_id == "BATCH-001" + assert result.decision_type == "filter_applied" + + +def test_valid_all_decision_types(): + for dt in VALID_DECISION_TYPES: + result = validate_pipeline_decision_audit(_valid_entry(decision_type=dt)) + assert result.passed, f"decision_type '{dt}' should be valid" + + +def test_valid_all_evidence_levels(): + for level in VALID_EVIDENCE_LEVELS - {1, 2}: + result = validate_pipeline_decision_audit(_valid_entry(evidence_level=level)) + assert result.passed + + +def test_valid_empty_alternatives(): + result = validate_pipeline_decision_audit(_valid_entry(alternatives_considered=[])) + assert result.passed + + +def test_valid_zero_affected_candidates(): + result = validate_pipeline_decision_audit(_valid_entry(affected_candidate_count=0)) + assert result.passed + + +# ── audit_id validation ─────────────────────────────────────────────────────── + + +def test_audit_id_missing_prefix_fails(): + result = validate_pipeline_decision_audit(_valid_entry(audit_id="001")) + assert not result.passed + assert any("AUD-" in e for e in result.errors) + + +def test_audit_id_wrong_prefix_fails(): + result = validate_pipeline_decision_audit(_valid_entry(audit_id="DSR-001")) + assert not result.passed + + +def test_audit_id_correct_prefix_passes(): + result = validate_pipeline_decision_audit(_valid_entry(audit_id="AUD-XYZ-99")) + assert result.passed + + +# ── date validation ─────────────────────────────────────────────────────────── + + +def test_invalid_date_fails(): + result = validate_pipeline_decision_audit(_valid_entry(decision_date="2026/07/09")) + assert not result.passed + assert any("YYYY-MM-DD" in e for e in result.errors) + + +# ── decision_type validation ────────────────────────────────────────────────── + + +def test_invalid_decision_type_fails(): + result = validate_pipeline_decision_audit(_valid_entry(decision_type="unknown_action")) + assert not result.passed + assert any("decision_type" in e for e in result.errors) + + +# ── description validation ──────────────────────────────────────────────────── + + +def test_empty_description_fails(): + result = validate_pipeline_decision_audit(_valid_entry(decision_description="")) + assert not result.passed + + +def test_description_too_long_fails(): + result = validate_pipeline_decision_audit( + _valid_entry(decision_description="X" * 501) + ) + assert not result.passed + assert any("500" in e for e in result.errors) + + +def test_description_exactly_500_passes(): + result = validate_pipeline_decision_audit( + _valid_entry(decision_description="X" * 500) + ) + assert result.passed + + +# ── rationale validation ────────────────────────────────────────────────────── + + +def test_empty_rationale_fails(): + result = validate_pipeline_decision_audit(_valid_entry(rationale="")) + assert not result.passed + + +def test_rationale_too_long_fails(): + result = validate_pipeline_decision_audit(_valid_entry(rationale="X" * 1001)) + assert not result.passed + assert any("1000" in e for e in result.errors) + + +# ── affected_candidate_count validation ─────────────────────────────────────── + + +def test_negative_candidate_count_fails(): + result = validate_pipeline_decision_audit(_valid_entry(affected_candidate_count=-1)) + assert not result.passed + + +# ── dry_lab_only constraint ─────────────────────────────────────────────────── + + +def test_dry_lab_only_false_fails(): + result = validate_pipeline_decision_audit(_valid_entry(dry_lab_only=False)) + assert not result.passed + assert any("dry_lab_only" in e for e in result.errors) + + +# ── warnings ───────────────────────────────────────────────────────────────── + + +def test_post_hoc_decision_warns(): + result = validate_pipeline_decision_audit(_valid_entry(pre_specified=False)) + assert result.passed + assert any("post-hoc" in w or "pre_specified" in w for w in result.warnings) + + +def test_empty_alternatives_warns(): + result = validate_pipeline_decision_audit(_valid_entry(alternatives_considered=[])) + assert result.passed + assert any("alternatives" in w for w in result.warnings) + + +def test_low_evidence_level_warns(): + result = validate_pipeline_decision_audit(_valid_entry(evidence_level=2)) + assert result.passed + assert any("low" in w or "limited" in w for w in result.warnings) + + +def test_zero_affected_candidates_warns(): + result = validate_pipeline_decision_audit(_valid_entry(affected_candidate_count=0)) + assert result.passed + assert any("0" in w or "no candidates" in w.lower() for w in result.warnings) + + +def test_full_pre_specified_entry_no_warnings(): + result = validate_pipeline_decision_audit(_valid_entry()) + assert result.passed + assert result.warnings == [] + + +# ── dict interface ──────────────────────────────────────────────────────────── + + +def test_dict_valid_passes(): + d = dict( + audit_id="AUD-D01", + batch_id="BATCH-D01", + pipeline_version="0.8.9", + decision_date="2026-07-09", + decision_type="threshold_chosen", + decision_description="Set novelty threshold to 0.30.", + rationale="Based on Wave 0 distribution analysis.", + alternatives_considered=["0.25", "0.35"], + affected_candidate_count=12, + evidence_level=3, + pre_specified=True, + reviewer="bob", + dry_lab_only=True, + ) + result = validate_pipeline_decision_audit_dict(d) + assert result.passed + + +def test_dict_missing_field_fails(): + d = dict( + audit_id="AUD-D02", + batch_id="BATCH-D02", + pipeline_version="0.8.9", + decision_date="2026-07-09", + decision_type="filter_applied", + decision_description="Applied filter.", + rationale="Pre-specified.", + alternatives_considered=[], + affected_candidate_count=5, + evidence_level=4, + # missing pre_specified + reviewer="alice", + ) + result = validate_pipeline_decision_audit_dict(d) + assert not result.passed + assert any("pre_specified" in e for e in result.errors) + + +def test_dict_dry_lab_only_defaults_true(): + d = dict( + audit_id="AUD-D03", + batch_id="BATCH-D03", + pipeline_version="0.8.9", + decision_date="2026-07-09", + decision_type="candidate_rejected", + decision_description="Rejected due to hemolysis flag.", + rationale="Safety policy requires rejection.", + alternatives_considered=["manual review"], + affected_candidate_count=3, + evidence_level=4, + pre_specified=True, + reviewer="alice", + ) + result = validate_pipeline_decision_audit_dict(d) + assert result.passed + assert result.dry_lab_only is True diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index b044f0a2..846d4892 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -6,7 +6,7 @@ # Note: 3 pre-existing collection errors in test_benchmark_calibration.py, # test_benchmark_charge_distribution.py, test_benchmark_cheap_enemies.py. # Raise threshold if legitimate tests are removed; lower threshold is okay. -BASELINE = 4020 +BASELINE = 4058 TOLERANCE = 0.05 # allow 5% fluctuation