From 53fcb0b70dc55d3d3fd5f6358a645f58200dfc2b Mon Sep 17 00:00:00 2001 From: OpenCode Date: Thu, 9 Jul 2026 22:28:47 +0700 Subject: [PATCH] feat: Phase K K3 pilot package completeness checker -- all artifacts validated before external lab submission --- Makefile | 7 +- docs/evidence/METRICS_CURRENT.md | 4 +- docs/evidence/PILOT_PACKAGE_GUIDE.md | 68 ++++ docs/research/ROADMAP.md | 20 ++ src/openamp_foundry/cli/commands/reports.py | 34 ++ src/openamp_foundry/cli/main.py | 23 +- src/openamp_foundry/evidence/pilot_package.py | 174 +++++++++++ tests/evidence/test_pilot_package.py | 293 ++++++++++++++++++ tests/test_test_count_regression.py | 2 +- 9 files changed, 621 insertions(+), 4 deletions(-) create mode 100644 docs/evidence/PILOT_PACKAGE_GUIDE.md create mode 100644 src/openamp_foundry/evidence/pilot_package.py create mode 100644 tests/evidence/test_pilot_package.py diff --git a/Makefile b/Makefile index 6116e234..1a3f1ff3 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report simulation-deprecation-check simulation-scope-check simulation-evidence-packet artifact-version candidate-manifest artifact-changelog integration-check adapter-author-check license-check artifact-compat-check contribution-check adoption-scorecard release-gate-check decision-log release-request-check coi-check rotation-plan-check security-report-check citation-check roadmap-sync-check advisory-review-check annual-review-check selection-rationale-check batch-priority-check +.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report simulation-deprecation-check simulation-scope-check simulation-evidence-packet artifact-version candidate-manifest artifact-changelog integration-check adapter-author-check license-check artifact-compat-check contribution-check adoption-scorecard release-gate-check decision-log release-request-check coi-check rotation-plan-check security-report-check citation-check roadmap-sync-check advisory-review-check annual-review-check selection-rationale-check batch-priority-check pilot-package-check PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3) PYTEST := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest) @@ -895,3 +895,8 @@ batch-priority-check: .venv/bin/openamp-foundry batch-priority-check \ --entry-json '{"priority_id":"PRI-2026-001","batch_id":"batch-2026-001","candidate_id":"OAMP-001","pipeline_version":"v0.8.0","priority_rank":1,"priority_score":0.87,"evidence_level":3,"synthesis_complexity":"low","novelty_tier":"high","primary_rationale":"Top ensemble score with selectivity > 2, low synthesis risk."}' @echo "Batch priority check complete." + +pilot-package-check: + .venv/bin/openamp-foundry pilot-package-check \ + --entry-json '{"package_id":"PKG-001","batch_id":"BATCH-001","submission_date":"2026-07-09","pipeline_version":"0.8.1","included_artifacts":["selection_rationale","batch_priority","evidence_certificate"],"missing_artifacts":[],"reviewer":"alice","approver":"bob","completeness_score":1.0,"ready_to_submit":true,"dry_lab_only":true}' \ + --format text diff --git a/docs/evidence/METRICS_CURRENT.md b/docs/evidence/METRICS_CURRENT.md index 082cbed8..230a330e 100644 --- a/docs/evidence/METRICS_CURRENT.md +++ b/docs/evidence/METRICS_CURRENT.md @@ -5,7 +5,9 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees > with this file, this file wins. Updated whenever benchmark/benchmark config changes. > -> **Last updated:** 2026-07-09 (Phase K K2 — batch experiment priority ranker — v0.8.0) +> **Last updated:** 2026-07-09 (Phase K K3 — pilot package completeness checker — v0.8.1) +> **New in v0.8.1:** Pilot package completeness checker (K3 — Loop 121) — docs/evidence/PILOT_PACKAGE_GUIDE.md with purpose, required field table (11 fields), mandatory artifact types table (3 types), valid artifact types, warnings, validation workflow, honest-use boundary. src/openamp_foundry/evidence/pilot_package.py with PilotPackageEntry dataclass (11 fields: package_id, batch_id, submission_date, pipeline_version, included_artifacts, missing_artifacts, reviewer, approver, completeness_score, ready_to_submit, dry_lab_only), PilotPackageResult dataclass (5 fields, dry_lab_only=True), MINIMUM_REQUIRED_ARTIFACTS (3), READINESS_SCORE_THRESHOLD (0.80), MANDATORY_ARTIFACT_TYPES (3: batch_priority, evidence_certificate, selection_rationale), VALID_ARTIFACT_TYPES (8 types), validate_pilot_package() (11 checks: PKG- prefix, non-empty batch_id/pipeline_version/reviewer/approver, YYYY-MM-DD submission_date, included_artifacts>=3, mandatory types covered, completeness_score 0.0-1.0, ready_to_submit must be False when score<0.80, dry_lab_only must be True; missing_artifacts non-empty warns, score<0.90 warns, same reviewer/approver warns), validate_pilot_package_dict() (10 required fields guard). CLI (openamp-foundry pilot-package-check) with --entry-json, --format text|json. make pilot-package-check target. Phase K K3: every pilot submission is now machine-validated for completeness. +> > **New in v0.8.0:** Batch experiment priority ranker (K2 — Loop 120) — docs/evidence/BATCH_PRIORITY_GUIDE.md with purpose, required field table (11 fields), how-to-validate instructions. src/openamp_foundry/evidence/batch_priority.py with BatchPriorityEntry dataclass (12 fields: priority_id, batch_id, candidate_id, pipeline_version, priority_rank, priority_score, evidence_level, synthesis_complexity, novelty_tier, primary_rationale, disqualifying_concerns, dry_lab_only), BatchPriorityResult dataclass (6 fields, dry_lab_only=True), VALID_SYNTHESIS_COMPLEXITIES (3: high, low, medium), VALID_NOVELTY_TIERS (3: high, low, medium), VALID_EVIDENCE_LEVELS (6: 1-6), validate_batch_priority() (11 checks: PRI- prefix, non-empty batch_id/candidate_id/pipeline_version, priority_rank>=1, priority_score 0.0-1.0, valid evidence_level, valid synthesis_complexity, valid novelty_tier, non-empty primary_rationale, dry_lab_only must be True; evidence_level<=2 warns, rank==1+high-complexity warns, priority_score<0.30 warns), validate_batch_priority_dict() (10 required fields guard). CLI (openamp-foundry batch-priority-check) with --entry-json, --format text|json. make batch-priority-check target. Phase K K2: synthesis wave ranking is now machine-validated. > > **New in v0.7.9:** Candidate selection rationale schema (K1 — Loop 119) — docs/evidence/SELECTION_RATIONALE_GUIDE.md with purpose, required field table (11 fields), evidence level table (6 levels from PROOF_LADDER.md), validation workflow. src/openamp_foundry/evidence/selection_rationale.py with SelectionRationaleEntry dataclass (11 fields: selection_id, batch_id, candidate_id, pipeline_version, selection_date, evidence_level, baseline_comparison, primary_criterion, safety_flags_checked, reviewer, dry_lab_only), SelectionRationaleResult dataclass (5 fields, dry_lab_only=True), VALID_EVIDENCE_LEVELS (6: 1-6), MINIMUM_SAFETY_FLAGS (1), validate_selection_rationale() (11 checks: SEL- prefix, non-empty batch_id/candidate_id/pipeline_version, YYYY-MM-DD date, valid evidence_level 1-6, non-empty baseline_comparison/primary_criterion, safety_flags_checked list with >=1 entry, non-empty reviewer, dry_lab_only must be True; evidence_level<=2 warns with synthesis caution), validate_selection_rationale_dict() (10 required fields guard). CLI (openamp-foundry selection-rationale-check) with --entry-json, --format text|json. make selection-rationale-check target. Phase K K1: every selection decision now has a machine-validated evidence trail. diff --git a/docs/evidence/PILOT_PACKAGE_GUIDE.md b/docs/evidence/PILOT_PACKAGE_GUIDE.md new file mode 100644 index 00000000..c3156e18 --- /dev/null +++ b/docs/evidence/PILOT_PACKAGE_GUIDE.md @@ -0,0 +1,68 @@ +# Pilot Package Completeness Guide + +A *pilot package* is the complete evidence bundle submitted to an external collaborating lab +before any synthesis or assay work begins. This guide defines what must be present, how to +validate completeness, and the honesty constraints that govern every package. + +## Purpose + +Pilot packages exist so external reviewers can: +- verify the provenance and evidence level of each candidate, +- confirm safety flags were checked, and +- reproduce the prioritisation logic from raw predictions. + +A package is only *ready to submit* when its completeness score reaches the threshold +defined in `pilot_package.py` (`READINESS_SCORE_THRESHOLD = 0.80`). + +## Required fields + +| Field | Type | Constraint | +|---|---|---| +| `package_id` | str | Must start with `"PKG-"` | +| `batch_id` | str | Non-empty | +| `submission_date` | str | ISO 8601 `YYYY-MM-DD` | +| `pipeline_version` | str | Non-empty | +| `included_artifacts` | list[str] | ≥ `MINIMUM_REQUIRED_ARTIFACTS` (3); must cover all `MANDATORY_ARTIFACT_TYPES` | +| `missing_artifacts` | list[str] | May be empty; each entry should name an absent artifact type | +| `reviewer` | str | Non-empty | +| `approver` | str | Non-empty | +| `completeness_score` | float | 0.0 – 1.0 | +| `ready_to_submit` | bool | Must be `False` when `completeness_score < READINESS_SCORE_THRESHOLD` | +| `dry_lab_only` | bool | Must be `True` | + +## Mandatory artifact types + +The following artifact types must appear in `included_artifacts` for a package to pass: + +| Artifact type | Why required | +|---|---| +| `selection_rationale` | Documents why each candidate was chosen. | +| `batch_priority` | Documents synthesis-wave rank with evidence level. | +| `evidence_certificate` | Signed evidence bundle for the batch. | + +## Valid artifact types + +`batch_priority`, `benchmark_card`, `candidate_manifest`, `evidence_certificate`, +`model_card`, `safety_assessment`, `selection_rationale`, `uncertainty_report` + +## Warnings + +- `completeness_score < 0.80` and `ready_to_submit=True` — hard error. +- `missing_artifacts` non-empty — warning: package has gaps. +- `completeness_score < 0.90` — warning: consider completing optional artifacts. +- reviewer equals approver — warning: same person reviewed and approved. + +## How to validate + +```bash +make pilot-package-check +# or +openamp-foundry pilot-package-check \ + --entry-json '{"package_id":"PKG-001","batch_id":"BATCH-001","submission_date":"2026-07-09","pipeline_version":"0.8.1","included_artifacts":["selection_rationale","batch_priority","evidence_certificate"],"missing_artifacts":[],"reviewer":"alice","approver":"bob","completeness_score":1.0,"ready_to_submit":true,"dry_lab_only":true}' \ + --format text +``` + +## Honest-use boundary + +Pilot packages are dry-lab outputs. The completeness score is a documentation metric, +not a biological confidence score. A complete package does not imply experimental success. diff --git a/docs/research/ROADMAP.md b/docs/research/ROADMAP.md index e7f0d380..6f5ed7bd 100644 --- a/docs/research/ROADMAP.md +++ b/docs/research/ROADMAP.md @@ -1,5 +1,25 @@ # Roadmap +## v0.8.1 — Loop 121: Phase K K3 — Pilot Package Completeness Checker + +`docs/evidence/PILOT_PACKAGE_GUIDE.md` with purpose, required field table (11 +fields), mandatory artifact types table (3 types: selection_rationale, +batch_priority, evidence_certificate), valid artifact types (8 types), warnings, +validation workflow, honest-use boundary. + +`src/openamp_foundry/evidence/pilot_package.py` with `PilotPackageEntry` +dataclass (11 fields, dry_lab_only=True enforced), `PilotPackageResult` +dataclass (5 fields, dry_lab_only=True), `MINIMUM_REQUIRED_ARTIFACTS` (3), +`READINESS_SCORE_THRESHOLD` (0.80), `MANDATORY_ARTIFACT_TYPES` (3: +batch_priority, evidence_certificate, selection_rationale), +`VALID_ARTIFACT_TYPES` (8 types), `validate_pilot_package()` (11 checks, 3 +warning conditions: missing artifacts, low completeness score, same +reviewer/approver), `validate_pilot_package_dict()` (10 required fields guard). + +CLI: `openamp-foundry pilot-package-check`. `make pilot-package-check` target. +**v0.8.1 milestone** — every pilot submission is machine-validated for +completeness before external lab submission. + ## v0.8.0 — Loop 120: Phase K K2 — Batch Experiment Priority Ranker ✓ (2026-07-09) `docs/evidence/BATCH_PRIORITY_GUIDE.md` with field table and validation workflow diff --git a/src/openamp_foundry/cli/commands/reports.py b/src/openamp_foundry/cli/commands/reports.py index d246e862..a8f9c97f 100644 --- a/src/openamp_foundry/cli/commands/reports.py +++ b/src/openamp_foundry/cli/commands/reports.py @@ -2885,3 +2885,37 @@ def _run_batch_priority_check(args: argparse.Namespace) -> int: print(" Batch priority validated. Synthesis wave ranking is auditable.") return 0 if result.passed else 3 + + +def _run_pilot_package_check(args: argparse.Namespace) -> int: + """Validate a pilot package completeness entry.""" + import json as _json + from openamp_foundry.evidence.pilot_package import validate_pilot_package_dict + + try: + d = _json.loads(args.entry_json) + except _json.JSONDecodeError as exc: + print(_json.dumps({"status": "error", "error": f"Invalid JSON: {exc}"})) + return 2 + + if not isinstance(d, dict): + print(_json.dumps({"status": "error", "error": "--entry-json must be a JSON object"})) + return 2 + + result = validate_pilot_package_dict(d) + output_format = getattr(args, "format", "text") + + if output_format == "json": + import dataclasses + print(_json.dumps(dataclasses.asdict(result), indent=2)) + else: + status = "PASS" if result.passed else "FAIL" + print(f"Pilot package {result.package_id} (batch {result.batch_id}): {status}") + for e in result.errors: + print(f" ERROR: {e}") + for w in result.warnings: + print(f" WARN: {w}") + if result.passed: + print(" Pilot package validated. All required artifacts present.") + + return 0 if result.passed else 3 diff --git a/src/openamp_foundry/cli/main.py b/src/openamp_foundry/cli/main.py index 2a5ec9de..5cd7eb3f 100644 --- a/src/openamp_foundry/cli/main.py +++ b/src/openamp_foundry/cli/main.py @@ -15,7 +15,7 @@ from openamp_foundry.cli.commands.reports import _run_security_report_check from openamp_foundry.cli.commands.reports import _run_citation_check from openamp_foundry.cli.commands.reports import _run_roadmap_sync_check -from openamp_foundry.cli.commands.reports import _run_advisory_review_check, _run_annual_review_check, _run_selection_rationale_check, _run_batch_priority_check +from openamp_foundry.cli.commands.reports import _run_advisory_review_check, _run_annual_review_check, _run_selection_rationale_check, _run_batch_priority_check, _run_pilot_package_check import argparse import json @@ -2017,6 +2017,24 @@ def build_parser() -> argparse.ArgumentParser: help="Output format (default: text).", ) + # ── Pilot package check (Phase K K3) ───────────────────────────── + ppc = sub.add_parser( + "pilot-package-check", + help=( + "Validate a pilot package completeness entry. Confirms all required " + "artifacts are present before external lab submission. Dry-lab only." + ), + ) + ppc.add_argument( + "--entry-json", type=str, required=True, + help="JSON string of a PilotPackageEntry dict (required).", + ) + ppc.add_argument( + "--format", type=str, default="text", + choices=["text", "json"], + help="Output format (default: text).", + ) + # ── Selection rationale check (Phase K K1) ─────────────────────── src2 = sub.add_parser( "selection-rationale-check", @@ -2328,6 +2346,9 @@ def main(argv: list[str] | None = None) -> int: if args.command == "batch-priority-check": return _run_batch_priority_check(args) + if args.command == "pilot-package-check": + return _run_pilot_package_check(args) + parser.error("unknown command") return 2 diff --git a/src/openamp_foundry/evidence/pilot_package.py b/src/openamp_foundry/evidence/pilot_package.py new file mode 100644 index 00000000..02d619be --- /dev/null +++ b/src/openamp_foundry/evidence/pilot_package.py @@ -0,0 +1,174 @@ +"""Pilot package completeness checker (Phase K K3). + +Validates that all required artifacts are present before an experiment +batch is submitted to an external collaborating lab. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import List + +MINIMUM_REQUIRED_ARTIFACTS: int = 3 +READINESS_SCORE_THRESHOLD: float = 0.80 + +MANDATORY_ARTIFACT_TYPES: set[str] = { + "batch_priority", + "evidence_certificate", + "selection_rationale", +} + +VALID_ARTIFACT_TYPES: set[str] = { + "batch_priority", + "benchmark_card", + "candidate_manifest", + "evidence_certificate", + "model_card", + "safety_assessment", + "selection_rationale", + "uncertainty_report", +} + +_DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$") + + +@dataclass +class PilotPackageEntry: + """One pilot package completeness record.""" + + package_id: str + batch_id: str + submission_date: str + pipeline_version: str + included_artifacts: List[str] + missing_artifacts: List[str] + reviewer: str + approver: str + completeness_score: float + ready_to_submit: bool + dry_lab_only: bool = True + + +@dataclass +class PilotPackageResult: + """Validation result for a PilotPackageEntry.""" + + package_id: str + batch_id: str + passed: bool + errors: List[str] + warnings: List[str] + dry_lab_only: bool = True + + +def validate_pilot_package(entry: PilotPackageEntry) -> PilotPackageResult: + """Validate a PilotPackageEntry. Returns a PilotPackageResult.""" + errors: List[str] = [] + warnings: List[str] = [] + + if not entry.package_id.startswith("PKG-"): + errors.append("package_id must start with 'PKG-'") + + if not entry.batch_id: + errors.append("batch_id must not be empty") + + if not _DATE_RE.match(entry.submission_date): + errors.append("submission_date must be YYYY-MM-DD") + + if not entry.pipeline_version: + errors.append("pipeline_version must not be empty") + + if len(entry.included_artifacts) < MINIMUM_REQUIRED_ARTIFACTS: + errors.append( + f"included_artifacts must have at least {MINIMUM_REQUIRED_ARTIFACTS} entries" + ) + + missing_mandatory = MANDATORY_ARTIFACT_TYPES - set(entry.included_artifacts) + if missing_mandatory: + errors.append( + f"included_artifacts is missing mandatory types: {sorted(missing_mandatory)}" + ) + + if not entry.reviewer: + errors.append("reviewer must not be empty") + + if not entry.approver: + errors.append("approver must not be empty") + + if not (0.0 <= entry.completeness_score <= 1.0): + errors.append("completeness_score must be between 0.0 and 1.0") + + if entry.ready_to_submit and entry.completeness_score < READINESS_SCORE_THRESHOLD: + errors.append( + f"ready_to_submit cannot be True when completeness_score " + f"({entry.completeness_score:.2f}) < {READINESS_SCORE_THRESHOLD}" + ) + + if entry.dry_lab_only is not True: + errors.append("dry_lab_only must be True") + + if entry.missing_artifacts: + warnings.append( + f"Package has {len(entry.missing_artifacts)} missing artifact(s): " + f"{entry.missing_artifacts}" + ) + + if 0.0 <= entry.completeness_score < 0.90: + warnings.append( + f"completeness_score {entry.completeness_score:.2f} is below 0.90; " + "consider completing optional artifacts before submission" + ) + + if entry.reviewer and entry.approver and entry.reviewer == entry.approver: + warnings.append( + "reviewer and approver are the same person; consider independent approval" + ) + + return PilotPackageResult( + package_id=entry.package_id, + batch_id=entry.batch_id, + passed=len(errors) == 0, + errors=errors, + warnings=warnings, + ) + + +def validate_pilot_package_dict(d: dict) -> PilotPackageResult: + """Validate a dict representation of a PilotPackageEntry.""" + required = [ + "package_id", + "batch_id", + "submission_date", + "pipeline_version", + "included_artifacts", + "missing_artifacts", + "reviewer", + "approver", + "completeness_score", + "ready_to_submit", + ] + for key in required: + if key not in d: + return PilotPackageResult( + package_id=d.get("package_id", ""), + batch_id=d.get("batch_id", ""), + passed=False, + errors=[f"missing required field: {key}"], + warnings=[], + ) + + entry = PilotPackageEntry( + package_id=d["package_id"], + batch_id=d["batch_id"], + submission_date=d["submission_date"], + pipeline_version=d["pipeline_version"], + included_artifacts=d["included_artifacts"], + missing_artifacts=d["missing_artifacts"], + reviewer=d["reviewer"], + approver=d["approver"], + completeness_score=d["completeness_score"], + ready_to_submit=d["ready_to_submit"], + dry_lab_only=d.get("dry_lab_only", True), + ) + return validate_pilot_package(entry) diff --git a/tests/evidence/test_pilot_package.py b/tests/evidence/test_pilot_package.py new file mode 100644 index 00000000..9679a2de --- /dev/null +++ b/tests/evidence/test_pilot_package.py @@ -0,0 +1,293 @@ +"""Tests for pilot package completeness checker (Phase K K3).""" + +import pytest +from openamp_foundry.evidence.pilot_package import ( + MANDATORY_ARTIFACT_TYPES, + MINIMUM_REQUIRED_ARTIFACTS, + READINESS_SCORE_THRESHOLD, + VALID_ARTIFACT_TYPES, + PilotPackageEntry, + PilotPackageResult, + validate_pilot_package, + validate_pilot_package_dict, +) + + +def _valid_entry(**kwargs) -> PilotPackageEntry: + defaults = dict( + package_id="PKG-001", + batch_id="BATCH-001", + submission_date="2026-07-09", + pipeline_version="0.8.1", + included_artifacts=[ + "selection_rationale", + "batch_priority", + "evidence_certificate", + ], + missing_artifacts=[], + reviewer="alice", + approver="bob", + completeness_score=1.0, + ready_to_submit=True, + dry_lab_only=True, + ) + defaults.update(kwargs) + return PilotPackageEntry(**defaults) + + +# ── Constants ──────────────────────────────────────────────────────────────── + + +def test_minimum_required_artifacts_is_three(): + assert MINIMUM_REQUIRED_ARTIFACTS == 3 + + +def test_readiness_score_threshold_is_0_80(): + assert READINESS_SCORE_THRESHOLD == 0.80 + + +def test_mandatory_artifact_types_present(): + assert "selection_rationale" in MANDATORY_ARTIFACT_TYPES + assert "batch_priority" in MANDATORY_ARTIFACT_TYPES + assert "evidence_certificate" in MANDATORY_ARTIFACT_TYPES + + +def test_valid_artifact_types_superset_of_mandatory(): + assert MANDATORY_ARTIFACT_TYPES.issubset(VALID_ARTIFACT_TYPES) + + +# ── Valid entry ─────────────────────────────────────────────────────────────── + + +def test_valid_entry_passes(): + result = validate_pilot_package(_valid_entry()) + assert result.passed + assert result.errors == [] + + +def test_result_has_dry_lab_only_true(): + result = validate_pilot_package(_valid_entry()) + assert result.dry_lab_only is True + + +def test_result_fields_match_entry(): + entry = _valid_entry() + result = validate_pilot_package(entry) + assert result.package_id == "PKG-001" + assert result.batch_id == "BATCH-001" + + +def test_valid_with_optional_artifacts(): + entry = _valid_entry( + included_artifacts=[ + "selection_rationale", + "batch_priority", + "evidence_certificate", + "model_card", + "safety_assessment", + ], + completeness_score=1.0, + ) + result = validate_pilot_package(entry) + assert result.passed + assert result.errors == [] + + +def test_valid_not_ready_to_submit(): + entry = _valid_entry( + completeness_score=0.75, + ready_to_submit=False, + ) + result = validate_pilot_package(entry) + assert result.passed + assert result.errors == [] + + +# ── package_id validation ───────────────────────────────────────────────────── + + +def test_package_id_missing_prefix_fails(): + result = validate_pilot_package(_valid_entry(package_id="001")) + assert not result.passed + assert any("PKG-" in e for e in result.errors) + + +def test_package_id_wrong_prefix_fails(): + result = validate_pilot_package(_valid_entry(package_id="BAT-001")) + assert not result.passed + + +def test_package_id_correct_prefix_passes(): + result = validate_pilot_package(_valid_entry(package_id="PKG-XYZ-99")) + assert result.passed + + +# ── date validation ─────────────────────────────────────────────────────────── + + +def test_invalid_date_format_fails(): + result = validate_pilot_package(_valid_entry(submission_date="09-07-2026")) + assert not result.passed + assert any("YYYY-MM-DD" in e for e in result.errors) + + +def test_valid_date_passes(): + result = validate_pilot_package(_valid_entry(submission_date="2026-01-01")) + assert result.passed + + +# ── included_artifacts validation ───────────────────────────────────────────── + + +def test_too_few_artifacts_fails(): + result = validate_pilot_package( + _valid_entry( + included_artifacts=["selection_rationale", "batch_priority"], + missing_artifacts=["evidence_certificate"], + ) + ) + assert not result.passed + assert any("at least" in e for e in result.errors) + + +def test_missing_mandatory_artifact_fails(): + result = validate_pilot_package( + _valid_entry( + included_artifacts=[ + "selection_rationale", + "batch_priority", + "benchmark_card", + ] + ) + ) + assert not result.passed + assert any("mandatory" in e for e in result.errors) + + +# ── completeness_score and ready_to_submit ──────────────────────────────────── + + +def test_score_below_threshold_with_ready_true_fails(): + result = validate_pilot_package( + _valid_entry(completeness_score=0.70, ready_to_submit=True) + ) + assert not result.passed + assert any("ready_to_submit" in e for e in result.errors) + + +def test_score_at_threshold_ready_true_passes(): + result = validate_pilot_package( + _valid_entry(completeness_score=0.80, ready_to_submit=True) + ) + assert result.passed + + +def test_score_out_of_range_fails(): + result = validate_pilot_package(_valid_entry(completeness_score=1.5)) + assert not result.passed + + +def test_score_negative_fails(): + result = validate_pilot_package(_valid_entry(completeness_score=-0.1)) + assert not result.passed + + +# ── dry_lab_only constraint ─────────────────────────────────────────────────── + + +def test_dry_lab_only_false_fails(): + result = validate_pilot_package(_valid_entry(dry_lab_only=False)) + assert not result.passed + assert any("dry_lab_only" in e for e in result.errors) + + +# ── warnings ───────────────────────────────────────────────────────────────── + + +def test_missing_artifacts_list_triggers_warning(): + result = validate_pilot_package( + _valid_entry( + missing_artifacts=["uncertainty_report"], + completeness_score=0.85, + ready_to_submit=True, + ) + ) + assert result.passed + assert any("missing" in w for w in result.warnings) + + +def test_score_below_90_triggers_warning(): + result = validate_pilot_package( + _valid_entry(completeness_score=0.85, ready_to_submit=True) + ) + assert result.passed + assert any("0.85" in w or "0.90" in w for w in result.warnings) + + +def test_same_reviewer_approver_triggers_warning(): + result = validate_pilot_package(_valid_entry(reviewer="alice", approver="alice")) + assert result.passed + assert any("same person" in w for w in result.warnings) + + +def test_perfect_score_no_warnings(): + result = validate_pilot_package(_valid_entry()) + assert result.passed + assert result.warnings == [] + + +# ── dict interface ──────────────────────────────────────────────────────────── + + +def test_dict_valid_passes(): + d = dict( + package_id="PKG-D01", + batch_id="BATCH-D01", + submission_date="2026-07-09", + pipeline_version="0.8.1", + included_artifacts=["selection_rationale", "batch_priority", "evidence_certificate"], + missing_artifacts=[], + reviewer="alice", + approver="bob", + completeness_score=1.0, + ready_to_submit=True, + dry_lab_only=True, + ) + result = validate_pilot_package_dict(d) + assert result.passed + + +def test_dict_missing_field_fails(): + d = dict( + package_id="PKG-D02", + batch_id="BATCH-D02", + submission_date="2026-07-09", + pipeline_version="0.8.1", + included_artifacts=["selection_rationale", "batch_priority", "evidence_certificate"], + missing_artifacts=[], + reviewer="alice", + # missing approver + completeness_score=1.0, + ready_to_submit=True, + ) + result = validate_pilot_package_dict(d) + assert not result.passed + assert any("approver" in e for e in result.errors) + + +def test_dict_inherits_dry_lab_only_default(): + d = dict( + package_id="PKG-D03", + batch_id="BATCH-D03", + submission_date="2026-07-09", + pipeline_version="0.8.1", + included_artifacts=["selection_rationale", "batch_priority", "evidence_certificate"], + missing_artifacts=[], + reviewer="alice", + approver="bob", + completeness_score=1.0, + ready_to_submit=True, + ) + result = validate_pilot_package_dict(d) + assert result.passed + assert result.dry_lab_only is True diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index 50b69ed4..466d5625 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -6,7 +6,7 @@ # Note: 3 pre-existing collection errors in test_benchmark_calibration.py, # test_benchmark_charge_distribution.py, test_benchmark_cheap_enemies.py. # Raise threshold if legitimate tests are removed; lower threshold is okay. -BASELINE = 3749 +BASELINE = 3780 TOLERANCE = 0.05 # allow 5% fluctuation