diff --git a/Makefile b/Makefile index 34b8cdfe..0d709541 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report simulation-deprecation-check simulation-scope-check simulation-evidence-packet artifact-version candidate-manifest artifact-changelog integration-check adapter-author-check license-check artifact-compat-check contribution-check adoption-scorecard release-gate-check decision-log release-request-check coi-check rotation-plan-check security-report-check citation-check roadmap-sync-check advisory-review-check annual-review-check selection-rationale-check batch-priority-check pilot-package-check calibration-intake-check uncertainty-report-check preprint-bundle-check reproducibility-manifest-check candidate-summary-card-check multi-candidate-comparison-check +.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report simulation-deprecation-check simulation-scope-check simulation-evidence-packet artifact-version candidate-manifest artifact-changelog integration-check adapter-author-check license-check artifact-compat-check contribution-check adoption-scorecard release-gate-check decision-log release-request-check coi-check rotation-plan-check security-report-check citation-check roadmap-sync-check advisory-review-check annual-review-check selection-rationale-check batch-priority-check pilot-package-check calibration-intake-check uncertainty-report-check preprint-bundle-check reproducibility-manifest-check candidate-summary-card-check multi-candidate-comparison-check dataset-release-check PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3) PYTEST := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest) @@ -930,3 +930,8 @@ multi-candidate-comparison-check: .venv/bin/openamp-foundry multi-candidate-comparison-check \ --entry-json '{"comparison_id":"CMP-001","batch_id":"BATCH-001","pipeline_version":"0.8.7","comparison_date":"2026-07-09","candidate_ids":["AMP-001","AMP-002","AMP-003"],"comparison_criteria":["predicted_mic","hemolysis_fraction","selectivity_index"],"top_candidate_id":"AMP-001","top_candidate_rationale":"Highest selectivity index with lowest hemolysis fraction.","evidence_level":4,"reviewer":"alice","dry_lab_only":true}' \ --format text + +dataset-release-check: + .venv/bin/openamp-foundry dataset-release-check \ + --entry-json '{"release_id":"DSR-001","dataset_name":"OpenAMP Wave 0.5 Candidates","dataset_version":"1.0.0","release_date":"2026-07-09","license_identifier":"CC-BY-4.0","data_sources":["UniProt AMP database","APD3","in-house pipeline"],"contains_sequences":true,"contains_activity_data":false,"dual_use_assessed":true,"usage_policy_url":"https://openamp.example.org/data-policy","contact_email":"data@openamp.example.org","release_approved":true,"dry_lab_only":true}' \ + --format text diff --git a/docs/evidence/DATASET_RELEASE_GUIDE.md b/docs/evidence/DATASET_RELEASE_GUIDE.md new file mode 100644 index 00000000..adc57cd2 --- /dev/null +++ b/docs/evidence/DATASET_RELEASE_GUIDE.md @@ -0,0 +1,59 @@ +# Dataset Release Package Guide + +A *dataset release package* record validates that an open dataset release meets all +data governance requirements before publication. This guide defines what a release +package must contain, how it is validated, and the honesty constraints that apply. + +## Purpose + +Before releasing a dataset (sequences, scores, negative results, benchmark data) +to the public, the release package ensures that: +- a data license has been chosen, +- a data usage policy has been documented, +- provenance of all included data is declared, +- dual-use risks have been assessed, +- the release has been approved by a reviewer. + +## Required fields + +| Field | Type | Constraint | +|---|---|---| +| `release_id` | str | Must start with `"DSR-"` | +| `dataset_name` | str | Non-empty | +| `dataset_version` | str | Non-empty | +| `release_date` | str | ISO 8601 `YYYY-MM-DD` | +| `license_identifier` | str | One of `VALID_LICENSE_IDENTIFIERS` | +| `data_sources` | list[str] | ≥ `MINIMUM_DATA_SOURCES` (1) entry | +| `contains_sequences` | bool | Required field | +| `contains_activity_data` | bool | Required field | +| `dual_use_assessed` | bool | Must be `True` | +| `usage_policy_url` | str | Non-empty | +| `contact_email` | str | Non-empty | +| `release_approved` | bool | Must be `True` | +| `dry_lab_only` | bool | Must be `True` | + +## Valid license identifiers + +`Apache-2.0`, `CC-BY-4.0`, `CC-BY-NC-4.0`, `MIT`, `CC0-1.0` + +## Warnings + +- `license_identifier == "CC-BY-NC-4.0"` — warning: non-commercial license may restrict some research uses. +- `contains_activity_data` is True and `dual_use_assessed` would be False — blocked at validation (error). +- `data_sources` has only 1 entry — warning: single data source; consider whether additional provenance is needed. + +## How to validate + +```bash +make dataset-release-check +# or +openamp-foundry dataset-release-check \ + --entry-json '{"release_id":"DSR-001","dataset_name":"OpenAMP Wave 0.5 Candidates","dataset_version":"1.0.0","release_date":"2026-07-09","license_identifier":"CC-BY-4.0","data_sources":["UniProt AMP database","APD3","in-house pipeline"],"contains_sequences":true,"contains_activity_data":false,"dual_use_assessed":true,"usage_policy_url":"https://openamp.example.org/data-policy","contact_email":"data@openamp.example.org","release_approved":true,"dry_lab_only":true}' \ + --format text +``` + +## Honest-use boundary + +Dataset release packages describe computational outputs only. Released datasets must +not claim experimental validation unless wet-lab confirmation has been independently +performed and documented. `dry_lab_only` must be `True`. diff --git a/docs/evidence/METRICS_CURRENT.md b/docs/evidence/METRICS_CURRENT.md index 50244931..183f7b90 100644 --- a/docs/evidence/METRICS_CURRENT.md +++ b/docs/evidence/METRICS_CURRENT.md @@ -5,7 +5,8 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees > with this file, this file wins. Updated whenever benchmark/benchmark config changes. > -> **Last updated:** 2026-07-09 (Phase L L4 — multi-candidate comparison schema — v0.8.7) +> **Last updated:** 2026-07-09 (Phase L L5 — dataset release package checker — v0.8.8) +> **New in v0.8.8:** Dataset release package checker (L5 — Loop 127) — docs/evidence/DATASET_RELEASE_GUIDE.md with purpose, required field table (13 fields), valid license identifiers (5), warnings, validation workflow, honest-use boundary. src/openamp_foundry/evidence/dataset_release.py with DatasetReleaseEntry dataclass (13 fields: release_id, dataset_name, dataset_version, release_date, license_identifier, data_sources, contains_sequences, contains_activity_data, dual_use_assessed, usage_policy_url, contact_email, release_approved, dry_lab_only), DatasetReleaseResult dataclass (4 fields: release_id, dataset_name, passed, errors, warnings, dry_lab_only=True), VALID_LICENSE_IDENTIFIERS (5: Apache-2.0, CC-BY-4.0, CC-BY-NC-4.0, CC0-1.0, MIT), MINIMUM_DATA_SOURCES (1), validate_dataset_release() (11 checks: DSR- prefix, non-empty dataset_name/dataset_version/usage_policy_url/contact_email, YYYY-MM-DD release_date, valid license_identifier, data_sources>=1, dual_use_assessed must be True, release_approved must be True, dry_lab_only must be True; CC-BY-NC-4.0 warns, single source warns), validate_dataset_release_dict() (12 required fields guard). CLI (openamp-foundry dataset-release-check) with --entry-json, --format text|json. make dataset-release-check target. Phase L L5 completes Phase L: open dataset releases now have machine-validated data governance checks. > **New in v0.8.7:** Multi-candidate comparison schema (L4 — Loop 126b) — docs/evidence/MULTI_CANDIDATE_COMPARISON_GUIDE.md with purpose, required field table (11 fields), minimum requirements (2 candidates, 2 criteria), warnings, validation workflow, honest-use boundary. src/openamp_foundry/evidence/multi_candidate_comparison.py with MultiCandidateComparisonEntry dataclass (11 fields: comparison_id, batch_id, pipeline_version, comparison_date, candidate_ids, comparison_criteria, top_candidate_id, top_candidate_rationale, evidence_level, reviewer, dry_lab_only), MultiCandidateComparisonResult dataclass (5 fields: comparison_id, batch_id, candidate_count, passed, errors, warnings, dry_lab_only=True), MINIMUM_CANDIDATES (2), MINIMUM_CRITERIA (2), RECOMMENDED_CRITERIA (3), MAX_RATIONALE_LENGTH (500), LARGE_CANDIDATE_SET_THRESHOLD (10), VALID_EVIDENCE_LEVELS (6: 1-6), validate_multi_candidate_comparison() (11 checks: CMP- prefix, non-empty batch_id/pipeline_version/reviewer, YYYY-MM-DD comparison_date, candidate_ids>=2, comparison_criteria>=2, top_candidate_id in candidate_ids, non-empty rationale<=500 chars, valid evidence_level, dry_lab_only must be True; evidence_level<=2 warns, candidate_count>10 warns, criteria_count<3 warns), validate_multi_candidate_comparison_dict() (10 required fields guard). CLI (openamp-foundry multi-candidate-comparison-check) with --entry-json, --format text|json. make multi-candidate-comparison-check target. Phase L L4: side-by-side candidate comparisons are now machine-validated for publication-ready supplementary tables. > **New in v0.8.6:** Candidate summary card schema (L3 — Loop 126) — docs/evidence/CANDIDATE_SUMMARY_CARD_GUIDE.md with purpose, required field table (12 fields), valid activity labels (5), valid amino acid set (20 standard), warnings, validation workflow, honest-use boundary. src/openamp_foundry/evidence/candidate_summary_card.py with CandidateSummaryCardEntry dataclass (12 fields: card_id, candidate_id, batch_id, pipeline_version, sequence, sequence_length, evidence_level, predicted_activity, safety_flags, selection_rationale_id, reviewer, dry_lab_only), CandidateSummaryCardResult dataclass (5 fields: card_id, candidate_id, sequence_length, passed, errors, warnings, dry_lab_only=True), VALID_ACTIVITY_LABELS (5: high_activity, inactive, low_activity, moderate_activity, uncertain), VALID_AMINO_ACIDS (20 standard one-letter codes), LONG_PEPTIDE_THRESHOLD (50), VALID_EVIDENCE_LEVELS (6: 1-6), validate_candidate_summary_card() (11 checks: CRD- prefix, non-empty candidate_id/batch_id/pipeline_version/reviewer, non-empty sequence with valid amino acids, sequence_length==len(sequence), valid evidence_level, valid predicted_activity, SEL- prefix on selection_rationale_id, dry_lab_only must be True; evidence_level<=2 warns, safety_flags non-empty warns, uncertain activity warns, length>50 warns), validate_candidate_summary_card_dict() (11 required fields guard). CLI (openamp-foundry candidate-summary-card-check) with --entry-json, --format text|json. make candidate-summary-card-check target. Phase L L3: every candidate now has a machine-validated publication-ready summary card. > **New in v0.8.5:** Reproducibility manifest schema (L2 — Loop 125) — docs/evidence/REPRODUCIBILITY_MANIFEST_GUIDE.md with purpose, required field table (11 fields), package checksums format, data checksums format, warnings, validation workflow, honest-use boundary. src/openamp_foundry/evidence/reproducibility_manifest.py with ReproducibilityManifestEntry dataclass (11 fields: manifest_id, batch_id, pipeline_version, run_date, python_version, package_checksums, data_checksums, random_seeds, hardware_summary, reviewer, dry_lab_only), ReproducibilityManifestResult dataclass (6 fields: manifest_id, batch_id, package_count, data_file_count, passed, errors, warnings, dry_lab_only=True), MINIMUM_PACKAGES (3), MINIMUM_DATA_FILES (1), RECOMMENDED_PACKAGES (5), validate_reproducibility_manifest() (10 checks: RPM- prefix, non-empty batch_id/pipeline_version/python_version/hardware_summary/reviewer, YYYY-MM-DD run_date, package_checksums>=3, data_checksums>=1, dry_lab_only must be True; empty random_seeds warns, package_count<5 warns, hardware_summary contains 'unknown' warns), validate_reproducibility_manifest_dict() (10 required fields guard). CLI (openamp-foundry reproducibility-manifest-check) with --entry-json, --format text|json. make reproducibility-manifest-check target. Phase L L2: every pipeline run now has a machine-validated reproducibility record. diff --git a/docs/research/NEXT_100_PR_MAP.md b/docs/research/NEXT_100_PR_MAP.md index b86f98f0..d206f622 100644 --- a/docs/research/NEXT_100_PR_MAP.md +++ b/docs/research/NEXT_100_PR_MAP.md @@ -207,10 +207,10 @@ Bridge the gap between evidence packages and published science. | PR | Task | Why it matters | Review class | |---:|---|---|---| | L1 | Add preprint evidence bundle schema (complete — Loop 124). | Ties K-phase artifacts into a submission-ready package. | B | -| L2 | Add reproducibility manifest schema. | Captures exact versions, checksums, seeds for full reproduction. | B | -| L3 | Add candidate summary card schema. | Publication-ready per-candidate structured summary. | B | -| L4 | Add multi-candidate comparison schema. | Structured comparison table for ≥2 candidates. | B | -| L5 | Add dataset release package checker. | Validates open dataset releases meet data governance. | B/D | +| L2 | Add reproducibility manifest schema (complete — Loop 125). | Captures exact versions, checksums, seeds for full reproduction. | B | +| L3 | Add candidate summary card schema (complete — Loop 126). | Publication-ready per-candidate structured summary. | B | +| L4 | Add multi-candidate comparison schema (complete — Loop 126b). | Structured comparison table for ≥2 candidates. | B | +| L5 | Add dataset release package checker (complete — Loop 127). | Validates open dataset releases meet data governance. | B/D | ## Prioritization rule diff --git a/docs/research/ROADMAP.md b/docs/research/ROADMAP.md index 4102052c..d0ddb4e1 100644 --- a/docs/research/ROADMAP.md +++ b/docs/research/ROADMAP.md @@ -1,5 +1,8 @@ # Roadmap +### v0.8.8 +- Phase L L5: dataset release package checker — validates that open dataset releases meet data governance requirements (license, provenance, dual-use assessment, release approval). DatasetReleaseEntry dataclass, validate_dataset_release(), CLI dataset-release-check. Completes Phase L. + ### v0.8.7 - Phase L L4: multi-candidate comparison schema — validates structured side-by-side comparisons of two or more candidates for publication-ready supplementary tables. MultiCandidateComparisonEntry dataclass, validate_multi_candidate_comparison(), CLI multi-candidate-comparison-check. diff --git a/src/openamp_foundry/cli/commands/reports.py b/src/openamp_foundry/cli/commands/reports.py index b7fff97b..e77a29ec 100644 --- a/src/openamp_foundry/cli/commands/reports.py +++ b/src/openamp_foundry/cli/commands/reports.py @@ -3124,3 +3124,37 @@ def _run_multi_candidate_comparison_check(args: argparse.Namespace) -> int: print(" Multi-candidate comparison validated. Ready for supplementary table.") return 0 if result.passed else 3 + + +def _run_dataset_release_check(args: argparse.Namespace) -> int: + """Validate a dataset release package entry.""" + import json as _json + from openamp_foundry.evidence.dataset_release import validate_dataset_release_dict + + try: + d = _json.loads(args.entry_json) + except _json.JSONDecodeError as exc: + print(_json.dumps({"status": "error", "error": f"Invalid JSON: {exc}"})) + return 2 + + if not isinstance(d, dict): + print(_json.dumps({"status": "error", "error": "--entry-json must be a JSON object"})) + return 2 + + result = validate_dataset_release_dict(d) + output_format = getattr(args, "format", "text") + + if output_format == "json": + import dataclasses + print(_json.dumps(dataclasses.asdict(result), indent=2)) + else: + status = "PASS" if result.passed else "FAIL" + print(f"Dataset release {result.release_id} ({result.dataset_name}): {status}") + for e in result.errors: + print(f" ERROR: {e}") + for w in result.warnings: + print(f" WARN: {w}") + if result.passed: + print(" Dataset release validated. Package meets data governance requirements.") + + return 0 if result.passed else 3 diff --git a/src/openamp_foundry/cli/main.py b/src/openamp_foundry/cli/main.py index 869affc9..1b0d6491 100644 --- a/src/openamp_foundry/cli/main.py +++ b/src/openamp_foundry/cli/main.py @@ -15,7 +15,7 @@ from openamp_foundry.cli.commands.reports import _run_security_report_check from openamp_foundry.cli.commands.reports import _run_citation_check from openamp_foundry.cli.commands.reports import _run_roadmap_sync_check -from openamp_foundry.cli.commands.reports import _run_advisory_review_check, _run_annual_review_check, _run_selection_rationale_check, _run_batch_priority_check, _run_pilot_package_check, _run_calibration_intake_check, _run_uncertainty_report_check, _run_preprint_bundle_check, _run_reproducibility_manifest_check, _run_candidate_summary_card_check, _run_multi_candidate_comparison_check +from openamp_foundry.cli.commands.reports import _run_advisory_review_check, _run_annual_review_check, _run_selection_rationale_check, _run_batch_priority_check, _run_pilot_package_check, _run_calibration_intake_check, _run_uncertainty_report_check, _run_preprint_bundle_check, _run_reproducibility_manifest_check, _run_candidate_summary_card_check, _run_multi_candidate_comparison_check, _run_dataset_release_check import argparse import json @@ -2143,6 +2143,24 @@ def build_parser() -> argparse.ArgumentParser: help="Output format (default: text).", ) + # ── Dataset release check (Phase L L5) ─────────────────────────── + drc = sub.add_parser( + "dataset-release-check", + help=( + "Validate a dataset release package. Confirms license, provenance, " + "dual-use assessment, and release approval. Dry-lab only." + ), + ) + drc.add_argument( + "--entry-json", type=str, required=True, + help="JSON string of a DatasetReleaseEntry dict (required).", + ) + drc.add_argument( + "--format", type=str, default="text", + choices=["text", "json"], + help="Output format (default: text).", + ) + # ── Selection rationale check (Phase K K1) ─────────────────────── src2 = sub.add_parser( "selection-rationale-check", @@ -2475,6 +2493,9 @@ def main(argv: list[str] | None = None) -> int: if args.command == "multi-candidate-comparison-check": return _run_multi_candidate_comparison_check(args) + if args.command == "dataset-release-check": + return _run_dataset_release_check(args) + parser.error("unknown command") return 2 diff --git a/src/openamp_foundry/evidence/dataset_release.py b/src/openamp_foundry/evidence/dataset_release.py new file mode 100644 index 00000000..48eba947 --- /dev/null +++ b/src/openamp_foundry/evidence/dataset_release.py @@ -0,0 +1,166 @@ +"""Dataset release package checker (Phase L L5). + +Validates that an open dataset release meets all data governance +requirements before publication. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import List + +VALID_LICENSE_IDENTIFIERS: set[str] = { + "Apache-2.0", + "CC-BY-4.0", + "CC-BY-NC-4.0", + "CC0-1.0", + "MIT", +} + +MINIMUM_DATA_SOURCES: int = 1 + +_DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$") + + +@dataclass +class DatasetReleaseEntry: + """One dataset release package record.""" + + release_id: str + dataset_name: str + dataset_version: str + release_date: str + license_identifier: str + data_sources: List[str] + contains_sequences: bool + contains_activity_data: bool + dual_use_assessed: bool + usage_policy_url: str + contact_email: str + release_approved: bool + dry_lab_only: bool = True + + +@dataclass +class DatasetReleaseResult: + """Validation result for a DatasetReleaseEntry.""" + + release_id: str + dataset_name: str + passed: bool + errors: List[str] + warnings: List[str] + dry_lab_only: bool = True + + +def validate_dataset_release(entry: DatasetReleaseEntry) -> DatasetReleaseResult: + """Validate a DatasetReleaseEntry. Returns a DatasetReleaseResult.""" + errors: List[str] = [] + warnings: List[str] = [] + + if not entry.release_id.startswith("DSR-"): + errors.append("release_id must start with 'DSR-'") + + if not entry.dataset_name: + errors.append("dataset_name must not be empty") + + if not entry.dataset_version: + errors.append("dataset_version must not be empty") + + if not _DATE_RE.match(entry.release_date): + errors.append("release_date must be YYYY-MM-DD") + + if entry.license_identifier not in VALID_LICENSE_IDENTIFIERS: + errors.append( + f"license_identifier '{entry.license_identifier}' is not valid; " + f"must be one of {sorted(VALID_LICENSE_IDENTIFIERS)}" + ) + + if len(entry.data_sources) < MINIMUM_DATA_SOURCES: + errors.append( + f"data_sources must have at least {MINIMUM_DATA_SOURCES} entry" + ) + + if not entry.dual_use_assessed: + errors.append( + "dual_use_assessed must be True; complete a dual-use risk assessment " + "before releasing any dataset" + ) + + if not entry.usage_policy_url: + errors.append("usage_policy_url must not be empty") + + if not entry.contact_email: + errors.append("contact_email must not be empty") + + if not entry.release_approved: + errors.append("release_approved must be True for a valid dataset release") + + if entry.dry_lab_only is not True: + errors.append("dry_lab_only must be True") + + if not errors: + if entry.license_identifier == "CC-BY-NC-4.0": + warnings.append( + "license_identifier is CC-BY-NC-4.0 (non-commercial); " + "this may restrict some academic and commercial research uses." + ) + + if len(entry.data_sources) == 1: + warnings.append( + "Only one data source declared; consider whether additional " + "provenance documentation is needed." + ) + + return DatasetReleaseResult( + release_id=entry.release_id, + dataset_name=entry.dataset_name, + passed=len(errors) == 0, + errors=errors, + warnings=warnings, + ) + + +def validate_dataset_release_dict(d: dict) -> DatasetReleaseResult: + """Validate a dict representation of a DatasetReleaseEntry.""" + required = [ + "release_id", + "dataset_name", + "dataset_version", + "release_date", + "license_identifier", + "data_sources", + "contains_sequences", + "contains_activity_data", + "dual_use_assessed", + "usage_policy_url", + "contact_email", + "release_approved", + ] + for key in required: + if key not in d: + return DatasetReleaseResult( + release_id=d.get("release_id", ""), + dataset_name=d.get("dataset_name", ""), + passed=False, + errors=[f"missing required field: {key}"], + warnings=[], + ) + + entry = DatasetReleaseEntry( + release_id=d["release_id"], + dataset_name=d["dataset_name"], + dataset_version=d["dataset_version"], + release_date=d["release_date"], + license_identifier=d["license_identifier"], + data_sources=d["data_sources"], + contains_sequences=d["contains_sequences"], + contains_activity_data=d["contains_activity_data"], + dual_use_assessed=d["dual_use_assessed"], + usage_policy_url=d["usage_policy_url"], + contact_email=d["contact_email"], + release_approved=d["release_approved"], + dry_lab_only=d.get("dry_lab_only", True), + ) + return validate_dataset_release(entry) diff --git a/tests/evidence/test_dataset_release.py b/tests/evidence/test_dataset_release.py new file mode 100644 index 00000000..86338708 --- /dev/null +++ b/tests/evidence/test_dataset_release.py @@ -0,0 +1,263 @@ +"""Tests for dataset release package checker (Phase L L5).""" + +import pytest +from openamp_foundry.evidence.dataset_release import ( + MINIMUM_DATA_SOURCES, + VALID_LICENSE_IDENTIFIERS, + DatasetReleaseEntry, + DatasetReleaseResult, + validate_dataset_release, + validate_dataset_release_dict, +) + + +def _valid_entry(**kwargs) -> DatasetReleaseEntry: + defaults = dict( + release_id="DSR-001", + dataset_name="OpenAMP Wave 0.5 Candidates", + dataset_version="1.0.0", + release_date="2026-07-09", + license_identifier="CC-BY-4.0", + data_sources=["UniProt AMP database", "APD3", "in-house pipeline"], + contains_sequences=True, + contains_activity_data=False, + dual_use_assessed=True, + usage_policy_url="https://openamp.example.org/data-policy", + contact_email="data@openamp.example.org", + release_approved=True, + dry_lab_only=True, + ) + defaults.update(kwargs) + return DatasetReleaseEntry(**defaults) + + +# ── Constants ──────────────────────────────────────────────────────────────── + + +def test_minimum_data_sources_is_one(): + assert MINIMUM_DATA_SOURCES == 1 + + +def test_valid_license_identifiers_count(): + assert len(VALID_LICENSE_IDENTIFIERS) == 5 + + +def test_cc_by_in_valid_licenses(): + assert "CC-BY-4.0" in VALID_LICENSE_IDENTIFIERS + + +def test_cc0_in_valid_licenses(): + assert "CC0-1.0" in VALID_LICENSE_IDENTIFIERS + + +# ── Valid entry ─────────────────────────────────────────────────────────────── + + +def test_valid_entry_passes(): + result = validate_dataset_release(_valid_entry()) + assert result.passed + assert result.errors == [] + + +def test_result_dry_lab_only_true(): + result = validate_dataset_release(_valid_entry()) + assert result.dry_lab_only is True + + +def test_result_fields_match(): + result = validate_dataset_release(_valid_entry()) + assert result.release_id == "DSR-001" + assert result.dataset_name == "OpenAMP Wave 0.5 Candidates" + + +def test_valid_all_licenses(): + for lic in VALID_LICENSE_IDENTIFIERS - {"CC-BY-NC-4.0"}: + result = validate_dataset_release(_valid_entry(license_identifier=lic)) + assert result.passed, f"license '{lic}' should be valid" + + +def test_valid_with_activity_data(): + result = validate_dataset_release( + _valid_entry(contains_activity_data=True, dual_use_assessed=True) + ) + assert result.passed + + +def test_valid_single_data_source(): + result = validate_dataset_release(_valid_entry(data_sources=["UniProt AMP database"])) + assert result.passed + + +# ── release_id validation ───────────────────────────────────────────────────── + + +def test_release_id_missing_prefix_fails(): + result = validate_dataset_release(_valid_entry(release_id="001")) + assert not result.passed + assert any("DSR-" in e for e in result.errors) + + +def test_release_id_wrong_prefix_fails(): + result = validate_dataset_release(_valid_entry(release_id="CMP-001")) + assert not result.passed + + +def test_release_id_correct_prefix_passes(): + result = validate_dataset_release(_valid_entry(release_id="DSR-XYZ-99")) + assert result.passed + + +# ── date validation ─────────────────────────────────────────────────────────── + + +def test_invalid_date_fails(): + result = validate_dataset_release(_valid_entry(release_date="09-07-2026")) + assert not result.passed + assert any("YYYY-MM-DD" in e for e in result.errors) + + +# ── license validation ──────────────────────────────────────────────────────── + + +def test_invalid_license_fails(): + result = validate_dataset_release(_valid_entry(license_identifier="GPL-3.0")) + assert not result.passed + assert any("license_identifier" in e for e in result.errors) + + +# ── dual_use_assessed constraint ────────────────────────────────────────────── + + +def test_dual_use_not_assessed_fails(): + result = validate_dataset_release(_valid_entry(dual_use_assessed=False)) + assert not result.passed + assert any("dual_use_assessed" in e for e in result.errors) + + +# ── release_approved constraint ─────────────────────────────────────────────── + + +def test_release_not_approved_fails(): + result = validate_dataset_release(_valid_entry(release_approved=False)) + assert not result.passed + assert any("release_approved" in e for e in result.errors) + + +# ── dry_lab_only constraint ─────────────────────────────────────────────────── + + +def test_dry_lab_only_false_fails(): + result = validate_dataset_release(_valid_entry(dry_lab_only=False)) + assert not result.passed + assert any("dry_lab_only" in e for e in result.errors) + + +# ── empty field constraints ─────────────────────────────────────────────────── + + +def test_empty_dataset_name_fails(): + result = validate_dataset_release(_valid_entry(dataset_name="")) + assert not result.passed + + +def test_empty_usage_policy_url_fails(): + result = validate_dataset_release(_valid_entry(usage_policy_url="")) + assert not result.passed + + +def test_empty_contact_email_fails(): + result = validate_dataset_release(_valid_entry(contact_email="")) + assert not result.passed + + +# ── data_sources validation ─────────────────────────────────────────────────── + + +def test_empty_data_sources_fails(): + result = validate_dataset_release(_valid_entry(data_sources=[])) + assert not result.passed + assert any("data_sources" in e for e in result.errors) + + +# ── warnings ───────────────────────────────────────────────────────────────── + + +def test_nc_license_warns(): + result = validate_dataset_release(_valid_entry(license_identifier="CC-BY-NC-4.0")) + assert result.passed + assert any("non-commercial" in w or "NC" in w for w in result.warnings) + + +def test_single_source_warns(): + result = validate_dataset_release(_valid_entry(data_sources=["UniProt"])) + assert result.passed + assert any("one data source" in w or "provenance" in w for w in result.warnings) + + +def test_multiple_sources_no_warning(): + result = validate_dataset_release(_valid_entry()) + assert result.passed + assert result.warnings == [] + + +# ── dict interface ──────────────────────────────────────────────────────────── + + +def test_dict_valid_passes(): + d = dict( + release_id="DSR-D01", + dataset_name="Test Dataset", + dataset_version="0.1.0", + release_date="2026-07-09", + license_identifier="CC-BY-4.0", + data_sources=["source_a", "source_b"], + contains_sequences=True, + contains_activity_data=False, + dual_use_assessed=True, + usage_policy_url="https://example.org/policy", + contact_email="test@example.org", + release_approved=True, + dry_lab_only=True, + ) + result = validate_dataset_release_dict(d) + assert result.passed + + +def test_dict_missing_field_fails(): + d = dict( + release_id="DSR-D02", + dataset_name="Test", + dataset_version="0.1.0", + release_date="2026-07-09", + license_identifier="CC-BY-4.0", + data_sources=["source_a"], + contains_sequences=True, + contains_activity_data=False, + dual_use_assessed=True, + usage_policy_url="https://example.org", + # missing contact_email + release_approved=True, + ) + result = validate_dataset_release_dict(d) + assert not result.passed + assert any("contact_email" in e for e in result.errors) + + +def test_dict_dry_lab_only_defaults_true(): + d = dict( + release_id="DSR-D03", + dataset_name="Test", + dataset_version="0.1.0", + release_date="2026-07-09", + license_identifier="MIT", + data_sources=["source_a", "source_b"], + contains_sequences=False, + contains_activity_data=False, + dual_use_assessed=True, + usage_policy_url="https://example.org", + contact_email="test@example.org", + release_approved=True, + ) + result = validate_dataset_release_dict(d) + assert result.passed + assert result.dry_lab_only is True diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index 5a4a0f0a..b044f0a2 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -6,7 +6,7 @@ # Note: 3 pre-existing collection errors in test_benchmark_calibration.py, # test_benchmark_charge_distribution.py, test_benchmark_cheap_enemies.py. # Raise threshold if legitimate tests are removed; lower threshold is okay. -BASELINE = 3984 +BASELINE = 4020 TOLERANCE = 0.05 # allow 5% fluctuation