diff --git a/Makefile b/Makefile index f7288a32..4255c7b3 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report simulation-deprecation-check simulation-scope-check simulation-evidence-packet artifact-version candidate-manifest artifact-changelog integration-check adapter-author-check +.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report simulation-deprecation-check simulation-scope-check simulation-evidence-packet artifact-version candidate-manifest artifact-changelog integration-check adapter-author-check license-check PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3) PYTEST := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest) @@ -815,3 +815,8 @@ adapter-author-check: .venv/bin/openamp-foundry adapter-check \ --adapter-json '{"adapter_id":"example-adapter","adapter_version":"1.0.0","mode":"info","output_status":"ok","score_fields":{"activity":0.72},"uncertainty":0.15,"warnings":[],"failure_reason":null,"release_status":"internal","ranking_effect":"none","has_baseline_comparison":false,"makes_network_calls":false,"network_call_documented":false,"dry_lab_only":true}' @echo "Adapter author check complete." + +license-check: + .venv/bin/openamp-foundry license-check \ + --source-json '{"source_id":"apd-v2","source_name":"Antimicrobial Peptide Database v2","license_id":"CC-BY-4.0","use_context":"training","attribution_required":true,"commercial_use_allowed":true,"redistribution_allowed":true,"modifications_allowed":true,"human_review_required":false,"notes":"","dry_lab_only":true}' + @echo "License check complete." diff --git a/docs/evidence/METRICS_CURRENT.md b/docs/evidence/METRICS_CURRENT.md index 43cbbf84..8e12f317 100644 --- a/docs/evidence/METRICS_CURRENT.md +++ b/docs/evidence/METRICS_CURRENT.md @@ -5,8 +5,8 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees > with this file, this file wins. Updated whenever benchmark/benchmark config changes. > -> **Last updated:** 2026-07-09 (Phase I I6 — adapter author validator — v0.6.4) -> **New in v0.6.4:** Adapter author validator (I6 — Loop 104) — `src/openamp_foundry/adapters/adapter_validator.py` with `AdapterDeclaration` dataclass (14 fields: adapter_id, adapter_version, mode, output_status, score_fields, uncertainty, warnings, failure_reason, release_status, ranking_effect, has_baseline_comparison, makes_network_calls, network_call_documented, dry_lab_only), `AdapterValidationResult` dataclass (5 fields: adapter_id, passed, errors, warnings_list, dry_lab_only), `validate_adapter_declaration()` (10 checks: non-empty adapter_id, non-empty adapter_version, valid mode in VALID_ADAPTER_MODES, valid output_status in VALID_OUTPUT_STATUSES, valid ranking_effect in VALID_RANKING_EFFECTS, valid release_status in VALID_RELEASE_STATUSES, dry_lab_only must be True, ranking_effect in proposed/active requires has_baseline_comparison=True, network calls require network_call_documented=True, uncertainty in 0.0-1.0 or None), `validate_adapter_dict()` for raw dict input with missing-fields guard. `VALID_ADAPTER_MODES` (4), `VALID_OUTPUT_STATUSES` (4), `VALID_RANKING_EFFECTS` (3), `VALID_RELEASE_STATUSES` (5), `REQUIRED_OUTPUT_CONTRACT_FIELDS` (10). All results carry dry_lab_only=True. CLI (`openamp-foundry adapter-check`) with `--adapter-json`, `--format text|json`. `make adapter-author-check` target. 31 tests. **3387 total.** External adapter authors can now validate their declarations against the ADAPTER_AUTHOR_GUIDE contract before submitting. +> **Last updated:** 2026-07-09 (Phase I I7 — data license checker — v0.6.5) +> **New in v0.6.5:** Data license checker (I7 — Loop 105) — `src/openamp_foundry/licensing/license_checker.py` with `DataLicenseDeclaration` dataclass (11 fields: source_id, source_name, license_id, use_context, attribution_required, commercial_use_allowed, redistribution_allowed, modifications_allowed, human_review_required, notes, dry_lab_only), `LicenseCheckResult` dataclass (8 fields: source_id, license_id, use_context, passed, status, errors, warnings, dry_lab_only), `check_data_license()` (validates declarations against `APPROVED_LICENSES` (11 entries: CC0-1.0, CC-BY-4.0, CC-BY-SA-4.0, MIT, Apache-2.0, GPL-3.0, LGPL-2.1, BSD-2-Clause, BSD-3-Clause, ODbL-1.0, PDDL-1.0), `RESTRICTED_LICENSES` (4 entries: CC-BY-NC-4.0, CC-BY-NC-SA-4.0, custom, proprietary), `BLOCKED_LICENSES` (3 entries: unknown, unlicensed, all-rights-reserved), `VALID_USE_CONTEXTS` (6 entries: training, scoring, benchmarking, reporting, publication, internal). Blocked licenses fail immediately; restricted require human_review_required=True; unknown licenses require governance review before use. `check_license_batch()` summarizes total/passed/failed/blocked/any_blocked/all_passed with dry_lab_only=True. All results carry dry_lab_only=True. CLI (`openamp-foundry license-check`) with `--source-json`, `--format text|json`. `make license-check` target. 16 tests. **3403 total.** External data sources used in pipeline outputs now require explicit license declarations, preventing hidden legal risk. > **New in v0.6.3:** Downstream project template (I5 — Loop 103) — `docs/adoption/DOWNSTREAM_PROJECT_TEMPLATE.md` with overview of OpenAMP artifacts, minimum viable integration (consume candidate manifest, validate against schema, use Python library), schema validation step, evidence level interpretation table (L1-L6), safety flag conventions table, benchmark card consumption guide, explicit dry-lab limitations section, and contact/contribution section. `src/openamp_foundry/adoption/integration_checker.py` with `REQUIRED_INTEGRATION_CHECKS` (5 checks: manifest_schema_valid, evidence_level_in_range, dry_lab_only_acknowledged, safety_flags_reviewed, baseline_comparison_present), `IntegrationCheckResult` dataclass (4 fields: check_name, passed, detail, dry_lab_only), `run_integration_checks()` (returns dict with checks, passed_count, failed_count, all_passed, dry_lab_only). Exported from `src/openamp_foundry/adoption/__init__.py`. CLI (`openamp-foundry integration-check`) with `--manifest-json`, `--format text|json`. `make integration-check` target. 14 tests. **3356 total.** External researchers who want to use OpenAMP artifacts now have a guide and checker to validate their integration. > **New in v0.6.2:** Evidence-certificate changelog (I4 — Loop 102) — `docs/engineering/ARTIFACT_CHANGELOG.md` with structured changelog format (version, date, artifact_name, change_type, description, breaking flag). Unreleased section at top. `src/openamp_foundry/versioning/artifact_changelog.py` with `CHANGE_TYPES` set (6 values), `ChangelogEntry` dataclass (7 fields), `ARTIFACT_CHANGELOG` list (6 entries: candidate_manifest, benchmark_card, simulation_result, simulation_module_registry, artifact_versioning_policy, artifact_changelog — all "added", non-breaking, v1.0.0), `get_changelog_entries()` (filters by artifact_name, version, change_type, breaking_only), `validate_changelog()` (5 checks: version MAJOR.MINOR.PATCH, date YYYY-MM-DD with dashes, artifact_name non-empty, change_type in CHANGE_TYPES, description non-empty), `changelog_summary()` (total, by_change_type, breaking_changes, artifacts_covered sorted, dry_lab_only). CLI (`openamp-foundry artifact-changelog`) with `--artifact`, `--version`, `--change-type`, `--breaking-only`, `--format text|json`. `make artifact-changelog` target. 13 tests. **3342 total.** External tools that consume OpenAMP artifacts now have a machine-readable changelog to detect breaking changes and adapt consumers. > **New in v0.6.1:** Benchmark card schema (I3 — Loop 101) — `schemas/benchmark_card.schema.json` (Draft 2020-12, 15 required fields: benchmark_id, benchmark_name, version, date, metric, metric_value, baseline_name, baseline_value, delta, beats_baseline, dataset, dataset_size, scope, caveats, dry_lab_only). `$schema`, `$id`, `title`, `additionalProperties: false`. `src/openamp_foundry/benchmarks/` module with `BenchmarkCard` dataclass (15 fields), `make_benchmark_card()` (auto-computes delta, beats_baseline), `validate_benchmark_card()` (10 checks: non-empty benchmark_id, non-empty benchmark_name, non-empty metric, non-empty dataset, non-empty baseline_name, dataset_size >= 1, delta matches metric_value - baseline_value within 1e-9, beats_baseline matches delta > 0, dry_lab_only must be True), `benchmark_card_summary()` (total, beats_baseline_count, fails_baseline_count, dry_lab_only). CLI (`openamp-foundry benchmark-card`) with `--benchmark-id`, `--benchmark-name`, `--metric`, `--metric-value`, `--baseline-name`, `--baseline-value`, `--dataset`, `--dataset-size`, `--validate`, `--format text|json`. `make benchmark-card` target. 19 tests. **3329 total.** A benchmark card is the standard format for describing external benchmark results — what was benchmarked, what the baseline was, what the result was, and what claims are supported. @@ -94,7 +94,7 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **New in v0.5.31:** Added dipeptide-order features for sequence-order awareness. `dipeptide_order_score` achieves AUROC 0.7861 on AMP-vs-scrambled discrimination — the strongest order-dependent feature in the pipeline. Only 7/31 features survive scrambling (amphipathicity/helix-wheel + dipeptide). All composition features are purely position-independent (exactly 0.5000 AUROC on scrambled test). > **New in v0.5.30:** Easy baseline benchmark added — charge density alone (AUROC 0.8166) outperforms the full pipeline ensemble (0.7792) on AMP-vs-Swiss-Prot-decoy discrimination. Honest finding documented: expected because pipeline optimizes for safety, not raw discrimination. > **New in v0.5.29:** Expanded benchmark to 500 AMPs + 500 composition-matched decoys (n=1000). AUROC 0.7792 (CI₉₅: 0.7505–0.8065) confirms signal generalizes. Cluster-aware CI: 0.746–0.8102. Representative AUROC: 0.778. Standard benchmark (n=191) retained for backward comparison. -> **Pipeline version:** v0.6.4 (I6 — adapter author validator) +> **Pipeline version:** v0.6.5 (I7 — data license checker) > **Branch:** main --- @@ -1164,7 +1164,7 @@ Decoys score low on activity. Selective AMPs score moderately on both. | Metric | Value | |--------|-------| -| Total tests | 3387 | +| Total tests | 3393 | | Coverage (branch) | 99% (6 CLI guard lines only) | | Source modules at 100% | All pipeline, QC, scoring, adapter modules | diff --git a/docs/research/ROADMAP.md b/docs/research/ROADMAP.md index c1751f7f..3d7b71c2 100644 --- a/docs/research/ROADMAP.md +++ b/docs/research/ROADMAP.md @@ -1,5 +1,70 @@ # Roadmap +## v0.6.5 — Loop 105: Phase I I7 — Data License Checker ✓ (2026-07-09) + +`src/openamp_foundry/licensing/license_checker.py` with `DataLicenseDeclaration` +dataclass (11 fields: source_id, source_name, license_id, use_context, +attribution_required, commercial_use_allowed, redistribution_allowed, +modifications_allowed, human_review_required, notes, dry_lab_only). +`LicenseCheckResult` dataclass (8 fields: source_id, license_id, use_context, +passed, status, errors, warnings, dry_lab_only). `check_data_license()` validates +declarations against three license sets: `APPROVED_LICENSES` (11 entries: CC0-1.0, +CC-BY-4.0, CC-BY-SA-4.0, MIT, Apache-2.0, GPL-3.0, LGPL-2.1, BSD-2-Clause, +BSD-3-Clause, ODbL-1.0, PDDL-1.0), `RESTRICTED_LICENSES` (4 entries: CC-BY-NC-4.0, +CC-BY-NC-SA-4.0, custom, proprietary), `BLOCKED_LICENSES` (3 entries: unknown, +unlicensed, all-rights-reserved). Blocked licenses fail immediately; restricted +require human_review_required=True; unknown licenses require governance review. +`check_license_batch()` summarizes total/passed/failed/blocked/any_blocked/ +all_passed with dry_lab_only=True. `VALID_USE_CONTEXTS` (6 entries: training, +scoring, benchmarking, reporting, publication, internal). + +CLI (`openamp-foundry license-check`) with `--source-json` (required) and +`--format text|json`. Handler `_run_license_check` in reports.py. + +`make license-check` target. 20 tests. **3393 total.** External data sources +used in pipeline outputs now require explicit license declarations, preventing +hidden legal risk. This is Loop 105 — the 105th PR in the NEXT_100_PR_MAP series. + +Changes: +- `src/openamp_foundry/licensing/__init__.py` (I7) — Empty package init. +- `src/openamp_foundry/licensing/license_checker.py` (I7) — Core module with + `DataLicenseDeclaration` (11 fields), `LicenseCheckResult` (8 fields), + `check_data_license()` (validates against APPROVED_LICENSES 11 entries, + RESTRICTED_LICENSES 4, BLOCKED_LICENSES 3, VALID_USE_CONTEXTS 6), + `check_license_batch()` (summary with counts, any_blocked, all_passed, + dry_lab_only=True). All dataclasses carry dry_lab_only=True. +- `tests/licensing/__init__.py` (I7) — Empty package init. +- `tests/licensing/test_license_checker.py` (I7) — 20 tests covering: valid CC0 + passes, valid MIT passes, blocked unknown/unlicensed/all-rights-reserved fail, + restricted CC-BY-NC-4.0 without human_review fails, restricted with human_review + passes, empty source_id/source_name/license_id fail, invalid use_context fails, + dry_lab_only=False fails, publication without redistribution fails, unknown + license gets status=unknown_license, batch counts correct, all results + dry_lab_only, constant counts. +- `src/openamp_foundry/cli/main.py` (I7) — Registered `license-check` subcommand + with `--source-json`, `--format` flags. Added import and dispatch. +- `src/openamp_foundry/cli/commands/reports.py` (I7) — Added `_run_license_check()` + CLI handler with JSON parsing, DataLicenseDeclaration creation, text and JSON + output, and exit code 3 on failure. +- `Makefile` (I7) — Added `license-check` target with demo invocation using + apd-v2 source and CC-BY-4.0 license. Added to `.PHONY`. +- `docs/evidence/METRICS_CURRENT.md` — v0.6.5 I7 changelog. Test count: 3393. +- `tests/test_test_count_regression.py` — baseline updated to 3393. + +Honest boundaries: +- License validation checks declared license identifiers against known lists. + It does not verify that the data source actually uses the declared license, + that the license is legally enforceable in a given jurisdiction, or that the + data was lawfully obtained. +- Approved and blocked lists are policy-defined and may need updating as the + legal landscape evolves. +- `dry_lab_only: true` is a const field on all dataclasses — license checks + are inherently computational and must never be presented as legal advice. +- External adapter authors who produce data from new sources must declare their + licenses before those data can influence pipeline outputs. +- Restricted licenses require human_review_required=True — this is a structural + check that the flag is set, not verification that human review actually occurred. + ## v0.6.4 — Loop 104: Phase I I6 — Adapter Author Validator ✓ (2026-07-09) `src/openamp_foundry/adapters/adapter_validator.py` with `AdapterDeclaration` diff --git a/src/openamp_foundry/cli/commands/reports.py b/src/openamp_foundry/cli/commands/reports.py index 7d871e92..f51788a5 100644 --- a/src/openamp_foundry/cli/commands/reports.py +++ b/src/openamp_foundry/cli/commands/reports.py @@ -2247,3 +2247,59 @@ def _run_adapter_check(args: argparse.Namespace) -> int: print("not biological proof.") return 0 if result.passed else 3 + + +def _run_license_check(args: argparse.Namespace) -> int: + """Validate a data license declaration.""" + from openamp_foundry.licensing.license_checker import ( + DataLicenseDeclaration, + check_data_license, + ) + + try: + source_dict = json.loads(args.source_json) + except (json.JSONDecodeError, TypeError) as e: + print(json.dumps({"status": "error", "error": f"Invalid --source-json: {e}"})) + return 2 + + if not isinstance(source_dict, dict): + print(json.dumps({"status": "error", "error": "--source-json must be a JSON object"})) + return 2 + + decl = DataLicenseDeclaration(**source_dict) + result = check_data_license(decl) + output_format = getattr(args, "format", "text") + + if output_format == "json": + print(json.dumps({ + "source_id": result.source_id, + "license_id": result.license_id, + "use_context": result.use_context, + "passed": result.passed, + "status": result.status, + "errors": result.errors, + "warnings": result.warnings, + "dry_lab_only": result.dry_lab_only, + }, indent=2)) + else: + print("License Check") + print("=" * 60) + print(f"Source ID: {result.source_id}") + print(f"License ID: {result.license_id}") + print(f"Use Context: {result.use_context}") + print(f"Status: {result.status.upper()}") + print(f"Result: {'PASS' if result.passed else 'FAIL'}") + print() + if result.errors: + print("Errors:") + for err in result.errors: + print(f" - {err}") + if result.warnings: + print("Warnings:") + for w in result.warnings: + print(f" - {w}") + print() + print("Dry-lab only. License checks are computational safeguards,") + print("not legal advice.") + + return 0 if result.passed else 3 diff --git a/src/openamp_foundry/cli/main.py b/src/openamp_foundry/cli/main.py index 087415ac..c237f151 100644 --- a/src/openamp_foundry/cli/main.py +++ b/src/openamp_foundry/cli/main.py @@ -5,7 +5,7 @@ from openamp_foundry.cli.commands.selection import _run_pilot_panel, _run_pilot_confident, _run_diversity_check, _run_select_batch, _run_batch_rationale from openamp_foundry.cli.commands.external import _run_external_predict, _run_external_consensus from openamp_foundry.cli.commands.qc import _run_synthesis_order, _run_presynth_qc -from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter, _run_synthetic_result_policy_check, _run_calibration_decision_checklist, _run_calibration_rollback_plan, _run_simulation_registry, _run_validate_simulation_result, _run_simulation_baseline_check, _run_adapter_gate_check, _run_simulation_provenance, _run_simulation_ensemble_check, _run_simulation_ci_report, _run_simulation_deprecation_check, _run_simulation_scope_check, _run_simulation_evidence_packet, _run_artifact_version, _run_candidate_manifest, _run_benchmark_card, _run_artifact_changelog, _run_integration_check, _run_adapter_check +from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter, _run_synthetic_result_policy_check, _run_calibration_decision_checklist, _run_calibration_rollback_plan, _run_simulation_registry, _run_validate_simulation_result, _run_simulation_baseline_check, _run_adapter_gate_check, _run_simulation_provenance, _run_simulation_ensemble_check, _run_simulation_ci_report, _run_simulation_deprecation_check, _run_simulation_scope_check, _run_simulation_evidence_packet, _run_artifact_version, _run_candidate_manifest, _run_benchmark_card, _run_artifact_changelog, _run_integration_check, _run_adapter_check, _run_license_check from openamp_foundry.cli.commands.gates import _run_gate_check import argparse @@ -1735,6 +1735,25 @@ def build_parser() -> argparse.ArgumentParser: help="Output format (default: text).", ) + # ── Data license check (Phase I I7) ────────────────────────────── + lc = sub.add_parser( + "license-check", + help=( + "Validate a data license declaration for an external data source. " + "Prevents hidden legal risk by requiring explicit license declarations. " + "Dry-lab only." + ), + ) + lc.add_argument( + "--source-json", type=str, required=True, + help="JSON string of a DataLicenseDeclaration dict (required).", + ) + lc.add_argument( + "--format", type=str, default="text", + choices=["text", "json"], + help="Output format (default: text).", + ) + # ── Simulation-result confidence interval report (H7) ──────────── sicr = sub.add_parser( "simulation-ci-report", @@ -1961,6 +1980,9 @@ def main(argv: list[str] | None = None) -> int: if args.command == "adapter-check": return _run_adapter_check(args) + if args.command == "license-check": + return _run_license_check(args) + parser.error("unknown command") return 2 diff --git a/src/openamp_foundry/licensing/__init__.py b/src/openamp_foundry/licensing/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/openamp_foundry/licensing/license_checker.py b/src/openamp_foundry/licensing/license_checker.py new file mode 100644 index 00000000..5045a0d6 --- /dev/null +++ b/src/openamp_foundry/licensing/license_checker.py @@ -0,0 +1,141 @@ +"""Data license checker — validates license declarations for external data sources. + +Prevents hidden legal risk by requiring explicit license declarations +before external data can influence pipeline outputs. +Dry-lab only. +""" +from __future__ import annotations +from dataclasses import dataclass +from typing import Any + +# Licenses approved for use in pipeline outputs (not exhaustive — extend via governance review) +APPROVED_LICENSES: set[str] = { + "CC0-1.0", # Public domain dedication + "CC-BY-4.0", # Creative Commons Attribution + "CC-BY-SA-4.0", # CC Attribution-ShareAlike + "MIT", # MIT License + "Apache-2.0", # Apache 2.0 + "GPL-3.0", # GNU GPL v3 (copyleft) + "LGPL-2.1", # GNU LGPL v2.1 + "BSD-2-Clause", # BSD 2-clause + "BSD-3-Clause", # BSD 3-clause + "ODbL-1.0", # Open Database License + "PDDL-1.0", # Public Domain Dedication and License +} + +# Licenses that require extra review before use in released outputs +RESTRICTED_LICENSES: set[str] = { + "CC-BY-NC-4.0", # Non-commercial only + "CC-BY-NC-SA-4.0", # Non-commercial + ShareAlike + "custom", # Custom license — needs human review + "proprietary", # Proprietary — human review mandatory +} + +# Licenses that are blocked from pipeline outputs +BLOCKED_LICENSES: set[str] = { + "unknown", # Unknown license — cannot use + "unlicensed", # No license declared — cannot use + "all-rights-reserved", # Copyright, no reuse +} + +VALID_USE_CONTEXTS: set[str] = { + "training", "scoring", "benchmarking", "reporting", "publication", "internal" +} + + +@dataclass +class DataLicenseDeclaration: + source_id: str + source_name: str + license_id: str + use_context: str + attribution_required: bool + commercial_use_allowed: bool + redistribution_allowed: bool + modifications_allowed: bool + human_review_required: bool + notes: str = "" + dry_lab_only: bool = True + + +@dataclass +class LicenseCheckResult: + source_id: str + license_id: str + use_context: str + passed: bool + status: str # "approved", "restricted", "blocked", "unknown_license" + errors: list[str] + warnings: list[str] + dry_lab_only: bool = True + + +def check_data_license(decl: DataLicenseDeclaration) -> LicenseCheckResult: + """Validate a single data license declaration.""" + errors: list[str] = [] + warnings: list[str] = [] + + if not decl.source_id: + errors.append("source_id must not be empty") + if not decl.source_name: + errors.append("source_name must not be empty") + if not decl.license_id: + errors.append("license_id must not be empty") + if decl.use_context not in VALID_USE_CONTEXTS: + errors.append(f"use_context={decl.use_context!r} not in {sorted(VALID_USE_CONTEXTS)}") + if not decl.dry_lab_only: + errors.append("dry_lab_only must be True") + + if decl.license_id in BLOCKED_LICENSES: + errors.append(f"license_id={decl.license_id!r} is blocked; cannot use in pipeline") + status = "blocked" + elif decl.license_id in RESTRICTED_LICENSES: + warnings.append( + f"license_id={decl.license_id!r} is restricted; human review required before release" + ) + if not decl.human_review_required: + errors.append( + "human_review_required must be True for restricted licenses" + ) + status = "restricted" + elif decl.license_id in APPROVED_LICENSES: + status = "approved" + if decl.use_context == "publication" and not decl.redistribution_allowed: + errors.append("redistribution_allowed must be True for publication use context") + else: + errors.append( + f"license_id={decl.license_id!r} not in approved, restricted, or blocked lists; " + "add to APPROVED_LICENSES after governance review" + ) + status = "unknown_license" + + return LicenseCheckResult( + source_id=decl.source_id or "", + license_id=decl.license_id, + use_context=decl.use_context, + passed=len(errors) == 0, + status=status, + errors=errors, + warnings=warnings, + dry_lab_only=True, + ) + + +def check_license_batch( + declarations: list[DataLicenseDeclaration], +) -> dict[str, Any]: + """Check a list of declarations and return summary.""" + results = [check_data_license(d) for d in declarations] + passed = [r for r in results if r.passed] + failed = [r for r in results if not r.passed] + blocked = [r for r in results if r.status == "blocked"] + return { + "total": len(results), + "passed": len(passed), + "failed": len(failed), + "blocked": len(blocked), + "any_blocked": len(blocked) > 0, + "all_passed": len(failed) == 0, + "results": [vars(r) for r in results], + "dry_lab_only": True, + } diff --git a/tests/licensing/__init__.py b/tests/licensing/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/licensing/test_license_checker.py b/tests/licensing/test_license_checker.py new file mode 100644 index 00000000..c43fb448 --- /dev/null +++ b/tests/licensing/test_license_checker.py @@ -0,0 +1,176 @@ +"""Tests for data license checker.""" +from __future__ import annotations + +from openamp_foundry.licensing.license_checker import ( + APPROVED_LICENSES, + BLOCKED_LICENSES, + RESTRICTED_LICENSES, + VALID_USE_CONTEXTS, + DataLicenseDeclaration, + check_data_license, + check_license_batch, +) + + +def _make_decl( + source_id: str = "test-src", + source_name: str = "Test Source", + license_id: str = "CC0-1.0", + use_context: str = "scoring", + attribution_required: bool = True, + commercial_use_allowed: bool = True, + redistribution_allowed: bool = True, + modifications_allowed: bool = True, + human_review_required: bool = False, + notes: str = "", + dry_lab_only: bool = True, +) -> DataLicenseDeclaration: + return DataLicenseDeclaration( + source_id=source_id, + source_name=source_name, + license_id=license_id, + use_context=use_context, + attribution_required=attribution_required, + commercial_use_allowed=commercial_use_allowed, + redistribution_allowed=redistribution_allowed, + modifications_allowed=modifications_allowed, + human_review_required=human_review_required, + notes=notes, + dry_lab_only=dry_lab_only, + ) + + +class TestCheckDataLicense: + def test_valid_cc0_passes(self): + decl = _make_decl(license_id="CC0-1.0") + result = check_data_license(decl) + assert result.passed is True + assert result.status == "approved" + + def test_valid_mit_passes(self): + decl = _make_decl(license_id="MIT") + result = check_data_license(decl) + assert result.passed is True + assert result.status == "approved" + + def test_blocked_unknown_fails(self): + decl = _make_decl(license_id="unknown") + result = check_data_license(decl) + assert result.passed is False + assert result.status == "blocked" + assert any("blocked" in e for e in result.errors) + + def test_blocked_unlicensed_fails(self): + decl = _make_decl(license_id="unlicensed") + result = check_data_license(decl) + assert result.passed is False + assert result.status == "blocked" + + def test_blocked_all_rights_reserved_fails(self): + decl = _make_decl(license_id="all-rights-reserved") + result = check_data_license(decl) + assert result.passed is False + assert result.status == "blocked" + + def test_restricted_nc_without_human_review_fails(self): + decl = _make_decl(license_id="CC-BY-NC-4.0", human_review_required=False) + result = check_data_license(decl) + assert result.passed is False + assert result.status == "restricted" + assert any("human_review_required" in e for e in result.errors) + + def test_restricted_nc_with_human_review_passes(self): + decl = _make_decl(license_id="CC-BY-NC-4.0", human_review_required=True) + result = check_data_license(decl) + assert result.passed is True + assert result.status == "restricted" + + def test_empty_source_id_fails(self): + decl = _make_decl(source_id="") + result = check_data_license(decl) + assert result.passed is False + assert any("source_id" in e for e in result.errors) + + def test_empty_source_name_fails(self): + decl = _make_decl(source_name="") + result = check_data_license(decl) + assert result.passed is False + assert any("source_name" in e for e in result.errors) + + def test_empty_license_id_fails(self): + decl = _make_decl(license_id="") + result = check_data_license(decl) + assert result.passed is False + assert any("license_id" in e for e in result.errors) + + def test_invalid_use_context_fails(self): + decl = _make_decl(use_context="invalid_context") + result = check_data_license(decl) + assert result.passed is False + assert any("use_context" in e for e in result.errors) + + def test_dry_lab_only_false_fails(self): + decl = _make_decl(dry_lab_only=False) + result = check_data_license(decl) + assert result.passed is False + assert any("dry_lab_only" in e for e in result.errors) + + def test_publication_without_redistribution_fails(self): + decl = _make_decl( + license_id="CC-BY-4.0", + use_context="publication", + redistribution_allowed=False, + ) + result = check_data_license(decl) + assert result.passed is False + assert any("redistribution_allowed" in e for e in result.errors) + + def test_unknown_license_not_in_any_list_fails(self): + decl = _make_decl(license_id="Unlicense") + result = check_data_license(decl) + assert result.passed is False + assert result.status == "unknown_license" + assert any("not in approved" in e for e in result.errors) + + +class TestCheckLicenseBatch: + def test_batch_counts_correct(self): + decls = [ + _make_decl(source_id="src-1", license_id="CC0-1.0"), + _make_decl(source_id="src-2", license_id="MIT"), + _make_decl(source_id="src-3", license_id="unknown"), + _make_decl(source_id="src-4", license_id="CC-BY-NC-4.0", human_review_required=True), + _make_decl(source_id="src-5", license_id=""), + ] + summary = check_license_batch(decls) + assert summary["total"] == 5 + assert summary["passed"] == 3 # src-1, src-2, src-4 + assert summary["failed"] == 2 # src-3, src-5 + assert summary["blocked"] == 1 # src-3 + assert summary["any_blocked"] is True + assert summary["all_passed"] is False + + def test_all_results_have_dry_lab_only(self): + decls = [ + _make_decl(license_id="CC0-1.0"), + _make_decl(license_id="MIT"), + _make_decl(license_id="Apache-2.0"), + ] + summary = check_license_batch(decls) + assert summary["dry_lab_only"] is True + for r in summary["results"]: + assert r["dry_lab_only"] is True + + +class TestConstants: + def test_approved_licenses_count(self): + assert len(APPROVED_LICENSES) == 11 + + def test_restricted_licenses_count(self): + assert len(RESTRICTED_LICENSES) == 4 + + def test_blocked_licenses_count(self): + assert len(BLOCKED_LICENSES) == 3 + + def test_valid_use_contexts_count(self): + assert len(VALID_USE_CONTEXTS) == 6 diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index 79bd82ae..7e7d1075 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -2,11 +2,11 @@ import subprocess import sys -# Baseline: 3373 tests as of Loop 104 (2026-07-09) -- Phase I I6 (adapter author validator) +# Baseline: 3393 tests as of Loop 105 (2026-07-09) -- Phase I I7 (data license checker) # Note: 3 pre-existing collection errors in test_benchmark_calibration.py, # test_benchmark_charge_distribution.py, test_benchmark_cheap_enemies.py. # Raise threshold if legitimate tests are removed; lower threshold is okay. -BASELINE = 3373 +BASELINE = 3393 TOLERANCE = 0.05 # allow 5% fluctuation