feat: Phase I I8 artifact compatibility tests -- prevents schema drift across artifact schemas

Makefile

@@ -1,4 +1,4 @@
-.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report simulation-deprecation-check simulation-scope-check simulation-evidence-packet artifact-version candidate-manifest artifact-changelog integration-check adapter-author-check license-check
+.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report simulation-deprecation-check simulation-scope-check simulation-evidence-packet artifact-version candidate-manifest artifact-changelog integration-check adapter-author-check license-check artifact-compat-check
 
 PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3)
 PYTEST  := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest)
@@ -820,3 +820,7 @@ license-check:
 	.venv/bin/openamp-foundry license-check \
 		--source-json '{"source_id":"apd-v2","source_name":"Antimicrobial Peptide Database v2","license_id":"CC-BY-4.0","use_context":"training","attribution_required":true,"commercial_use_allowed":true,"redistribution_allowed":true,"modifications_allowed":true,"human_review_required":false,"notes":"","dry_lab_only":true}'
 	@echo "License check complete."
+
+artifact-compat-check:
+	.venv/bin/openamp-foundry artifact-compat-check
+	@echo "Artifact compatibility check complete."

docs/evidence/METRICS_CURRENT.md

@@ -5,7 +5,8 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak
 > **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees
 > with this file, this file wins. Updated whenever benchmark/benchmark config changes.
 >
-> **Last updated:** 2026-07-09 (Phase I I7 — data license checker — v0.6.5)
+> **Last updated:** 2026-07-09 (Phase I I8 — artifact compatibility tests — v0.6.6)
+> **New in v0.6.6:** Artifact compatibility checker (I8 — Loop 106) — `src/openamp_foundry/compatibility/artifact_compatibility.py` with `SchemaCompatibilityResult` dataclass (5 fields: schema_name, schema_path, passed, errors, warnings, dry_lab_only=True), `UNIVERSAL_REQUIRED_FIELDS` (set: dry_lab_only, version), `CONVENTION_CHECKS` (dict: dry_lab_only → boolean const true, version → string matching MAJOR.MINOR.PATCH, evidence_level → integer in 1-6 when present). `check_schema_conventions()` validates each schema against universal conventions (universal fields present in properties and required, dry_lab_only const:true, version type:string, evidence_level type:integer min:1 max:6, additionalProperties:false warning, Draft 2020-12 $schema warning). `run_compatibility_check()` scans all schemas/*.schema.json and returns total/passed/failed/all_passed/results/dry_lab_only. CLI (`openamp-foundry artifact-compat-check`) with `--schemas-dir`, `--format text|json`. `make artifact-compat-check` target. 20 tests. **3413 total.** Cross-artifact reference compatibility, shared field conventions, and structural invariants are now automatically checked across all schema files, preventing drift between artifact versions.
 > **New in v0.6.5:** Data license checker (I7 — Loop 105) — `src/openamp_foundry/licensing/license_checker.py` with `DataLicenseDeclaration` dataclass (11 fields: source_id, source_name, license_id, use_context, attribution_required, commercial_use_allowed, redistribution_allowed, modifications_allowed, human_review_required, notes, dry_lab_only), `LicenseCheckResult` dataclass (8 fields: source_id, license_id, use_context, passed, status, errors, warnings, dry_lab_only), `check_data_license()` (validates declarations against `APPROVED_LICENSES` (11 entries: CC0-1.0, CC-BY-4.0, CC-BY-SA-4.0, MIT, Apache-2.0, GPL-3.0, LGPL-2.1, BSD-2-Clause, BSD-3-Clause, ODbL-1.0, PDDL-1.0), `RESTRICTED_LICENSES` (4 entries: CC-BY-NC-4.0, CC-BY-NC-SA-4.0, custom, proprietary), `BLOCKED_LICENSES` (3 entries: unknown, unlicensed, all-rights-reserved), `VALID_USE_CONTEXTS` (6 entries: training, scoring, benchmarking, reporting, publication, internal). Blocked licenses fail immediately; restricted require human_review_required=True; unknown licenses require governance review before use. `check_license_batch()` summarizes total/passed/failed/blocked/any_blocked/all_passed with dry_lab_only=True. All results carry dry_lab_only=True. CLI (`openamp-foundry license-check`) with `--source-json`, `--format text|json`. `make license-check` target. 16 tests. **3403 total.** External data sources used in pipeline outputs now require explicit license declarations, preventing hidden legal risk.
 > **New in v0.6.3:** Downstream project template (I5 — Loop 103) — `docs/adoption/DOWNSTREAM_PROJECT_TEMPLATE.md` with overview of OpenAMP artifacts, minimum viable integration (consume candidate manifest, validate against schema, use Python library), schema validation step, evidence level interpretation table (L1-L6), safety flag conventions table, benchmark card consumption guide, explicit dry-lab limitations section, and contact/contribution section. `src/openamp_foundry/adoption/integration_checker.py` with `REQUIRED_INTEGRATION_CHECKS` (5 checks: manifest_schema_valid, evidence_level_in_range, dry_lab_only_acknowledged, safety_flags_reviewed, baseline_comparison_present), `IntegrationCheckResult` dataclass (4 fields: check_name, passed, detail, dry_lab_only), `run_integration_checks()` (returns dict with checks, passed_count, failed_count, all_passed, dry_lab_only). Exported from `src/openamp_foundry/adoption/__init__.py`. CLI (`openamp-foundry integration-check`) with `--manifest-json`, `--format text|json`. `make integration-check` target. 14 tests. **3356 total.** External researchers who want to use OpenAMP artifacts now have a guide and checker to validate their integration.
 > **New in v0.6.2:** Evidence-certificate changelog (I4 — Loop 102) — `docs/engineering/ARTIFACT_CHANGELOG.md` with structured changelog format (version, date, artifact_name, change_type, description, breaking flag). Unreleased section at top. `src/openamp_foundry/versioning/artifact_changelog.py` with `CHANGE_TYPES` set (6 values), `ChangelogEntry` dataclass (7 fields), `ARTIFACT_CHANGELOG` list (6 entries: candidate_manifest, benchmark_card, simulation_result, simulation_module_registry, artifact_versioning_policy, artifact_changelog — all "added", non-breaking, v1.0.0), `get_changelog_entries()` (filters by artifact_name, version, change_type, breaking_only), `validate_changelog()` (5 checks: version MAJOR.MINOR.PATCH, date YYYY-MM-DD with dashes, artifact_name non-empty, change_type in CHANGE_TYPES, description non-empty), `changelog_summary()` (total, by_change_type, breaking_changes, artifacts_covered sorted, dry_lab_only). CLI (`openamp-foundry artifact-changelog`) with `--artifact`, `--version`, `--change-type`, `--breaking-only`, `--format text|json`. `make artifact-changelog` target. 13 tests. **3342 total.** External tools that consume OpenAMP artifacts now have a machine-readable changelog to detect breaking changes and adapt consumers.
@@ -94,7 +95,7 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak
 > **New in v0.5.31:** Added dipeptide-order features for sequence-order awareness. `dipeptide_order_score` achieves AUROC 0.7861 on AMP-vs-scrambled discrimination — the strongest order-dependent feature in the pipeline. Only 7/31 features survive scrambling (amphipathicity/helix-wheel + dipeptide). All composition features are purely position-independent (exactly 0.5000 AUROC on scrambled test).
 > **New in v0.5.30:** Easy baseline benchmark added — charge density alone (AUROC 0.8166) outperforms the full pipeline ensemble (0.7792) on AMP-vs-Swiss-Prot-decoy discrimination. Honest finding documented: expected because pipeline optimizes for safety, not raw discrimination.
 > **New in v0.5.29:** Expanded benchmark to 500 AMPs + 500 composition-matched decoys (n=1000). AUROC 0.7792 (CI₉₅: 0.7505–0.8065) confirms signal generalizes. Cluster-aware CI: 0.746–0.8102. Representative AUROC: 0.778. Standard benchmark (n=191) retained for backward comparison.
-> **Pipeline version:** v0.6.5 (I7 — data license checker)
+> **Pipeline version:** v0.6.6 (I8 — artifact compatibility tests)
 > **Branch:** main
 
 ---
@@ -1164,7 +1165,7 @@ Decoys score low on activity. Selective AMPs score moderately on both.
 
 | Metric | Value |
 |--------|-------|
-| Total tests | 3393 |
+| Total tests | 3413 |
 | Coverage (branch) | 99% (6 CLI guard lines only) |
 | Source modules at 100% | All pipeline, QC, scoring, adapter modules |
 

docs/research/ROADMAP.md

@@ -1,5 +1,45 @@
 # Roadmap
 
+## v0.6.6 — Loop 106: Phase I I8 — Artifact Compatibility Tests ✓ (2026-07-09)
+
+`src/openamp_foundry/compatibility/artifact_compatibility.py` with
+`SchemaCompatibilityResult` dataclass (5 fields: schema_name, schema_path, passed,
+errors, warnings, dry_lab_only=True). `UNIVERSAL_REQUIRED_FIELDS` (set: dry_lab_only,
+version). `CONVENTION_CHECKS` (dict: dry_lab_only → boolean const true, version →
+string matching MAJOR.MINOR.PATCH, evidence_level → integer in 1-6 when present).
+`check_schema_conventions()` validates each schema against universal conventions.
+`run_compatibility_check()` scans all schemas/*.schema.json and returns
+total/passed/failed/all_passed/results/dry_lab_only.
+
+CLI (`openamp-foundry artifact-compat-check`) with `--schemas-dir` (default:
+schemas/) and `--format text|json`. Handler `_run_artifact_compat_check` in
+reports.py.
+
+`make artifact-compat-check` target. 20 tests. **3413 total.** Cross-artifact
+schema compatibility is automatically checked across all schema files, preventing
+drift between artifact versions.
+
+Changes:
+- `src/openamp_foundry/compatibility/__init__.py` (I8) — Empty package init.
+- `src/openamp_foundry/compatibility/artifact_compatibility.py` (I8) — Core module
+  with `SchemaCompatibilityResult` (5 fields, dry_lab_only=True default),
+  `UNIVERSAL_REQUIRED_FIELDS`, `CONVENTION_CHECKS`, `check_schema_conventions()`,
+  `run_compatibility_check()`.
+- `tests/compatibility/__init__.py` (I8) — Empty package init.
+- `tests/compatibility/test_artifact_compatibility.py` (I8) — 20 tests covering
+  candidate_manifest/benchmark_card passes, simulation_result detected (known
+  dry_lab_context convention difference), synthetic schemas with missing/misformed
+  fields, run_compatibility_check dict shape, SchemaCompatibilityResult defaults,
+  additionalProperties/$schema warnings.
+- `src/openamp_foundry/cli/main.py` (I8) — Registered `artifact-compat-check`
+  subcommand with `--schemas-dir`, `--format` flags. Added import and dispatch.
+- `src/openamp_foundry/cli/commands/reports.py` (I8) — Added
+  `_run_artifact_compat_check()` CLI handler with text and JSON output, exit code 3
+  on failure.
+- `Makefile` (I8) — Added `artifact-compat-check` target. Added to `.PHONY`.
+- `docs/evidence/METRICS_CURRENT.md` (I8) — Updated last updated, new in v0.6.6,
+  pipeline version, test count (3393→3413).
+
 ## v0.6.5 — Loop 105: Phase I I7 — Data License Checker ✓ (2026-07-09)
 
 `src/openamp_foundry/licensing/license_checker.py` with `DataLicenseDeclaration`

src/openamp_foundry/cli/commands/reports.py

@@ -2303,3 +2303,37 @@ def _run_license_check(args: argparse.Namespace) -> int:
         print("not legal advice.")
 
     return 0 if result.passed else 3
+
+
+def _run_artifact_compat_check(args: argparse.Namespace) -> int:
+    """Run cross-artifact schema compatibility checks."""
+    from openamp_foundry.compatibility.artifact_compatibility import (
+        run_compatibility_check,
+    )
+
+    schemas_dir = Path(args.schemas_dir) if args.schemas_dir else None
+    report = run_compatibility_check(schemas_dir)
+    output_format = getattr(args, "format", "text")
+
+    if output_format == "json":
+        print(json.dumps(report, indent=2))
+    else:
+        print("Artifact Compatibility Check")
+        print("=" * 60)
+        print(f"Total schemas:  {report['total']}")
+        print(f"Passed:         {report['passed']}")
+        print(f"Failed:         {report['failed']}")
+        print(f"All passed:     {report['all_passed']}")
+        print()
+        if not report["all_passed"]:
+            print("Failures:")
+            for r in report["results"]:
+                if not r["passed"]:
+                    print(f"  Schema: {r['schema_name']}")
+                    for err in r["errors"]:
+                        print(f"    - {err}")
+        print()
+        print("Dry-lab only. Schema compatibility checks prevent drift")
+        print("between artifact versions.")
+
+    return 0 if report["all_passed"] else 3

src/openamp_foundry/cli/main.py

@@ -5,7 +5,7 @@
 from openamp_foundry.cli.commands.selection import _run_pilot_panel, _run_pilot_confident, _run_diversity_check, _run_select_batch, _run_batch_rationale
 from openamp_foundry.cli.commands.external import _run_external_predict, _run_external_consensus
 from openamp_foundry.cli.commands.qc import _run_synthesis_order, _run_presynth_qc
-from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter, _run_synthetic_result_policy_check, _run_calibration_decision_checklist, _run_calibration_rollback_plan, _run_simulation_registry, _run_validate_simulation_result, _run_simulation_baseline_check, _run_adapter_gate_check, _run_simulation_provenance, _run_simulation_ensemble_check, _run_simulation_ci_report, _run_simulation_deprecation_check, _run_simulation_scope_check, _run_simulation_evidence_packet, _run_artifact_version, _run_candidate_manifest, _run_benchmark_card, _run_artifact_changelog, _run_integration_check, _run_adapter_check, _run_license_check
+from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter, _run_synthetic_result_policy_check, _run_calibration_decision_checklist, _run_calibration_rollback_plan, _run_simulation_registry, _run_validate_simulation_result, _run_simulation_baseline_check, _run_adapter_gate_check, _run_simulation_provenance, _run_simulation_ensemble_check, _run_simulation_ci_report, _run_simulation_deprecation_check, _run_simulation_scope_check, _run_simulation_evidence_packet, _run_artifact_version, _run_candidate_manifest, _run_benchmark_card, _run_artifact_changelog, _run_integration_check, _run_adapter_check, _run_license_check, _run_artifact_compat_check
 from openamp_foundry.cli.commands.gates import _run_gate_check
 
 import argparse
@@ -1754,6 +1754,25 @@ def build_parser() -> argparse.ArgumentParser:
         help="Output format (default: text).",
     )
 
+    # ── Artifact compatibility check (Phase I I8) ────────────────────
+    acc = sub.add_parser(
+        "artifact-compat-check",
+        help=(
+            "Run cross-artifact schema compatibility checks. Validates that "
+            "published artifact schemas share consistent conventions and "
+            "remain mutually compatible as they evolve. Dry-lab only."
+        ),
+    )
+    acc.add_argument(
+        "--schemas-dir", type=str, default=None,
+        help="Path to schemas directory (default: schemas/).",
+    )
+    acc.add_argument(
+        "--format", type=str, default="text",
+        choices=["text", "json"],
+        help="Output format (default: text).",
+    )
+
     # ── Simulation-result confidence interval report (H7) ────────────
     sicr = sub.add_parser(
         "simulation-ci-report",
@@ -1983,6 +2002,9 @@ def main(argv: list[str] | None = None) -> int:
     if args.command == "license-check":
         return _run_license_check(args)
 
+    if args.command == "artifact-compat-check":
+        return _run_artifact_compat_check(args)
+
     parser.error("unknown command")
     return 2