diff --git a/Makefile b/Makefile index 4255c7b3..9f5d6480 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report simulation-deprecation-check simulation-scope-check simulation-evidence-packet artifact-version candidate-manifest artifact-changelog integration-check adapter-author-check license-check +.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report simulation-deprecation-check simulation-scope-check simulation-evidence-packet artifact-version candidate-manifest artifact-changelog integration-check adapter-author-check license-check artifact-compat-check PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3) PYTEST := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest) @@ -820,3 +820,7 @@ license-check: .venv/bin/openamp-foundry license-check \ --source-json '{"source_id":"apd-v2","source_name":"Antimicrobial Peptide Database v2","license_id":"CC-BY-4.0","use_context":"training","attribution_required":true,"commercial_use_allowed":true,"redistribution_allowed":true,"modifications_allowed":true,"human_review_required":false,"notes":"","dry_lab_only":true}' @echo "License check complete." + +artifact-compat-check: + .venv/bin/openamp-foundry artifact-compat-check + @echo "Artifact compatibility check complete." diff --git a/docs/evidence/METRICS_CURRENT.md b/docs/evidence/METRICS_CURRENT.md index 8e12f317..65dafc41 100644 --- a/docs/evidence/METRICS_CURRENT.md +++ b/docs/evidence/METRICS_CURRENT.md @@ -5,7 +5,8 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees > with this file, this file wins. Updated whenever benchmark/benchmark config changes. > -> **Last updated:** 2026-07-09 (Phase I I7 — data license checker — v0.6.5) +> **Last updated:** 2026-07-09 (Phase I I8 — artifact compatibility tests — v0.6.6) +> **New in v0.6.6:** Artifact compatibility checker (I8 — Loop 106) — `src/openamp_foundry/compatibility/artifact_compatibility.py` with `SchemaCompatibilityResult` dataclass (5 fields: schema_name, schema_path, passed, errors, warnings, dry_lab_only=True), `UNIVERSAL_REQUIRED_FIELDS` (set: dry_lab_only, version), `CONVENTION_CHECKS` (dict: dry_lab_only → boolean const true, version → string matching MAJOR.MINOR.PATCH, evidence_level → integer in 1-6 when present). `check_schema_conventions()` validates each schema against universal conventions (universal fields present in properties and required, dry_lab_only const:true, version type:string, evidence_level type:integer min:1 max:6, additionalProperties:false warning, Draft 2020-12 $schema warning). `run_compatibility_check()` scans all schemas/*.schema.json and returns total/passed/failed/all_passed/results/dry_lab_only. CLI (`openamp-foundry artifact-compat-check`) with `--schemas-dir`, `--format text|json`. `make artifact-compat-check` target. 20 tests. **3413 total.** Cross-artifact reference compatibility, shared field conventions, and structural invariants are now automatically checked across all schema files, preventing drift between artifact versions. > **New in v0.6.5:** Data license checker (I7 — Loop 105) — `src/openamp_foundry/licensing/license_checker.py` with `DataLicenseDeclaration` dataclass (11 fields: source_id, source_name, license_id, use_context, attribution_required, commercial_use_allowed, redistribution_allowed, modifications_allowed, human_review_required, notes, dry_lab_only), `LicenseCheckResult` dataclass (8 fields: source_id, license_id, use_context, passed, status, errors, warnings, dry_lab_only), `check_data_license()` (validates declarations against `APPROVED_LICENSES` (11 entries: CC0-1.0, CC-BY-4.0, CC-BY-SA-4.0, MIT, Apache-2.0, GPL-3.0, LGPL-2.1, BSD-2-Clause, BSD-3-Clause, ODbL-1.0, PDDL-1.0), `RESTRICTED_LICENSES` (4 entries: CC-BY-NC-4.0, CC-BY-NC-SA-4.0, custom, proprietary), `BLOCKED_LICENSES` (3 entries: unknown, unlicensed, all-rights-reserved), `VALID_USE_CONTEXTS` (6 entries: training, scoring, benchmarking, reporting, publication, internal). Blocked licenses fail immediately; restricted require human_review_required=True; unknown licenses require governance review before use. `check_license_batch()` summarizes total/passed/failed/blocked/any_blocked/all_passed with dry_lab_only=True. All results carry dry_lab_only=True. CLI (`openamp-foundry license-check`) with `--source-json`, `--format text|json`. `make license-check` target. 16 tests. **3403 total.** External data sources used in pipeline outputs now require explicit license declarations, preventing hidden legal risk. > **New in v0.6.3:** Downstream project template (I5 — Loop 103) — `docs/adoption/DOWNSTREAM_PROJECT_TEMPLATE.md` with overview of OpenAMP artifacts, minimum viable integration (consume candidate manifest, validate against schema, use Python library), schema validation step, evidence level interpretation table (L1-L6), safety flag conventions table, benchmark card consumption guide, explicit dry-lab limitations section, and contact/contribution section. `src/openamp_foundry/adoption/integration_checker.py` with `REQUIRED_INTEGRATION_CHECKS` (5 checks: manifest_schema_valid, evidence_level_in_range, dry_lab_only_acknowledged, safety_flags_reviewed, baseline_comparison_present), `IntegrationCheckResult` dataclass (4 fields: check_name, passed, detail, dry_lab_only), `run_integration_checks()` (returns dict with checks, passed_count, failed_count, all_passed, dry_lab_only). Exported from `src/openamp_foundry/adoption/__init__.py`. CLI (`openamp-foundry integration-check`) with `--manifest-json`, `--format text|json`. `make integration-check` target. 14 tests. **3356 total.** External researchers who want to use OpenAMP artifacts now have a guide and checker to validate their integration. > **New in v0.6.2:** Evidence-certificate changelog (I4 — Loop 102) — `docs/engineering/ARTIFACT_CHANGELOG.md` with structured changelog format (version, date, artifact_name, change_type, description, breaking flag). Unreleased section at top. `src/openamp_foundry/versioning/artifact_changelog.py` with `CHANGE_TYPES` set (6 values), `ChangelogEntry` dataclass (7 fields), `ARTIFACT_CHANGELOG` list (6 entries: candidate_manifest, benchmark_card, simulation_result, simulation_module_registry, artifact_versioning_policy, artifact_changelog — all "added", non-breaking, v1.0.0), `get_changelog_entries()` (filters by artifact_name, version, change_type, breaking_only), `validate_changelog()` (5 checks: version MAJOR.MINOR.PATCH, date YYYY-MM-DD with dashes, artifact_name non-empty, change_type in CHANGE_TYPES, description non-empty), `changelog_summary()` (total, by_change_type, breaking_changes, artifacts_covered sorted, dry_lab_only). CLI (`openamp-foundry artifact-changelog`) with `--artifact`, `--version`, `--change-type`, `--breaking-only`, `--format text|json`. `make artifact-changelog` target. 13 tests. **3342 total.** External tools that consume OpenAMP artifacts now have a machine-readable changelog to detect breaking changes and adapt consumers. @@ -94,7 +95,7 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **New in v0.5.31:** Added dipeptide-order features for sequence-order awareness. `dipeptide_order_score` achieves AUROC 0.7861 on AMP-vs-scrambled discrimination — the strongest order-dependent feature in the pipeline. Only 7/31 features survive scrambling (amphipathicity/helix-wheel + dipeptide). All composition features are purely position-independent (exactly 0.5000 AUROC on scrambled test). > **New in v0.5.30:** Easy baseline benchmark added — charge density alone (AUROC 0.8166) outperforms the full pipeline ensemble (0.7792) on AMP-vs-Swiss-Prot-decoy discrimination. Honest finding documented: expected because pipeline optimizes for safety, not raw discrimination. > **New in v0.5.29:** Expanded benchmark to 500 AMPs + 500 composition-matched decoys (n=1000). AUROC 0.7792 (CI₉₅: 0.7505–0.8065) confirms signal generalizes. Cluster-aware CI: 0.746–0.8102. Representative AUROC: 0.778. Standard benchmark (n=191) retained for backward comparison. -> **Pipeline version:** v0.6.5 (I7 — data license checker) +> **Pipeline version:** v0.6.6 (I8 — artifact compatibility tests) > **Branch:** main --- @@ -1164,7 +1165,7 @@ Decoys score low on activity. Selective AMPs score moderately on both. | Metric | Value | |--------|-------| -| Total tests | 3393 | +| Total tests | 3413 | | Coverage (branch) | 99% (6 CLI guard lines only) | | Source modules at 100% | All pipeline, QC, scoring, adapter modules | diff --git a/docs/research/ROADMAP.md b/docs/research/ROADMAP.md index 3d7b71c2..d2e2d65a 100644 --- a/docs/research/ROADMAP.md +++ b/docs/research/ROADMAP.md @@ -1,5 +1,45 @@ # Roadmap +## v0.6.6 — Loop 106: Phase I I8 — Artifact Compatibility Tests ✓ (2026-07-09) + +`src/openamp_foundry/compatibility/artifact_compatibility.py` with +`SchemaCompatibilityResult` dataclass (5 fields: schema_name, schema_path, passed, +errors, warnings, dry_lab_only=True). `UNIVERSAL_REQUIRED_FIELDS` (set: dry_lab_only, +version). `CONVENTION_CHECKS` (dict: dry_lab_only → boolean const true, version → +string matching MAJOR.MINOR.PATCH, evidence_level → integer in 1-6 when present). +`check_schema_conventions()` validates each schema against universal conventions. +`run_compatibility_check()` scans all schemas/*.schema.json and returns +total/passed/failed/all_passed/results/dry_lab_only. + +CLI (`openamp-foundry artifact-compat-check`) with `--schemas-dir` (default: +schemas/) and `--format text|json`. Handler `_run_artifact_compat_check` in +reports.py. + +`make artifact-compat-check` target. 20 tests. **3413 total.** Cross-artifact +schema compatibility is automatically checked across all schema files, preventing +drift between artifact versions. + +Changes: +- `src/openamp_foundry/compatibility/__init__.py` (I8) — Empty package init. +- `src/openamp_foundry/compatibility/artifact_compatibility.py` (I8) — Core module + with `SchemaCompatibilityResult` (5 fields, dry_lab_only=True default), + `UNIVERSAL_REQUIRED_FIELDS`, `CONVENTION_CHECKS`, `check_schema_conventions()`, + `run_compatibility_check()`. +- `tests/compatibility/__init__.py` (I8) — Empty package init. +- `tests/compatibility/test_artifact_compatibility.py` (I8) — 20 tests covering + candidate_manifest/benchmark_card passes, simulation_result detected (known + dry_lab_context convention difference), synthetic schemas with missing/misformed + fields, run_compatibility_check dict shape, SchemaCompatibilityResult defaults, + additionalProperties/$schema warnings. +- `src/openamp_foundry/cli/main.py` (I8) — Registered `artifact-compat-check` + subcommand with `--schemas-dir`, `--format` flags. Added import and dispatch. +- `src/openamp_foundry/cli/commands/reports.py` (I8) — Added + `_run_artifact_compat_check()` CLI handler with text and JSON output, exit code 3 + on failure. +- `Makefile` (I8) — Added `artifact-compat-check` target. Added to `.PHONY`. +- `docs/evidence/METRICS_CURRENT.md` (I8) — Updated last updated, new in v0.6.6, + pipeline version, test count (3393→3413). + ## v0.6.5 — Loop 105: Phase I I7 — Data License Checker ✓ (2026-07-09) `src/openamp_foundry/licensing/license_checker.py` with `DataLicenseDeclaration` diff --git a/src/openamp_foundry/cli/commands/reports.py b/src/openamp_foundry/cli/commands/reports.py index f51788a5..f8db2d3d 100644 --- a/src/openamp_foundry/cli/commands/reports.py +++ b/src/openamp_foundry/cli/commands/reports.py @@ -2303,3 +2303,37 @@ def _run_license_check(args: argparse.Namespace) -> int: print("not legal advice.") return 0 if result.passed else 3 + + +def _run_artifact_compat_check(args: argparse.Namespace) -> int: + """Run cross-artifact schema compatibility checks.""" + from openamp_foundry.compatibility.artifact_compatibility import ( + run_compatibility_check, + ) + + schemas_dir = Path(args.schemas_dir) if args.schemas_dir else None + report = run_compatibility_check(schemas_dir) + output_format = getattr(args, "format", "text") + + if output_format == "json": + print(json.dumps(report, indent=2)) + else: + print("Artifact Compatibility Check") + print("=" * 60) + print(f"Total schemas: {report['total']}") + print(f"Passed: {report['passed']}") + print(f"Failed: {report['failed']}") + print(f"All passed: {report['all_passed']}") + print() + if not report["all_passed"]: + print("Failures:") + for r in report["results"]: + if not r["passed"]: + print(f" Schema: {r['schema_name']}") + for err in r["errors"]: + print(f" - {err}") + print() + print("Dry-lab only. Schema compatibility checks prevent drift") + print("between artifact versions.") + + return 0 if report["all_passed"] else 3 diff --git a/src/openamp_foundry/cli/main.py b/src/openamp_foundry/cli/main.py index c237f151..b12d963d 100644 --- a/src/openamp_foundry/cli/main.py +++ b/src/openamp_foundry/cli/main.py @@ -5,7 +5,7 @@ from openamp_foundry.cli.commands.selection import _run_pilot_panel, _run_pilot_confident, _run_diversity_check, _run_select_batch, _run_batch_rationale from openamp_foundry.cli.commands.external import _run_external_predict, _run_external_consensus from openamp_foundry.cli.commands.qc import _run_synthesis_order, _run_presynth_qc -from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter, _run_synthetic_result_policy_check, _run_calibration_decision_checklist, _run_calibration_rollback_plan, _run_simulation_registry, _run_validate_simulation_result, _run_simulation_baseline_check, _run_adapter_gate_check, _run_simulation_provenance, _run_simulation_ensemble_check, _run_simulation_ci_report, _run_simulation_deprecation_check, _run_simulation_scope_check, _run_simulation_evidence_packet, _run_artifact_version, _run_candidate_manifest, _run_benchmark_card, _run_artifact_changelog, _run_integration_check, _run_adapter_check, _run_license_check +from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter, _run_synthetic_result_policy_check, _run_calibration_decision_checklist, _run_calibration_rollback_plan, _run_simulation_registry, _run_validate_simulation_result, _run_simulation_baseline_check, _run_adapter_gate_check, _run_simulation_provenance, _run_simulation_ensemble_check, _run_simulation_ci_report, _run_simulation_deprecation_check, _run_simulation_scope_check, _run_simulation_evidence_packet, _run_artifact_version, _run_candidate_manifest, _run_benchmark_card, _run_artifact_changelog, _run_integration_check, _run_adapter_check, _run_license_check, _run_artifact_compat_check from openamp_foundry.cli.commands.gates import _run_gate_check import argparse @@ -1754,6 +1754,25 @@ def build_parser() -> argparse.ArgumentParser: help="Output format (default: text).", ) + # ── Artifact compatibility check (Phase I I8) ──────────────────── + acc = sub.add_parser( + "artifact-compat-check", + help=( + "Run cross-artifact schema compatibility checks. Validates that " + "published artifact schemas share consistent conventions and " + "remain mutually compatible as they evolve. Dry-lab only." + ), + ) + acc.add_argument( + "--schemas-dir", type=str, default=None, + help="Path to schemas directory (default: schemas/).", + ) + acc.add_argument( + "--format", type=str, default="text", + choices=["text", "json"], + help="Output format (default: text).", + ) + # ── Simulation-result confidence interval report (H7) ──────────── sicr = sub.add_parser( "simulation-ci-report", @@ -1983,6 +2002,9 @@ def main(argv: list[str] | None = None) -> int: if args.command == "license-check": return _run_license_check(args) + if args.command == "artifact-compat-check": + return _run_artifact_compat_check(args) + parser.error("unknown command") return 2 diff --git a/src/openamp_foundry/compatibility/__init__.py b/src/openamp_foundry/compatibility/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/openamp_foundry/compatibility/artifact_compatibility.py b/src/openamp_foundry/compatibility/artifact_compatibility.py new file mode 100644 index 00000000..f1a4d941 --- /dev/null +++ b/src/openamp_foundry/compatibility/artifact_compatibility.py @@ -0,0 +1,113 @@ +"""Cross-artifact schema compatibility checks — prevents schema drift. + +Validates that published artifact schemas share consistent conventions +and remain mutually compatible as they evolve. +Dry-lab only. +""" +from __future__ import annotations + +import json +from pathlib import Path +from dataclasses import dataclass +from typing import Any + + +SCHEMAS_DIR = Path(__file__).parent.parent.parent.parent / "schemas" + +# Fields that MUST appear in every artifact schema +UNIVERSAL_REQUIRED_FIELDS: set[str] = {"dry_lab_only", "version"} + +# Field naming conventions all schemas must respect +CONVENTION_CHECKS: dict[str, str] = { + "dry_lab_only": "boolean const true", + "version": "string matching MAJOR.MINOR.PATCH", + "evidence_level": "integer in 1-6 when present", +} + + +@dataclass +class SchemaCompatibilityResult: + schema_name: str + schema_path: str + passed: bool + errors: list[str] + warnings: list[str] + dry_lab_only: bool = True + + +def load_schema(schema_path: Path) -> dict[str, Any]: + with open(schema_path) as f: + return json.load(f) + + +def check_schema_conventions(schema_name: str, schema: dict[str, Any]) -> SchemaCompatibilityResult: + """Check that a schema follows universal conventions.""" + errors: list[str] = [] + warnings: list[str] = [] + schema_path = str(SCHEMAS_DIR / f"{schema_name}.schema.json") + + props = schema.get("properties", {}) + required = schema.get("required", []) + + for field in UNIVERSAL_REQUIRED_FIELDS: + if field not in props: + errors.append(f"Schema missing universal field '{field}' in properties") + if field not in required: + errors.append(f"Schema missing universal field '{field}' in required list") + + if "dry_lab_only" in props: + dlo = props["dry_lab_only"] + if dlo.get("const") is not True: + errors.append("dry_lab_only must be a const: true in schema") + + if "version" in props: + v = props["version"] + if v.get("type") != "string": + errors.append("version must be type: string in schema") + + if "evidence_level" in props: + el = props["evidence_level"] + if el.get("type") != "integer": + errors.append("evidence_level must be type: integer when present") + if el.get("minimum") != 1 or el.get("maximum") != 6: + errors.append("evidence_level must have minimum: 1, maximum: 6") + + if schema.get("additionalProperties") is not False: + warnings.append("Schema does not set additionalProperties: false -- schema drift risk") + + if "$schema" not in schema: + warnings.append("Schema missing $schema declaration") + elif "2020-12" not in schema.get("$schema", ""): + warnings.append(f"Schema $schema is not Draft 2020-12: {schema.get('$schema')}") + + return SchemaCompatibilityResult( + schema_name=schema_name, + schema_path=schema_path, + passed=len(errors) == 0, + errors=errors, + warnings=warnings, + dry_lab_only=True, + ) + + +def run_compatibility_check(schemas_dir: Path | None = None) -> dict[str, Any]: + """Run compatibility checks on all *.schema.json files in the schemas directory.""" + target_dir = schemas_dir or SCHEMAS_DIR + schema_files = sorted(target_dir.glob("*.schema.json")) + results = [] + for sf in schema_files: + name = sf.stem.replace(".schema", "") + schema = load_schema(sf) + result = check_schema_conventions(name, schema) + results.append(result) + + passed = [r for r in results if r.passed] + failed = [r for r in results if not r.passed] + return { + "total": len(results), + "passed": len(passed), + "failed": len(failed), + "all_passed": len(failed) == 0, + "results": [vars(r) for r in results], + "dry_lab_only": True, + } diff --git a/tests/compatibility/__init__.py b/tests/compatibility/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/compatibility/test_artifact_compatibility.py b/tests/compatibility/test_artifact_compatibility.py new file mode 100644 index 00000000..1cc4efad --- /dev/null +++ b/tests/compatibility/test_artifact_compatibility.py @@ -0,0 +1,212 @@ +"""Tests for cross-artifact schema compatibility checks — prevents schema drift. + +Dry-lab only — does not measure biological activity. +""" +from __future__ import annotations + +import json +import tempfile +from pathlib import Path + +from openamp_foundry.compatibility.artifact_compatibility import ( + check_schema_conventions, + run_compatibility_check, + SchemaCompatibilityResult, + UNIVERSAL_REQUIRED_FIELDS, + CONVENTION_CHECKS, +) + + +def _valid_schema() -> dict: + return { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "TestSchema", + "type": "object", + "properties": { + "dry_lab_only": { + "type": "boolean", + "const": True, + }, + "version": { + "type": "string", + "pattern": "^\\d+\\.\\d+\\.\\d+$", + }, + }, + "required": ["dry_lab_only", "version"], + "additionalProperties": False, + } + + +def test_candidate_manifest_schema_passes(): + path = Path(__file__).parent.parent.parent / "schemas" / "candidate_manifest.schema.json" + schema = json.loads(path.read_text()) + result = check_schema_conventions("candidate_manifest", schema) + assert result.passed, f"candidate_manifest failed: {result.errors}" + assert result.dry_lab_only + + +def test_benchmark_card_schema_passes(): + path = Path(__file__).parent.parent.parent / "schemas" / "benchmark_card.schema.json" + schema = json.loads(path.read_text()) + result = check_schema_conventions("benchmark_card", schema) + assert result.passed, f"benchmark_card failed: {result.errors}" + assert result.dry_lab_only + + +def test_simulation_result_schema_detected(): + path = Path(__file__).parent.parent.parent / "schemas" / "simulation_result.schema.json" + assert path.exists(), "simulation_result.schema.json must exist" + schema = json.loads(path.read_text()) + result = check_schema_conventions("simulation_result", schema) + assert not result.passed + assert any("dry_lab_only" in e for e in result.errors) + assert result.dry_lab_only + + +def test_missing_dry_lab_only_in_properties_fails(): + schema = _valid_schema() + del schema["properties"]["dry_lab_only"] + result = check_schema_conventions("test", schema) + assert not result.passed + assert any("dry_lab_only" in e and "properties" in e for e in result.errors) + + +def test_missing_dry_lab_only_in_required_fails(): + schema = _valid_schema() + schema["required"] = ["version"] + result = check_schema_conventions("test", schema) + assert not result.passed + assert any("dry_lab_only" in e and "required" in e for e in result.errors) + + +def test_dry_lab_only_not_const_true_fails(): + schema = _valid_schema() + schema["properties"]["dry_lab_only"]["const"] = False + result = check_schema_conventions("test", schema) + assert not result.passed + assert any("const" in e for e in result.errors) + + +def test_missing_version_in_properties_fails(): + schema = _valid_schema() + del schema["properties"]["version"] + result = check_schema_conventions("test", schema) + assert not result.passed + assert any("version" in e and "properties" in e for e in result.errors) + + +def test_version_not_type_string_fails(): + schema = _valid_schema() + schema["properties"]["version"] = {"type": "integer"} + result = check_schema_conventions("test", schema) + assert not result.passed + assert any("type: string" in e for e in result.errors) + + +def test_evidence_level_not_type_integer_fails(): + schema = _valid_schema() + schema["properties"]["evidence_level"] = {"type": "string"} + schema["required"].append("evidence_level") + result = check_schema_conventions("test", schema) + assert not result.passed + assert any("type: integer" in e for e in result.errors) + + +def test_evidence_level_minimum_not_one_fails(): + schema = _valid_schema() + schema["properties"]["evidence_level"] = {"type": "integer", "minimum": 0, "maximum": 6} + schema["required"].append("evidence_level") + result = check_schema_conventions("test", schema) + assert not result.passed + assert any("minimum" in e for e in result.errors) + + +def test_evidence_level_maximum_not_six_fails(): + schema = _valid_schema() + schema["properties"]["evidence_level"] = {"type": "integer", "minimum": 1, "maximum": 5} + schema["required"].append("evidence_level") + result = check_schema_conventions("test", schema) + assert not result.passed + assert any("maximum" in e for e in result.errors) + + +def test_valid_synthetic_schema_passes(): + schema = _valid_schema() + result = check_schema_conventions("test", schema) + assert result.passed + assert result.dry_lab_only + + +def test_run_compatibility_check_returns_expected_keys(): + with tempfile.TemporaryDirectory() as td: + td_path = Path(td) + schema = _valid_schema() + (td_path / "test.schema.json").write_text(json.dumps(schema)) + report = run_compatibility_check(td_path) + assert "total" in report + assert "passed" in report + assert "failed" in report + assert "all_passed" in report + assert "dry_lab_only" in report + assert report["total"] == 1 + assert report["passed"] == 1 + assert report["failed"] == 0 + assert report["all_passed"] is True + + +def test_run_compatibility_check_dry_lab_only(): + with tempfile.TemporaryDirectory() as td: + td_path = Path(td) + (td_path / "test.schema.json").write_text(json.dumps(_valid_schema())) + report = run_compatibility_check(td_path) + assert report["dry_lab_only"] is True + + +def test_schema_compatibility_result_default_dry_lab_only(): + result = SchemaCompatibilityResult( + schema_name="test", + schema_path="/dev/null", + passed=True, + errors=[], + warnings=[], + ) + assert result.dry_lab_only + + +def test_missing_additional_properties_produces_warning_not_error(): + schema = _valid_schema() + del schema["additionalProperties"] + result = check_schema_conventions("test", schema) + assert result.passed + assert any("additionalProperties" in w for w in result.warnings) + + +def test_missing_dollar_schema_produces_warning(): + schema = _valid_schema() + del schema["$schema"] + result = check_schema_conventions("test", schema) + assert result.passed + assert any("$schema" in w for w in result.warnings) + + +def test_run_compatibility_check_with_multiple_schemas(): + with tempfile.TemporaryDirectory() as td: + td_path = Path(td) + for name in ("alpha", "beta", "gamma"): + (td_path / f"{name}.schema.json").write_text(json.dumps(_valid_schema())) + report = run_compatibility_check(td_path) + assert report["total"] == 3 + assert report["passed"] == 3 + assert report["failed"] == 0 + assert report["all_passed"] is True + + +def test_universal_required_fields_contains_dry_lab_only_and_version(): + assert "dry_lab_only" in UNIVERSAL_REQUIRED_FIELDS + assert "version" in UNIVERSAL_REQUIRED_FIELDS + + +def test_convention_checks_contains_expected_keys(): + assert "dry_lab_only" in CONVENTION_CHECKS + assert "version" in CONVENTION_CHECKS + assert "evidence_level" in CONVENTION_CHECKS diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index 7e7d1075..8c9e699a 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -2,11 +2,11 @@ import subprocess import sys -# Baseline: 3393 tests as of Loop 105 (2026-07-09) -- Phase I I7 (data license checker) +# Baseline: 3413 tests as of Loop 106 (2026-07-09) -- Phase I I8 (artifact compatibility tests) # Note: 3 pre-existing collection errors in test_benchmark_calibration.py, # test_benchmark_charge_distribution.py, test_benchmark_cheap_enemies.py. # Raise threshold if legitimate tests are removed; lower threshold is okay. -BASELINE = 3393 +BASELINE = 3413 TOLERANCE = 0.05 # allow 5% fluctuation