diff --git a/Makefile b/Makefile index 8ee7c966..bd001908 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report simulation-deprecation-check simulation-scope-check simulation-evidence-packet +.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report simulation-deprecation-check simulation-scope-check simulation-evidence-packet artifact-version PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3) PYTEST := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest) @@ -776,3 +776,7 @@ simulation-evidence-packet: --claimed-level 2 \ --baseline-beaten false @echo "Evidence packet complete." + +artifact-version: + .venv/bin/openamp-foundry artifact-version --list + @echo "Artifact version list complete." diff --git a/docs/engineering/ARTIFACT_VERSIONING_POLICY.md b/docs/engineering/ARTIFACT_VERSIONING_POLICY.md new file mode 100644 index 00000000..b50857a6 --- /dev/null +++ b/docs/engineering/ARTIFACT_VERSIONING_POLICY.md @@ -0,0 +1,161 @@ +# Artifact Versioning Policy + +## Purpose + +External users need stability guarantees for schemas, evidence certificates, +and candidate manifests. Without a declared policy, any schema change can +silently break downstream consumers. + +This document defines a structured versioning policy covering which artifacts +are versioned, the version format, breaking change rules, deprecation +timelines, and stability tiers. + +## Scope + +The following artifact types are versioned: + +- **Schemas** (`schemas/*.schema.json`) — JSON Schema definitions for all + machine-readable artifacts +- **Evidence certificates** — candidate evidence certificate outputs +- **Candidate manifests** — batch-level candidate manifests +- **Provenance records** — simulation provenance chain records +- **Benchmark cards** — machine-readable benchmark summaries +- **Model cards** — model metadata and capability descriptions + +## Version format + +All versioned artifacts use **MAJOR.MINOR.PATCH** (semantic versioning): + +``` +MAJOR.MINOR.PATCH +``` + +Where each component is a non-negative integer. + +### Major version + +Increment when a breaking change is made (see definition below). + +### Minor version + +Increment when functionality is added in a backward-compatible manner. + +### Patch version + +Increment when backward-compatible bug fixes, typo corrections, or +documentation improvements are made. + +## Breaking change definition + +A change is **breaking** if it would cause a valid artifact from the +previous version to fail validation, including: + +- Removing or renaming a required field +- Changing a field's type in an incompatible way +- Adding a new required field +- Changing an enum constraint in a way that invalidates previously valid + values (unless the change adds values only) +- Changing the structural composition rules of an artifact +- Changing the interpretation or meaning of an existing field + +## Non-breaking changes + +The following changes are NOT breaking: + +- Adding optional fields +- Relaxing constraints (e.g., making a required field optional) +- Adding new enum values without removing existing ones +- Adding new metadata fields +- Improving validation error messages +- Adding documentation +- Adding examples + +## Deprecation timeline + +When a schema field, enum value, or artifact must be removed: + +1. The deprecation is announced in the CHANGELOG entry for the minor + version in which deprecation begins. +2. The deprecated element remains valid for at least **one full minor + version cycle** after deprecation announcement. +3. Removal happens in the next MAJOR version, or after at least one + minor version has passed since deprecation. +4. Deprecated elements SHOULD be clearly marked in the schema with a + `deprecated` annotation or equivalent. + +## Schema `$id` policy + +Every stable schema MUST have a `$id` URI in the format: + +``` +https://openamp.org/schemas// +``` + +The `$id` MUST be updated when the MAJOR or MINOR version changes. +Schemas without an `$id` are considered experimental. + +## Changelog requirement + +Every schema version change MUST have a corresponding entry in +`docs/engineering/ARTIFACT_CHANGELOG.md`, recording: + +- Date of change +- Schema name +- Old version and new version +- Type of change (MAJOR, MINOR, PATCH) +- Summary of what changed and why + +## Stability tiers + +### Tier 1 — Stable + +Schemas that have a `$id` URI and are fully documented. These require a +MAJOR version bump for breaking changes. External consumers can rely on +Tier 1 artifacts for production workflows. + +Requirements: +- `$id` URI set and versioned +- Documented in changelog +- Full test coverage for valid/invalid states +- Approval required for MAJOR version changes + +### Tier 2 — Experimental + +Schemas without a `$id` URI or in draft status. These may change without +notice but MUST be labeled `"experimental"` in their metadata or +documentation. External consumers should not depend on experimental +schemas for production use. + +Requirements: +- Labeled as experimental +- Documented with known limitations +- May change at any time without deprecation notice + +### Tier 3 — Internal + +Artifacts not intended for external use. These carry no stability +promise. Internal tools and agents may change them freely. + +Requirements: +- Clearly marked as internal +- Not documented for external consumption +- No deprecation notice required for changes + +## Dry-lab-only statement + +Artifact versioning applies to computational artifacts only. Versioning +does not change the computational or dry-lab nature of the results. All +artifacts remain dry-lab hypotheses requiring wet-lab validation. + +Version numbers describe schema and format compatibility, not biological +validity, safety, or efficacy. + +## Compliance + +All new schemas added to `schemas/` MUST declare a stability tier. The +default tier for new schemas without explicit documentation is +**Tier 2 (Experimental)**. + +Existing schemas added before this policy was adopted are Tier 1 +(Stable) starting at version 1.0.0 if they have a `$id` field and +are in active use by the pipeline. diff --git a/docs/evidence/METRICS_CURRENT.md b/docs/evidence/METRICS_CURRENT.md index 493117f8..f1efb716 100644 --- a/docs/evidence/METRICS_CURRENT.md +++ b/docs/evidence/METRICS_CURRENT.md @@ -5,8 +5,8 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees > with this file, this file wins. Updated whenever benchmark/benchmark config changes. > -> **Last updated:** 2026-07-09 (Phase H H10 — simulation-evidence packet assembler — v0.5.98) -> **New in v0.5.98:** Simulation-evidence packet assembler (H10 — Phase H capstone) — `SimulationEvidencePacket` dataclass (12 fields: module_id, result, requested_scopes, claimed_evidence_level, baseline_beaten, deprecation_check, scope_check, baseline_check, adapter_gate, effective_evidence_level, all_checks_passed, failure_reasons, dry_lab_only). `assemble_evidence_packet()` runs all Phase H sub-checks (deprecation, scope, baseline, adapter gate) and assembles into a single auditable packet with all_checks_passed (True only if not deprecation_check["is_blocked"], scope_check["is_fully_covered"], not baseline_check["capped"], and adapter_gate["passed"]), failure_reasons (human-readable list of what failed), and effective_evidence_level (from baseline_check). `evidence_packet_summary()` returns compact dict with module_id, claimed_evidence_level, effective_evidence_level, all_checks_passed, failure_reasons, dry_lab_only. CLI (`openamp-foundry simulation-evidence-packet`) with `--module-id`, `--result-json`, `--requested-scopes`, `--claimed-level`, `--baseline-beaten`, `--format text|json`. `make simulation-evidence-packet` target. 16+ tests. **3272 total.** This is the capstone of Phase H — it assembles all the individual simulation discipline checks into a single auditable evidence packet showing exactly why a simulation result is trustworthy (or not) enough to support a given evidence level claim. +> **Last updated:** 2026-07-09 (Phase I I1 — artifact versioning policy — v0.5.99) +> **New in v0.5.99:** Artifact versioning policy (I1 — starts Phase I) — `docs/engineering/ARTIFACT_VERSIONING_POLICY.md` with structured versioning policy covering scope, SemVer format, breaking/non-breaking change definitions, deprecation timeline (at least one minor version warning), schema `$id` policy, changelog requirement, and three stability tiers (stable/experimental/internal). `src/openamp_foundry/versioning/` module with `ArtifactVersionInfo` dataclass (7 fields), `STABILITY_TIERS` dict, `VERSIONED_ARTIFACTS` list (6 entries covering candidate, lab_result, run_manifest, external_review_packet, safety_release_decision, simulation_result at v1.0.0), `get_artifact_version()`, `list_versioned_artifacts()`, `validate_version_format()`, `artifact_version_summary()`. CLI (`openamp-foundry artifact-version`) with `--list`, `--show`, `--tier`, `--format text|json`. `make artifact-version` target. 19 tests. **3291 total.** Starts Phase I (interoperability) — external users now have stability guarantees for schemas, evidence certificates, and candidate manifests. > **New in v0.5.97:** Simulation-scope coverage checker (H9) — `ScopeCoverageResult` dataclass (8 fields: module_id, requested_scopes, module_scopes, covered, uncovered, coverage_fraction, is_fully_covered, dry_lab_only). `check_scope_coverage()` looks up module in registry, computes covered (intersection) and uncovered (requested scopes not in module scopes), coverage_fraction = len(covered)/len(requested_scopes) if requested else 1.0. `check_result_scope()` uses conservative intersection of registry scopes and result.scope as effective module_scopes. `scope_coverage_report()` returns full dict. CLI (`openamp-foundry simulation-scope-check`) with `--module-id`, `--requested-scopes`, `--format text|json`. `make simulation-scope-check` target. 17 tests. **3255 total.** A simulation module may cover only some biological scopes. If a candidate is evaluated for a scope the module doesn't cover, that result must be flagged as out-of-scope rather than silently trusted. > **New in v0.5.96:** Simulation-module deprecation enforcer (H8) — `DeprecationCheckResult` dataclass (5 fields: module_id, status, is_blocked, block_reason, dry_lab_only). `BLOCKED_STATUSES = {"deprecated", "unavailable"}`. `check_module_deprecation()` looks up module in registry: not-found returns is_blocked=True status="unknown"; deprecated/unavailable returns is_blocked=True with reason; active/experimental returns is_blocked=False. `enforce_deprecation()` filters list[SimulationResult] to only non-blocked modules, returns total_input/passed/blocked/blocked_modules/passed_results/checks/dry_lab_only. `run_deprecation_check_batch()` bulk-checks module_ids with total/blocked/allowed/any_blocked/results/dry_lab_only. CLI (`openamp-foundry simulation-deprecation-check`) with `--module-ids`, `--format text|json`. `make simulation-deprecation-check` target. 21 tests. **3238 total.** Deprecated simulation modules must not be used in production scoring — the enforcer prevents stale or unreliable modules from tainting evidence packets. > **New in v0.5.95:** Simulation-result confidence interval reporter (H7) — `ci_reporter.py` with `ScoreCI` dataclass (9 fields: module_id, score_key, point_estimate, uncertainty, ci_lower, ci_upper, ci_width, overlaps_with, dry_lab_only). `compute_score_ci()` builds CIs from SimulationResult.scores[score_key] ± uncertainty, returns None if score_key missing. `compare_cis()` pairwise checks overlap condition (a_lo <= b_hi and b_lo <= a_hi), returns new list with overlaps_with populated (no in-place mutation). `ci_report()` produces full report with n_results, cis list, any_overlap flag, and dry_lab_only=True. CLI (`openamp-foundry simulation-ci-report`) with `--results-json`, `--score-key`, `--format text|json`. `make simulation-ci-report` target. 16 tests. **3217 total.** Raw scores without uncertainty ranges make it impossible to judge whether two candidates are distinguishable. The CI reporter makes uncertainty explicit and auditable. @@ -89,7 +89,7 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **New in v0.5.31:** Added dipeptide-order features for sequence-order awareness. `dipeptide_order_score` achieves AUROC 0.7861 on AMP-vs-scrambled discrimination — the strongest order-dependent feature in the pipeline. Only 7/31 features survive scrambling (amphipathicity/helix-wheel + dipeptide). All composition features are purely position-independent (exactly 0.5000 AUROC on scrambled test). > **New in v0.5.30:** Easy baseline benchmark added — charge density alone (AUROC 0.8166) outperforms the full pipeline ensemble (0.7792) on AMP-vs-Swiss-Prot-decoy discrimination. Honest finding documented: expected because pipeline optimizes for safety, not raw discrimination. > **New in v0.5.29:** Expanded benchmark to 500 AMPs + 500 composition-matched decoys (n=1000). AUROC 0.7792 (CI₉₅: 0.7505–0.8065) confirms signal generalizes. Cluster-aware CI: 0.746–0.8102. Representative AUROC: 0.778. Standard benchmark (n=191) retained for backward comparison. -> **Pipeline version:** v0.5.98 (H10 — simulation-evidence packet assembler) +> **Pipeline version:** v0.5.99 (I1 — artifact versioning policy) > **Branch:** main --- @@ -1159,7 +1159,7 @@ Decoys score low on activity. Selective AMPs score moderately on both. | Metric | Value | |--------|-------| -| Total tests | 3255 | +| Total tests | 3291 | | Coverage (branch) | 99% (6 CLI guard lines only) | | Source modules at 100% | All pipeline, QC, scoring modules | diff --git a/docs/research/ROADMAP.md b/docs/research/ROADMAP.md index d0f01ae0..37b6f548 100644 --- a/docs/research/ROADMAP.md +++ b/docs/research/ROADMAP.md @@ -51,6 +51,73 @@ Honest boundaries: not invalidate results; it highlights them for human review. - All CI reports are dry-lab only and must not be presented as biological proof. +## v0.5.99 — Loop 99: Phase I I1 — Artifact Versioning Policy ✓ (2026-07-09) + +`docs/engineering/ARTIFACT_VERSIONING_POLICY.md` with structured versioning +policy covering scope (schemas, evidence certificates, candidate manifests, +provenance records, benchmark cards, model cards), version format +(MAJOR.MINOR.PATCH), breaking/non-breaking change definitions, deprecation +timeline (at least one minor version warning), schema `$id` policy, changelog +requirement, and three stability tiers (Tier 1 stable, Tier 2 experimental, +Tier 3 internal). Dry-lab-only statement included. + +`src/openamp_foundry/versioning/` module with `ArtifactVersionInfo` dataclass +(7 fields), `STABILITY_TIERS` dict (3 tiers), `VERSIONED_ARTIFACTS` list +(6 entries: candidate, lab_result, run_manifest, external_review_packet, +safety_release_decision, simulation_result — all at v1.0.0; 5 stable, +1 experimental). `get_artifact_version()`, `list_versioned_artifacts()` with +tier filtering, `validate_version_format()` (regex MAJOR.MINOR.PATCH), +`artifact_version_summary()` with total/by_tier/stable_count/experimental_count/ +dry_lab_only. + +CLI (`openamp-foundry artifact-version`) with `--list`, `--show `, +`--tier `, `--format text|json`. +`make artifact-version` target. + +19 tests. **3291 total.** Starts Phase I (interoperability and adoption) +— external users now have stability guarantees for schemas, evidence +certificates, and candidate manifests. + +Changes: +- `docs/engineering/ARTIFACT_VERSIONING_POLICY.md` (I1) — Full versioning + policy document with scope, SemVer, breaking/non-breaking rules, + deprecation timeline, $id policy, changelog requirement, stability tiers, + dry-lab-only statement. +- `src/openamp_foundry/versioning/__init__.py` (I1) — Package init, + exports all versioning symbols. +- `src/openamp_foundry/versioning/artifact_version.py` (I1) — Core module + with `STABILITY_TIERS` dict, `ArtifactVersionInfo` dataclass, + `VERSIONED_ARTIFACTS` list (6 entries), `get_artifact_version()`, + `list_versioned_artifacts()`, `validate_version_format()`, + `artifact_version_summary()`. +- `src/openamp_foundry/cli/main.py` — Registered `artifact-version` + subcommand with `--list`, `--show`, `--tier`, `--format` flags. + Added import and dispatch. +- `src/openamp_foundry/cli/commands/reports.py` — Added + `_run_artifact_version()` CLI handler with text and JSON output, + artifact lookup, tier filtering, and error handling. +- `Makefile` — Added `artifact-version` target. Added to `.PHONY`. +- `tests/versioning/__init__.py` — Empty package init. +- `tests/versioning/test_artifact_version.py` — 19 tests covering: + at least 5 entries, all valid versions, valid stability tiers, no duplicate + names, get known/unknown, list all/filtered, validate valid/invalid + versions, summary dry_lab_only, summary total, stable/experimental counts, + by_tier sum. +- `docs/evidence/METRICS_CURRENT.md` — v0.5.99 I1 changelog. Test count: 3291. +- `tests/test_test_count_regression.py` — baseline updated to 3291. + +Honest boundaries: +- Artifact versioning describes schema and format compatibility, not + biological validity, safety, or efficacy. +- The version registry is manually maintained; entries must be updated when + schemas change. +- `validate_version_format()` checks structural validity only — it does not + enforce semantic version ordering or compatibility rules. +- Stability tiers are policy declarations, not machine-enforced guarantees. + A Tier 1 artifact may still have undiscovered bugs or incompatibilities. +- All artifacts remain dry-lab hypotheses requiring wet-lab validation + regardless of version number. + ## v0.5.98 — Loop 98: Phase H H10 — Simulation-Evidence Packet Assembler ✓ (2026-07-09) `SimulationEvidencePacket` dataclass (12 fields: module_id, result, diff --git a/src/openamp_foundry/cli/commands/reports.py b/src/openamp_foundry/cli/commands/reports.py index c4acc32d..0d5281f1 100644 --- a/src/openamp_foundry/cli/commands/reports.py +++ b/src/openamp_foundry/cli/commands/reports.py @@ -1855,3 +1855,84 @@ def _run_simulation_evidence_packet(args: argparse.Namespace) -> int: print("not biological proof.") return 0 if packet.all_checks_passed else 3 + + +def _run_artifact_version(args: argparse.Namespace) -> int: + """Display artifact versioning information.""" + from openamp_foundry.versioning.artifact_version import ( + STABILITY_TIERS, + VERSIONED_ARTIFACTS, + get_artifact_version, + list_versioned_artifacts, + artifact_version_summary, + ) + + show_name = getattr(args, "show", None) + tier_filter = getattr(args, "tier", None) + output_format = getattr(args, "format", "text") + + if show_name: + entry = get_artifact_version(show_name) + if entry is None: + print(json.dumps({ + "status": "error", + "error": f"Unknown artifact '{show_name}'. " + f"Use --list to see available artifacts.", + })) + return 3 + if output_format == "json": + print(json.dumps({ + "artifact_name": entry.artifact_name, + "version": entry.version, + "stability_tier": entry.stability_tier, + "schema_id": entry.schema_id, + "description": entry.description, + "is_breaking_change": entry.is_breaking_change, + "notes": entry.notes, + }, indent=2)) + else: + tier_desc = STABILITY_TIERS.get(entry.stability_tier, entry.stability_tier) + print(f"Artifact: {entry.artifact_name}") + print(f" Version: {entry.version}") + print(f" Stability tier: {tier_desc}") + if entry.schema_id: + print(f" Schema $id: {entry.schema_id}") + print(f" Description: {entry.description}") + print(f" Breaking change: {entry.is_breaking_change}") + if entry.notes: + print(f" Notes: {entry.notes}") + return 0 + + entries = list_versioned_artifacts(tier=tier_filter) + summary = artifact_version_summary() + + if output_format == "json": + print(json.dumps({ + "artifacts": [ + { + "artifact_name": e.artifact_name, + "version": e.version, + "stability_tier": e.stability_tier, + "schema_id": e.schema_id, + "description": e.description, + } + for e in entries + ], + "summary": summary, + }, indent=2)) + else: + print("Artifact Version Registry") + print("=" * 60) + print(f"Total artifacts: {summary['total']}") + print(f"By tier: {dict(sorted(summary['by_tier'].items()))}") + print(f"Dry-lab only: {summary['dry_lab_only']}") + print() + for entry in entries: + sid = f" $id: {entry.schema_id}" if entry.schema_id else "" + print(f" [{entry.stability_tier:12}] {entry.artifact_name:30s} v{entry.version}{sid}") + print() + print("Dry-lab only. Artifact versioning describes schema compatibility,") + print("not biological validity, safety, or efficacy.") + print("See docs/engineering/ARTIFACT_VERSIONING_POLICY.md for policy details.") + + return 0 diff --git a/src/openamp_foundry/cli/main.py b/src/openamp_foundry/cli/main.py index 23dad6f8..56f195f5 100644 --- a/src/openamp_foundry/cli/main.py +++ b/src/openamp_foundry/cli/main.py @@ -5,7 +5,7 @@ from openamp_foundry.cli.commands.selection import _run_pilot_panel, _run_pilot_confident, _run_diversity_check, _run_select_batch, _run_batch_rationale from openamp_foundry.cli.commands.external import _run_external_predict, _run_external_consensus from openamp_foundry.cli.commands.qc import _run_synthesis_order, _run_presynth_qc -from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter, _run_synthetic_result_policy_check, _run_calibration_decision_checklist, _run_calibration_rollback_plan, _run_simulation_registry, _run_validate_simulation_result, _run_simulation_baseline_check, _run_adapter_gate_check, _run_simulation_provenance, _run_simulation_ensemble_check, _run_simulation_ci_report, _run_simulation_deprecation_check, _run_simulation_scope_check, _run_simulation_evidence_packet +from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter, _run_synthetic_result_policy_check, _run_calibration_decision_checklist, _run_calibration_rollback_plan, _run_simulation_registry, _run_validate_simulation_result, _run_simulation_baseline_check, _run_adapter_gate_check, _run_simulation_provenance, _run_simulation_ensemble_check, _run_simulation_ci_report, _run_simulation_deprecation_check, _run_simulation_scope_check, _run_simulation_evidence_packet, _run_artifact_version from openamp_foundry.cli.commands.gates import _run_gate_check import argparse @@ -1598,6 +1598,33 @@ def build_parser() -> argparse.ArgumentParser: help="Output format (default: text).", ) + # ── Artifact versioning (Phase I I1) ───────────────────────────── + av = sub.add_parser( + "artifact-version", + help=( + "Display artifact versioning information for schemas and " + "machine-readable artifacts. Dry-lab only." + ), + ) + av.add_argument( + "--list", action="store_true", default=False, dest="list_artifacts", + help="List all versioned artifacts (default action).", + ) + av.add_argument( + "--show", type=str, default=None, metavar="ARTIFACT_NAME", + help="Show details for a specific artifact by name.", + ) + av.add_argument( + "--tier", type=str, default=None, + choices=["stable", "experimental", "internal"], + help="Filter artifacts by stability tier.", + ) + av.add_argument( + "--format", type=str, default="text", + choices=["text", "json"], + help="Output format (default: text).", + ) + # ── Simulation-result confidence interval report (H7) ──────────── sicr = sub.add_parser( "simulation-ci-report", @@ -1806,6 +1833,9 @@ def main(argv: list[str] | None = None) -> int: if args.command == "simulation-evidence-packet": return _run_simulation_evidence_packet(args) + if args.command == "artifact-version": + return _run_artifact_version(args) + parser.error("unknown command") return 2 diff --git a/src/openamp_foundry/versioning/__init__.py b/src/openamp_foundry/versioning/__init__.py new file mode 100644 index 00000000..5838a4cd --- /dev/null +++ b/src/openamp_foundry/versioning/__init__.py @@ -0,0 +1,19 @@ +from openamp_foundry.versioning.artifact_version import ( + STABILITY_TIERS, + ArtifactVersionInfo, + VERSIONED_ARTIFACTS, + get_artifact_version, + list_versioned_artifacts, + validate_version_format, + artifact_version_summary, +) + +__all__ = [ + "STABILITY_TIERS", + "ArtifactVersionInfo", + "VERSIONED_ARTIFACTS", + "get_artifact_version", + "list_versioned_artifacts", + "validate_version_format", + "artifact_version_summary", +] diff --git a/src/openamp_foundry/versioning/artifact_version.py b/src/openamp_foundry/versioning/artifact_version.py new file mode 100644 index 00000000..a76574fb --- /dev/null +++ b/src/openamp_foundry/versioning/artifact_version.py @@ -0,0 +1,110 @@ +from __future__ import annotations + +import re +from dataclasses import dataclass, field + +STABILITY_TIERS: dict[str, str] = { + "stable": "Tier 1 — requires MAJOR bump for breaking changes", + "experimental": "Tier 2 — may change without notice, must be labeled", + "internal": "Tier 3 — no stability promise, not for external use", +} + + +@dataclass +class ArtifactVersionInfo: + artifact_name: str + version: str + stability_tier: str + schema_id: str | None + description: str + is_breaking_change: bool = False + notes: str = "" + + +VERSIONED_ARTIFACTS: list[ArtifactVersionInfo] = [ + ArtifactVersionInfo( + artifact_name="candidate", + version="1.0.0", + stability_tier="stable", + schema_id=None, + description="Candidate evidence certificate — per-candidate scoring and evidence output", + is_breaking_change=False, + notes="Core schema used in all pipeline runs. No $id set, but in active production use.", + ), + ArtifactVersionInfo( + artifact_name="lab_result", + version="1.0.0", + stability_tier="stable", + schema_id=None, + description="Lab assay result — machine-readable outcome for active-learning ingestion", + is_breaking_change=False, + notes="Required for calibration intake loop. No $id set, but in active production use.", + ), + ArtifactVersionInfo( + artifact_name="run_manifest", + version="1.0.0", + stability_tier="stable", + schema_id=None, + description="Run manifest — pipeline run provenance with input hashes and output paths", + is_breaking_change=False, + notes="Provenance record for reproducibility. No $id set, but in active production use.", + ), + ArtifactVersionInfo( + artifact_name="external_review_packet", + version="1.0.0", + stability_tier="stable", + schema_id="https://openamp-foundry.org/schemas/external_review_packet.schema.json", + description="External review packet — complete machine-checkable review artifact for lab partners", + is_breaking_change=False, + notes="Has $id URI. Used for external-facing review packets.", + ), + ArtifactVersionInfo( + artifact_name="safety_release_decision", + version="1.0.0", + stability_tier="stable", + schema_id="https://openamp-foundry.org/schemas/safety_release_decision.schema.json", + description="Safety release decision — human-gated release decision with safety checks", + is_breaking_change=False, + notes="Has $id URI. Human review required for release decisions.", + ), + ArtifactVersionInfo( + artifact_name="simulation_result", + version="1.0.0", + stability_tier="experimental", + schema_id="https://openamp.ai/schemas/simulation_result.schema.json", + description="Simulation result — virtual assay module output with uncertainty", + is_breaking_change=False, + notes="Has $id URI. Experimental due to ongoing simulation module development.", + ), +] + + +def get_artifact_version(artifact_name: str) -> ArtifactVersionInfo | None: + for entry in VERSIONED_ARTIFACTS: + if entry.artifact_name == artifact_name: + return entry + return None + + +def list_versioned_artifacts(tier: str | None = None) -> list[ArtifactVersionInfo]: + if tier is None: + return list(VERSIONED_ARTIFACTS) + return [entry for entry in VERSIONED_ARTIFACTS if entry.stability_tier == tier] + + +def validate_version_format(version: str) -> bool: + pattern = r"^\d+\.\d+\.\d+$" + return bool(re.match(pattern, version)) + + +def artifact_version_summary() -> dict: + by_tier: dict[str, int] = {} + for entry in VERSIONED_ARTIFACTS: + by_tier[entry.stability_tier] = by_tier.get(entry.stability_tier, 0) + 1 + return { + "total": len(VERSIONED_ARTIFACTS), + "by_tier": by_tier, + "stable_count": len(list_versioned_artifacts("stable")), + "experimental_count": len(list_versioned_artifacts("experimental")), + "dry_lab_only": True, + } diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index ba7978cd..af7e4c30 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -2,9 +2,9 @@ import subprocess import sys -# Baseline: 3272 tests as of Loop 98 (2026-07-09) -- Phase H H10 (simulation-evidence packet assembler) +# Baseline: 3291 tests as of Loop 99 (2026-07-09) -- Phase I I1 (artifact versioning policy) # Raise threshold if legitimate tests are removed; lower threshold is okay. -BASELINE = 3272 +BASELINE = 3291 TOLERANCE = 0.05 # allow 5% fluctuation diff --git a/tests/versioning/__init__.py b/tests/versioning/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/versioning/test_artifact_version.py b/tests/versioning/test_artifact_version.py new file mode 100644 index 00000000..a464ddfe --- /dev/null +++ b/tests/versioning/test_artifact_version.py @@ -0,0 +1,103 @@ +"""Tests for artifact versioning module.""" +import pytest + +from openamp_foundry.versioning.artifact_version import ( + STABILITY_TIERS, + ArtifactVersionInfo, + VERSIONED_ARTIFACTS, + get_artifact_version, + list_versioned_artifacts, + validate_version_format, + artifact_version_summary, +) + + +class TestVERSIONED_ARTIFACTS: + def test_at_least_5_entries(self): + assert len(VERSIONED_ARTIFACTS) >= 5 + + def test_all_versions_valid_format(self): + for entry in VERSIONED_ARTIFACTS: + assert validate_version_format(entry.version), ( + f"{entry.artifact_name} has invalid version: {entry.version}" + ) + + def test_all_stability_tiers_valid(self): + valid_tiers = set(STABILITY_TIERS.keys()) + for entry in VERSIONED_ARTIFACTS: + assert entry.stability_tier in valid_tiers, ( + f"{entry.artifact_name} has invalid tier: {entry.stability_tier}" + ) + + def test_no_duplicate_artifact_names(self): + names = [entry.artifact_name for entry in VERSIONED_ARTIFACTS] + assert len(names) == len(set(names)), "Duplicate artifact_name found" + + +class TestGetArtifactVersion: + def test_known_name_returns_entry(self): + entry = get_artifact_version("candidate") + assert entry is not None + assert entry.artifact_name == "candidate" + + def test_unknown_name_returns_none(self): + entry = get_artifact_version("nonexistent_schema") + assert entry is None + + +class TestListVersionedArtifacts: + def test_no_filter_returns_all(self): + all_entries = list_versioned_artifacts() + assert len(all_entries) == len(VERSIONED_ARTIFACTS) + + def test_tier_stable_filters_correctly(self): + stable = list_versioned_artifacts(tier="stable") + for entry in stable: + assert entry.stability_tier == "stable" + assert len(stable) > 0 + + def test_tier_experimental_filters_correctly(self): + experimental = list_versioned_artifacts(tier="experimental") + for entry in experimental: + assert entry.stability_tier == "experimental" + + +class TestValidateVersionFormat: + def test_valid_semver(self): + assert validate_version_format("1.0.0") is True + + def test_missing_patch(self): + assert validate_version_format("1.0") is False + + def test_non_numeric_components(self): + assert validate_version_format("a.b.c") is False + + def test_large_numbers(self): + assert validate_version_format("999.999.999") is True + + def test_leading_zeros(self): + assert validate_version_format("01.0.0") is True + + +class TestArtifactVersionSummary: + def test_dry_lab_only_true(self): + summary = artifact_version_summary() + assert summary["dry_lab_only"] is True + + def test_total_correct(self): + summary = artifact_version_summary() + assert summary["total"] == len(VERSIONED_ARTIFACTS) + + def test_stable_count_matches(self): + summary = artifact_version_summary() + expected_stable = len(list_versioned_artifacts(tier="stable")) + assert summary["stable_count"] == expected_stable + + def test_experimental_count_matches(self): + summary = artifact_version_summary() + expected_exp = len(list_versioned_artifacts(tier="experimental")) + assert summary["experimental_count"] == expected_exp + + def test_by_tier_sums_to_total(self): + summary = artifact_version_summary() + assert sum(summary["by_tier"].values()) == summary["total"]