diff --git a/Makefile b/Makefile index b1190e88..6554e5b8 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report simulation-deprecation-check simulation-scope-check simulation-evidence-packet artifact-version candidate-manifest +.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report simulation-deprecation-check simulation-scope-check simulation-evidence-packet artifact-version candidate-manifest artifact-changelog PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3) PYTEST := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest) @@ -801,3 +801,7 @@ candidate-manifest: --source-modules membrane_proxy \ --validate @echo "Candidate manifest complete." + +artifact-changelog: + .venv/bin/openamp-foundry artifact-changelog + @echo "Artifact changelog complete." diff --git a/docs/engineering/ARTIFACT_CHANGELOG.md b/docs/engineering/ARTIFACT_CHANGELOG.md new file mode 100644 index 00000000..12ffa04c --- /dev/null +++ b/docs/engineering/ARTIFACT_CHANGELOG.md @@ -0,0 +1,38 @@ +# Artifact Changelog + +> Machine-readable changelog for evidence certificates and computational +> artifacts. External tools can parse this document to detect breaking +> changes and adapt consumers. +> +> All entries are for dry-lab computational artifacts only — they describe +> schema and format changes, not biological findings. + +## Unreleased + +- (no pending changes) + +## [1.0.0] - 2026-07-09 + +### Added + +- **candidate_manifest** — initial schema (14 required fields: candidate_id, + sequence, evidence_level, scopes, scores, uncertainty, source_modules, + calibration_set, safety_flags, provenance_run_id, dry_lab_only, version, + created_at, notes). Phase I I2. Entry type: added. Breaking: false. + +- **benchmark_card** — initial schema (15 required fields: benchmark_id, + benchmark_name, version, date, metric, metric_value, baseline_name, + baseline_value, delta, beats_baseline, dataset, dataset_size, scope, + caveats, dry_lab_only). Phase I I3. Entry type: added. Breaking: false. + +- **simulation_result** — initial schema (8 required fields: module, version, + scope, scores, uncertainty, calibration_set, validated_against, notes). + Phase H H2. Entry type: added. Breaking: false. + +- **simulation_module_registry** — initial schema with module metadata + (module_id, name, description, status, evidence_level, baseline_comparison, + scope, maintainer, notes). Phase H H1. Entry type: added. Breaking: false. + +- **artifact_versioning_policy** — initial versioning policy document + covering stability tiers, SemVer format, breaking change rules, and + deprecation timeline. Phase I I1. Entry type: added. Breaking: false. diff --git a/docs/evidence/METRICS_CURRENT.md b/docs/evidence/METRICS_CURRENT.md index aa7ded22..b2c44cc6 100644 --- a/docs/evidence/METRICS_CURRENT.md +++ b/docs/evidence/METRICS_CURRENT.md @@ -5,7 +5,8 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees > with this file, this file wins. Updated whenever benchmark/benchmark config changes. > -> **Last updated:** 2026-07-09 (Phase I I3 — benchmark card schema — v0.6.1) +> **Last updated:** 2026-07-09 (Phase I I4 — evidence-certificate changelog — v0.6.2) +> **New in v0.6.2:** Evidence-certificate changelog (I4 — Loop 102) — `docs/engineering/ARTIFACT_CHANGELOG.md` with structured changelog format (version, date, artifact_name, change_type, description, breaking flag). Unreleased section at top. `src/openamp_foundry/versioning/artifact_changelog.py` with `CHANGE_TYPES` set (6 values), `ChangelogEntry` dataclass (7 fields), `ARTIFACT_CHANGELOG` list (6 entries: candidate_manifest, benchmark_card, simulation_result, simulation_module_registry, artifact_versioning_policy, artifact_changelog — all "added", non-breaking, v1.0.0), `get_changelog_entries()` (filters by artifact_name, version, change_type, breaking_only), `validate_changelog()` (5 checks: version MAJOR.MINOR.PATCH, date YYYY-MM-DD with dashes, artifact_name non-empty, change_type in CHANGE_TYPES, description non-empty), `changelog_summary()` (total, by_change_type, breaking_changes, artifacts_covered sorted, dry_lab_only). CLI (`openamp-foundry artifact-changelog`) with `--artifact`, `--version`, `--change-type`, `--breaking-only`, `--format text|json`. `make artifact-changelog` target. 13 tests. **3342 total.** External tools that consume OpenAMP artifacts now have a machine-readable changelog to detect breaking changes and adapt consumers. > **New in v0.6.1:** Benchmark card schema (I3 — Loop 101) — `schemas/benchmark_card.schema.json` (Draft 2020-12, 15 required fields: benchmark_id, benchmark_name, version, date, metric, metric_value, baseline_name, baseline_value, delta, beats_baseline, dataset, dataset_size, scope, caveats, dry_lab_only). `$schema`, `$id`, `title`, `additionalProperties: false`. `src/openamp_foundry/benchmarks/` module with `BenchmarkCard` dataclass (15 fields), `make_benchmark_card()` (auto-computes delta, beats_baseline), `validate_benchmark_card()` (10 checks: non-empty benchmark_id, non-empty benchmark_name, non-empty metric, non-empty dataset, non-empty baseline_name, dataset_size >= 1, delta matches metric_value - baseline_value within 1e-9, beats_baseline matches delta > 0, dry_lab_only must be True), `benchmark_card_summary()` (total, beats_baseline_count, fails_baseline_count, dry_lab_only). CLI (`openamp-foundry benchmark-card`) with `--benchmark-id`, `--benchmark-name`, `--metric`, `--metric-value`, `--baseline-name`, `--baseline-value`, `--dataset`, `--dataset-size`, `--validate`, `--format text|json`. `make benchmark-card` target. 19 tests. **3329 total.** A benchmark card is the standard format for describing external benchmark results — what was benchmarked, what the baseline was, what the result was, and what claims are supported. > **New in v0.6.0:** Candidate manifest schema (I2 — Loop 100) — `schemas/candidate_manifest.schema.json` (Draft 2020-12, 14 required fields: candidate_id, sequence, evidence_level, scopes, scores, uncertainty, source_modules, calibration_set, safety_flags, provenance_run_id, dry_lab_only, version, created_at, notes). `$schema`, `$id`, `title`, `description`, `additionalProperties: false`. `src/openamp_foundry/manifests/` module with `CandidateManifest` dataclass (14 fields), `make_candidate_manifest()`, `validate_candidate_manifest()` (8 checks: non-empty candidate_id, non-empty sequence, evidence_level 1-6, non-empty scopes, uncertainty 0.0-1.0, non-empty source_modules, dry_lab_only must be True, version MAJOR.MINOR.PATCH), `manifest_to_dict()`, `manifest_summary()` (total, by_evidence_level, with_safety_flags, dry_lab_only). CLI (`openamp-foundry candidate-manifest`) with `--candidate-id`, `--sequence`, `--evidence-level`, `--scopes`, `--scores-json`, `--uncertainty`, `--source-modules`, `--validate`, `--format text|json`. `make candidate-manifest` target. 19 tests. **3310 total.** A candidate manifest is the core interoperable artifact — it describes a dry-lab candidate (sequence, scores, evidence level, scopes, safety flags, provenance) in a machine-readable format that external tools can consume without the full OpenAMP stack. > **New in v0.5.99:** Artifact versioning policy (I1 — starts Phase I) — `docs/engineering/ARTIFACT_VERSIONING_POLICY.md` with structured versioning policy covering scope, SemVer format, breaking/non-breaking change definitions, deprecation timeline (at least one minor version warning), schema `$id` policy, changelog requirement, and three stability tiers (stable/experimental/internal). `src/openamp_foundry/versioning/` module with `ArtifactVersionInfo` dataclass (7 fields), `STABILITY_TIERS` dict, `VERSIONED_ARTIFACTS` list (6 entries covering candidate, lab_result, run_manifest, external_review_packet, safety_release_decision, simulation_result at v1.0.0), `get_artifact_version()`, `list_versioned_artifacts()`, `validate_version_format()`, `artifact_version_summary()`. CLI (`openamp-foundry artifact-version`) with `--list`, `--show`, `--tier`, `--format text|json`. `make artifact-version` target. 19 tests. **3291 total.** Starts Phase I (interoperability) — external users now have stability guarantees for schemas, evidence certificates, and candidate manifests. @@ -91,7 +92,7 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **New in v0.5.31:** Added dipeptide-order features for sequence-order awareness. `dipeptide_order_score` achieves AUROC 0.7861 on AMP-vs-scrambled discrimination — the strongest order-dependent feature in the pipeline. Only 7/31 features survive scrambling (amphipathicity/helix-wheel + dipeptide). All composition features are purely position-independent (exactly 0.5000 AUROC on scrambled test). > **New in v0.5.30:** Easy baseline benchmark added — charge density alone (AUROC 0.8166) outperforms the full pipeline ensemble (0.7792) on AMP-vs-Swiss-Prot-decoy discrimination. Honest finding documented: expected because pipeline optimizes for safety, not raw discrimination. > **New in v0.5.29:** Expanded benchmark to 500 AMPs + 500 composition-matched decoys (n=1000). AUROC 0.7792 (CI₉₅: 0.7505–0.8065) confirms signal generalizes. Cluster-aware CI: 0.746–0.8102. Representative AUROC: 0.778. Standard benchmark (n=191) retained for backward comparison. -> **Pipeline version:** v0.6.1 (I3 — benchmark card schema) +> **Pipeline version:** v0.6.2 (I4 — evidence-certificate changelog) > **Branch:** main --- @@ -1161,7 +1162,7 @@ Decoys score low on activity. Selective AMPs score moderately on both. | Metric | Value | |--------|-------| -| Total tests | 3291 | +| Total tests | 3342 | | Coverage (branch) | 99% (6 CLI guard lines only) | | Source modules at 100% | All pipeline, QC, scoring modules | diff --git a/docs/research/ROADMAP.md b/docs/research/ROADMAP.md index 48ca809c..d1953129 100644 --- a/docs/research/ROADMAP.md +++ b/docs/research/ROADMAP.md @@ -1,5 +1,69 @@ # Roadmap +## v0.6.2 — Loop 102: Phase I I4 — Evidence-Certificate Changelog ✓ (2026-07-09) + +`docs/engineering/ARTIFACT_CHANGELOG.md` with structured changelog format +(version, date, artifact_name, change_type, description, breaking flag). +Unreleased section at top for pending changes. 5 entries covering recent +Phase H and I additions (candidate_manifest, benchmark_card, +simulation_result, simulation_module_registry, artifact_versioning_policy) +plus the changelog itself — all as "added", non-breaking, v1.0.0. + +`src/openamp_foundry/versioning/artifact_changelog.py` with `CHANGE_TYPES` +set (6 values: added, changed, deprecated, removed, fixed, security), +`ChangelogEntry` dataclass (7 fields: version, date, artifact_name, +change_type, description, breaking, notes), `ARTIFACT_CHANGELOG` list +(6 entries), `get_changelog_entries()` (filters by artifact_name, version, +change_type, breaking_only), `validate_changelog()` (5 checks: version +MAJOR.MINOR.PATCH, date YYYY-MM-DD with dashes, artifact_name non-empty, +change_type in CHANGE_TYPES, description non-empty), `changelog_summary()` +(total, by_change_type, breaking_changes, artifacts_covered sorted, +dry_lab_only). + +CLI (`openamp-foundry artifact-changelog`) with `--artifact`, `--version`, +`--change-type`, `--breaking-only`, `--format text|json`. +`make artifact-changelog` target. + +13 tests. **3342 total.** External tools that consume OpenAMP artifacts +now have a machine-readable changelog to detect breaking changes and adapt +consumers. This is Loop 102 — the 102nd PR in the NEXT_100_PR_MAP series. + +Changes: +- `docs/engineering/ARTIFACT_CHANGELOG.md` (I4) — Structured changelog + document with versioned entries, Unreleased section, and dry-lab-only + disclaimer. +- `src/openamp_foundry/versioning/artifact_changelog.py` (I4) — Core + module with `CHANGE_TYPES`, `ChangelogEntry` dataclass, + `ARTIFACT_CHANGELOG`, `get_changelog_entries()`, `validate_changelog()`, + `changelog_summary()`. +- `src/openamp_foundry/versioning/__init__.py` (I4) — Exported + `CHANGE_TYPES`, `ChangelogEntry`, `ARTIFACT_CHANGELOG`, + `get_changelog_entries`, `validate_changelog`, `changelog_summary`. +- `src/openamp_foundry/cli/main.py` (I4) — Registered `artifact-changelog` + subcommand with `--artifact`, `--version`, `--change-type`, + `--breaking-only`, `--format` flags. Added import and dispatch. +- `src/openamp_foundry/cli/commands/reports.py` (I4) — Added + `_run_artifact_changelog()` CLI handler with JSON and text output, + filtering, validation display, and summary. +- `Makefile` (I4) — Added `artifact-changelog` target with `.PHONY`. +- `tests/versioning/test_artifact_changelog.py` (I4) — 13 tests covering: + at least 5 entries, all pass validation, no filter returns all, filter + by artifact_name, breaking_only, change_type, catches empty artifact_name, + invalid change_type, invalid version format, summary total, dry_lab_only, + breaking_changes count, artifacts_covered sorted. +- `docs/evidence/METRICS_CURRENT.md` — v0.6.2 I4 changelog. Test count: 3342. +- `tests/test_test_count_regression.py` — baseline updated to 3342. + +Honest boundaries: +- Changelog entries describe schema and format changes only — they do not + measure biological activity, safety, or clinical value. +- All current entries are "added" and non-breaking; breaking changes are + expected in future major versions. +- The changelog is a computational artifact — it records version history, + not biological findings. +- `dry_lab_only: true` applies to all entries — changelogs are inherently + dry-lab and must never be presented as validated biological findings. + ## v0.6.1 — Loop 101: Phase I I3 — Benchmark Card Schema ✓ (2026-07-09) `schemas/benchmark_card.schema.json` (Draft 2020-12, 15 required fields: diff --git a/src/openamp_foundry/cli/commands/reports.py b/src/openamp_foundry/cli/commands/reports.py index d05cb3ed..9060b5c1 100644 --- a/src/openamp_foundry/cli/commands/reports.py +++ b/src/openamp_foundry/cli/commands/reports.py @@ -2087,3 +2087,79 @@ def _run_benchmark_card(args: argparse.Namespace) -> int: if validation_errors: return 3 return 0 + + +def _run_artifact_changelog(args: argparse.Namespace) -> int: + """Display evidence-certificate changelog entries.""" + from openamp_foundry.versioning.artifact_changelog import ( + ARTIFACT_CHANGELOG, + get_changelog_entries, + validate_changelog, + changelog_summary, + ) + + artifact = getattr(args, "artifact", None) + version = getattr(args, "version", None) + change_type = getattr(args, "change_type", None) + breaking_only = getattr(args, "breaking_only", False) + output_format = getattr(args, "format", "text") + + entries = get_changelog_entries( + artifact_name=artifact, + version=version, + change_type=change_type, + breaking_only=breaking_only, + ) + + summary = changelog_summary() + validation_errors = validate_changelog() + + if output_format == "json": + result = { + "entries": [ + { + "version": e.version, + "date": e.date, + "artifact_name": e.artifact_name, + "change_type": e.change_type, + "description": e.description, + "breaking": e.breaking, + "notes": e.notes, + } + for e in entries + ], + "summary": summary, + "valid": len(validation_errors) == 0, + } + if validation_errors: + result["validation_errors"] = validation_errors + print(json.dumps(result, indent=2)) + else: + print("Artifact Changelog") + print("=" * 60) + print(f"Total entries: {summary['total']}") + print(f"By change type: {dict(sorted(summary['by_change_type'].items()))}") + print(f"Breaking: {summary['breaking_changes']}") + print(f"Artifacts: {', '.join(summary['artifacts_covered'])}") + print(f"Dry-lab only: {summary['dry_lab_only']}") + if validation_errors: + print(f"Validation: {len(validation_errors)} error(s)") + else: + print(f"Validation: passed") + print() + for entry in entries: + breaking_label = " [BREAKING]" if entry.breaking else "" + print(f" v{entry.version:8s} {entry.date} " + f"[{entry.change_type:10s}]{breaking_label}") + print(f" {'':18s} {entry.artifact_name}") + print(f" {'':18s} {entry.description[:80]}") + if entry.notes: + print(f" {'':18s} Notes: {entry.notes[:80]}") + print() + + print("Dry-lab only. Changelog entries describe schema and format") + print("changes, not biological validity, safety, or efficacy.") + + if validation_errors: + return 3 + return 0 diff --git a/src/openamp_foundry/cli/main.py b/src/openamp_foundry/cli/main.py index 292f3a1c..5f777510 100644 --- a/src/openamp_foundry/cli/main.py +++ b/src/openamp_foundry/cli/main.py @@ -5,7 +5,7 @@ from openamp_foundry.cli.commands.selection import _run_pilot_panel, _run_pilot_confident, _run_diversity_check, _run_select_batch, _run_batch_rationale from openamp_foundry.cli.commands.external import _run_external_predict, _run_external_consensus from openamp_foundry.cli.commands.qc import _run_synthesis_order, _run_presynth_qc -from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter, _run_synthetic_result_policy_check, _run_calibration_decision_checklist, _run_calibration_rollback_plan, _run_simulation_registry, _run_validate_simulation_result, _run_simulation_baseline_check, _run_adapter_gate_check, _run_simulation_provenance, _run_simulation_ensemble_check, _run_simulation_ci_report, _run_simulation_deprecation_check, _run_simulation_scope_check, _run_simulation_evidence_packet, _run_artifact_version, _run_candidate_manifest, _run_benchmark_card +from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter, _run_synthetic_result_policy_check, _run_calibration_decision_checklist, _run_calibration_rollback_plan, _run_simulation_registry, _run_validate_simulation_result, _run_simulation_baseline_check, _run_adapter_gate_check, _run_simulation_provenance, _run_simulation_ensemble_check, _run_simulation_ci_report, _run_simulation_deprecation_check, _run_simulation_scope_check, _run_simulation_evidence_packet, _run_artifact_version, _run_candidate_manifest, _run_benchmark_card, _run_artifact_changelog from openamp_foundry.cli.commands.gates import _run_gate_check import argparse @@ -1625,6 +1625,38 @@ def build_parser() -> argparse.ArgumentParser: help="Output format (default: text).", ) + # ── Artifact changelog (Phase I I4) ────────────────────────────── + acl = sub.add_parser( + "artifact-changelog", + help=( + "Display the evidence-certificate changelog — a structured " + "record of artifact additions, changes, deprecations, and " + "removals for backward compatibility. Dry-lab only." + ), + ) + acl.add_argument( + "--artifact", type=str, default=None, + help="Filter by artifact name.", + ) + acl.add_argument( + "--version", type=str, default=None, + help="Filter by version (MAJOR.MINOR.PATCH).", + ) + acl.add_argument( + "--change-type", type=str, default=None, + choices=["added", "changed", "deprecated", "removed", "fixed", "security"], + help="Filter by change type.", + ) + acl.add_argument( + "--breaking-only", action="store_true", default=False, + help="Show only breaking changes.", + ) + acl.add_argument( + "--format", type=str, default="text", + choices=["text", "json"], + help="Output format (default: text).", + ) + # ── Candidate manifest (Phase I I2) ────────────────────────────── cm = sub.add_parser( "candidate-manifest", @@ -1874,6 +1906,9 @@ def main(argv: list[str] | None = None) -> int: if args.command == "artifact-version": return _run_artifact_version(args) + if args.command == "artifact-changelog": + return _run_artifact_changelog(args) + if args.command == "candidate-manifest": return _run_candidate_manifest(args) diff --git a/src/openamp_foundry/versioning/__init__.py b/src/openamp_foundry/versioning/__init__.py index 5838a4cd..5d6f2c94 100644 --- a/src/openamp_foundry/versioning/__init__.py +++ b/src/openamp_foundry/versioning/__init__.py @@ -1,3 +1,12 @@ +from openamp_foundry.versioning.artifact_changelog import ( + CHANGE_TYPES, + ChangelogEntry, + ARTIFACT_CHANGELOG, + get_changelog_entries, + validate_changelog, + changelog_summary, +) + from openamp_foundry.versioning.artifact_version import ( STABILITY_TIERS, ArtifactVersionInfo, @@ -16,4 +25,10 @@ "list_versioned_artifacts", "validate_version_format", "artifact_version_summary", + "CHANGE_TYPES", + "ChangelogEntry", + "ARTIFACT_CHANGELOG", + "get_changelog_entries", + "validate_changelog", + "changelog_summary", ] diff --git a/src/openamp_foundry/versioning/artifact_changelog.py b/src/openamp_foundry/versioning/artifact_changelog.py new file mode 100644 index 00000000..fe1e7cb2 --- /dev/null +++ b/src/openamp_foundry/versioning/artifact_changelog.py @@ -0,0 +1,153 @@ +from __future__ import annotations + +import re +from dataclasses import dataclass + +CHANGE_TYPES: set[str] = {"added", "changed", "deprecated", "removed", "fixed", "security"} + + +@dataclass +class ChangelogEntry: + version: str + date: str + artifact_name: str + change_type: str + description: str + breaking: bool = False + notes: str = "" + + +ARTIFACT_CHANGELOG: list[ChangelogEntry] = [ + ChangelogEntry( + version="1.0.0", + date="2026-07-09", + artifact_name="candidate_manifest", + change_type="added", + description=( + "Initial schema with 14 required fields: candidate_id, sequence, " + "evidence_level, scopes, scores, uncertainty, source_modules, " + "calibration_set, safety_flags, provenance_run_id, dry_lab_only, " + "version, created_at, notes. Phase I I2." + ), + breaking=False, + notes="Core interoperable artifact for dry-lab candidates.", + ), + ChangelogEntry( + version="1.0.0", + date="2026-07-09", + artifact_name="benchmark_card", + change_type="added", + description=( + "Initial schema with 15 required fields: benchmark_id, " + "benchmark_name, version, date, metric, metric_value, " + "baseline_name, baseline_value, delta, beats_baseline, dataset, " + "dataset_size, scope, caveats, dry_lab_only. Phase I I3." + ), + breaking=False, + notes="Standard format for external benchmark results.", + ), + ChangelogEntry( + version="1.0.0", + date="2026-07-09", + artifact_name="simulation_result", + change_type="added", + description=( + "Initial schema with 8 required fields: module, version, scope, " + "scores, uncertainty, calibration_set, validated_against, notes. " + "Phase H H2." + ), + breaking=False, + notes="Virtual assay module output with uncertainty.", + ), + ChangelogEntry( + version="1.0.0", + date="2026-07-09", + artifact_name="simulation_module_registry", + change_type="added", + description=( + "Initial schema with module metadata fields: module_id, name, " + "description, status, evidence_level, baseline_comparison, scope, " + "maintainer, notes. Phase H H1." + ), + breaking=False, + notes="Registry of virtual assay modules with status and evidence level.", + ), + ChangelogEntry( + version="1.0.0", + date="2026-07-09", + artifact_name="artifact_versioning_policy", + change_type="added", + description=( + "Initial versioning policy document with stability tiers, " + "SemVer format, breaking change rules, and deprecation " + "timeline. Phase I I1." + ), + breaking=False, + notes="External users now have stability guarantees.", + ), + ChangelogEntry( + version="1.0.0", + date="2026-07-09", + artifact_name="artifact_changelog", + change_type="added", + description=( + "Initial evidence-certificate changelog with structured " + "entries for all versioned artifacts. Phase I I4." + ), + breaking=False, + notes="Backward compatibility for external artifact consumers.", + ), +] + + +def get_changelog_entries( + artifact_name: str | None = None, + version: str | None = None, + change_type: str | None = None, + breaking_only: bool = False, +) -> list[ChangelogEntry]: + result = list(ARTIFACT_CHANGELOG) + if artifact_name is not None: + result = [e for e in result if e.artifact_name == artifact_name] + if version is not None: + result = [e for e in result if e.version == version] + if change_type is not None: + result = [e for e in result if e.change_type == change_type] + if breaking_only: + result = [e for e in result if e.breaking] + return result + + +def validate_changelog() -> list[str]: + errors: list[str] = [] + version_pattern = re.compile(r"^\d+\.\d+\.\d+$") + for i, entry in enumerate(ARTIFACT_CHANGELOG): + if not version_pattern.match(entry.version): + errors.append(f"Entry {i}: invalid version format '{entry.version}'") + if not entry.date or "-" not in entry.date: + errors.append(f"Entry {i}: invalid date format '{entry.date}'") + if not entry.artifact_name: + errors.append(f"Entry {i}: empty artifact_name") + if entry.change_type not in CHANGE_TYPES: + errors.append(f"Entry {i}: invalid change_type '{entry.change_type}'") + if not entry.description: + errors.append(f"Entry {i}: empty description") + return errors + + +def changelog_summary() -> dict: + by_change_type: dict[str, int] = {} + breaking_count = 0 + artifacts: set[str] = set() + for entry in ARTIFACT_CHANGELOG: + by_change_type[entry.change_type] = by_change_type.get(entry.change_type, 0) + 1 + if entry.breaking: + breaking_count += 1 + artifacts.add(entry.artifact_name) + return { + "total": len(ARTIFACT_CHANGELOG), + "by_change_type": by_change_type, + "breaking_changes": breaking_count, + "artifacts_covered": sorted(artifacts), + "dry_lab_only": True, + } diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index e23cae65..4e39053c 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -2,9 +2,9 @@ import subprocess import sys -# Baseline: 3329 tests as of Loop 101 (2026-07-09) -- Phase I I3 (benchmark card schema) +# Baseline: 3342 tests as of Loop 102 (2026-07-09) -- Phase I I4 (evidence-certificate changelog) # Raise threshold if legitimate tests are removed; lower threshold is okay. -BASELINE = 3329 +BASELINE = 3342 TOLERANCE = 0.05 # allow 5% fluctuation diff --git a/tests/versioning/test_artifact_changelog.py b/tests/versioning/test_artifact_changelog.py new file mode 100644 index 00000000..cda98cfb --- /dev/null +++ b/tests/versioning/test_artifact_changelog.py @@ -0,0 +1,116 @@ +"""Tests for artifact_changelog module.""" +from openamp_foundry.versioning.artifact_changelog import ( + CHANGE_TYPES, + ARTIFACT_CHANGELOG, + ChangelogEntry, + get_changelog_entries, + validate_changelog, + changelog_summary, +) + + +def test_has_at_least_5_entries(): + assert len(ARTIFACT_CHANGELOG) >= 5 + + +def test_all_entries_pass_validation(): + errors = validate_changelog() + assert errors == [], f"Validation errors: {errors}" + + +def test_get_changelog_entries_no_filter(): + entries = get_changelog_entries() + assert len(entries) == len(ARTIFACT_CHANGELOG) + + +def test_get_changelog_entries_filters_by_artifact_name(): + entries = get_changelog_entries(artifact_name="benchmark_card") + assert all(e.artifact_name == "benchmark_card" for e in entries) + assert len(entries) >= 1 + + +def test_get_changelog_entries_breaking_only(): + entries = get_changelog_entries(breaking_only=True) + assert all(e.breaking for e in entries) + + +def test_get_changelog_entries_change_type(): + entries = get_changelog_entries(change_type="added") + assert all(e.change_type == "added" for e in entries) + assert len(entries) >= 1 + + +def test_validate_changelog_catches_empty_artifact_name(): + bad_entry = ChangelogEntry( + version="1.0.0", + date="2026-01-01", + artifact_name="", + change_type="added", + description="test", + ) + original = list(ARTIFACT_CHANGELOG) + ARTIFACT_CHANGELOG.append(bad_entry) + try: + errors = validate_changelog() + assert any("empty artifact_name" in err for err in errors) + finally: + ARTIFACT_CHANGELOG.clear() + ARTIFACT_CHANGELOG.extend(original) + + +def test_validate_changelog_catches_invalid_change_type(): + bad_entry = ChangelogEntry( + version="1.0.0", + date="2026-01-01", + artifact_name="test", + change_type="invalid_type", + description="test", + ) + original = list(ARTIFACT_CHANGELOG) + ARTIFACT_CHANGELOG.append(bad_entry) + try: + errors = validate_changelog() + assert any("invalid change_type" in err for err in errors) + finally: + ARTIFACT_CHANGELOG.clear() + ARTIFACT_CHANGELOG.extend(original) + + +def test_validate_changelog_catches_invalid_version(): + bad_entry = ChangelogEntry( + version="not-a-version", + date="2026-01-01", + artifact_name="test", + change_type="added", + description="test", + ) + original = list(ARTIFACT_CHANGELOG) + ARTIFACT_CHANGELOG.append(bad_entry) + try: + errors = validate_changelog() + assert any("invalid version format" in err for err in errors) + finally: + ARTIFACT_CHANGELOG.clear() + ARTIFACT_CHANGELOG.extend(original) + + +def test_changelog_summary_total_correct(): + summary = changelog_summary() + assert summary["total"] == len(ARTIFACT_CHANGELOG) + + +def test_changelog_summary_dry_lab_only(): + summary = changelog_summary() + assert summary["dry_lab_only"] is True + + +def test_changelog_summary_breaking_changes(): + summary = changelog_summary() + expected = sum(1 for e in ARTIFACT_CHANGELOG if e.breaking) + assert summary["breaking_changes"] == expected + + +def test_changelog_summary_artifacts_covered_sorted(): + summary = changelog_summary() + assert summary["artifacts_covered"] == sorted(summary["artifacts_covered"]) + assert len(summary["artifacts_covered"]) >= 1