diff --git a/src/openamp_foundry/evidence/release_manifest.py b/src/openamp_foundry/evidence/release_manifest.py new file mode 100644 index 00000000..961117a6 --- /dev/null +++ b/src/openamp_foundry/evidence/release_manifest.py @@ -0,0 +1,207 @@ +""" +Machine-readable release manifest schema. + +I8: Downstream tools can parse what was released. +Records what candidate IDs, evidence certificates, benchmark cards, +and schema versions were included in a named release to external reviewers. +Release manifests use the RMF- prefix and are dry-lab-only. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +RELEASE_MANIFEST_ID_PREFIX: str = "RMF-" +DRY_LAB_ONLY_REQUIRED: bool = True +MIN_CANDIDATE_IDS: int = 1 +MIN_SCHEMA_VERSION_LENGTH: int = 3 + +VALID_RELEASE_STATUSES: frozenset[str] = frozenset({ + "draft", + "under_review", + "approved", + "released", + "retracted", +}) + + +@dataclass +class ReleaseManifest: + """ + Machine-readable record of what was included in a named candidate release. + + Every field is present so downstream tools can verify receipt and + reconstruct exactly what was shared with an external reviewer. + """ + + manifest_id: str + release_name: str + generated_at: str + pipeline_version: str + git_sha: str + schema_version: str + candidate_ids: list[str] + evidence_certificate_ids: list[str] + benchmark_card_ids: list[str] + is_dry_lab_only: bool + total_candidates: int + contact: str + release_status: str + notes: str + + +@dataclass +class ManifestValidationResult: + """Result of validating a ReleaseManifest.""" + + is_valid: bool + violations: list[str] + warnings: list[str] + manifest_id: str + validation_summary: str + + +def validate_release_manifest(manifest: ReleaseManifest) -> ManifestValidationResult: + """ + Validate a ReleaseManifest against all integrity rules. + + Rules: + 1. manifest_id must start with 'RMF-' + 2. manifest_id must not be empty beyond the prefix + 3. release_name must not be empty + 4. generated_at must not be empty + 5. pipeline_version must not be empty + 6. git_sha must not be empty + 7. schema_version must be at least MIN_SCHEMA_VERSION_LENGTH characters + 8. candidate_ids must have at least MIN_CANDIDATE_IDS entries + 9. total_candidates must equal len(candidate_ids) + 10. is_dry_lab_only must be True + 11. contact must not be empty + 12. release_status must be in VALID_RELEASE_STATUSES + 13. candidate_ids must contain no duplicates + 14. evidence_certificate_ids must contain no duplicates + """ + violations: list[str] = [] + warnings: list[str] = [] + + if not manifest.manifest_id.startswith(RELEASE_MANIFEST_ID_PREFIX): + violations.append( + f"manifest_id must start with '{RELEASE_MANIFEST_ID_PREFIX}', " + f"got: {manifest.manifest_id!r}" + ) + + if len(manifest.manifest_id) <= len(RELEASE_MANIFEST_ID_PREFIX): + violations.append( + "manifest_id must have content after the prefix." + ) + + if not manifest.release_name.strip(): + violations.append("release_name must not be empty.") + + if not manifest.generated_at.strip(): + violations.append("generated_at must not be empty.") + + if not manifest.pipeline_version.strip(): + violations.append("pipeline_version must not be empty.") + + if not manifest.git_sha.strip(): + violations.append("git_sha must not be empty.") + + if len(manifest.schema_version.strip()) < MIN_SCHEMA_VERSION_LENGTH: + violations.append( + f"schema_version must be at least {MIN_SCHEMA_VERSION_LENGTH} characters, " + f"got: {manifest.schema_version!r}" + ) + + if len(manifest.candidate_ids) < MIN_CANDIDATE_IDS: + violations.append( + f"candidate_ids must have at least {MIN_CANDIDATE_IDS} entry, " + f"got {len(manifest.candidate_ids)}." + ) + + if manifest.total_candidates != len(manifest.candidate_ids): + violations.append( + f"total_candidates ({manifest.total_candidates}) must equal " + f"len(candidate_ids) ({len(manifest.candidate_ids)})." + ) + + if not manifest.is_dry_lab_only: + violations.append( + "is_dry_lab_only must be True. Release manifests are dry-lab artifacts only." + ) + + if not manifest.contact.strip(): + violations.append("contact must not be empty.") + + if manifest.release_status not in VALID_RELEASE_STATUSES: + violations.append( + f"release_status must be one of {sorted(VALID_RELEASE_STATUSES)}, " + f"got: {manifest.release_status!r}" + ) + + if len(manifest.candidate_ids) != len(set(manifest.candidate_ids)): + violations.append("candidate_ids must not contain duplicates.") + + if len(manifest.evidence_certificate_ids) != len(set(manifest.evidence_certificate_ids)): + violations.append("evidence_certificate_ids must not contain duplicates.") + + if not manifest.evidence_certificate_ids: + warnings.append( + "evidence_certificate_ids is empty. Consider linking certificates for auditability." + ) + + if not manifest.benchmark_card_ids: + warnings.append( + "benchmark_card_ids is empty. Consider linking BMC- cards for transparency." + ) + + is_valid = len(violations) == 0 + if is_valid: + summary = ( + f"ReleaseManifest {manifest.manifest_id} is valid: " + f"{manifest.total_candidates} candidate(s), " + f"status={manifest.release_status}." + ) + else: + summary = ( + f"ReleaseManifest {manifest.manifest_id} has {len(violations)} violation(s): " + + "; ".join(violations[:2]) + + ("..." if len(violations) > 2 else "") + ) + + return ManifestValidationResult( + is_valid=is_valid, + violations=violations, + warnings=warnings, + manifest_id=manifest.manifest_id, + validation_summary=summary, + ) + + +def format_release_manifest(manifest: ReleaseManifest) -> str: + """Return a human-readable summary of a ReleaseManifest.""" + lines: list[str] = [ + f"Release Manifest: {manifest.manifest_id}", + f" Release name: {manifest.release_name}", + f" Status: {manifest.release_status}", + f" Generated at: {manifest.generated_at}", + f" Pipeline version: {manifest.pipeline_version}", + f" Git SHA: {manifest.git_sha}", + f" Schema version: {manifest.schema_version}", + f" Dry-lab only: {manifest.is_dry_lab_only}", + f" Candidates ({manifest.total_candidates}):", + ] + for cid in manifest.candidate_ids: + lines.append(f" - {cid}") + if manifest.evidence_certificate_ids: + lines.append(f" Evidence certificates ({len(manifest.evidence_certificate_ids)}):") + for eid in manifest.evidence_certificate_ids: + lines.append(f" - {eid}") + if manifest.benchmark_card_ids: + lines.append(f" Benchmark cards ({len(manifest.benchmark_card_ids)}):") + for bid in manifest.benchmark_card_ids: + lines.append(f" - {bid}") + lines.append(f" Contact: {manifest.contact}") + if manifest.notes: + lines.append(f" Notes: {manifest.notes}") + return "\n".join(lines) diff --git a/tests/evidence/test_release_manifest.py b/tests/evidence/test_release_manifest.py new file mode 100644 index 00000000..3c0a55df --- /dev/null +++ b/tests/evidence/test_release_manifest.py @@ -0,0 +1,332 @@ +"""Tests for I8 machine-readable release manifest schema.""" + +from __future__ import annotations + +import pytest + +from openamp_foundry.evidence.release_manifest import ( + MIN_CANDIDATE_IDS, + MIN_SCHEMA_VERSION_LENGTH, + RELEASE_MANIFEST_ID_PREFIX, + VALID_RELEASE_STATUSES, + ManifestValidationResult, + ReleaseManifest, + format_release_manifest, + validate_release_manifest, +) + + +def _make_manifest(**kwargs) -> ReleaseManifest: + defaults = dict( + manifest_id="RMF-2026-001", + release_name="Wave 0.5 External Review Release", + generated_at="2026-01-01T00:00:00Z", + pipeline_version="openamp-foundry==0.1.0", + git_sha="abc1234def5678901234567890123456789abcde", + schema_version="1.0.0", + candidate_ids=["CAND-001", "CAND-002"], + evidence_certificate_ids=["EVC-001", "EVC-002"], + benchmark_card_ids=["BMC-0001"], + is_dry_lab_only=True, + total_candidates=2, + contact="reviewer@lab.example", + release_status="under_review", + notes="First external review release.", + ) + defaults.update(kwargs) + return ReleaseManifest(**defaults) + + +class TestReleaseManifestConstants: + def test_prefix_starts_with_rmf(self): + assert RELEASE_MANIFEST_ID_PREFIX == "RMF-" + + def test_dry_lab_only_required(self): + from openamp_foundry.evidence.release_manifest import DRY_LAB_ONLY_REQUIRED + assert DRY_LAB_ONLY_REQUIRED is True + + def test_min_candidate_ids(self): + assert MIN_CANDIDATE_IDS >= 1 + + def test_min_schema_version_length(self): + assert MIN_SCHEMA_VERSION_LENGTH >= 3 + + def test_valid_statuses_is_frozenset(self): + assert isinstance(VALID_RELEASE_STATUSES, frozenset) + + def test_valid_statuses_includes_draft(self): + assert "draft" in VALID_RELEASE_STATUSES + + def test_valid_statuses_includes_released(self): + assert "released" in VALID_RELEASE_STATUSES + + def test_valid_statuses_includes_retracted(self): + assert "retracted" in VALID_RELEASE_STATUSES + + def test_valid_statuses_includes_approved(self): + assert "approved" in VALID_RELEASE_STATUSES + + def test_valid_statuses_includes_under_review(self): + assert "under_review" in VALID_RELEASE_STATUSES + + +class TestReleaseManifestDataclass: + def test_creates_successfully(self): + m = _make_manifest() + assert m.manifest_id == "RMF-2026-001" + + def test_candidate_ids_stored(self): + m = _make_manifest(candidate_ids=["A", "B", "C"]) + assert m.candidate_ids == ["A", "B", "C"] + + def test_is_dry_lab_only_stored(self): + m = _make_manifest() + assert m.is_dry_lab_only is True + + def test_evidence_certificate_ids_stored(self): + m = _make_manifest(evidence_certificate_ids=["EVC-X"]) + assert m.evidence_certificate_ids == ["EVC-X"] + + def test_benchmark_card_ids_stored(self): + m = _make_manifest(benchmark_card_ids=["BMC-9999"]) + assert m.benchmark_card_ids == ["BMC-9999"] + + def test_release_status_stored(self): + m = _make_manifest(release_status="released") + assert m.release_status == "released" + + +class TestValidateReleaseManifest: + def test_valid_manifest_passes(self): + m = _make_manifest() + result = validate_release_manifest(m) + assert result.is_valid is True + + def test_valid_result_no_violations(self): + m = _make_manifest() + result = validate_release_manifest(m) + assert result.violations == [] + + def test_result_is_manifest_validation_result(self): + m = _make_manifest() + result = validate_release_manifest(m) + assert isinstance(result, ManifestValidationResult) + + def test_manifest_id_in_result(self): + m = _make_manifest() + result = validate_release_manifest(m) + assert result.manifest_id == "RMF-2026-001" + + def test_summary_nonempty(self): + m = _make_manifest() + result = validate_release_manifest(m) + assert len(result.validation_summary) > 0 + + def test_bad_prefix_fails(self): + m = _make_manifest(manifest_id="BAD-001") + result = validate_release_manifest(m) + assert not result.is_valid + assert any("RMF-" in v for v in result.violations) + + def test_prefix_only_id_fails(self): + m = _make_manifest(manifest_id="RMF-") + result = validate_release_manifest(m) + assert not result.is_valid + + def test_empty_release_name_fails(self): + m = _make_manifest(release_name="") + result = validate_release_manifest(m) + assert not result.is_valid + + def test_whitespace_release_name_fails(self): + m = _make_manifest(release_name=" ") + result = validate_release_manifest(m) + assert not result.is_valid + + def test_empty_generated_at_fails(self): + m = _make_manifest(generated_at="") + result = validate_release_manifest(m) + assert not result.is_valid + + def test_empty_pipeline_version_fails(self): + m = _make_manifest(pipeline_version="") + result = validate_release_manifest(m) + assert not result.is_valid + + def test_empty_git_sha_fails(self): + m = _make_manifest(git_sha="") + result = validate_release_manifest(m) + assert not result.is_valid + + def test_short_schema_version_fails(self): + m = _make_manifest(schema_version="1") + result = validate_release_manifest(m) + assert not result.is_valid + + def test_schema_version_two_chars_fails(self): + m = _make_manifest(schema_version="1.") + result = validate_release_manifest(m) + assert not result.is_valid + + def test_schema_version_three_chars_passes(self): + m = _make_manifest(schema_version="1.0") + result = validate_release_manifest(m) + assert result.is_valid + + def test_empty_candidate_ids_fails(self): + m = _make_manifest(candidate_ids=[], total_candidates=0) + result = validate_release_manifest(m) + assert not result.is_valid + + def test_mismatched_total_candidates_fails(self): + m = _make_manifest(candidate_ids=["X"], total_candidates=99) + result = validate_release_manifest(m) + assert not result.is_valid + + def test_dry_lab_false_fails(self): + m = _make_manifest(is_dry_lab_only=False) + result = validate_release_manifest(m) + assert not result.is_valid + + def test_empty_contact_fails(self): + m = _make_manifest(contact="") + result = validate_release_manifest(m) + assert not result.is_valid + + def test_invalid_release_status_fails(self): + m = _make_manifest(release_status="published_to_nature") + result = validate_release_manifest(m) + assert not result.is_valid + + def test_duplicate_candidate_ids_fails(self): + m = _make_manifest( + candidate_ids=["CAND-001", "CAND-001"], + total_candidates=2, + ) + result = validate_release_manifest(m) + assert not result.is_valid + + def test_duplicate_evidence_ids_fails(self): + m = _make_manifest( + evidence_certificate_ids=["EVC-001", "EVC-001"], + ) + result = validate_release_manifest(m) + assert not result.is_valid + + def test_empty_evidence_ids_warns(self): + m = _make_manifest(evidence_certificate_ids=[]) + result = validate_release_manifest(m) + assert result.is_valid + assert len(result.warnings) > 0 + + def test_empty_benchmark_ids_warns(self): + m = _make_manifest(benchmark_card_ids=[]) + result = validate_release_manifest(m) + assert result.is_valid + assert len(result.warnings) > 0 + + def test_all_release_statuses_pass(self): + for status in VALID_RELEASE_STATUSES: + m = _make_manifest(release_status=status) + result = validate_release_manifest(m) + assert result.is_valid, f"Status {status!r} should pass" + + def test_draft_status_passes(self): + m = _make_manifest(release_status="draft") + result = validate_release_manifest(m) + assert result.is_valid + + def test_retracted_status_passes(self): + m = _make_manifest(release_status="retracted") + result = validate_release_manifest(m) + assert result.is_valid + + def test_many_candidates_pass(self): + ids = [f"CAND-{i:04d}" for i in range(50)] + m = _make_manifest(candidate_ids=ids, total_candidates=50) + result = validate_release_manifest(m) + assert result.is_valid + + def test_single_candidate_passes(self): + m = _make_manifest(candidate_ids=["CAND-001"], total_candidates=1) + result = validate_release_manifest(m) + assert result.is_valid + + def test_violations_list_is_list(self): + m = _make_manifest(manifest_id="BAD-001") + result = validate_release_manifest(m) + assert isinstance(result.violations, list) + + def test_warnings_list_is_list(self): + m = _make_manifest() + result = validate_release_manifest(m) + assert isinstance(result.warnings, list) + + def test_valid_summary_contains_candidate_count(self): + m = _make_manifest() + result = validate_release_manifest(m) + assert "2" in result.validation_summary + + def test_invalid_summary_contains_violations_count(self): + m = _make_manifest(manifest_id="BAD-001", is_dry_lab_only=False) + result = validate_release_manifest(m) + assert "violation" in result.validation_summary.lower() + + +class TestFormatReleaseManifest: + def setup_method(self): + self.manifest = _make_manifest() + self.formatted = format_release_manifest(self.manifest) + + def test_returns_string(self): + assert isinstance(self.formatted, str) + + def test_contains_manifest_id(self): + assert "RMF-2026-001" in self.formatted + + def test_contains_release_name(self): + assert "Wave 0.5" in self.formatted + + def test_contains_status(self): + assert "under_review" in self.formatted + + def test_contains_pipeline_version(self): + assert "openamp-foundry" in self.formatted + + def test_contains_git_sha(self): + assert "abc1234" in self.formatted + + def test_contains_schema_version(self): + assert "1.0.0" in self.formatted + + def test_contains_dry_lab_flag(self): + assert "True" in self.formatted + + def test_contains_candidate_count(self): + assert "2" in self.formatted + + def test_contains_candidate_ids(self): + assert "CAND-001" in self.formatted + assert "CAND-002" in self.formatted + + def test_contains_evidence_ids(self): + assert "EVC-001" in self.formatted + + def test_contains_benchmark_ids(self): + assert "BMC-0001" in self.formatted + + def test_contains_contact(self): + assert "reviewer@lab.example" in self.formatted + + def test_contains_notes(self): + assert "First external review" in self.formatted + + def test_no_evidence_ids_omits_section(self): + m = _make_manifest(evidence_certificate_ids=[]) + text = format_release_manifest(m) + assert "Evidence certificates" not in text + + def test_no_benchmark_ids_omits_section(self): + m = _make_manifest(benchmark_card_ids=[]) + text = format_release_manifest(m) + assert "Benchmark cards" not in text