From b3310113f02a331278e68ac2ff4d26c436bc270a Mon Sep 17 00:00:00 2001 From: Saverio Mazza Date: Fri, 2 Oct 2026 08:25:47 +0200 Subject: [PATCH 1/2] fix(quality): reject missing release evidence and incomplete approvals --- CHANGELOG.md | 8 +++ HANDOVER.md | 26 +++++++ benchmarks/compare_quality.py | 16 ++--- benchmarks/quality_gate.py | 81 ++++++++++++++++------ docs/quality_benchmarks.md | 21 ++++++ scripts/verify_release.py | 42 ++++++++++- tests/integration/test_release_verifier.py | 58 +++++++++++++++- tests/unit/test_quality_gate.py | 71 +++++++++++++++++-- 8 files changed, 279 insertions(+), 44 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8600206..90c6333 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ All notable changes follow Keep a Changelog and Semantic Versioning. ## [Unreleased] +### Fixed + +- Reject quality-gate approval when external evaluation is absent or its F1 is invalid, + and reject missing or incomparable critical-entity counts. Report which entities + actually have evaluated support. +- Require every quality criterion to pass explicitly when verifying a release report, + rejecting incomplete reports and legacy skipped external evaluations. + ## [1.34.0] - 2026-09-30 ### Added diff --git a/HANDOVER.md b/HANDOVER.md index 091c165..bb5a1fe 100644 --- a/HANDOVER.md +++ b/HANDOVER.md @@ -399,3 +399,29 @@ verification unless they are intentionally retained as a user-approved release a - Push only when explicitly requested. - A version is published only after its matching tag, successful release workflow, PyPI artifact, and GitHub release exist. Until then, keep changes under `[Unreleased]`. + + +## Required quality evidence review (2026-10-02) + +Three regression tests reproduced approval with absent external evidence, a +missing critical entity, and an infinite external F1. The gate now rejects those +inputs, compares critical annotation support, and distinguishes represented from +unrepresented entities. The release verifier checks all five criteria and rejects +legacy skipped external evaluations. Per-entity F1 uses the equivalent count +formula, avoiding division by zero when TP is zero and FP/FN are nonzero. + +Observed verification using the frozen lockfile: + +- Quality gate, comparator and release-verifier tests: 40 passed. +- Suite excluding ONNX/BIO modules with `--no-cov`: 510 passed; 1 Tesseract skip. +- Pre-commit, mypy, strict docs build, package build and release verifier passed. +- Full coverage and supported-platform acceptance remain pending CI. + +No detector, held-out sample, published score or quality tolerance was changed. +The remaining difference between the roadmap's strict exact-boundary improvement +protocol and the existing overlap-based tolerance gate requires a separate +methodology change. This patch must not be described as proving that protocol. + +Reusable lesson assessment: missing measurements cannot count as passing release +criteria. Explicit negative tests now enforce that invariant at generation and +consumption of gate reports. diff --git a/benchmarks/compare_quality.py b/benchmarks/compare_quality.py index 0413069..b8e2204 100644 --- a/benchmarks/compare_quality.py +++ b/benchmarks/compare_quality.py @@ -146,22 +146,14 @@ def compare_results( b_tp = b_et.get("true_positives", 0) b_fp = b_et.get("false_positives", 0) b_fn = b_et.get("false_negatives", 0) - b_f1 = ( - (2 * (b_tp / (b_tp + b_fp)) * (b_tp / (b_tp + b_fn))) - / ((b_tp / (b_tp + b_fp)) + (b_tp / (b_tp + b_fn))) - if (b_tp + b_fp > 0 and b_tp + b_fn > 0) - else 0.0 - ) + b_denominator = 2 * b_tp + b_fp + b_fn + b_f1 = 2 * b_tp / b_denominator if b_denominator else 0.0 c_tp = c_et.get("true_positives", 0) c_fp = c_et.get("false_positives", 0) c_fn = c_et.get("false_negatives", 0) - c_f1 = ( - (2 * (c_tp / (c_tp + c_fp)) * (c_tp / (c_tp + c_fn))) - / ((c_tp / (c_tp + c_fp)) + (c_tp / (c_tp + c_fn))) - if (c_tp + c_fp > 0 and c_tp + c_fn > 0) - else 0.0 - ) + c_denominator = 2 * c_tp + c_fp + c_fn + c_f1 = 2 * c_tp / c_denominator if c_denominator else 0.0 per_entity_shifts[et] = { "delta_tp": c_tp - b_tp, diff --git a/benchmarks/quality_gate.py b/benchmarks/quality_gate.py index 5225977..a3e6aed 100644 --- a/benchmarks/quality_gate.py +++ b/benchmarks/quality_gate.py @@ -13,6 +13,7 @@ import argparse import json import logging +import math from dataclasses import dataclass from pathlib import Path from typing import Any @@ -200,15 +201,27 @@ def evaluate_quality_gate( base_per_entity = baseline_record.get("per_entity", {}) cand_per_entity = candidate_record.get("per_entity", {}) critical_regressions: list[dict[str, Any]] = [] + missing_critical_evidence: list[str] = [] + checked_entities: list[str] = [] - for ent in HIGH_RISK_ENTITY_TYPES: + for ent in sorted(HIGH_RISK_ENTITY_TYPES): if ent in base_per_entity: b_et = base_per_entity[ent] c_et = cand_per_entity.get(ent, {}) - b_tp = b_et.get("true_positives", 0) - b_fn = b_et.get("false_negatives", 0) - c_tp = c_et.get("true_positives", 0) - c_fn = c_et.get("false_negatives", 0) + b_tp = b_et.get("true_positives") + b_fn = b_et.get("false_negatives") + c_tp = c_et.get("true_positives") + c_fn = c_et.get("false_negatives") + counts = (b_tp, b_fn, c_tp, c_fn) + if not all(isinstance(n, int) and not isinstance(n, bool) and n >= 0 for n in counts): + missing_critical_evidence.append(ent) + continue + if b_tp + b_fn != c_tp + c_fn: + missing_critical_evidence.append(ent) + continue + if b_tp + b_fn == 0: + continue + checked_entities.append(ent) b_rec = b_tp / (b_tp + b_fn) if (b_tp + b_fn) > 0 else 1.0 c_rec = c_tp / (c_tp + c_fn) if (c_tp + c_fn) > 0 else 1.0 @@ -224,55 +237,79 @@ def evaluate_quality_gate( } ) - critical_ok = len(critical_regressions) == 0 + critical_ok = ( + not critical_regressions and not missing_critical_evidence and bool(checked_entities) + ) if not critical_ok: - reason = ( - f"Recall regressed for {len(critical_regressions)} high-risk identifiers: " - f"{[r['entity_type'] for r in critical_regressions]}" - ) + if missing_critical_evidence or not checked_entities: + reason = "Critical identifier recall evidence is missing, invalid or incomparable." + else: + reason = ( + f"Recall regressed for {len(critical_regressions)} high-risk identifiers: " + f"{[r['entity_type'] for r in critical_regressions]}" + ) failure_reasons.append(reason) results["critical_entities_floor"] = QualityGateCriteriaResult( criterion_name="critical_entities_floor", passed=False, - details={"regressed_entities": critical_regressions}, + details={ + "regressed_entities": critical_regressions, + "missing_or_incomparable_entities": missing_critical_evidence, + }, failure_reason=reason, ) else: results["critical_entities_floor"] = QualityGateCriteriaResult( criterion_name="critical_entities_floor", passed=True, - details={"protected_entities_checked": list(HIGH_RISK_ENTITY_TYPES)}, + details={ + "protected_entities_checked": checked_entities, + "unrepresented_entities": sorted(HIGH_RISK_ENTITY_TYPES - set(checked_entities)), + }, ) # 5. Independent External Generalization Track (PIIMB) if external_eval_record is not None: ext_micro = external_eval_record.get("character_micro_metrics", {}) - ext_f1 = ext_micro.get("f1", 0.0) - ext_ok = ext_f1 >= external_f1_floor + ext_f1 = ext_micro.get("f1") if isinstance(ext_micro, dict) else None + valid_score = ( + isinstance(ext_f1, (int, float)) + and not isinstance(ext_f1, bool) + and math.isfinite(ext_f1) + and 0 <= ext_f1 <= 1 + ) + ext_ok = valid_score and isinstance(ext_f1, (int, float)) and ext_f1 >= external_f1_floor if not ext_ok: reason = ( - f"External benchmark (PIIMB) character F1 {ext_f1:.4f} is below floor " - f"{external_f1_floor:.4f}." + "External benchmark character F1 is missing, invalid or below the required floor." ) failure_reasons.append(reason) results["external_generalization"] = QualityGateCriteriaResult( criterion_name="external_generalization", passed=False, - details={"external_f1": ext_f1, "required_floor": external_f1_floor}, + details={ + "external_f1": ext_f1 if valid_score else None, + "required_floor": external_f1_floor, + }, failure_reason=reason, ) else: results["external_generalization"] = QualityGateCriteriaResult( criterion_name="external_generalization", passed=True, - details={"external_f1": ext_f1, "required_floor": external_f1_floor}, + details={ + "external_f1": ext_f1 if valid_score else None, + "required_floor": external_f1_floor, + }, ) else: - # Informational pass if external track was not evaluated in this specific run + reason = "Independent external generalization evidence is required for shipment." + failure_reasons.append(reason) results["external_generalization"] = QualityGateCriteriaResult( criterion_name="external_generalization", - passed=True, + passed=False, details={"status": "not_evaluated_in_current_run"}, + failure_reason=reason, ) all_passed = all(r.passed for r in results.values()) @@ -304,7 +341,7 @@ def main() -> int: "--external", type=Path, default=None, - help="Optional path to external generalization artifact JSON (PIIMB).", + help="External generalization artifact JSON (PIIMB), required for a passing gate.", ) parser.add_argument( "--output", @@ -317,7 +354,7 @@ def main() -> int: baseline_data = json.loads(args.baseline.read_text(encoding="utf-8")) candidate_data = json.loads(args.candidate.read_text(encoding="utf-8")) external_data = None - if args.external and args.external.exists(): + if args.external is not None: external_data = json.loads(args.external.read_text(encoding="utf-8")) report = evaluate_quality_gate( diff --git a/docs/quality_benchmarks.md b/docs/quality_benchmarks.md index dcaaeda..819b269 100644 --- a/docs/quality_benchmarks.md +++ b/docs/quality_benchmarks.md @@ -146,3 +146,24 @@ Evaluated against the pinned validation split (`a785eb528e28be2693c3718a27e06697 See also the formal [Model Card](model_card.md) for full architecture specifications and hardware profiles. + + +## Required evidence for a quality-gate report + +A release-quality evaluation needs an external evaluation record. Omitting it +produces `REJECT`; a missing file supplied through `--external` is an error. +External character F1 must be a finite number between zero and one and meet the +configured floor. Critical-entity comparisons require matching annotation +support and explicit nonnegative integer TP/FN counts. The report distinguishes +entities with observed support from those not represented in that evaluation. + +`verify_release.py --quality-gate-report` requires all five criteria to be +present and explicitly passing. A top-level `passed: true` flag is insufficient, +and legacy reports that treated an unevaluated external track as passing are +rejected. Regenerate those reports with external evidence rather than editing +approval flags. + +These checks validate evidence presence and consistency. They do not authenticate +an artifact, prove that a corpus is independent, or replace the roadmap's stricter +experimental protocol. This change does not alter detector behavior or establish +an improvement in the published quality baseline. diff --git a/scripts/verify_release.py b/scripts/verify_release.py index 8b21d44..2a88dbf 100644 --- a/scripts/verify_release.py +++ b/scripts/verify_release.py @@ -2,6 +2,7 @@ import email.message import email.parser import json +import math import os import re import tarfile @@ -159,9 +160,46 @@ def verify_quality_gate_report(report_path: Path) -> None: if not report_path.is_file(): raise ValueError(f"quality gate report not found at {report_path}") data = json.loads(report_path.read_text(encoding="utf-8")) - if not data.get("passed", False) or data.get("recommendation") != "SHIP": - summary = data.get("summary_reasons", ["unspecified failure"]) + if ( + not isinstance(data, dict) + or data.get("passed") is not True + or data.get("recommendation") != "SHIP" + ): + summary = ( + data.get("summary_reasons", ["unspecified failure"]) + if isinstance(data, dict) + else ["invalid report"] + ) raise ValueError(f"quality gate failed: {summary}") + required = { + "pinned_provenance", + "paired_f1_delta", + "precision_protection", + "critical_entities_floor", + "external_generalization", + } + criteria = data.get("criteria") + if not isinstance(criteria, dict) or not required <= criteria.keys(): + raise ValueError("quality gate report is missing required criteria") + for name in sorted(required): + criterion = criteria[name] + if not isinstance(criterion, dict) or criterion.get("passed") is not True: + raise ValueError(f"quality gate criterion did not pass: {name}") + external_details = criteria["external_generalization"].get("details") + if ( + not isinstance(external_details, dict) + or external_details.get("status") == "not_evaluated_in_current_run" + or "external_f1" not in external_details + ): + raise ValueError("quality gate report lacks external evaluation evidence") + external_f1 = external_details["external_f1"] + if ( + not isinstance(external_f1, (int, float)) + or isinstance(external_f1, bool) + or not math.isfinite(external_f1) + or not 0 <= external_f1 <= 1 + ): + raise ValueError("quality gate report has invalid external evaluation evidence") rec = data.get("recommendation") print(f"verified quality gate report: candidate approved for shipment ({rec})") diff --git a/tests/integration/test_release_verifier.py b/tests/integration/test_release_verifier.py index 19aa4c3..e98db3d 100644 --- a/tests/integration/test_release_verifier.py +++ b/tests/integration/test_release_verifier.py @@ -1,5 +1,6 @@ import email.message import io +import json import tarfile import zipfile from pathlib import Path @@ -152,7 +153,7 @@ def test_verify_quality_gate_report_passing( tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: report = tmp_path / "gate.json" - report.write_text('{"passed": true, "recommendation": "SHIP"}', encoding="utf-8") + report.write_text(json.dumps(_complete_gate_report()), encoding="utf-8") verify_quality_gate_report(report) assert "approved for shipment" in capsys.readouterr().out @@ -171,3 +172,58 @@ def test_verify_quality_gate_report_missing_file(tmp_path: Path) -> None: missing = tmp_path / "nonexistent.json" with pytest.raises(ValueError, match="quality gate report not found"): verify_quality_gate_report(missing) + + +def _complete_gate_report() -> dict[str, object]: + return { + "passed": True, + "recommendation": "SHIP", + "criteria": { + name: {"passed": True, "details": {"external_f1": 0.82}} + for name in ( + "pinned_provenance", + "paired_f1_delta", + "precision_protection", + "critical_entities_floor", + "external_generalization", + ) + }, + } + + +@pytest.mark.parametrize( + "criterion", + [ + "pinned_provenance", + "paired_f1_delta", + "precision_protection", + "critical_entities_floor", + "external_generalization", + ], +) +@pytest.mark.parametrize("state", ["missing", "failed", "string"]) +def test_verify_quality_gate_report_checks_each_criterion( + tmp_path: Path, criterion: str, state: str +) -> None: + data = _complete_gate_report() + criteria = data["criteria"] + assert isinstance(criteria, dict) + if state == "missing": + del criteria[criterion] + else: + criteria[criterion]["passed"] = False if state == "failed" else "true" + path = tmp_path / "gate.json" + path.write_text(json.dumps(data), encoding="utf-8") + with pytest.raises(ValueError, match="quality gate"): + verify_quality_gate_report(path) + + +def test_verify_quality_gate_rejects_legacy_skipped_external_track(tmp_path: Path) -> None: + data = _complete_gate_report() + criteria = data["criteria"] + assert isinstance(criteria, dict) + criteria["external_generalization"]["details"] = {"status": "not_evaluated_in_current_run"} + path = tmp_path / "gate.json" + path.write_text(json.dumps(data), encoding="utf-8") + with pytest.raises(ValueError, match="external evaluation evidence"): + verify_quality_gate_report(path) diff --git a/tests/unit/test_quality_gate.py b/tests/unit/test_quality_gate.py index d2fb200..480fcd9 100644 --- a/tests/unit/test_quality_gate.py +++ b/tests/unit/test_quality_gate.py @@ -24,7 +24,7 @@ def _make_dummy_eval_record( "PAYMENT_CARD": { "true_positives": int(10 * card_recall), "false_positives": 0, - "false_negatives": int(10 * (1 - card_recall)), + "false_negatives": 10 - int(10 * card_recall), }, }, "error_categories": { @@ -42,10 +42,10 @@ def _make_dummy_eval_record( "source": "wiki", "tp": {"EMAIL": 5, "PAYMENT_CARD": int(5 * card_recall)}, "fp": {"EMAIL": 0, "PAYMENT_CARD": 0}, - "fn": {"EMAIL": 0, "PAYMENT_CARD": int(5 * (1 - card_recall))}, + "fn": {"EMAIL": 0, "PAYMENT_CARD": 5 - int(5 * card_recall)}, "exact_tp": {"EMAIL": 5, "PAYMENT_CARD": int(5 * card_recall)}, "exact_fp": {"EMAIL": 0, "PAYMENT_CARD": 0}, - "exact_fn": {"EMAIL": 0, "PAYMENT_CARD": int(5 * (1 - card_recall))}, + "exact_fn": {"EMAIL": 0, "PAYMENT_CARD": 5 - int(5 * card_recall)}, "error_categories": {}, }, { @@ -55,10 +55,10 @@ def _make_dummy_eval_record( "source": "wiki", "tp": {"EMAIL": 5, "PAYMENT_CARD": int(5 * card_recall)}, "fp": {"EMAIL": 1, "PAYMENT_CARD": 0}, - "fn": {"EMAIL": 1, "PAYMENT_CARD": int(5 * (1 - card_recall))}, + "fn": {"EMAIL": 1, "PAYMENT_CARD": 5 - int(5 * card_recall)}, "exact_tp": {"EMAIL": 5, "PAYMENT_CARD": int(5 * card_recall)}, "exact_fp": {"EMAIL": 1, "PAYMENT_CARD": 0}, - "exact_fn": {"EMAIL": 1, "PAYMENT_CARD": int(5 * (1 - card_recall))}, + "exact_fn": {"EMAIL": 1, "PAYMENT_CARD": 5 - int(5 * card_recall)}, "error_categories": {}, }, ], @@ -69,7 +69,9 @@ def test_quality_gate_passes_healthy_candidate() -> None: base = _make_dummy_eval_record(f1=0.83, precision=0.86) cand = _make_dummy_eval_record(f1=0.84, precision=0.865) - report = evaluate_quality_gate(base, cand, num_resamples=50) + report = evaluate_quality_gate( + base, cand, {"character_micro_metrics": {"f1": 0.82}}, num_resamples=50 + ) assert report.passed assert report.recommendation == "SHIP" assert report.criteria_results["pinned_provenance"].passed @@ -97,7 +99,9 @@ def test_quality_gate_rejects_critical_identifier_recall_regression() -> None: assert not report.passed assert report.recommendation == "REJECT" assert not report.criteria_results["critical_entities_floor"].passed - assert "Recall regressed for 1 high-risk" in report.summary_reasons[0] + assert "Recall regressed for 1 high-risk" in ( + report.criteria_results["critical_entities_floor"].failure_reason or "" + ) def test_quality_gate_rejects_unpinned_provenance() -> None: @@ -121,3 +125,56 @@ def test_quality_gate_verifies_external_generalization_floor() -> None: report_fail = evaluate_quality_gate(base, cand, external_eval_record=ext_fail, num_resamples=50) assert not report_fail.passed assert not report_fail.criteria_results["external_generalization"].passed + + +def test_quality_gate_rejects_missing_external_evidence() -> None: + report = evaluate_quality_gate( + _make_dummy_eval_record(), _make_dummy_eval_record(), num_resamples=10 + ) + assert not report.passed + assert not report.criteria_results["external_generalization"].passed + + +def test_quality_gate_rejects_missing_critical_entity_counts() -> None: + base = _make_dummy_eval_record() + candidate = _make_dummy_eval_record() + del candidate["per_entity"]["PAYMENT_CARD"] + report = evaluate_quality_gate( + base, candidate, {"character_micro_metrics": {"f1": 0.82}}, num_resamples=10 + ) + assert not report.passed + assert not report.criteria_results["critical_entities_floor"].passed + + +def test_quality_gate_rejects_invalid_external_scores() -> None: + for f1 in (float("inf"), float("nan"), -0.1, 1.1, True, "0.82", None): + report = evaluate_quality_gate( + _make_dummy_eval_record(), + _make_dummy_eval_record(), + {"character_micro_metrics": {"f1": f1}}, + num_resamples=10, + ) + assert not report.passed + assert not report.criteria_results["external_generalization"].passed + + +def test_zero_critical_recall_is_rejected_without_division_error() -> None: + base = _make_dummy_eval_record() + candidate = _make_dummy_eval_record(card_recall=0.0) + candidate["per_entity"]["PAYMENT_CARD"]["false_positives"] = 1 + report = evaluate_quality_gate( + base, candidate, {"character_micro_metrics": {"f1": 0.82}}, num_resamples=10 + ) + assert not report.passed + assert not report.criteria_results["critical_entities_floor"].passed + + +def test_quality_gate_rejects_changed_critical_support() -> None: + base = _make_dummy_eval_record() + candidate = _make_dummy_eval_record() + candidate["per_entity"]["EMAIL"]["false_negatives"] = 0 + report = evaluate_quality_gate( + base, candidate, {"character_micro_metrics": {"f1": 0.82}}, num_resamples=10 + ) + assert not report.passed + assert not report.criteria_results["critical_entities_floor"].passed From c9d43edf95dc174567ed16c45c916b8f7396bea7 Mon Sep 17 00:00:00 2001 From: Paolo Mazza Date: Fri, 9 Oct 2026 11:58:06 +0000 Subject: [PATCH 2/2] fix: verify and audit the optional service extra --- CHANGELOG.md | 4 ++++ scripts/audit_extras.py | 2 +- scripts/audit_install.py | 4 ++++ scripts/verify_release.py | 4 +++- tests/integration/test_release_verifier.py | 13 +++++++++++++ 5 files changed, 25 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 93ec177..979c70b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ All notable changes follow Keep a Changelog and Semantic Versioning. - Optional bounded ONNX case recovery for lowercase PERSON candidates using high-confidence re-inference and constrained BIO decoding. Disabled by default; no name dictionaries or fabricated spans. - Python-owned Lambda HTTP service with verified model artifacts, required gateway authentication, ONNX-only person detection, and explicit inference failure. Hosted processing disables coreference expansion so each name occurrence requires model confirmation. +### Fixed +- Include the hosted-service extra in release verification and isolated installation + audits, while keeping service frameworks out of the dependency-free base import. + ## [1.36.0] - 2026-10-09 ### Added diff --git a/scripts/audit_extras.py b/scripts/audit_extras.py index 4c2cf20..7f8d028 100644 --- a/scripts/audit_extras.py +++ b/scripts/audit_extras.py @@ -6,7 +6,7 @@ import tempfile from pathlib import Path -DOCUMENTED_EXTRAS = ("html", "ml", "ocr", "office", "pdf", "remote") +DOCUMENTED_EXTRAS = ("html", "ml", "ocr", "office", "pdf", "remote", "service") def audit_extras(wheel_path: Path, python_executable: str | None = None) -> None: diff --git a/scripts/audit_install.py b/scripts/audit_install.py index c51b2c7..41fe382 100644 --- a/scripts/audit_install.py +++ b/scripts/audit_install.py @@ -11,6 +11,9 @@ "aiohttp", "docling", "docx", + "fastapi", + "pydantic", + "uvicorn", "httpx", "onnxruntime", "openpyxl", @@ -19,6 +22,7 @@ } EXPECTED_BASE_REQUIREMENTS: frozenset[str] = frozenset() EXTRA_CHECKS: dict[str, tuple[str, frozenset[str]]] = { + "service": ("pseudonymize.service", frozenset({"fastapi", "pydantic", "uvicorn"})), "remote": ("pseudonymize.backends.remote", frozenset({"httpx"})), "html": ("pseudonymize.html_xml", frozenset()), "office": ("pseudonymize.inspection.office", frozenset({"docx", "openpyxl"})), diff --git a/scripts/verify_release.py b/scripts/verify_release.py index 8b21d44..847f1f8 100644 --- a/scripts/verify_release.py +++ b/scripts/verify_release.py @@ -23,7 +23,9 @@ } EXPECTED_DEVELOPMENT_CLASSIFIER = "Development Status :: 5 - Production/Stable" EXPECTED_BASE_REQUIREMENTS: frozenset[str] = frozenset() -EXPECTED_EXTRAS: frozenset[str] = frozenset({"ml", "office", "pdf", "ocr", "remote", "html"}) +EXPECTED_EXTRAS: frozenset[str] = frozenset( + {"ml", "office", "pdf", "ocr", "remote", "html", "service"} +) REQUIRED_SDIST_FILES = frozenset( { "CHANGELOG.md", diff --git a/tests/integration/test_release_verifier.py b/tests/integration/test_release_verifier.py index 19aa4c3..5647aba 100644 --- a/tests/integration/test_release_verifier.py +++ b/tests/integration/test_release_verifier.py @@ -1,6 +1,7 @@ import email.message import io import tarfile +import tomllib import zipfile from pathlib import Path @@ -171,3 +172,15 @@ def test_verify_quality_gate_report_missing_file(tmp_path: Path) -> None: missing = tmp_path / "nonexistent.json" with pytest.raises(ValueError, match="quality gate report not found"): verify_quality_gate_report(missing) + + +def test_declared_extras_have_release_and_install_audits() -> None: + from scripts.audit_extras import DOCUMENTED_EXTRAS + from scripts.audit_install import EXTRA_CHECKS + + project_file = Path(__file__).resolve().parents[2] / "pyproject.toml" + with project_file.open("rb") as stream: + declared = set(tomllib.load(stream)["project"]["optional-dependencies"]) + assert declared == EXPECTED_EXTRAS + assert declared == set(DOCUMENTED_EXTRAS) + assert declared == set(EXTRA_CHECKS)