diff --git a/CHANGELOG.md b/CHANGELOG.md index 979c70b..8cfc25f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,12 @@ All notable changes follow Keep a Changelog and Semantic Versioning. - Python-owned Lambda HTTP service with verified model artifacts, required gateway authentication, ONNX-only person detection, and explicit inference failure. Hosted processing disables coreference expansion so each name occurrence requires model confirmation. ### Fixed + +- Reject quality-gate approval when external evaluation is absent or its F1 is invalid, + and reject missing or incomparable critical-entity counts. Report which entities + actually have evaluated support. +- Require every quality criterion to pass explicitly when verifying a release report, + rejecting incomplete reports and legacy skipped external evaluations. - Include the hosted-service extra in release verification and isolated installation audits, while keeping service frameworks out of the dependency-free base import. diff --git a/HANDOVER.md b/HANDOVER.md index a602c9a..7b9bf35 100644 --- a/HANDOVER.md +++ b/HANDOVER.md @@ -405,3 +405,29 @@ verification unless they are intentionally retained as a user-approved release a - Push only when explicitly requested. - A version is published only after its matching tag, successful release workflow, PyPI artifact, and GitHub release exist. Until then, keep changes under `[Unreleased]`. + + +## Required quality evidence review (2026-10-02) + +Three regression tests reproduced approval with absent external evidence, a +missing critical entity, and an infinite external F1. The gate now rejects those +inputs, compares critical annotation support, and distinguishes represented from +unrepresented entities. The release verifier checks all five criteria and rejects +legacy skipped external evaluations. Per-entity F1 uses the equivalent count +formula, avoiding division by zero when TP is zero and FP/FN are nonzero. + +Observed verification using the frozen lockfile: + +- Quality gate, comparator and release-verifier tests: 40 passed. +- Suite excluding ONNX/BIO modules with `--no-cov`: 510 passed; 1 Tesseract skip. +- Pre-commit, mypy, strict docs build, package build and release verifier passed. +- Full coverage and supported-platform acceptance remain pending CI. + +No detector, held-out sample, published score or quality tolerance was changed. +The remaining difference between the roadmap's strict exact-boundary improvement +protocol and the existing overlap-based tolerance gate requires a separate +methodology change. This patch must not be described as proving that protocol. + +Reusable lesson assessment: missing measurements cannot count as passing release +criteria. Explicit negative tests now enforce that invariant at generation and +consumption of gate reports. diff --git a/benchmarks/compare_quality.py b/benchmarks/compare_quality.py index 0413069..b8e2204 100644 --- a/benchmarks/compare_quality.py +++ b/benchmarks/compare_quality.py @@ -146,22 +146,14 @@ def compare_results( b_tp = b_et.get("true_positives", 0) b_fp = b_et.get("false_positives", 0) b_fn = b_et.get("false_negatives", 0) - b_f1 = ( - (2 * (b_tp / (b_tp + b_fp)) * (b_tp / (b_tp + b_fn))) - / ((b_tp / (b_tp + b_fp)) + (b_tp / (b_tp + b_fn))) - if (b_tp + b_fp > 0 and b_tp + b_fn > 0) - else 0.0 - ) + b_denominator = 2 * b_tp + b_fp + b_fn + b_f1 = 2 * b_tp / b_denominator if b_denominator else 0.0 c_tp = c_et.get("true_positives", 0) c_fp = c_et.get("false_positives", 0) c_fn = c_et.get("false_negatives", 0) - c_f1 = ( - (2 * (c_tp / (c_tp + c_fp)) * (c_tp / (c_tp + c_fn))) - / ((c_tp / (c_tp + c_fp)) + (c_tp / (c_tp + c_fn))) - if (c_tp + c_fp > 0 and c_tp + c_fn > 0) - else 0.0 - ) + c_denominator = 2 * c_tp + c_fp + c_fn + c_f1 = 2 * c_tp / c_denominator if c_denominator else 0.0 per_entity_shifts[et] = { "delta_tp": c_tp - b_tp, diff --git a/benchmarks/quality_gate.py b/benchmarks/quality_gate.py index 5225977..a3e6aed 100644 --- a/benchmarks/quality_gate.py +++ b/benchmarks/quality_gate.py @@ -13,6 +13,7 @@ import argparse import json import logging +import math from dataclasses import dataclass from pathlib import Path from typing import Any @@ -200,15 +201,27 @@ def evaluate_quality_gate( base_per_entity = baseline_record.get("per_entity", {}) cand_per_entity = candidate_record.get("per_entity", {}) critical_regressions: list[dict[str, Any]] = [] + missing_critical_evidence: list[str] = [] + checked_entities: list[str] = [] - for ent in HIGH_RISK_ENTITY_TYPES: + for ent in sorted(HIGH_RISK_ENTITY_TYPES): if ent in base_per_entity: b_et = base_per_entity[ent] c_et = cand_per_entity.get(ent, {}) - b_tp = b_et.get("true_positives", 0) - b_fn = b_et.get("false_negatives", 0) - c_tp = c_et.get("true_positives", 0) - c_fn = c_et.get("false_negatives", 0) + b_tp = b_et.get("true_positives") + b_fn = b_et.get("false_negatives") + c_tp = c_et.get("true_positives") + c_fn = c_et.get("false_negatives") + counts = (b_tp, b_fn, c_tp, c_fn) + if not all(isinstance(n, int) and not isinstance(n, bool) and n >= 0 for n in counts): + missing_critical_evidence.append(ent) + continue + if b_tp + b_fn != c_tp + c_fn: + missing_critical_evidence.append(ent) + continue + if b_tp + b_fn == 0: + continue + checked_entities.append(ent) b_rec = b_tp / (b_tp + b_fn) if (b_tp + b_fn) > 0 else 1.0 c_rec = c_tp / (c_tp + c_fn) if (c_tp + c_fn) > 0 else 1.0 @@ -224,55 +237,79 @@ def evaluate_quality_gate( } ) - critical_ok = len(critical_regressions) == 0 + critical_ok = ( + not critical_regressions and not missing_critical_evidence and bool(checked_entities) + ) if not critical_ok: - reason = ( - f"Recall regressed for {len(critical_regressions)} high-risk identifiers: " - f"{[r['entity_type'] for r in critical_regressions]}" - ) + if missing_critical_evidence or not checked_entities: + reason = "Critical identifier recall evidence is missing, invalid or incomparable." + else: + reason = ( + f"Recall regressed for {len(critical_regressions)} high-risk identifiers: " + f"{[r['entity_type'] for r in critical_regressions]}" + ) failure_reasons.append(reason) results["critical_entities_floor"] = QualityGateCriteriaResult( criterion_name="critical_entities_floor", passed=False, - details={"regressed_entities": critical_regressions}, + details={ + "regressed_entities": critical_regressions, + "missing_or_incomparable_entities": missing_critical_evidence, + }, failure_reason=reason, ) else: results["critical_entities_floor"] = QualityGateCriteriaResult( criterion_name="critical_entities_floor", passed=True, - details={"protected_entities_checked": list(HIGH_RISK_ENTITY_TYPES)}, + details={ + "protected_entities_checked": checked_entities, + "unrepresented_entities": sorted(HIGH_RISK_ENTITY_TYPES - set(checked_entities)), + }, ) # 5. Independent External Generalization Track (PIIMB) if external_eval_record is not None: ext_micro = external_eval_record.get("character_micro_metrics", {}) - ext_f1 = ext_micro.get("f1", 0.0) - ext_ok = ext_f1 >= external_f1_floor + ext_f1 = ext_micro.get("f1") if isinstance(ext_micro, dict) else None + valid_score = ( + isinstance(ext_f1, (int, float)) + and not isinstance(ext_f1, bool) + and math.isfinite(ext_f1) + and 0 <= ext_f1 <= 1 + ) + ext_ok = valid_score and isinstance(ext_f1, (int, float)) and ext_f1 >= external_f1_floor if not ext_ok: reason = ( - f"External benchmark (PIIMB) character F1 {ext_f1:.4f} is below floor " - f"{external_f1_floor:.4f}." + "External benchmark character F1 is missing, invalid or below the required floor." ) failure_reasons.append(reason) results["external_generalization"] = QualityGateCriteriaResult( criterion_name="external_generalization", passed=False, - details={"external_f1": ext_f1, "required_floor": external_f1_floor}, + details={ + "external_f1": ext_f1 if valid_score else None, + "required_floor": external_f1_floor, + }, failure_reason=reason, ) else: results["external_generalization"] = QualityGateCriteriaResult( criterion_name="external_generalization", passed=True, - details={"external_f1": ext_f1, "required_floor": external_f1_floor}, + details={ + "external_f1": ext_f1 if valid_score else None, + "required_floor": external_f1_floor, + }, ) else: - # Informational pass if external track was not evaluated in this specific run + reason = "Independent external generalization evidence is required for shipment." + failure_reasons.append(reason) results["external_generalization"] = QualityGateCriteriaResult( criterion_name="external_generalization", - passed=True, + passed=False, details={"status": "not_evaluated_in_current_run"}, + failure_reason=reason, ) all_passed = all(r.passed for r in results.values()) @@ -304,7 +341,7 @@ def main() -> int: "--external", type=Path, default=None, - help="Optional path to external generalization artifact JSON (PIIMB).", + help="External generalization artifact JSON (PIIMB), required for a passing gate.", ) parser.add_argument( "--output", @@ -317,7 +354,7 @@ def main() -> int: baseline_data = json.loads(args.baseline.read_text(encoding="utf-8")) candidate_data = json.loads(args.candidate.read_text(encoding="utf-8")) external_data = None - if args.external and args.external.exists(): + if args.external is not None: external_data = json.loads(args.external.read_text(encoding="utf-8")) report = evaluate_quality_gate( diff --git a/docs/quality_benchmarks.md b/docs/quality_benchmarks.md index dcaaeda..819b269 100644 --- a/docs/quality_benchmarks.md +++ b/docs/quality_benchmarks.md @@ -146,3 +146,24 @@ Evaluated against the pinned validation split (`a785eb528e28be2693c3718a27e06697 See also the formal [Model Card](model_card.md) for full architecture specifications and hardware profiles. + + +## Required evidence for a quality-gate report + +A release-quality evaluation needs an external evaluation record. Omitting it +produces `REJECT`; a missing file supplied through `--external` is an error. +External character F1 must be a finite number between zero and one and meet the +configured floor. Critical-entity comparisons require matching annotation +support and explicit nonnegative integer TP/FN counts. The report distinguishes +entities with observed support from those not represented in that evaluation. + +`verify_release.py --quality-gate-report` requires all five criteria to be +present and explicitly passing. A top-level `passed: true` flag is insufficient, +and legacy reports that treated an unevaluated external track as passing are +rejected. Regenerate those reports with external evidence rather than editing +approval flags. + +These checks validate evidence presence and consistency. They do not authenticate +an artifact, prove that a corpus is independent, or replace the roadmap's stricter +experimental protocol. This change does not alter detector behavior or establish +an improvement in the published quality baseline. diff --git a/scripts/verify_release.py b/scripts/verify_release.py index 847f1f8..61fbab1 100644 --- a/scripts/verify_release.py +++ b/scripts/verify_release.py @@ -2,6 +2,7 @@ import email.message import email.parser import json +import math import os import re import tarfile @@ -161,9 +162,46 @@ def verify_quality_gate_report(report_path: Path) -> None: if not report_path.is_file(): raise ValueError(f"quality gate report not found at {report_path}") data = json.loads(report_path.read_text(encoding="utf-8")) - if not data.get("passed", False) or data.get("recommendation") != "SHIP": - summary = data.get("summary_reasons", ["unspecified failure"]) + if ( + not isinstance(data, dict) + or data.get("passed") is not True + or data.get("recommendation") != "SHIP" + ): + summary = ( + data.get("summary_reasons", ["unspecified failure"]) + if isinstance(data, dict) + else ["invalid report"] + ) raise ValueError(f"quality gate failed: {summary}") + required = { + "pinned_provenance", + "paired_f1_delta", + "precision_protection", + "critical_entities_floor", + "external_generalization", + } + criteria = data.get("criteria") + if not isinstance(criteria, dict) or not required <= criteria.keys(): + raise ValueError("quality gate report is missing required criteria") + for name in sorted(required): + criterion = criteria[name] + if not isinstance(criterion, dict) or criterion.get("passed") is not True: + raise ValueError(f"quality gate criterion did not pass: {name}") + external_details = criteria["external_generalization"].get("details") + if ( + not isinstance(external_details, dict) + or external_details.get("status") == "not_evaluated_in_current_run" + or "external_f1" not in external_details + ): + raise ValueError("quality gate report lacks external evaluation evidence") + external_f1 = external_details["external_f1"] + if ( + not isinstance(external_f1, (int, float)) + or isinstance(external_f1, bool) + or not math.isfinite(external_f1) + or not 0 <= external_f1 <= 1 + ): + raise ValueError("quality gate report has invalid external evaluation evidence") rec = data.get("recommendation") print(f"verified quality gate report: candidate approved for shipment ({rec})") diff --git a/tests/integration/test_release_verifier.py b/tests/integration/test_release_verifier.py index 5647aba..2d8e1d3 100644 --- a/tests/integration/test_release_verifier.py +++ b/tests/integration/test_release_verifier.py @@ -1,5 +1,6 @@ import email.message import io +import json import tarfile import tomllib import zipfile @@ -153,7 +154,7 @@ def test_verify_quality_gate_report_passing( tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: report = tmp_path / "gate.json" - report.write_text('{"passed": true, "recommendation": "SHIP"}', encoding="utf-8") + report.write_text(json.dumps(_complete_gate_report()), encoding="utf-8") verify_quality_gate_report(report) assert "approved for shipment" in capsys.readouterr().out @@ -174,6 +175,61 @@ def test_verify_quality_gate_report_missing_file(tmp_path: Path) -> None: verify_quality_gate_report(missing) +def _complete_gate_report() -> dict[str, object]: + return { + "passed": True, + "recommendation": "SHIP", + "criteria": { + name: {"passed": True, "details": {"external_f1": 0.82}} + for name in ( + "pinned_provenance", + "paired_f1_delta", + "precision_protection", + "critical_entities_floor", + "external_generalization", + ) + }, + } + + +@pytest.mark.parametrize( + "criterion", + [ + "pinned_provenance", + "paired_f1_delta", + "precision_protection", + "critical_entities_floor", + "external_generalization", + ], +) +@pytest.mark.parametrize("state", ["missing", "failed", "string"]) +def test_verify_quality_gate_report_checks_each_criterion( + tmp_path: Path, criterion: str, state: str +) -> None: + data = _complete_gate_report() + criteria = data["criteria"] + assert isinstance(criteria, dict) + if state == "missing": + del criteria[criterion] + else: + criteria[criterion]["passed"] = False if state == "failed" else "true" + path = tmp_path / "gate.json" + path.write_text(json.dumps(data), encoding="utf-8") + with pytest.raises(ValueError, match="quality gate"): + verify_quality_gate_report(path) + + +def test_verify_quality_gate_rejects_legacy_skipped_external_track(tmp_path: Path) -> None: + data = _complete_gate_report() + criteria = data["criteria"] + assert isinstance(criteria, dict) + criteria["external_generalization"]["details"] = {"status": "not_evaluated_in_current_run"} + path = tmp_path / "gate.json" + path.write_text(json.dumps(data), encoding="utf-8") + with pytest.raises(ValueError, match="external evaluation evidence"): + verify_quality_gate_report(path) + + def test_declared_extras_have_release_and_install_audits() -> None: from scripts.audit_extras import DOCUMENTED_EXTRAS from scripts.audit_install import EXTRA_CHECKS diff --git a/tests/unit/test_quality_gate.py b/tests/unit/test_quality_gate.py index d2fb200..480fcd9 100644 --- a/tests/unit/test_quality_gate.py +++ b/tests/unit/test_quality_gate.py @@ -24,7 +24,7 @@ def _make_dummy_eval_record( "PAYMENT_CARD": { "true_positives": int(10 * card_recall), "false_positives": 0, - "false_negatives": int(10 * (1 - card_recall)), + "false_negatives": 10 - int(10 * card_recall), }, }, "error_categories": { @@ -42,10 +42,10 @@ def _make_dummy_eval_record( "source": "wiki", "tp": {"EMAIL": 5, "PAYMENT_CARD": int(5 * card_recall)}, "fp": {"EMAIL": 0, "PAYMENT_CARD": 0}, - "fn": {"EMAIL": 0, "PAYMENT_CARD": int(5 * (1 - card_recall))}, + "fn": {"EMAIL": 0, "PAYMENT_CARD": 5 - int(5 * card_recall)}, "exact_tp": {"EMAIL": 5, "PAYMENT_CARD": int(5 * card_recall)}, "exact_fp": {"EMAIL": 0, "PAYMENT_CARD": 0}, - "exact_fn": {"EMAIL": 0, "PAYMENT_CARD": int(5 * (1 - card_recall))}, + "exact_fn": {"EMAIL": 0, "PAYMENT_CARD": 5 - int(5 * card_recall)}, "error_categories": {}, }, { @@ -55,10 +55,10 @@ def _make_dummy_eval_record( "source": "wiki", "tp": {"EMAIL": 5, "PAYMENT_CARD": int(5 * card_recall)}, "fp": {"EMAIL": 1, "PAYMENT_CARD": 0}, - "fn": {"EMAIL": 1, "PAYMENT_CARD": int(5 * (1 - card_recall))}, + "fn": {"EMAIL": 1, "PAYMENT_CARD": 5 - int(5 * card_recall)}, "exact_tp": {"EMAIL": 5, "PAYMENT_CARD": int(5 * card_recall)}, "exact_fp": {"EMAIL": 1, "PAYMENT_CARD": 0}, - "exact_fn": {"EMAIL": 1, "PAYMENT_CARD": int(5 * (1 - card_recall))}, + "exact_fn": {"EMAIL": 1, "PAYMENT_CARD": 5 - int(5 * card_recall)}, "error_categories": {}, }, ], @@ -69,7 +69,9 @@ def test_quality_gate_passes_healthy_candidate() -> None: base = _make_dummy_eval_record(f1=0.83, precision=0.86) cand = _make_dummy_eval_record(f1=0.84, precision=0.865) - report = evaluate_quality_gate(base, cand, num_resamples=50) + report = evaluate_quality_gate( + base, cand, {"character_micro_metrics": {"f1": 0.82}}, num_resamples=50 + ) assert report.passed assert report.recommendation == "SHIP" assert report.criteria_results["pinned_provenance"].passed @@ -97,7 +99,9 @@ def test_quality_gate_rejects_critical_identifier_recall_regression() -> None: assert not report.passed assert report.recommendation == "REJECT" assert not report.criteria_results["critical_entities_floor"].passed - assert "Recall regressed for 1 high-risk" in report.summary_reasons[0] + assert "Recall regressed for 1 high-risk" in ( + report.criteria_results["critical_entities_floor"].failure_reason or "" + ) def test_quality_gate_rejects_unpinned_provenance() -> None: @@ -121,3 +125,56 @@ def test_quality_gate_verifies_external_generalization_floor() -> None: report_fail = evaluate_quality_gate(base, cand, external_eval_record=ext_fail, num_resamples=50) assert not report_fail.passed assert not report_fail.criteria_results["external_generalization"].passed + + +def test_quality_gate_rejects_missing_external_evidence() -> None: + report = evaluate_quality_gate( + _make_dummy_eval_record(), _make_dummy_eval_record(), num_resamples=10 + ) + assert not report.passed + assert not report.criteria_results["external_generalization"].passed + + +def test_quality_gate_rejects_missing_critical_entity_counts() -> None: + base = _make_dummy_eval_record() + candidate = _make_dummy_eval_record() + del candidate["per_entity"]["PAYMENT_CARD"] + report = evaluate_quality_gate( + base, candidate, {"character_micro_metrics": {"f1": 0.82}}, num_resamples=10 + ) + assert not report.passed + assert not report.criteria_results["critical_entities_floor"].passed + + +def test_quality_gate_rejects_invalid_external_scores() -> None: + for f1 in (float("inf"), float("nan"), -0.1, 1.1, True, "0.82", None): + report = evaluate_quality_gate( + _make_dummy_eval_record(), + _make_dummy_eval_record(), + {"character_micro_metrics": {"f1": f1}}, + num_resamples=10, + ) + assert not report.passed + assert not report.criteria_results["external_generalization"].passed + + +def test_zero_critical_recall_is_rejected_without_division_error() -> None: + base = _make_dummy_eval_record() + candidate = _make_dummy_eval_record(card_recall=0.0) + candidate["per_entity"]["PAYMENT_CARD"]["false_positives"] = 1 + report = evaluate_quality_gate( + base, candidate, {"character_micro_metrics": {"f1": 0.82}}, num_resamples=10 + ) + assert not report.passed + assert not report.criteria_results["critical_entities_floor"].passed + + +def test_quality_gate_rejects_changed_critical_support() -> None: + base = _make_dummy_eval_record() + candidate = _make_dummy_eval_record() + candidate["per_entity"]["EMAIL"]["false_negatives"] = 0 + report = evaluate_quality_gate( + base, candidate, {"character_micro_metrics": {"f1": 0.82}}, num_resamples=10 + ) + assert not report.passed + assert not report.criteria_results["critical_entities_floor"].passed