fix(bench): count matches correctly and keep diagnostics value-safe

CHANGELOG.md

@@ -15,6 +15,10 @@ All notable changes follow Keep a Changelog and Semantic Versioning.
   actually have evaluated support.
 - Require every quality criterion to pass explicitly when verifying a release report,
   rejecting incomplete reports and legacy skipped external evaluations.
+
+- Correct one-to-one match accounting in the local benchmark, including empty and
+  disjoint inputs. Explanations now report offsets without copying source values.
+- Keep optional ML imports and root logging configuration out of benchmark imports.
 - Include the hosted-service extra in release verification and isolated installation
   audits, while keeping service frameworks out of the dependency-free base import.
 

HANDOVER.md

@@ -431,3 +431,25 @@ methodology change. This patch must not be described as proving that protocol.
 Reusable lesson assessment: missing measurements cannot count as passing release
 criteria. Explicit negative tests now enforce that invariant at generation and
 consumption of gate reports.
+
+## Local benchmark accounting review (2026-10-02)
+
+`pseudonymize.bench._match` updated `matched_truth` outside the acceptance loop.
+This produced duplicate detection matches, undercounted true positives, and an
+UnboundLocalError for empty/disjoint inputs. Six new cases failed before the fix.
+The correction restores one-to-one matching; explanation logs now contain row,
+type and offsets instead of source values. Optional ML imports are lazy and the
+root logger is configured only by the CLI entry point.
+
+Observed verification using the frozen lockfile:
+
+- Local benchmark and evaluator tests: 13 passed, including exact expected
+  aggregate metrics for a corpus with true positives, false positives and misses.
+- Suite excluding ONNX/BIO modules with `--no-cov`: 496 passed; 1 Tesseract skip.
+- Pre-commit, mypy, strict docs build, package build and release verifier passed.
+- Full coverage and supported-platform verification are pending CI.
+
+This repairs the local benchmark harness; it does not improve detection accuracy
+or revise the separate published evaluation baseline. Reusable lesson assessment:
+assert match cardinality and zero-match cases instead of only exercising the
+benchmark command. Those invariants are now covered by regression tests.

src/pseudonymize/bench.py

@@ -4,14 +4,12 @@
 from pathlib import Path
 from typing import Any
 
-from pseudonymize.backends.ml.onnx import LocalONNXPIIBackend
 from pseudonymize.engine import Pseudonymizer
 from pseudonymize.result import EntityType
 
 # We duplicate the minimum necessary parts from evaluate_quality.py
 # to make it completely stand-alone and zero-dependency (other than what's already in core).
 
-logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
 logger = logging.getLogger("pseudonymize.bench")
 
 SUPPORTED_LABELS = {
@@ -50,7 +48,7 @@ def _match(
         if detection_index in matched_detections or truth_index in matched_truth:
             continue
         matched_detections.add(detection_index)
-    matched_truth.add(truth_index)
+        matched_truth.add(truth_index)
     return matched_detections, matched_truth
 
 
@@ -78,6 +76,8 @@ def evaluate(
 
     engine = Pseudonymizer()
     if use_ml:
+        from pseudonymize.backends.ml.onnx import LocalONNXPIIBackend
+
         CACHE_DIR = Path(".cache/pseudonymize-tests/models/multilang-pii-ner-ml")
         onnx_model_path = CACHE_DIR / "model_int8.onnx"
         tokenizer_path = CACHE_DIR / "tokenizer.json"
@@ -135,7 +135,7 @@ def evaluate(
                 entity_type = EntityType[label]
                 fn_per_type[entity_type] += 1
                 if explain:
-                    logger.info(f"FN: '{text[g_start:g_end]}' should be {label}")
+                    logger.info("FN: row=%d type=%s start=%d end=%d", count, label, g_start, g_end)
 
         # Unmatched Detections (False Positives)
         for index, (d_start, d_end, entity_type) in enumerate(detections):
@@ -145,7 +145,13 @@ def evaluate(
             fp_per_type[entity_type] += 1
             false_positives += 1
             if explain:
-                logger.info(f"FP: '{text[d_start:d_end]}' as {entity_type.value}")
+                logger.info(
+                    "FP: row=%d type=%s start=%d end=%d",
+                    count,
+                    entity_type.value,
+                    d_start,
+                    d_end,
+                )
 
         count += 1
 
@@ -195,6 +201,7 @@ def evaluate(
 
 
 def main() -> None:
+    logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
     parser = argparse.ArgumentParser(
         description="Local evaluation harness for pseudonymize quality testing."
     )
@@ -210,7 +217,7 @@ def main() -> None:
     parser.add_argument(
         "--explain",
         action="store_true",
-        help="Log details of all False Positives and False Negatives.",
+        help="Log row numbers, entity types and offsets for errors without source values.",
     )
     args = parser.parse_args()
 

tests/unit/test_bench.py

@@ -27,3 +27,93 @@ def test_bench_main(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
     )
     monkeypatch.setattr("sys.argv", ["pseudonymize.bench", str(sample_file), "--ml"])
     main()
+
+
+@pytest.mark.parametrize(
+    "detections,truth",
+    [
+        ([], []),
+        ([(0, 3, "EMAIL")], []),
+        ([], [(0, 3, "EMAIL")]),
+        ([(0, 3, "EMAIL")], [(4, 7, "EMAIL")]),
+    ],
+)
+def test_match_empty_and_disjoint_spans(
+    detections: list[tuple[int, int, str]], truth: list[tuple[int, int, str]]
+) -> None:
+    from pseudonymize.bench import _match
+    from pseudonymize.result import EntityType
+
+    assert _match(
+        [(start, end, EntityType(kind)) for start, end, kind in detections], truth, True
+    ) == (set(), set())
+
+
+def test_match_is_one_to_one() -> None:
+    from pseudonymize.bench import _match
+    from pseudonymize.result import EntityType
+
+    assert _match(
+        [(0, 8, EntityType.EMAIL), (1, 7, EntityType.EMAIL)], [(0, 8, "EMAIL")], True
+    ) == ({0}, {0})
+    assert _match([(0, 8, EntityType.EMAIL)], [(0, 8, "EMAIL"), (1, 7, "EMAIL")], True) == (
+        {0},
+        {0},
+    )
+    assert _match(
+        [(0, 3, EntityType.EMAIL), (5, 8, EntityType.PHONE)],
+        [(0, 3, "EMAIL"), (5, 8, "PHONE")],
+        True,
+    ) == ({0, 1}, {0, 1})
+    assert _match([(0, 3, EntityType.EMAIL)], [(0, 3, "PHONE")], True) == (set(), set())
+    assert _match([(0, 3, EntityType.EMAIL)], [(0, 3, "PHONE")], False) == ({0}, {0})
+
+
+def test_benchmark_metrics_and_explanations(
+    tmp_path: Path, caplog: pytest.LogCaptureFixture
+) -> None:
+    import json
+    import logging
+
+    email = "synthetic@example.com"
+    missing = "Syntheticname"
+    rows = [
+        {"source_text": email, "privacy_mask": [{"start": 0, "end": len(email), "label": "EMAIL"}]},
+        {"source_text": email, "privacy_mask": []},
+        {
+            "source_text": missing,
+            "privacy_mask": [{"start": 0, "end": len(missing), "label": "PERSON"}],
+        },
+        {"source_text": "nothing sensitive here", "privacy_mask": []},
+    ]
+    sample = tmp_path / "corpus.jsonl"
+    sample.write_text("\n".join(json.dumps(row) for row in rows), encoding="utf-8")
+    with caplog.at_level(logging.INFO, logger="pseudonymize.bench"):
+        evaluate(sample, explain=True)
+    assert "True Positives:  1" in caplog.text
+    assert "False Positives: 1" in caplog.text
+    assert "False Negatives: 1" in caplog.text
+    assert "Precision:       0.5000" in caplog.text
+    assert "Recall:          0.5000" in caplog.text
+    assert "F1 Score:        0.5000" in caplog.text
+    assert email not in caplog.text
+    assert missing not in caplog.text
+
+
+def test_import_does_not_load_optional_ml_or_configure_logging() -> None:
+    import subprocess
+    import sys
+
+    subprocess.run(
+        [
+            sys.executable,
+            "-c",
+            "import logging, sys; before = list(logging.getLogger().handlers); "
+            "import pseudonymize.bench; "
+            "assert logging.getLogger().handlers == before; "
+            "assert not {'onnxruntime', 'tokenizers', 'numpy'} & sys.modules.keys()",
+        ],
+        check=True,
+        capture_output=True,
+        timeout=30,
+    )