From 510fb1ee2fc90e9e0a25f239651242b83f358a7d Mon Sep 17 00:00:00 2001 From: Saverio Mazza Date: Fri, 2 Oct 2026 08:13:10 +0200 Subject: [PATCH 1/2] perf(coreference): replace growing regex alternations with token lookup --- CHANGELOG.md | 5 ++ HANDOVER.md | 30 ++++++++++++ benchmarks/benchmark_coreference.py | 46 ++++++++++++++++++ src/pseudonymize/coreference.py | 14 +++--- tests/unit/test_coreference.py | 73 +++++++++++++++++++++++++++++ 5 files changed, 161 insertions(+), 7 deletions(-) create mode 100644 benchmarks/benchmark_coreference.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 8600206..fbc4ce1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes follow Keep a Changelog and Semantic Versioning. ## [Unreleased] +### Changed + +- Scan coreference words once per block using dictionary lookup instead of rebuilding + a regular expression containing every learned token. Preserve case and word boundaries. + ## [1.34.0] - 2026-09-30 ### Added diff --git a/HANDOVER.md b/HANDOVER.md index 091c165..cd45017 100644 --- a/HANDOVER.md +++ b/HANDOVER.md @@ -399,3 +399,33 @@ verification unless they are intentionally retained as a user-approved release a - Push only when explicitly requested. - A version is published only after its matching tag, successful release workflow, PyPI artifact, and GitHub release exist. Until then, keep changes under `[Unreleased]`. + + +## Coreference lookup review (2026-10-02) + +Learned coreference tokens are alphabetic words. A single Unicode word scan with +hash-table lookup replaces per-block sorting and regex alternation construction. +Case sensitivity, offsets, confidence, token updates, and ambiguous-word exclusions +are unchanged. Added generative equivalence checks against the previous matching +algorithm and a benchmark including that algorithm as a reference. + +Observed verification with the frozen lockfile on Python 3.14.2: + +- Coreference tests: 6 passed, including Unicode and generated boundary cases. +- Suite excluding `test_onnx.py` and `test_bio_decoder.py`, with `--no-cov`: + 492 passed, 1 skipped because Tesseract is unavailable. +- Pre-commit, strict mypy, strict docs build, package build, release verifier: + passed. Mypy includes the new benchmark (156 source files). +- Reproducible performance command: + `uv run --frozen pytest benchmarks/benchmark_coreference.py --no-cov + --benchmark-only --benchmark-max-time=0.1 --benchmark-json=benchmark.json`. + Six benchmark cases passed. Timings are machine-dependent and are not a gate + or a claim of improved model quality. Small scopes need not be faster. +- Full ONNX/coverage and supported-platform verification remain pending. A full + suite attempt on the sibling Bloom branch terminated during ONNX with exit 137; + that failure is not evidence of this branch passing those tests. + +Reusable lesson assessment: match learned literal words through word scanning +and dictionary lookup when semantics allow it; preserve a boundary-equivalence +test before replacing a dynamically constructed regex. No new global policy is +needed for this local algorithm substitution. diff --git a/benchmarks/benchmark_coreference.py b/benchmarks/benchmark_coreference.py new file mode 100644 index 0000000..ed9eb72 --- /dev/null +++ b/benchmarks/benchmark_coreference.py @@ -0,0 +1,46 @@ +"""Compare scope lookup costs without changing detector or model quality.""" + +import re + +import pytest + +from pseudonymize.coreference import CoreferenceGraph +from pseudonymize.result import Detection, EntityType + + +def _suffix(value: int) -> str: + result = "" + while True: + result += chr(97 + value % 26) + value //= 26 + if not value: + return result + + +def _alternation_reference(graph: CoreferenceGraph, text: str) -> list[Detection]: + words = sorted(graph.tokens, key=len, reverse=True) + pattern = r"\b(?:" + "|".join(map(re.escape, words)) + r")\b" + results = [] + for match in re.finditer(pattern, text): + entity_type, confidence = graph.tokens[match.group()] + results.append( + Detection(entity_type, match.start(), match.end(), confidence, "coreference") + ) + return results + + +@pytest.mark.parametrize("size", [100, 1_000, 10_000]) +@pytest.mark.parametrize("implementation", ["alternation", "lookup"]) +def test_coreference_scope_lookup(benchmark: object, size: int, implementation: str) -> None: + graph = CoreferenceGraph() + for index in range(size): + word = "Person" + _suffix(index) + graph.add_detections([Detection(EntityType.PERSON, 0, len(word), 0.99, "source")], word) + text = "Ordinary unrelated words. " * 40 + "Persona Personb." + expected = _alternation_reference(graph, text) + assert graph.detect(text) == expected # noqa: S101 + if implementation == "alternation": + result = benchmark(_alternation_reference, graph, text) # type: ignore[operator] + else: + result = benchmark(graph.detect, text) # type: ignore[operator] + assert result == expected # noqa: S101 diff --git a/src/pseudonymize/coreference.py b/src/pseudonymize/coreference.py index b0954cd..422f216 100644 --- a/src/pseudonymize/coreference.py +++ b/src/pseudonymize/coreference.py @@ -155,13 +155,13 @@ def detect(self, text: str) -> list[Detection]: return [] results = [] - # Sort by length descending to match longest first (though they are words) - words = sorted(self.tokens.keys(), key=len, reverse=True) - pattern = r"\b(?:" + "|".join(map(re.escape, words)) + r")\b" - - for match in re.finditer(pattern, text): - token = match.group(0) - ent_type, conf = self.tokens[token] + # Learned tokens are alphabetic words. Scanning once avoids rebuilding an + # alternation proportional to the entire scope for every content block. + for match in re.finditer(r"\b\w+\b", text): + learned = self.tokens.get(match.group(0)) + if learned is None: + continue + ent_type, conf = learned results.append(Detection(ent_type, match.start(), match.end(), conf, "coreference")) return results diff --git a/tests/unit/test_coreference.py b/tests/unit/test_coreference.py index a2814e7..dffe14d 100644 --- a/tests/unit/test_coreference.py +++ b/tests/unit/test_coreference.py @@ -1,3 +1,8 @@ +import re + +from hypothesis import given +from hypothesis import strategies as st + from pseudonymize.engine import Pseudonymizer from pseudonymize.policy import Policy from pseudonymize.result import EntityType @@ -104,3 +109,71 @@ def detect(self, text: str) -> list[Detection]: # Independent second call has its own fresh scope; 'Cooper' alone must not be linked res2 = engine.process("Cooper was absent from the subsequent panel.") assert "Cooper was absent" in res2.text + + +def test_token_lookup_preserves_unicode_boundaries_and_case() -> None: + from pseudonymize.coreference import CoreferenceGraph + from pseudonymize.result import Detection + + graph = CoreferenceGraph() + source = "Élodie Smith O\u2019Connor" + graph.add_detections([Detection(EntityType.PERSON, 0, len(source), 0.99, "source")], source) + text = "Élodie, Smith's Connor-Smith; smith Smithson _Smith Smith2 2Smith Élodié." + found = graph.detect(text) + assert [text[d.start : d.end] for d in found] == [ + "Élodie", + "Smith", + "Connor", + "Smith", + "Élodie", + ] + assert all(d.entity_type is EntityType.PERSON and d.confidence == 0.99 for d in found) + + +def test_lookup_observes_new_tokens_and_confidence_updates() -> None: + from pseudonymize.coreference import CoreferenceGraph + from pseudonymize.result import Detection + + graph = CoreferenceGraph() + assert graph.detect("Smith") == [] + graph.add_detections([Detection(EntityType.PERSON, 0, 5, 0.96, "source")], "Smith") + assert graph.detect("Smith")[0].confidence == 0.96 + graph.add_detections([Detection(EntityType.PERSON, 0, 5, 0.99, "source")], "Smith") + assert graph.detect("Smith")[0].confidence == 0.99 + + +@given( + st.lists( + st.sampled_from( + [ + "Smith", + "Smithson", + "Élodie", + "Connor", + "smith", + "東京", + "_", + "2", + " ", + "-", + "'", + "\n", + "\u0301", + "🙂", + ] + ), + max_size=100, + ).map("".join) +) +def test_lookup_matches_original_word_boundary_semantics(text: str) -> None: + from pseudonymize.coreference import CoreferenceGraph + from pseudonymize.result import Detection + + graph = CoreferenceGraph() + source = "Smith Smithson Élodie Connor" + graph.add_detections([Detection(EntityType.PERSON, 0, len(source), 0.99, "source")], source) + pattern = ( + r"\b(?:" + "|".join(map(re.escape, sorted(graph.tokens, key=len, reverse=True))) + r")\b" + ) + expected = [(m.start(), m.end()) for m in re.finditer(pattern, text)] + assert [(d.start, d.end) for d in graph.detect(text)] == expected From c9d43edf95dc174567ed16c45c916b8f7396bea7 Mon Sep 17 00:00:00 2001 From: Paolo Mazza Date: Fri, 9 Oct 2026 11:58:06 +0000 Subject: [PATCH 2/2] fix: verify and audit the optional service extra --- CHANGELOG.md | 4 ++++ scripts/audit_extras.py | 2 +- scripts/audit_install.py | 4 ++++ scripts/verify_release.py | 4 +++- tests/integration/test_release_verifier.py | 13 +++++++++++++ 5 files changed, 25 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 93ec177..979c70b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ All notable changes follow Keep a Changelog and Semantic Versioning. - Optional bounded ONNX case recovery for lowercase PERSON candidates using high-confidence re-inference and constrained BIO decoding. Disabled by default; no name dictionaries or fabricated spans. - Python-owned Lambda HTTP service with verified model artifacts, required gateway authentication, ONNX-only person detection, and explicit inference failure. Hosted processing disables coreference expansion so each name occurrence requires model confirmation. +### Fixed +- Include the hosted-service extra in release verification and isolated installation + audits, while keeping service frameworks out of the dependency-free base import. + ## [1.36.0] - 2026-10-09 ### Added diff --git a/scripts/audit_extras.py b/scripts/audit_extras.py index 4c2cf20..7f8d028 100644 --- a/scripts/audit_extras.py +++ b/scripts/audit_extras.py @@ -6,7 +6,7 @@ import tempfile from pathlib import Path -DOCUMENTED_EXTRAS = ("html", "ml", "ocr", "office", "pdf", "remote") +DOCUMENTED_EXTRAS = ("html", "ml", "ocr", "office", "pdf", "remote", "service") def audit_extras(wheel_path: Path, python_executable: str | None = None) -> None: diff --git a/scripts/audit_install.py b/scripts/audit_install.py index c51b2c7..41fe382 100644 --- a/scripts/audit_install.py +++ b/scripts/audit_install.py @@ -11,6 +11,9 @@ "aiohttp", "docling", "docx", + "fastapi", + "pydantic", + "uvicorn", "httpx", "onnxruntime", "openpyxl", @@ -19,6 +22,7 @@ } EXPECTED_BASE_REQUIREMENTS: frozenset[str] = frozenset() EXTRA_CHECKS: dict[str, tuple[str, frozenset[str]]] = { + "service": ("pseudonymize.service", frozenset({"fastapi", "pydantic", "uvicorn"})), "remote": ("pseudonymize.backends.remote", frozenset({"httpx"})), "html": ("pseudonymize.html_xml", frozenset()), "office": ("pseudonymize.inspection.office", frozenset({"docx", "openpyxl"})), diff --git a/scripts/verify_release.py b/scripts/verify_release.py index 8b21d44..847f1f8 100644 --- a/scripts/verify_release.py +++ b/scripts/verify_release.py @@ -23,7 +23,9 @@ } EXPECTED_DEVELOPMENT_CLASSIFIER = "Development Status :: 5 - Production/Stable" EXPECTED_BASE_REQUIREMENTS: frozenset[str] = frozenset() -EXPECTED_EXTRAS: frozenset[str] = frozenset({"ml", "office", "pdf", "ocr", "remote", "html"}) +EXPECTED_EXTRAS: frozenset[str] = frozenset( + {"ml", "office", "pdf", "ocr", "remote", "html", "service"} +) REQUIRED_SDIST_FILES = frozenset( { "CHANGELOG.md", diff --git a/tests/integration/test_release_verifier.py b/tests/integration/test_release_verifier.py index 19aa4c3..5647aba 100644 --- a/tests/integration/test_release_verifier.py +++ b/tests/integration/test_release_verifier.py @@ -1,6 +1,7 @@ import email.message import io import tarfile +import tomllib import zipfile from pathlib import Path @@ -171,3 +172,15 @@ def test_verify_quality_gate_report_missing_file(tmp_path: Path) -> None: missing = tmp_path / "nonexistent.json" with pytest.raises(ValueError, match="quality gate report not found"): verify_quality_gate_report(missing) + + +def test_declared_extras_have_release_and_install_audits() -> None: + from scripts.audit_extras import DOCUMENTED_EXTRAS + from scripts.audit_install import EXTRA_CHECKS + + project_file = Path(__file__).resolve().parents[2] / "pyproject.toml" + with project_file.open("rb") as stream: + declared = set(tomllib.load(stream)["project"]["optional-dependencies"]) + assert declared == EXPECTED_EXTRAS + assert declared == set(DOCUMENTED_EXTRAS) + assert declared == set(EXTRA_CHECKS)