diff --git a/CHANGELOG.md b/CHANGELOG.md index 6df5fb3..fdcb482 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,12 @@ All notable changes follow Keep a Changelog and Semantic Versioning. - Align README aliases with the emitted token format and execute its Python examples in an integration test. Clarify overlap benchmark scoring and streaming limitations. + +### Changed + +- Scan coreference words once per block using dictionary lookup instead of rebuilding + a regular expression containing every learned token. Preserve case and word boundaries. +### Fixed - Include the hosted-service extra in release verification and isolated installation audits, while keeping service frameworks out of the dependency-free base import. diff --git a/HANDOVER.md b/HANDOVER.md index cf5d7b3..ca66531 100644 --- a/HANDOVER.md +++ b/HANDOVER.md @@ -500,3 +500,32 @@ Observed verification with the frozen lockfile: Reusable lesson assessment: executable onboarding assertions catch token-contract drift that prose review misses. This is enforced by the new integration test. + +## Coreference lookup review (2026-10-02) + +Learned coreference tokens are alphabetic words. A single Unicode word scan with +hash-table lookup replaces per-block sorting and regex alternation construction. +Case sensitivity, offsets, confidence, token updates, and ambiguous-word exclusions +are unchanged. Added generative equivalence checks against the previous matching +algorithm and a benchmark including that algorithm as a reference. + +Observed verification with the frozen lockfile on Python 3.14.2: + +- Coreference tests: 6 passed, including Unicode and generated boundary cases. +- Suite excluding `test_onnx.py` and `test_bio_decoder.py`, with `--no-cov`: + 492 passed, 1 skipped because Tesseract is unavailable. +- Pre-commit, strict mypy, strict docs build, package build, release verifier: + passed. Mypy includes the new benchmark (156 source files). +- Reproducible performance command: + `uv run --frozen pytest benchmarks/benchmark_coreference.py --no-cov + --benchmark-only --benchmark-max-time=0.1 --benchmark-json=benchmark.json`. + Six benchmark cases passed. Timings are machine-dependent and are not a gate + or a claim of improved model quality. Small scopes need not be faster. +- Full ONNX/coverage and supported-platform verification remain pending. A full + suite attempt on the sibling Bloom branch terminated during ONNX with exit 137; + that failure is not evidence of this branch passing those tests. + +Reusable lesson assessment: match learned literal words through word scanning +and dictionary lookup when semantics allow it; preserve a boundary-equivalence +test before replacing a dynamically constructed regex. No new global policy is +needed for this local algorithm substitution. diff --git a/benchmarks/benchmark_coreference.py b/benchmarks/benchmark_coreference.py new file mode 100644 index 0000000..ed9eb72 --- /dev/null +++ b/benchmarks/benchmark_coreference.py @@ -0,0 +1,46 @@ +"""Compare scope lookup costs without changing detector or model quality.""" + +import re + +import pytest + +from pseudonymize.coreference import CoreferenceGraph +from pseudonymize.result import Detection, EntityType + + +def _suffix(value: int) -> str: + result = "" + while True: + result += chr(97 + value % 26) + value //= 26 + if not value: + return result + + +def _alternation_reference(graph: CoreferenceGraph, text: str) -> list[Detection]: + words = sorted(graph.tokens, key=len, reverse=True) + pattern = r"\b(?:" + "|".join(map(re.escape, words)) + r")\b" + results = [] + for match in re.finditer(pattern, text): + entity_type, confidence = graph.tokens[match.group()] + results.append( + Detection(entity_type, match.start(), match.end(), confidence, "coreference") + ) + return results + + +@pytest.mark.parametrize("size", [100, 1_000, 10_000]) +@pytest.mark.parametrize("implementation", ["alternation", "lookup"]) +def test_coreference_scope_lookup(benchmark: object, size: int, implementation: str) -> None: + graph = CoreferenceGraph() + for index in range(size): + word = "Person" + _suffix(index) + graph.add_detections([Detection(EntityType.PERSON, 0, len(word), 0.99, "source")], word) + text = "Ordinary unrelated words. " * 40 + "Persona Personb." + expected = _alternation_reference(graph, text) + assert graph.detect(text) == expected # noqa: S101 + if implementation == "alternation": + result = benchmark(_alternation_reference, graph, text) # type: ignore[operator] + else: + result = benchmark(graph.detect, text) # type: ignore[operator] + assert result == expected # noqa: S101 diff --git a/src/pseudonymize/coreference.py b/src/pseudonymize/coreference.py index b0954cd..422f216 100644 --- a/src/pseudonymize/coreference.py +++ b/src/pseudonymize/coreference.py @@ -155,13 +155,13 @@ def detect(self, text: str) -> list[Detection]: return [] results = [] - # Sort by length descending to match longest first (though they are words) - words = sorted(self.tokens.keys(), key=len, reverse=True) - pattern = r"\b(?:" + "|".join(map(re.escape, words)) + r")\b" - - for match in re.finditer(pattern, text): - token = match.group(0) - ent_type, conf = self.tokens[token] + # Learned tokens are alphabetic words. Scanning once avoids rebuilding an + # alternation proportional to the entire scope for every content block. + for match in re.finditer(r"\b\w+\b", text): + learned = self.tokens.get(match.group(0)) + if learned is None: + continue + ent_type, conf = learned results.append(Detection(ent_type, match.start(), match.end(), conf, "coreference")) return results diff --git a/tests/unit/test_coreference.py b/tests/unit/test_coreference.py index 3f657cd..341e370 100644 --- a/tests/unit/test_coreference.py +++ b/tests/unit/test_coreference.py @@ -1,3 +1,8 @@ +import re + +from hypothesis import given +from hypothesis import strategies as st + from pseudonymize.engine import Pseudonymizer from pseudonymize.policy import Policy from pseudonymize.result import EntityType @@ -113,3 +118,71 @@ def detect(self, text: str) -> list[Detection]: # Independent second call has its own fresh scope; 'Cooper' alone must not be linked res2 = engine.process("Cooper was absent from the subsequent panel.") assert "Cooper was absent" in res2.text + + +def test_token_lookup_preserves_unicode_boundaries_and_case() -> None: + from pseudonymize.coreference import CoreferenceGraph + from pseudonymize.result import Detection + + graph = CoreferenceGraph() + source = "Élodie Smith O\u2019Connor" + graph.add_detections([Detection(EntityType.PERSON, 0, len(source), 0.99, "source")], source) + text = "Élodie, Smith's Connor-Smith; smith Smithson _Smith Smith2 2Smith Élodié." + found = graph.detect(text) + assert [text[d.start : d.end] for d in found] == [ + "Élodie", + "Smith", + "Connor", + "Smith", + "Élodie", + ] + assert all(d.entity_type is EntityType.PERSON and d.confidence == 0.99 for d in found) + + +def test_lookup_observes_new_tokens_and_confidence_updates() -> None: + from pseudonymize.coreference import CoreferenceGraph + from pseudonymize.result import Detection + + graph = CoreferenceGraph() + assert graph.detect("Smith") == [] + graph.add_detections([Detection(EntityType.PERSON, 0, 5, 0.96, "source")], "Smith") + assert graph.detect("Smith")[0].confidence == 0.96 + graph.add_detections([Detection(EntityType.PERSON, 0, 5, 0.99, "source")], "Smith") + assert graph.detect("Smith")[0].confidence == 0.99 + + +@given( + st.lists( + st.sampled_from( + [ + "Smith", + "Smithson", + "Élodie", + "Connor", + "smith", + "東京", + "_", + "2", + " ", + "-", + "'", + "\n", + "\u0301", + "🙂", + ] + ), + max_size=100, + ).map("".join) +) +def test_lookup_matches_original_word_boundary_semantics(text: str) -> None: + from pseudonymize.coreference import CoreferenceGraph + from pseudonymize.result import Detection + + graph = CoreferenceGraph() + source = "Smith Smithson Élodie Connor" + graph.add_detections([Detection(EntityType.PERSON, 0, len(source), 0.99, "source")], source) + pattern = ( + r"\b(?:" + "|".join(map(re.escape, sorted(graph.tokens, key=len, reverse=True))) + r")\b" + ) + expected = [(m.start(), m.end()) for m in re.finditer(pattern, text)] + assert [(d.start, d.end) for d in graph.detect(text)] == expected