perf(coreference): avoid rebuilding growing regex alternations

CHANGELOG.md

@@ -25,6 +25,12 @@ All notable changes follow Keep a Changelog and Semantic Versioning.
 
 - Align README aliases with the emitted token format and execute its Python examples
   in an integration test. Clarify overlap benchmark scoring and streaming limitations.
+
+### Changed
+
+- Scan coreference words once per block using dictionary lookup instead of rebuilding
+  a regular expression containing every learned token. Preserve case and word boundaries.
+### Fixed
 - Include the hosted-service extra in release verification and isolated installation
   audits, while keeping service frameworks out of the dependency-free base import.
 

HANDOVER.md

@@ -500,3 +500,32 @@ Observed verification with the frozen lockfile:
 
 Reusable lesson assessment: executable onboarding assertions catch token-contract
   drift that prose review misses. This is enforced by the new integration test.
+
+## Coreference lookup review (2026-10-02)
+
+Learned coreference tokens are alphabetic words. A single Unicode word scan with
+hash-table lookup replaces per-block sorting and regex alternation construction.
+Case sensitivity, offsets, confidence, token updates, and ambiguous-word exclusions
+are unchanged. Added generative equivalence checks against the previous matching
+algorithm and a benchmark including that algorithm as a reference.
+
+Observed verification with the frozen lockfile on Python 3.14.2:
+
+- Coreference tests: 6 passed, including Unicode and generated boundary cases.
+- Suite excluding `test_onnx.py` and `test_bio_decoder.py`, with `--no-cov`:
+  492 passed, 1 skipped because Tesseract is unavailable.
+- Pre-commit, strict mypy, strict docs build, package build, release verifier:
+  passed. Mypy includes the new benchmark (156 source files).
+- Reproducible performance command:
+  `uv run --frozen pytest benchmarks/benchmark_coreference.py --no-cov
+  --benchmark-only --benchmark-max-time=0.1 --benchmark-json=benchmark.json`.
+  Six benchmark cases passed. Timings are machine-dependent and are not a gate
+  or a claim of improved model quality. Small scopes need not be faster.
+- Full ONNX/coverage and supported-platform verification remain pending. A full
+  suite attempt on the sibling Bloom branch terminated during ONNX with exit 137;
+  that failure is not evidence of this branch passing those tests.
+
+Reusable lesson assessment: match learned literal words through word scanning
+and dictionary lookup when semantics allow it; preserve a boundary-equivalence
+test before replacing a dynamically constructed regex. No new global policy is
+needed for this local algorithm substitution.

benchmarks/benchmark_coreference.py

@@ -0,0 +1,46 @@
+"""Compare scope lookup costs without changing detector or model quality."""
+
+import re
+
+import pytest
+
+from pseudonymize.coreference import CoreferenceGraph
+from pseudonymize.result import Detection, EntityType
+
+
+def _suffix(value: int) -> str:
+    result = ""
+    while True:
+        result += chr(97 + value % 26)
+        value //= 26
+        if not value:
+            return result
+
+
+def _alternation_reference(graph: CoreferenceGraph, text: str) -> list[Detection]:
+    words = sorted(graph.tokens, key=len, reverse=True)
+    pattern = r"\b(?:" + "|".join(map(re.escape, words)) + r")\b"
+    results = []
+    for match in re.finditer(pattern, text):
+        entity_type, confidence = graph.tokens[match.group()]
+        results.append(
+            Detection(entity_type, match.start(), match.end(), confidence, "coreference")
+        )
+    return results
+
+
+@pytest.mark.parametrize("size", [100, 1_000, 10_000])
+@pytest.mark.parametrize("implementation", ["alternation", "lookup"])
+def test_coreference_scope_lookup(benchmark: object, size: int, implementation: str) -> None:
+    graph = CoreferenceGraph()
+    for index in range(size):
+        word = "Person" + _suffix(index)
+        graph.add_detections([Detection(EntityType.PERSON, 0, len(word), 0.99, "source")], word)
+    text = "Ordinary unrelated words. " * 40 + "Persona Personb."
+    expected = _alternation_reference(graph, text)
+    assert graph.detect(text) == expected  # noqa: S101
+    if implementation == "alternation":
+        result = benchmark(_alternation_reference, graph, text)  # type: ignore[operator]
+    else:
+        result = benchmark(graph.detect, text)  # type: ignore[operator]
+    assert result == expected  # noqa: S101

src/pseudonymize/coreference.py

@@ -155,13 +155,13 @@ def detect(self, text: str) -> list[Detection]:
             return []
 
         results = []
-        # Sort by length descending to match longest first (though they are words)
-        words = sorted(self.tokens.keys(), key=len, reverse=True)
-        pattern = r"\b(?:" + "|".join(map(re.escape, words)) + r")\b"
-
-        for match in re.finditer(pattern, text):
-            token = match.group(0)
-            ent_type, conf = self.tokens[token]
+        # Learned tokens are alphabetic words. Scanning once avoids rebuilding an
+        # alternation proportional to the entire scope for every content block.
+        for match in re.finditer(r"\b\w+\b", text):
+            learned = self.tokens.get(match.group(0))
+            if learned is None:
+                continue
+            ent_type, conf = learned
             results.append(Detection(ent_type, match.start(), match.end(), conf, "coreference"))
 
         return results

tests/unit/test_coreference.py

@@ -1,3 +1,8 @@
+import re
+
+from hypothesis import given
+from hypothesis import strategies as st
+
 from pseudonymize.engine import Pseudonymizer
 from pseudonymize.policy import Policy
 from pseudonymize.result import EntityType
@@ -113,3 +118,71 @@ def detect(self, text: str) -> list[Detection]:
     # Independent second call has its own fresh scope; 'Cooper' alone must not be linked
     res2 = engine.process("Cooper was absent from the subsequent panel.")
     assert "Cooper was absent" in res2.text
+
+
+def test_token_lookup_preserves_unicode_boundaries_and_case() -> None:
+    from pseudonymize.coreference import CoreferenceGraph
+    from pseudonymize.result import Detection
+
+    graph = CoreferenceGraph()
+    source = "Élodie Smith O\u2019Connor"
+    graph.add_detections([Detection(EntityType.PERSON, 0, len(source), 0.99, "source")], source)
+    text = "Élodie, Smith's Connor-Smith; smith Smithson _Smith Smith2 2Smith Élodié."
+    found = graph.detect(text)
+    assert [text[d.start : d.end] for d in found] == [
+        "Élodie",
+        "Smith",
+        "Connor",
+        "Smith",
+        "Élodie",
+    ]
+    assert all(d.entity_type is EntityType.PERSON and d.confidence == 0.99 for d in found)
+
+
+def test_lookup_observes_new_tokens_and_confidence_updates() -> None:
+    from pseudonymize.coreference import CoreferenceGraph
+    from pseudonymize.result import Detection
+
+    graph = CoreferenceGraph()
+    assert graph.detect("Smith") == []
+    graph.add_detections([Detection(EntityType.PERSON, 0, 5, 0.96, "source")], "Smith")
+    assert graph.detect("Smith")[0].confidence == 0.96
+    graph.add_detections([Detection(EntityType.PERSON, 0, 5, 0.99, "source")], "Smith")
+    assert graph.detect("Smith")[0].confidence == 0.99
+
+
+@given(
+    st.lists(
+        st.sampled_from(
+            [
+                "Smith",
+                "Smithson",
+                "Élodie",
+                "Connor",
+                "smith",
+                "東京",
+                "_",
+                "2",
+                " ",
+                "-",
+                "'",
+                "\n",
+                "\u0301",
+                "🙂",
+            ]
+        ),
+        max_size=100,
+    ).map("".join)
+)
+def test_lookup_matches_original_word_boundary_semantics(text: str) -> None:
+    from pseudonymize.coreference import CoreferenceGraph
+    from pseudonymize.result import Detection
+
+    graph = CoreferenceGraph()
+    source = "Smith Smithson Élodie Connor"
+    graph.add_detections([Detection(EntityType.PERSON, 0, len(source), 0.99, "source")], source)
+    pattern = (
+        r"\b(?:" + "|".join(map(re.escape, sorted(graph.tokens, key=len, reverse=True))) + r")\b"
+    )
+    expected = [(m.start(), m.end()) for m in re.finditer(pattern, text)]
+    assert [(d.start, d.end) for d in graph.detect(text)] == expected