From 51fa094b1a2964480b184829226c5b536f8be89d Mon Sep 17 00:00:00 2001 From: Saverio Mazza Date: Fri, 2 Oct 2026 08:10:50 +0200 Subject: [PATCH 1/2] fix(memory): invalidate Bloom membership cache after insertion --- CHANGELOG.md | 5 +++++ HANDOVER.md | 25 ++++++++++++++++++++++ src/pseudonymize/memory/bloom.py | 9 +++++++- tests/unit/test_bloom.py | 36 ++++++++++++++++++++++++++++++++ 4 files changed, 74 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8600206..6612b48 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes follow Keep a Changelog and Semantic Versioning. ## [Unreleased] +### Fixed + +- Invalidate Bloom filter membership caches after insertion so earlier misses cannot hide + newly added entries. Validate filter parameters and allocate at least one bit. + ## [1.34.0] - 2026-09-30 ### Added diff --git a/HANDOVER.md b/HANDOVER.md index 091c165..955ccf4 100644 --- a/HANDOVER.md +++ b/HANDOVER.md @@ -399,3 +399,28 @@ verification unless they are intentionally retained as a user-approved release a - Push only when explicitly requested. - A version is published only after its matching tag, successful release workflow, PyPI artifact, and GitHub release exist. Until then, keep changes under `[Unreleased]`. + + +## Bloom filter correctness review (2026-10-02) + +The mutation regression was reproduced before fixing it: a cached negative lookup +remained negative after insertion. Small filters could also allocate zero bits. +The change clears membership cache entries after insertion, validates parameters, +and rounds the bit count upward. No dependency or detection threshold changes. + +Observed verification on Python 3.14.2 with the frozen lockfile: + +- Focused Bloom and gazetteer tests: 11 passed. +- `uv run --frozen pre-commit run --all-files`: passed. +- `uv run --frozen mypy`: passed, 155 source files. +- `uv run --frozen mkdocs build --strict`: passed. +- `uv build` and `uv run --frozen python scripts/verify_release.py`: passed. +- `uv run --frozen pytest --ignore=tests/unit/backends/test_onnx.py + --ignore=tests/unit/backends/test_bio_decoder.py --no-cov`: 493 passed, + 1 skipped because Tesseract was unavailable. +- The full suite terminated with exit 137 during ONNX tests. Full coverage and the + cross-platform matrix remain unverified locally; the partial run is not a + substitute for either gate. + +Reusable lesson assessment: mutation must invalidate cached negative membership +results. The regression test records that invariant in its owning implementation. diff --git a/src/pseudonymize/memory/bloom.py b/src/pseudonymize/memory/bloom.py index d394e61..a790fa5 100644 --- a/src/pseudonymize/memory/bloom.py +++ b/src/pseudonymize/memory/bloom.py @@ -8,6 +8,10 @@ class BloomFilter: __slots__ = ("_bit_array", "_contains_cached", "_hash_count", "_size") def __init__(self, capacity: int, error_rate: float = 0.001) -> None: + if capacity <= 0: + raise ValueError("capacity must be positive") + if not 0 < error_rate < 1: + raise ValueError("error_rate must be between 0 and 1") self._size = self._get_size(capacity, error_rate) self._hash_count = self._get_hash_count(self._size, capacity) self._bit_array = bytearray((self._size + 7) // 8) @@ -41,6 +45,9 @@ def add(self, item: str) -> None: digest = int.from_bytes(h.digest(), "big") % self._size self._bit_array[digest // 8] |= 1 << (digest % 8) + # Cached misses become stale whenever the bit array changes. + self._contains_cached.cache_clear() + def _raw_contains(self, item: str) -> bool: item_bytes = item.encode("utf-8") base = hashlib.sha256(item_bytes) @@ -58,7 +65,7 @@ def __contains__(self, item: str) -> bool: @staticmethod def _get_size(n: int, p: float) -> int: m = -(n * math.log(p)) / (math.log(2) ** 2) - return int(m) + return max(1, math.ceil(m)) @staticmethod def _get_hash_count(m: int, n: int) -> int: diff --git a/tests/unit/test_bloom.py b/tests/unit/test_bloom.py index ace3c63..3e45ad7 100644 --- a/tests/unit/test_bloom.py +++ b/tests/unit/test_bloom.py @@ -10,3 +10,39 @@ def test_bloom_filter() -> None: assert "Jonathan" not in bf assert "Doe" not in bf + + +def test_cached_miss_is_invalidated_after_add() -> None: + bf = BloomFilter(100) + for word in ("new-entry", "Élodie", "東京", ""): + assert word not in bf + bf.add(word) + assert word in bf + bf.add(word) + assert word in bf + + +def test_mutation_preserves_previous_members() -> None: + bf = BloomFilter.from_words(iter(["alpha", "beta"])) + assert "alpha" in bf + assert "gamma" not in bf + bf.add("gamma") + assert all(word in bf for word in ("alpha", "beta", "gamma")) + + +def test_small_filter_has_at_least_one_bit() -> None: + bf = BloomFilter(1, error_rate=0.99) + assert "entry" not in bf + bf.add("entry") + assert "entry" in bf + + +def test_invalid_filter_parameters() -> None: + import pytest + + for capacity in (0, -1): + with pytest.raises(ValueError, match="capacity must be positive"): + BloomFilter(capacity) + for rate in (0.0, -0.1, 1.0, 2.0, float("nan"), float("inf")): + with pytest.raises(ValueError, match="error_rate must be between 0 and 1"): + BloomFilter(10, rate) From c9d43edf95dc174567ed16c45c916b8f7396bea7 Mon Sep 17 00:00:00 2001 From: Paolo Mazza Date: Fri, 9 Oct 2026 11:58:06 +0000 Subject: [PATCH 2/2] fix: verify and audit the optional service extra --- CHANGELOG.md | 4 ++++ scripts/audit_extras.py | 2 +- scripts/audit_install.py | 4 ++++ scripts/verify_release.py | 4 +++- tests/integration/test_release_verifier.py | 13 +++++++++++++ 5 files changed, 25 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 93ec177..979c70b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ All notable changes follow Keep a Changelog and Semantic Versioning. - Optional bounded ONNX case recovery for lowercase PERSON candidates using high-confidence re-inference and constrained BIO decoding. Disabled by default; no name dictionaries or fabricated spans. - Python-owned Lambda HTTP service with verified model artifacts, required gateway authentication, ONNX-only person detection, and explicit inference failure. Hosted processing disables coreference expansion so each name occurrence requires model confirmation. +### Fixed +- Include the hosted-service extra in release verification and isolated installation + audits, while keeping service frameworks out of the dependency-free base import. + ## [1.36.0] - 2026-10-09 ### Added diff --git a/scripts/audit_extras.py b/scripts/audit_extras.py index 4c2cf20..7f8d028 100644 --- a/scripts/audit_extras.py +++ b/scripts/audit_extras.py @@ -6,7 +6,7 @@ import tempfile from pathlib import Path -DOCUMENTED_EXTRAS = ("html", "ml", "ocr", "office", "pdf", "remote") +DOCUMENTED_EXTRAS = ("html", "ml", "ocr", "office", "pdf", "remote", "service") def audit_extras(wheel_path: Path, python_executable: str | None = None) -> None: diff --git a/scripts/audit_install.py b/scripts/audit_install.py index c51b2c7..41fe382 100644 --- a/scripts/audit_install.py +++ b/scripts/audit_install.py @@ -11,6 +11,9 @@ "aiohttp", "docling", "docx", + "fastapi", + "pydantic", + "uvicorn", "httpx", "onnxruntime", "openpyxl", @@ -19,6 +22,7 @@ } EXPECTED_BASE_REQUIREMENTS: frozenset[str] = frozenset() EXTRA_CHECKS: dict[str, tuple[str, frozenset[str]]] = { + "service": ("pseudonymize.service", frozenset({"fastapi", "pydantic", "uvicorn"})), "remote": ("pseudonymize.backends.remote", frozenset({"httpx"})), "html": ("pseudonymize.html_xml", frozenset()), "office": ("pseudonymize.inspection.office", frozenset({"docx", "openpyxl"})), diff --git a/scripts/verify_release.py b/scripts/verify_release.py index 8b21d44..847f1f8 100644 --- a/scripts/verify_release.py +++ b/scripts/verify_release.py @@ -23,7 +23,9 @@ } EXPECTED_DEVELOPMENT_CLASSIFIER = "Development Status :: 5 - Production/Stable" EXPECTED_BASE_REQUIREMENTS: frozenset[str] = frozenset() -EXPECTED_EXTRAS: frozenset[str] = frozenset({"ml", "office", "pdf", "ocr", "remote", "html"}) +EXPECTED_EXTRAS: frozenset[str] = frozenset( + {"ml", "office", "pdf", "ocr", "remote", "html", "service"} +) REQUIRED_SDIST_FILES = frozenset( { "CHANGELOG.md", diff --git a/tests/integration/test_release_verifier.py b/tests/integration/test_release_verifier.py index 19aa4c3..5647aba 100644 --- a/tests/integration/test_release_verifier.py +++ b/tests/integration/test_release_verifier.py @@ -1,6 +1,7 @@ import email.message import io import tarfile +import tomllib import zipfile from pathlib import Path @@ -171,3 +172,15 @@ def test_verify_quality_gate_report_missing_file(tmp_path: Path) -> None: missing = tmp_path / "nonexistent.json" with pytest.raises(ValueError, match="quality gate report not found"): verify_quality_gate_report(missing) + + +def test_declared_extras_have_release_and_install_audits() -> None: + from scripts.audit_extras import DOCUMENTED_EXTRAS + from scripts.audit_install import EXTRA_CHECKS + + project_file = Path(__file__).resolve().parents[2] / "pyproject.toml" + with project_file.open("rb") as stream: + declared = set(tomllib.load(stream)["project"]["optional-dependencies"]) + assert declared == EXPECTED_EXTRAS + assert declared == set(DOCUMENTED_EXTRAS) + assert declared == set(EXTRA_CHECKS)