diff --git a/tests/_stand_in_tokenizer.py b/tests/_stand_in_tokenizer.py new file mode 100644 index 0000000..e24e27c --- /dev/null +++ b/tests/_stand_in_tokenizer.py @@ -0,0 +1,51 @@ +"""A deterministic stand-in for the BGE-M3 tokenizer the chunker uses. + +The chunker needs a tokenizer to count tokens, window long proofs and +truncate statements. Its tests check chunk structure (pairing, kinds, +ids, window mechanics), not BGE-M3's subword splits, so they run against +this instead of a 2 GB model download. Real-tokenizer behaviour is pinned +separately by ``requires_model`` tests. + +A token is a run of up to four non-space characters, which keeps counts +in the same range as BGE-M3's on English and LaTeX text. Ids are stable +within an instance, and character offsets are real, so windowing and +truncation slice the text exactly as they would with the real tokenizer. +""" + +from __future__ import annotations + +import re + +_TOKEN = re.compile(r"\S{1,4}") + + +class StandInTokenizer: + """The three calls ``ingest.chunker`` makes on a Hugging Face tokenizer.""" + + def __init__(self) -> None: + self._ids: dict[str, int] = {} + self._pieces: list[str] = [] + + def _id(self, piece: str) -> int: + if piece not in self._ids: + self._ids[piece] = len(self._pieces) + self._pieces.append(piece) + return self._ids[piece] + + def __call__( + self, + text: str, + add_special_tokens: bool = False, + return_offsets_mapping: bool = False, + ) -> dict[str, list]: + matches = list(_TOKEN.finditer(text)) + out: dict[str, list] = {"input_ids": [self._id(m.group()) for m in matches]} + if return_offsets_mapping: + out["offset_mapping"] = [(m.start(), m.end()) for m in matches] + return out + + def encode(self, text: str, add_special_tokens: bool = False) -> list[int]: + return self(text)["input_ids"] + + def decode(self, token_ids: list[int], skip_special_tokens: bool = True) -> str: + return " ".join(self._pieces[i] for i in token_ids) diff --git a/tests/conftest.py b/tests/conftest.py index 82b35fa..7b91c67 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -289,6 +289,24 @@ def rerank(request: pytest.FixtureRequest) -> bool: return rerank_set +@pytest.fixture(autouse=True) +def _stand_in_chunker_tokenizer(request, monkeypatch): + """Give the chunker a deterministic tokenizer instead of BGE-M3's. + + Without this, every test that runs the chunker needs the BGE-M3 + tokenizer in the local Hugging Face cache or network access to fetch + it, and fails otherwise. ``requires_model`` tests are left alone so + they exercise the real tokenizer. + """ + if request.node.get_closest_marker("requires_model") is not None: + yield + return + from tests._stand_in_tokenizer import StandInTokenizer + + monkeypatch.setattr("ingest.chunker._tokenizer", StandInTokenizer()) + yield + + @pytest.fixture(autouse=True) def _patched_store_stats_path(tmp_path, monkeypatch): """Redirect ``ingest.store.STORE_STATS_PATH`` into ``tmp_path``. diff --git a/tests/test_chunker_real_tokenizer.py b/tests/test_chunker_real_tokenizer.py new file mode 100644 index 0000000..3b1f52a --- /dev/null +++ b/tests/test_chunker_real_tokenizer.py @@ -0,0 +1,49 @@ +"""Chunker tests that depend on token counts, re-run on the real BGE-M3 +tokenizer. + +Every other test gives the chunker a stand-in tokenizer +(``tests/_stand_in_tokenizer.py``). These classes re-run the ones whose +assertions are about token budgets, windows, truncation and the golden +fixtures, so the real tokenizer's behaviour stays pinned. They need the +tokenizer in the Hugging Face cache: ``pytest -m requires_model``. +""" + +from __future__ import annotations + +import pytest + +from tests import test_chunker as stand_in_suite + +pytestmark = pytest.mark.requires_model + + +class TestRealFixtureSuite(stand_in_suite.TestFixtureSuite): + pass + + +class TestRealProofWindowSplitting(stand_in_suite.TestProofWindowSplitting): + pass + + +class TestRealStatementTokenBudget(stand_in_suite.TestStatementTokenBudget): + pass + + +class TestRealBudgetBump(stand_in_suite.TestB2BudgetBumpTakesEffect): + pass + + +class TestRealStmtTruncationFlag(stand_in_suite.TestF5StmtTruncationFlag): + pass + + +class TestRealCharOffsetWindowing(stand_in_suite.TestF6CharOffsetWindowing): + pass + + +def test_runs_on_the_real_tokenizer(): + """Guards the guard: if the stand-in leaked in here, the classes above + would re-test the stand-in and pin nothing.""" + from ingest.chunker import _get_tokenizer + + assert type(_get_tokenizer()).__name__ != "StandInTokenizer" diff --git a/tests/test_textbook_chunker.py b/tests/test_textbook_chunker.py index ac6d020..21fc9ae 100644 --- a/tests/test_textbook_chunker.py +++ b/tests/test_textbook_chunker.py @@ -4,9 +4,8 @@ a synthetic LaTeXML-shaped HTML5 fixture into a tmp notebook layout, patch `ingest.textbook_chunker.NOTEBOOKS_BASE`, run `chunk_textbook`. -The tokenizer (BGE-M3 vocab, ~5 MB cached) is loaded by the reused -`_count_tokens` primitive — same as the arXiv chunker tests, which run -without a `requires_model` marker. +Token counting runs on the stand-in tokenizer ``tests/conftest.py`` +installs, as in the arXiv chunker tests. """ from __future__ import annotations