Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 51 additions & 0 deletions tests/_stand_in_tokenizer.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
"""A deterministic stand-in for the BGE-M3 tokenizer the chunker uses.

The chunker needs a tokenizer to count tokens, window long proofs and
truncate statements. Its tests check chunk structure (pairing, kinds,
ids, window mechanics), not BGE-M3's subword splits, so they run against
this instead of a 2 GB model download. Real-tokenizer behaviour is pinned
separately by ``requires_model`` tests.

A token is a run of up to four non-space characters, which keeps counts
in the same range as BGE-M3's on English and LaTeX text. Ids are stable
within an instance, and character offsets are real, so windowing and
truncation slice the text exactly as they would with the real tokenizer.
"""

from __future__ import annotations

import re

_TOKEN = re.compile(r"\S{1,4}")


class StandInTokenizer:
"""The three calls ``ingest.chunker`` makes on a Hugging Face tokenizer."""

def __init__(self) -> None:
self._ids: dict[str, int] = {}
self._pieces: list[str] = []

def _id(self, piece: str) -> int:
if piece not in self._ids:
self._ids[piece] = len(self._pieces)
self._pieces.append(piece)
return self._ids[piece]

def __call__(
self,
text: str,
add_special_tokens: bool = False,
return_offsets_mapping: bool = False,
) -> dict[str, list]:
matches = list(_TOKEN.finditer(text))
out: dict[str, list] = {"input_ids": [self._id(m.group()) for m in matches]}
if return_offsets_mapping:
out["offset_mapping"] = [(m.start(), m.end()) for m in matches]
return out

def encode(self, text: str, add_special_tokens: bool = False) -> list[int]:
return self(text)["input_ids"]

def decode(self, token_ids: list[int], skip_special_tokens: bool = True) -> str:
return " ".join(self._pieces[i] for i in token_ids)
18 changes: 18 additions & 0 deletions tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -289,6 +289,24 @@ def rerank(request: pytest.FixtureRequest) -> bool:
return rerank_set


@pytest.fixture(autouse=True)
def _stand_in_chunker_tokenizer(request, monkeypatch):
"""Give the chunker a deterministic tokenizer instead of BGE-M3's.

Without this, every test that runs the chunker needs the BGE-M3
tokenizer in the local Hugging Face cache or network access to fetch
it, and fails otherwise. ``requires_model`` tests are left alone so
they exercise the real tokenizer.
"""
if request.node.get_closest_marker("requires_model") is not None:
yield
return
from tests._stand_in_tokenizer import StandInTokenizer

monkeypatch.setattr("ingest.chunker._tokenizer", StandInTokenizer())
yield


@pytest.fixture(autouse=True)
def _patched_store_stats_path(tmp_path, monkeypatch):
"""Redirect ``ingest.store.STORE_STATS_PATH`` into ``tmp_path``.
Expand Down
49 changes: 49 additions & 0 deletions tests/test_chunker_real_tokenizer.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
"""Chunker tests that depend on token counts, re-run on the real BGE-M3
tokenizer.

Every other test gives the chunker a stand-in tokenizer
(``tests/_stand_in_tokenizer.py``). These classes re-run the ones whose
assertions are about token budgets, windows, truncation and the golden
fixtures, so the real tokenizer's behaviour stays pinned. They need the
tokenizer in the Hugging Face cache: ``pytest -m requires_model``.
"""

from __future__ import annotations

import pytest

from tests import test_chunker as stand_in_suite

pytestmark = pytest.mark.requires_model


class TestRealFixtureSuite(stand_in_suite.TestFixtureSuite):
pass


class TestRealProofWindowSplitting(stand_in_suite.TestProofWindowSplitting):
pass


class TestRealStatementTokenBudget(stand_in_suite.TestStatementTokenBudget):
pass


class TestRealBudgetBump(stand_in_suite.TestB2BudgetBumpTakesEffect):
pass


class TestRealStmtTruncationFlag(stand_in_suite.TestF5StmtTruncationFlag):
pass


class TestRealCharOffsetWindowing(stand_in_suite.TestF6CharOffsetWindowing):
pass


def test_runs_on_the_real_tokenizer():
"""Guards the guard: if the stand-in leaked in here, the classes above
would re-test the stand-in and pin nothing."""
from ingest.chunker import _get_tokenizer

assert type(_get_tokenizer()).__name__ != "StandInTokenizer"
5 changes: 2 additions & 3 deletions tests/test_textbook_chunker.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,8 @@
a synthetic LaTeXML-shaped HTML5 fixture into a tmp notebook layout,
patch `ingest.textbook_chunker.NOTEBOOKS_BASE`, run `chunk_textbook`.

The tokenizer (BGE-M3 vocab, ~5 MB cached) is loaded by the reused
`_count_tokens` primitive — same as the arXiv chunker tests, which run
without a `requires_model` marker.
Token counting runs on the stand-in tokenizer ``tests/conftest.py``
installs, as in the arXiv chunker tests.
"""

from __future__ import annotations
Expand Down
Loading