fix(retrieval): unicode-aware tokenization; empty chunk text is never a relevant hit

rag-platform/libs/rag/evaluation/retrieval_benchmark.py

@@ -72,7 +72,11 @@ def to_summary(self) -> str:
 def _is_relevant(chunk_id: str, text: str, sample: RetrievalEvalSample) -> bool:
     if chunk_id in sample.relevant_chunk_ids:
         return True
-    text_lower = text.lower()
+    text_lower = text.lower().strip()
+    if not text_lower:
+        # A chunk without text cannot evidence relevance: "" would
+        # substring-match every reference and inflate recall/MRR/NDCG.
+        return False
     for ref in sample.relevant_texts:
         ref_lower = ref.lower().strip()
         if ref_lower and (ref_lower in text_lower or text_lower in ref_lower):

rag-platform/libs/rag/pipeline/embedders.py

@@ -3,18 +3,12 @@
 from __future__ import annotations
 
 import math
-import re
 from collections import Counter
 from typing import List
 
 import numpy as np
 
-
-_TOKEN_RE = re.compile(r"[a-z0-9]+")
-
-
-def _tokenize(text: str) -> List[str]:
-    return _TOKEN_RE.findall(text.lower())
+from libs.rag.retrieval.keyword import tokenize as _tokenize
 
 
 class TfidfEmbedder:

rag-platform/libs/rag/retrieval/keyword.py

@@ -4,21 +4,156 @@
 
 import logging
 import math
-import re
 import time
+import unicodedata
 from collections import Counter
 from typing import Any, Dict, List, Optional, Tuple
 
 from .models import RetrievedChunk, RetrievalResult
 
 logger = logging.getLogger(__name__)
 
-TOKEN_PATTERN = re.compile(r"[a-z0-9]+")
+# Scripts whose writing omits spaces between words (Han ideographs,
+# Japanese kana, Thai, Lao, Khmer, Burmese): a run of them cannot rely on
+# word boundaries, so they tokenize per character plus overlapping bigrams
+# so multi-character queries still match adjacent characters.
+_NGRAM_SCRIPTS = frozenset({"CJK", "HIRAGANA", "KATAKANA", "THAI", "LAO", "KHMER", "MYANMAR"})
+
+# Kanji and kana mix inside single Japanese words ("食べる"), so contiguous
+# segments of these scripts tokenize as one run - otherwise no bigram spans
+# the boundary and a punctuated decoy scores the same as the real word.
+_JAPANESE_SCRIPTS = frozenset({"CJK", "HIRAGANA", "KATAKANA"})
+
+# Japanese iteration/prolonged marks whose Unicode names fall outside the
+# script buckets ("IDEOGRAPHIC ITERATION MARK", "KATAKANA-HIRAGANA
+# PROLONGED SOUND MARK"); they inherit the surrounding script so "人々" and
+# "ユーザー" stay single segments. As tokens they only ever repeat the
+# previous character, so the n-gram path keeps them inside bigrams but
+# never emits them as standalone unigrams. U+3007 is deliberately absent:
+# it is IDEOGRAPHIC NUMBER ZERO (a real character, classified CJK below),
+# not a repetition mark.
+_INHERIT_CODEPOINTS = frozenset(
+    {"\u3005", "\u3006", "\u30fc", "\u309d", "\u309e", "\u30fd", "\u30fe"}
+)
+
+
+def _clusters(seg: str) -> List[str]:
+    """Split into base+marks clusters so n-grams never detach a vowel sign
+    or tone mark from its base (Thai "กิ" must not emit a bare "ิ")."""
+    clusters: List[str] = []
+    for ch in seg:
+        if clusters and _is_mark(ch):
+            clusters[-1] += ch
+        else:
+            clusters.append(ch)
+    return clusters
+
+# Characters that join the current token: letters and digits from any
+# script, plus combining marks (Devanagari vowel signs, Arabic harakat,
+# etc. - NFC leaves many of them decomposed, and dropping them collapses
+# distinct words like "की" / "का" / "कि" onto one token).
+def _is_mark(ch: str) -> bool:
+    return unicodedata.category(ch).startswith("M")
+
+
+def _script_of(ch: str) -> Optional[str]:
+    """Script family of a character; None for script-neutral characters."""
+    if ch == "\u3007":
+        # IDEOGRAPHIC NUMBER ZERO is a real character, not a mark.
+        return "CJK"
+    if ch.isdigit() or _is_mark(ch) or ch in _INHERIT_CODEPOINTS:
+        # Digits, combining marks, and Japanese iteration/prolonged marks
+        # inherit the surrounding script.
+        return None
+    name = unicodedata.name(ch, "")
+    return name.split(" ")[0] if name else None
+
+
+def _emit_word(word: str, tokens: List[str]) -> None:
+    """Emit a matched word, splitting at script boundaries.
+
+    Each script segment tokenizes its own way: CJK segments emit each
+    character (meaning is per-character and the text is not space-separated)
+    plus overlapping bigrams so a multi-character query phrase matches
+    adjacent characters in the document; every other script stays whole, so
+    "Python東京" and "Python으로" both still yield "python".
+    """
+    segment = ""
+    segment_script: Optional[str] = None
+    segments: List[Tuple[str, str]] = []
+    for ch in word:
+        script = _script_of(ch)
+        if script is None:
+            # Digits and unattributed characters inherit the current script -
+            # except digits beside (or opening a word before) a non-Latin
+            # script, which form their own whole token so "إصدار2026" and
+            # "2026東京" both keep a searchable "2026".
+            if ch.isdigit() and segment_script not in ("LATIN", "DIGIT"):
+                script = "DIGIT"
+            else:
+                segment += ch
+                continue
+        if segment_script is None:
+            segment_script = script
+        if script != segment_script:
+            segments.append((segment, segment_script))
+            segment = ch
+            segment_script = script
+        else:
+            segment += ch
+    if segment:
+        segments.append((segment, segment_script or ""))
+    # Merge contiguous Japanese-script segments so n-grams span kanji/kana
+    # boundaries.
+    runs: List[List[Any]] = []
+    for seg, script in segments:
+        if runs and script in _JAPANESE_SCRIPTS and runs[-1][1] in _JAPANESE_SCRIPTS:
+            runs[-1][0] += seg
+        else:
+            runs.append([seg, script])
+    for seg, script in runs:
+        if script in _NGRAM_SCRIPTS:
+            clusters = _clusters(seg)
+            # Iteration/prolonged marks (々, ー) keep adjacency via bigrams
+            # but are never standalone terms - every Japanese word shares
+            # them, so as unigrams they false-match unrelated words.
+            tokens.extend(c for c in clusters if c not in _INHERIT_CODEPOINTS)
+            tokens.extend(
+                clusters[k] + clusters[k + 1] for k in range(len(clusters) - 1)
+            )
+        else:
+            tokens.append(seg)
 
 
 def tokenize(text: str) -> List[str]:
-    """Lowercase alphanumeric tokenization."""
-    return TOKEN_PATTERN.findall(text.lower())
+    """Casefolded, NFKC-normalized, unicode-aware tokenization.
+
+    NFKC first: extraction pipelines often produce decomposed or
+    compatibility-form Unicode, and a precomposed plain query must still
+    match ("Mu\u0308nchen" == "München", "Ｐｙｔｈｏｎ" == "Python").
+    """
+    tokens: List[str] = []
+    # Unicode default caseless matching is NFKC(casefold(NFKC(s))): the
+    # first pass turns compatibility forms into ordinary characters
+    # (full-width Latin, half-width katakana, squared unit symbols like
+    # "㎒" -> "MHz") so case folding reaches every letter, and the second
+    # settles anything case folding decomposed.
+    normalized = unicodedata.normalize("NFKC", unicodedata.normalize("NFKC", text).casefold())
+    i = 0
+    n = len(normalized)
+    while i < n:
+        if not normalized[i].isalnum():
+            # Tokens start on letters/digits only: a detached combining mark
+            # (e.g. an emoji variation selector) must not become a token, or
+            # every emoji collapses onto one shared, meaningless token.
+            i += 1
+            continue
+        j = i + 1
+        while j < n and (normalized[j].isalnum() or _is_mark(normalized[j])):
+            j += 1
+        _emit_word(normalized[i:j], tokens)
+        i = j
+    return tokens
 
 
 class BM25Index:

rag-platform/libs/rag/retrieval/tests/test_retrieval.py

@@ -13,6 +13,8 @@
 from libs.rag.evaluation.models import EvaluationDataset, EvaluationSample
 from libs.rag.evaluation.retrieval_benchmark import (
     RetrievalBenchmark,
+    RetrievalEvalSample,
+    _is_relevant,
     mrr,
     ndcg_at_k,
     precision_at_k,
@@ -75,6 +77,144 @@ def test_basic(self):
         assert tokenize("Hello World 123") == ["hello", "world", "123"]
 
 
+
+
+class TestTokenizeUnicode:
+    def test_accented_word_stays_whole(self):
+        assert tokenize("München") == ["münchen"]
+
+    def test_casefold_maps_sharp_s(self):
+        assert tokenize("Straße") == ["strasse"]
+
+    def test_cjk_text_produces_tokens(self):
+        tokens = tokenize("東京")
+        assert "東" in tokens
+        assert "京" in tokens
+        assert "東京" in tokens
+
+    def test_mixed_script_run_keeps_latin_token(self):
+        tokens = tokenize("Python東京")
+        assert "python" in tokens
+        assert "東" in tokens
+        assert "京" in tokens
+
+    def test_mixed_script_cjk_first(self):
+        tokens = tokenize("使用Python开发")
+        assert "python" in tokens
+        assert "使" in tokens and "用" in tokens
+
+    def test_canonical_equivalence_matches(self):
+        decomposed = tokenize("Mu\u0308nchen")
+        precomposed = tokenize("München")
+        assert decomposed == precomposed == ["münchen"]
+
+    def test_mixed_latin_hangul_run_splits(self):
+        tokens = tokenize("Python으로")
+        assert "python" in tokens
+        assert "으로" in tokens
+
+    def test_combining_marks_stay_attached(self):
+        # Devanagari vowel signs remain combining marks after NFC; dropping
+        # them would collapse these distinct words onto one token.
+        assert tokenize("की") == ["की"]
+        tokens = set()
+        for word in ("की", "का", "कि", "कु"):
+            tokens.update(tokenize(word))
+        assert tokens == {"की", "का", "कि", "कु"}
+
+    def test_thai_ngrams_keep_marks_attached(self):
+        tokens = tokenize("กิ")
+        assert tokens == ["กิ"]
+        # A bare vowel mark must never be a shared term.
+        idx = BM25Index([("c1", "มิ")])
+        assert idx.search("กิ") == []
+
+    def test_japanese_iteration_marks_keep_adjacency(self):
+        tokens = tokenize("人々")
+        assert "人々" in tokens
+        kana = tokenize("ユーザー")
+        assert "ユー" in kana and "ザー" in kana
+
+    def test_iteration_marks_are_not_standalone_unigrams(self):
+        idx = BM25Index([("d", "スーパー")])
+        assert idx.search("ユーザー") == []
+        assert "々" not in tokenize("人々")
+        assert "ー" not in tokenize("ユーザー")
+
+    def test_digit_runs_beside_non_latin_scripts(self):
+        tokens = tokenize("إصدار2026")
+        assert "2026" in tokens
+        idx = BM25Index([("d", "إصدار2026")])
+        assert idx.search("2026") != []
+        # Latin words keep their digits inside the token.
+        assert "bm25" in tokenize("BM25")
+
+    def test_kanji_kana_boundary_keeps_bigram(self):
+        assert "食べ" in tokenize("食べる")
+        assert "食べ" not in tokenize("食、べる")
+
+    def test_compatibility_forms_match_plain_query(self):
+        assert tokenize("Ｐｙｔｈｏｎ") == ["python"]
+        idx = BM25Index([("d", "Ｐｙｔｈｏｎ ３")])
+        assert idx.search("Python") != []
+
+    def test_detached_combining_marks_form_no_token(self):
+        # Emoji variation selectors are combining marks; unattached they must
+        # not tokenize, or all emoji collapse onto one shared token.
+        assert tokenize("❤️") == []
+        assert tokenize("☀️") == []
+        heart = set(tokenize("I ❤️ pizza"))
+        sun = set(tokenize("☀️ beach"))
+        assert heart.isdisjoint(sun & {"️"})
+
+    def test_marks_inherit_script_inside_words(self):
+        # A mark with no precomposed form must not split a Latin run.
+        assert tokenize("a\u0330b") == ["a\u0330b"]
+
+    def test_digit_runs_next_to_cjk_stay_whole(self):
+        tokens = tokenize("東京2026年")
+        assert "2026" in tokens
+
+    def test_spaceless_thai_matches_via_ngrams(self):
+        doc_tokens = set(tokenize("กรุงเทพมหานครเป็นเมืองหลวงของประเทศไทย"))
+        query_tokens = set(tokenize("ประเทศไทย"))
+        assert query_tokens <= doc_tokens
+
+    def test_digits_stay_inside_words(self):
+        assert tokenize("BM25 k1=1.5") == ["bm25", "k1", "1", "5"]
+
+    def test_cjk_phrase_emits_bigram(self):
+        assert "東京" in tokenize("東京は日本の首都です")
+        # The same characters separated do NOT form the phrase token.
+        assert "東京" not in tokenize("東と京")
+
+    def test_bm25_finds_cjk_document(self):
+        index = BM25Index([("doc-1", "東京は日本の首都です")])
+        results = index.search("東京", top_k=1)
+        assert [r[0] for r in results] == ["doc-1"]
+
+    def test_bm25_finds_accented_document(self):
+        index = BM25Index(
+            [("doc-1", "München ist eine Stadt"), ("doc-2", "Berlin ist eine Stadt")]
+        )
+        results = index.search("München", top_k=1)
+        assert results[0][0] == "doc-1"
+
+
+class TestIsRelevantEmptyText:
+    def test_missing_chunk_text_is_not_a_relevant_hit(self):
+        sample = RetrievalEvalSample(
+            question="q",
+            relevant_chunk_ids=set(),
+            relevant_texts=["alpha is the source context"],
+        )
+        assert _is_relevant("missing-text-chunk", "", sample) is False
+
+    def test_missing_chunk_text_still_counts_by_id(self):
+        sample = RetrievalEvalSample(question="q", relevant_chunk_ids={"c1"})
+        assert _is_relevant("c1", "", sample) is True
+
+
 class TestBM25Index:
     def test_search_returns_relevant(self):
         index = BM25Index(CORPUS)
@@ -240,3 +380,37 @@ def test_benchmark_comparison(self):
         assert "dense" in table
         assert "keyword" in table
         assert "hybrid" in table
+
+
+def test_leading_digit_runs_before_non_latin_scripts():
+    # Digits that OPEN a word beside a non-Latin script must also form their
+    # own token: "2026إصدار" and "2026東京" stay searchable by "2026".
+    assert "2026" in tokenize("2026إصدار")
+    assert "2026" in tokenize("2026東京")
+    # Latin-leading words keep their digits: "bm25" stays one token.
+    assert "bm25" in tokenize("bm25")
+
+
+def test_compatibility_forms_normalize_before_casefold():
+    # NFKC after casefold leaves compatibility letters unfolded: ㎒
+    # decomposes to "MHz" and must fold to the same token as "MHz".
+    assert tokenize("㎒") == tokenize("MHz")
+    assert tokenize("𝐀") == tokenize("a")
+
+
+def test_ideographic_zero_is_a_searchable_unigram():
+    # 〇 is IDEOGRAPHIC NUMBER ZERO, not an iteration mark: "二〇二六" must
+    # answer a "〇" query.
+    assert "〇" in tokenize("二〇二六")
+    assert "二〇" in tokenize("二〇二六")
+
+
+def test_kana_iteration_marks_are_not_standalone_unigrams():
+    # ゝゞヽヾ repeat the previous kana; as shared unigrams they false-match
+    # unrelated words ("さゝ" vs "くゝ").
+    assert "ゝ" not in tokenize("さゝ")
+    assert "ゞ" not in tokenize("さゞ")
+    assert "ヽ" not in tokenize("サヽ")
+    assert "ヾ" not in tokenize("サヾ")
+    # Adjacency is still searchable via the bigram.
+    assert "さゝ" in tokenize("さゝ")