diff --git a/rag-platform/libs/rag/evaluation/retrieval_benchmark.py b/rag-platform/libs/rag/evaluation/retrieval_benchmark.py index 3fa8df0..27d6969 100644 --- a/rag-platform/libs/rag/evaluation/retrieval_benchmark.py +++ b/rag-platform/libs/rag/evaluation/retrieval_benchmark.py @@ -72,7 +72,11 @@ def to_summary(self) -> str: def _is_relevant(chunk_id: str, text: str, sample: RetrievalEvalSample) -> bool: if chunk_id in sample.relevant_chunk_ids: return True - text_lower = text.lower() + text_lower = text.lower().strip() + if not text_lower: + # A chunk without text cannot evidence relevance: "" would + # substring-match every reference and inflate recall/MRR/NDCG. + return False for ref in sample.relevant_texts: ref_lower = ref.lower().strip() if ref_lower and (ref_lower in text_lower or text_lower in ref_lower): diff --git a/rag-platform/libs/rag/pipeline/embedders.py b/rag-platform/libs/rag/pipeline/embedders.py index 8ff8767..ada43c3 100644 --- a/rag-platform/libs/rag/pipeline/embedders.py +++ b/rag-platform/libs/rag/pipeline/embedders.py @@ -3,18 +3,12 @@ from __future__ import annotations import math -import re from collections import Counter from typing import List import numpy as np - -_TOKEN_RE = re.compile(r"[a-z0-9]+") - - -def _tokenize(text: str) -> List[str]: - return _TOKEN_RE.findall(text.lower()) +from libs.rag.retrieval.keyword import tokenize as _tokenize class TfidfEmbedder: diff --git a/rag-platform/libs/rag/retrieval/keyword.py b/rag-platform/libs/rag/retrieval/keyword.py index acff495..afb4cf1 100644 --- a/rag-platform/libs/rag/retrieval/keyword.py +++ b/rag-platform/libs/rag/retrieval/keyword.py @@ -4,8 +4,8 @@ import logging import math -import re import time +import unicodedata from collections import Counter from typing import Any, Dict, List, Optional, Tuple @@ -13,12 +13,147 @@ logger = logging.getLogger(__name__) -TOKEN_PATTERN = re.compile(r"[a-z0-9]+") +# Scripts whose writing omits spaces between words (Han ideographs, +# Japanese kana, Thai, Lao, Khmer, Burmese): a run of them cannot rely on +# word boundaries, so they tokenize per character plus overlapping bigrams +# so multi-character queries still match adjacent characters. +_NGRAM_SCRIPTS = frozenset({"CJK", "HIRAGANA", "KATAKANA", "THAI", "LAO", "KHMER", "MYANMAR"}) + +# Kanji and kana mix inside single Japanese words ("食べる"), so contiguous +# segments of these scripts tokenize as one run - otherwise no bigram spans +# the boundary and a punctuated decoy scores the same as the real word. +_JAPANESE_SCRIPTS = frozenset({"CJK", "HIRAGANA", "KATAKANA"}) + +# Japanese iteration/prolonged marks whose Unicode names fall outside the +# script buckets ("IDEOGRAPHIC ITERATION MARK", "KATAKANA-HIRAGANA +# PROLONGED SOUND MARK"); they inherit the surrounding script so "人々" and +# "ユーザー" stay single segments. As tokens they only ever repeat the +# previous character, so the n-gram path keeps them inside bigrams but +# never emits them as standalone unigrams. U+3007 is deliberately absent: +# it is IDEOGRAPHIC NUMBER ZERO (a real character, classified CJK below), +# not a repetition mark. +_INHERIT_CODEPOINTS = frozenset( + {"\u3005", "\u3006", "\u30fc", "\u309d", "\u309e", "\u30fd", "\u30fe"} +) + + +def _clusters(seg: str) -> List[str]: + """Split into base+marks clusters so n-grams never detach a vowel sign + or tone mark from its base (Thai "กิ" must not emit a bare "ิ").""" + clusters: List[str] = [] + for ch in seg: + if clusters and _is_mark(ch): + clusters[-1] += ch + else: + clusters.append(ch) + return clusters + +# Characters that join the current token: letters and digits from any +# script, plus combining marks (Devanagari vowel signs, Arabic harakat, +# etc. - NFC leaves many of them decomposed, and dropping them collapses +# distinct words like "की" / "का" / "कि" onto one token). +def _is_mark(ch: str) -> bool: + return unicodedata.category(ch).startswith("M") + + +def _script_of(ch: str) -> Optional[str]: + """Script family of a character; None for script-neutral characters.""" + if ch == "\u3007": + # IDEOGRAPHIC NUMBER ZERO is a real character, not a mark. + return "CJK" + if ch.isdigit() or _is_mark(ch) or ch in _INHERIT_CODEPOINTS: + # Digits, combining marks, and Japanese iteration/prolonged marks + # inherit the surrounding script. + return None + name = unicodedata.name(ch, "") + return name.split(" ")[0] if name else None + + +def _emit_word(word: str, tokens: List[str]) -> None: + """Emit a matched word, splitting at script boundaries. + + Each script segment tokenizes its own way: CJK segments emit each + character (meaning is per-character and the text is not space-separated) + plus overlapping bigrams so a multi-character query phrase matches + adjacent characters in the document; every other script stays whole, so + "Python東京" and "Python으로" both still yield "python". + """ + segment = "" + segment_script: Optional[str] = None + segments: List[Tuple[str, str]] = [] + for ch in word: + script = _script_of(ch) + if script is None: + # Digits and unattributed characters inherit the current script - + # except digits beside (or opening a word before) a non-Latin + # script, which form their own whole token so "إصدار2026" and + # "2026東京" both keep a searchable "2026". + if ch.isdigit() and segment_script not in ("LATIN", "DIGIT"): + script = "DIGIT" + else: + segment += ch + continue + if segment_script is None: + segment_script = script + if script != segment_script: + segments.append((segment, segment_script)) + segment = ch + segment_script = script + else: + segment += ch + if segment: + segments.append((segment, segment_script or "")) + # Merge contiguous Japanese-script segments so n-grams span kanji/kana + # boundaries. + runs: List[List[Any]] = [] + for seg, script in segments: + if runs and script in _JAPANESE_SCRIPTS and runs[-1][1] in _JAPANESE_SCRIPTS: + runs[-1][0] += seg + else: + runs.append([seg, script]) + for seg, script in runs: + if script in _NGRAM_SCRIPTS: + clusters = _clusters(seg) + # Iteration/prolonged marks (々, ー) keep adjacency via bigrams + # but are never standalone terms - every Japanese word shares + # them, so as unigrams they false-match unrelated words. + tokens.extend(c for c in clusters if c not in _INHERIT_CODEPOINTS) + tokens.extend( + clusters[k] + clusters[k + 1] for k in range(len(clusters) - 1) + ) + else: + tokens.append(seg) def tokenize(text: str) -> List[str]: - """Lowercase alphanumeric tokenization.""" - return TOKEN_PATTERN.findall(text.lower()) + """Casefolded, NFKC-normalized, unicode-aware tokenization. + + NFKC first: extraction pipelines often produce decomposed or + compatibility-form Unicode, and a precomposed plain query must still + match ("Mu\u0308nchen" == "München", "Ｐｙｔｈｏｎ" == "Python"). + """ + tokens: List[str] = [] + # Unicode default caseless matching is NFKC(casefold(NFKC(s))): the + # first pass turns compatibility forms into ordinary characters + # (full-width Latin, half-width katakana, squared unit symbols like + # "㎒" -> "MHz") so case folding reaches every letter, and the second + # settles anything case folding decomposed. + normalized = unicodedata.normalize("NFKC", unicodedata.normalize("NFKC", text).casefold()) + i = 0 + n = len(normalized) + while i < n: + if not normalized[i].isalnum(): + # Tokens start on letters/digits only: a detached combining mark + # (e.g. an emoji variation selector) must not become a token, or + # every emoji collapses onto one shared, meaningless token. + i += 1 + continue + j = i + 1 + while j < n and (normalized[j].isalnum() or _is_mark(normalized[j])): + j += 1 + _emit_word(normalized[i:j], tokens) + i = j + return tokens class BM25Index: diff --git a/rag-platform/libs/rag/retrieval/tests/test_retrieval.py b/rag-platform/libs/rag/retrieval/tests/test_retrieval.py index e3ac9fb..8ef7873 100644 --- a/rag-platform/libs/rag/retrieval/tests/test_retrieval.py +++ b/rag-platform/libs/rag/retrieval/tests/test_retrieval.py @@ -13,6 +13,8 @@ from libs.rag.evaluation.models import EvaluationDataset, EvaluationSample from libs.rag.evaluation.retrieval_benchmark import ( RetrievalBenchmark, + RetrievalEvalSample, + _is_relevant, mrr, ndcg_at_k, precision_at_k, @@ -75,6 +77,144 @@ def test_basic(self): assert tokenize("Hello World 123") == ["hello", "world", "123"] + + +class TestTokenizeUnicode: + def test_accented_word_stays_whole(self): + assert tokenize("München") == ["münchen"] + + def test_casefold_maps_sharp_s(self): + assert tokenize("Straße") == ["strasse"] + + def test_cjk_text_produces_tokens(self): + tokens = tokenize("東京") + assert "東" in tokens + assert "京" in tokens + assert "東京" in tokens + + def test_mixed_script_run_keeps_latin_token(self): + tokens = tokenize("Python東京") + assert "python" in tokens + assert "東" in tokens + assert "京" in tokens + + def test_mixed_script_cjk_first(self): + tokens = tokenize("使用Python开发") + assert "python" in tokens + assert "使" in tokens and "用" in tokens + + def test_canonical_equivalence_matches(self): + decomposed = tokenize("Mu\u0308nchen") + precomposed = tokenize("München") + assert decomposed == precomposed == ["münchen"] + + def test_mixed_latin_hangul_run_splits(self): + tokens = tokenize("Python으로") + assert "python" in tokens + assert "으로" in tokens + + def test_combining_marks_stay_attached(self): + # Devanagari vowel signs remain combining marks after NFC; dropping + # them would collapse these distinct words onto one token. + assert tokenize("की") == ["की"] + tokens = set() + for word in ("की", "का", "कि", "कु"): + tokens.update(tokenize(word)) + assert tokens == {"की", "का", "कि", "कु"} + + def test_thai_ngrams_keep_marks_attached(self): + tokens = tokenize("กิ") + assert tokens == ["กิ"] + # A bare vowel mark must never be a shared term. + idx = BM25Index([("c1", "มิ")]) + assert idx.search("กิ") == [] + + def test_japanese_iteration_marks_keep_adjacency(self): + tokens = tokenize("人々") + assert "人々" in tokens + kana = tokenize("ユーザー") + assert "ユー" in kana and "ザー" in kana + + def test_iteration_marks_are_not_standalone_unigrams(self): + idx = BM25Index([("d", "スーパー")]) + assert idx.search("ユーザー") == [] + assert "々" not in tokenize("人々") + assert "ー" not in tokenize("ユーザー") + + def test_digit_runs_beside_non_latin_scripts(self): + tokens = tokenize("إصدار2026") + assert "2026" in tokens + idx = BM25Index([("d", "إصدار2026")]) + assert idx.search("2026") != [] + # Latin words keep their digits inside the token. + assert "bm25" in tokenize("BM25") + + def test_kanji_kana_boundary_keeps_bigram(self): + assert "食べ" in tokenize("食べる") + assert "食べ" not in tokenize("食、べる") + + def test_compatibility_forms_match_plain_query(self): + assert tokenize("Ｐｙｔｈｏｎ") == ["python"] + idx = BM25Index([("d", "Ｐｙｔｈｏｎ ３")]) + assert idx.search("Python") != [] + + def test_detached_combining_marks_form_no_token(self): + # Emoji variation selectors are combining marks; unattached they must + # not tokenize, or all emoji collapse onto one shared token. + assert tokenize("❤️") == [] + assert tokenize("☀️") == [] + heart = set(tokenize("I ❤️ pizza")) + sun = set(tokenize("☀️ beach")) + assert heart.isdisjoint(sun & {"️"}) + + def test_marks_inherit_script_inside_words(self): + # A mark with no precomposed form must not split a Latin run. + assert tokenize("a\u0330b") == ["a\u0330b"] + + def test_digit_runs_next_to_cjk_stay_whole(self): + tokens = tokenize("東京2026年") + assert "2026" in tokens + + def test_spaceless_thai_matches_via_ngrams(self): + doc_tokens = set(tokenize("กรุงเทพมหานครเป็นเมืองหลวงของประเทศไทย")) + query_tokens = set(tokenize("ประเทศไทย")) + assert query_tokens <= doc_tokens + + def test_digits_stay_inside_words(self): + assert tokenize("BM25 k1=1.5") == ["bm25", "k1", "1", "5"] + + def test_cjk_phrase_emits_bigram(self): + assert "東京" in tokenize("東京は日本の首都です") + # The same characters separated do NOT form the phrase token. + assert "東京" not in tokenize("東と京") + + def test_bm25_finds_cjk_document(self): + index = BM25Index([("doc-1", "東京は日本の首都です")]) + results = index.search("東京", top_k=1) + assert [r[0] for r in results] == ["doc-1"] + + def test_bm25_finds_accented_document(self): + index = BM25Index( + [("doc-1", "München ist eine Stadt"), ("doc-2", "Berlin ist eine Stadt")] + ) + results = index.search("München", top_k=1) + assert results[0][0] == "doc-1" + + +class TestIsRelevantEmptyText: + def test_missing_chunk_text_is_not_a_relevant_hit(self): + sample = RetrievalEvalSample( + question="q", + relevant_chunk_ids=set(), + relevant_texts=["alpha is the source context"], + ) + assert _is_relevant("missing-text-chunk", "", sample) is False + + def test_missing_chunk_text_still_counts_by_id(self): + sample = RetrievalEvalSample(question="q", relevant_chunk_ids={"c1"}) + assert _is_relevant("c1", "", sample) is True + + class TestBM25Index: def test_search_returns_relevant(self): index = BM25Index(CORPUS) @@ -240,3 +380,37 @@ def test_benchmark_comparison(self): assert "dense" in table assert "keyword" in table assert "hybrid" in table + + +def test_leading_digit_runs_before_non_latin_scripts(): + # Digits that OPEN a word beside a non-Latin script must also form their + # own token: "2026إصدار" and "2026東京" stay searchable by "2026". + assert "2026" in tokenize("2026إصدار") + assert "2026" in tokenize("2026東京") + # Latin-leading words keep their digits: "bm25" stays one token. + assert "bm25" in tokenize("bm25") + + +def test_compatibility_forms_normalize_before_casefold(): + # NFKC after casefold leaves compatibility letters unfolded: ㎒ + # decomposes to "MHz" and must fold to the same token as "MHz". + assert tokenize("㎒") == tokenize("MHz") + assert tokenize("𝐀") == tokenize("a") + + +def test_ideographic_zero_is_a_searchable_unigram(): + # 〇 is IDEOGRAPHIC NUMBER ZERO, not an iteration mark: "二〇二六" must + # answer a "〇" query. + assert "〇" in tokenize("二〇二六") + assert "二〇" in tokenize("二〇二六") + + +def test_kana_iteration_marks_are_not_standalone_unigrams(): + # ゝゞヽヾ repeat the previous kana; as shared unigrams they false-match + # unrelated words ("さゝ" vs "くゝ"). + assert "ゝ" not in tokenize("さゝ") + assert "ゞ" not in tokenize("さゞ") + assert "ヽ" not in tokenize("サヽ") + assert "ヾ" not in tokenize("サヾ") + # Adjacency is still searchable via the bigram. + assert "さゝ" in tokenize("さゝ")