diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 50efefa..de6ed85 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -33,7 +33,7 @@ Repository = "https://github.com/The-Interdependency/edcmbone" Issues = "https://github.com/The-Interdependency/edcmbone/issues" [tool.pytest.ini_options] -testpaths = ["../tests"] +testpaths = ["../Tests", "../tests"] [tool.hatch.build.targets.wheel] packages = ["src/edcmbone"] diff --git a/backend/src/edcmbone/parser/turns_rounds.py b/backend/src/edcmbone/parser/turns_rounds.py index 96b63e0..a6a673b 100644 --- a/backend/src/edcmbone/parser/turns_rounds.py +++ b/backend/src/edcmbone/parser/turns_rounds.py @@ -211,8 +211,9 @@ def _group_into_rounds(turns, strategy="cycle"): # Tokenizer — word + punctuation split # --------------------------------------------------------------------------- -# Split into word-runs and individual punctuation characters -_WORD_RE = re.compile(r"[A-Za-z]+(?:'[A-Za-z]+)*|[0-9]+|[^\w\s]") +# Split into word-runs and individual punctuation characters. +# Preserve hyphenated compounds as one token before punctuation handling. +_WORD_RE = re.compile(r"[A-Za-z0-9]+(?:-[A-Za-z0-9]+)+|[A-Za-z]+(?:'[A-Za-z]+)*|[0-9]+|[^\w\s]") def _raw_tokens(text): @@ -247,6 +248,11 @@ def __init__(self, canon: CanonLoader): # but we access via the public API for correctness) self._word_cache = {} self._affix_cache = {} + self._valid_stems = { + entry["word"].lower() + for entry in canon.all_words() + if entry.get("primary") != "S" + } def _make_bone(self, surface, normalized, bone_type, entry): return BoneToken( @@ -298,8 +304,7 @@ def classify_sequence(self, raw_tokens): for pre in self._prefixes: if lower.startswith(pre) and len(lower) - len(pre) >= 2: residual = lower[len(pre):] - residual_entry = self._canon.lookup_word(residual) - if (not residual_entry) or residual_entry.get("primary") == "S": + if residual not in self._valid_stems: continue affix_key = pre + "-" if affix_key not in self._affix_cache: @@ -317,8 +322,7 @@ def classify_sequence(self, raw_tokens): for suf in self._suffixes: if lower.endswith(suf) and len(lower) - len(suf) >= 2: residual = lower[:-len(suf)] - residual_entry = self._canon.lookup_word(residual) - if (not residual_entry) or residual_entry.get("primary") == "S": + if residual not in self._valid_stems: continue affix_key = "-" + suf if affix_key not in self._affix_cache: diff --git a/core/operator/matcher.py b/core/operator/matcher.py index 2a76f6c..81767aa 100644 --- a/core/operator/matcher.py +++ b/core/operator/matcher.py @@ -65,6 +65,10 @@ def match_affixes(tok: str, prefix_map: Dict[str, str], suffix_map: Dict[str, st """ Longest-match-first; prefix then suffix; emit ALL matched affixes. Returns (families_emitted, residual_root). + + When valid_stems is provided, validation is deferred until after all affixes + are stripped. This allows multi-affix words like "redoing" (re+do+ing) to work + correctly even when intermediate forms like "doing" are not in valid_stems. """ t = normalize_text_for_matching(tok) fams: List[str] = [] @@ -78,8 +82,6 @@ def match_affixes(tok: str, prefix_map: Dict[str, str], suffix_map: Dict[str, st for p in pref_list: if root.startswith(p) and len(root) > len(p): residual = root[len(p):] - if valid_stems is not None and residual not in valid_stems: - continue fams.append(prefix_map[p]) root = residual changed = True @@ -92,11 +94,14 @@ def match_affixes(tok: str, prefix_map: Dict[str, str], suffix_map: Dict[str, st for s in suf_list: if root.endswith(s) and len(root) > len(s): residual = root[:-len(s)] - if valid_stems is not None and residual not in valid_stems: - continue fams.append(suffix_map[s]) root = residual changed = True break + # Validate final root against valid_stems if provided. + # If validation fails, return no affixes (treat as unmatched). + if valid_stems is not None and fams and root not in valid_stems: + return [], t + return fams, root diff --git a/tests/test_affix_residual_validation.py b/tests/test_affix_residual_validation.py index cde1230..4ab77f8 100644 --- a/tests/test_affix_residual_validation.py +++ b/tests/test_affix_residual_validation.py @@ -19,13 +19,29 @@ def test_backend_parser_affix_does_not_fire_on_invalid_residuals(): assert all((not hasattr(x, "bone_type") or x.bone_type != "affix") for x in out) -def test_backend_parser_affix_positive_cases_still_emit(): +def test_backend_parser_affix_positive_cases_still_emit_for_canon_valid_stems(): canon = CanonLoader() c = _BoneClassifier(canon) + + # Guaranteed by current canon word inventory: "redo" -> residual "do" exists. out = c.classify_sequence(["redo"]) assert [getattr(x, "bone_type", None) for x in out] == ["affix"] +def test_backend_parser_examples_unhappy_and_linking_are_canon_dependent(): + canon = CanonLoader() + c = _BoneClassifier(canon) + + # These examples are morphologically valid in English, but backend affix emission + # depends on whether residual stems are present in the canon word index. + # This test documents that parser behavior remains canon-driven, not heuristic. + for tok, residual in (("unhappy", "happy"), ("linking", "link")): + out = c.classify_sequence([tok])[0] + emits_affix = getattr(out, "bone_type", None) == "affix" + residual_in_canon = canon.lookup_word(residual) is not None + assert emits_affix == residual_in_canon + + def test_core_matcher_affix_respects_valid_stems_negative_and_positive(): prefix_map = {"un": "P", "re": "K"} suffix_map = {"ing": "K"} @@ -36,3 +52,29 @@ def test_core_matcher_affix_respects_valid_stems_negative_and_positive(): fams2, root2 = match_affixes("uncle", prefix_map, suffix_map, valid_stems=valid_stems) assert fams2 == [] and root2 == "uncle" + + +def test_core_matcher_multi_affix_words_validate_final_root_only(): + """ + Regression test for issue where intermediate residuals were validated too early. + For "redoing" (re+do+ing), the intermediate "doing" is not a valid stem, but + the final root "do" is. Validation should be deferred until all affixes are stripped. + """ + prefix_map = {"re": "K", "un": "P"} + suffix_map = {"ing": "K", "ed": "K"} + valid_stems = {"do", "happy"} + + # "redoing" -> strip "re" -> "doing" -> strip "ing" -> "do" (valid!) + fams, root = match_affixes("redoing", prefix_map, suffix_map, valid_stems=valid_stems) + assert fams == ["K", "K"], f"Expected ['K', 'K'] but got {fams}" + assert root == "do", f"Expected 'do' but got {root}" + + # "unhappying" -> strip "un" -> "happying" -> strip "ing" -> "happy" (valid!) + fams2, root2 = match_affixes("unhappying", prefix_map, suffix_map, valid_stems=valid_stems) + assert fams2 == ["P", "K"], f"Expected ['P', 'K'] but got {fams2}" + assert root2 == "happy", f"Expected 'happy' but got {root2}" + + # "rethinking" -> strip "re" -> "thinking" -> strip "ing" -> "think" (NOT valid!) + fams3, root3 = match_affixes("rethinking", prefix_map, suffix_map, valid_stems=valid_stems) + assert fams3 == [], f"Expected [] but got {fams3}" + assert root3 == "rethinking", f"Expected 'rethinking' but got {root3}" diff --git a/tests/test_apostrophe_normalization_and_tokenization.py b/tests/test_apostrophe_normalization_and_tokenization.py index c5d2d2e..e865278 100644 --- a/tests/test_apostrophe_normalization_and_tokenization.py +++ b/tests/test_apostrophe_normalization_and_tokenization.py @@ -5,7 +5,9 @@ def _parser_word_re(): - src = Path("backend/src/edcmbone/parser/turns_rounds.py").read_text() + repo_root = Path(__file__).resolve().parents[1] + src_path = repo_root / "backend" / "src" / "edcmbone" / "parser" / "turns_rounds.py" + src = src_path.read_text() m = re.search(r"_WORD_RE\s*=\s*re\.compile\(r\"([^\"]+)\"\)", src) assert m, "Could not locate _WORD_RE in parser/turns_rounds.py" return re.compile(m.group(1)) @@ -32,3 +34,8 @@ def test_parser_word_re_keeps_smart_contractions_whole_after_normalization(): normalized = normalize_text_for_matching("don’t can’t it’s") tokens = _parser_raw_tokens_like_impl(normalized) assert tokens == ["don't", "can't", "it's"] + + +def test_parser_word_re_keeps_hyphenated_compounds_as_one_surface_token(): + tokens = _parser_raw_tokens_like_impl("state-of-the-art") + assert tokens == ["state-of-the-art"]