Fix #49: block false-positive affix bones with canon stem validation

backend/pyproject.toml

@@ -33,7 +33,7 @@ Repository = "https://github.com/The-Interdependency/edcmbone"
 Issues = "https://github.com/The-Interdependency/edcmbone/issues"
 
 [tool.pytest.ini_options]
-testpaths = ["../tests"]
+testpaths = ["../Tests", "../tests"]
 
 [tool.hatch.build.targets.wheel]
 packages = ["src/edcmbone"]

backend/src/edcmbone/parser/turns_rounds.py

@@ -211,8 +211,9 @@ def _group_into_rounds(turns, strategy="cycle"):
 # Tokenizer — word + punctuation split
 # ---------------------------------------------------------------------------
 
-# Split into word-runs and individual punctuation characters
-_WORD_RE = re.compile(r"[A-Za-z]+(?:'[A-Za-z]+)*|[0-9]+|[^\w\s]")
+# Split into word-runs and individual punctuation characters.
+# Preserve hyphenated compounds as one token before punctuation handling.
+_WORD_RE = re.compile(r"[A-Za-z0-9]+(?:-[A-Za-z0-9]+)+|[A-Za-z]+(?:'[A-Za-z]+)*|[0-9]+|[^\w\s]")
 
 
 def _raw_tokens(text):
@@ -247,6 +248,11 @@ def __init__(self, canon: CanonLoader):
         # but we access via the public API for correctness)
         self._word_cache = {}
         self._affix_cache = {}
+        self._valid_stems = {
+            entry["word"].lower()
+            for entry in canon.all_words()
+            if entry.get("primary") != "S"
+        }
 
     def _make_bone(self, surface, normalized, bone_type, entry):
         return BoneToken(
@@ -298,8 +304,7 @@ def classify_sequence(self, raw_tokens):
             for pre in self._prefixes:
                 if lower.startswith(pre) and len(lower) - len(pre) >= 2:
                     residual = lower[len(pre):]
-                    residual_entry = self._canon.lookup_word(residual)
-                    if (not residual_entry) or residual_entry.get("primary") == "S":
+                    if residual not in self._valid_stems:
                         continue
                     affix_key = pre + "-"
                     if affix_key not in self._affix_cache:
@@ -317,8 +322,7 @@ def classify_sequence(self, raw_tokens):
             for suf in self._suffixes:
                 if lower.endswith(suf) and len(lower) - len(suf) >= 2:
                     residual = lower[:-len(suf)]
-                    residual_entry = self._canon.lookup_word(residual)
-                    if (not residual_entry) or residual_entry.get("primary") == "S":
+                    if residual not in self._valid_stems:
                         continue
                     affix_key = "-" + suf
                     if affix_key not in self._affix_cache:

core/operator/matcher.py

@@ -65,6 +65,10 @@ def match_affixes(tok: str, prefix_map: Dict[str, str], suffix_map: Dict[str, st
     """
     Longest-match-first; prefix then suffix; emit ALL matched affixes.
     Returns (families_emitted, residual_root).
+
+    When valid_stems is provided, validation is deferred until after all affixes
+    are stripped. This allows multi-affix words like "redoing" (re+do+ing) to work
+    correctly even when intermediate forms like "doing" are not in valid_stems.
     """
     t = normalize_text_for_matching(tok)
     fams: List[str] = []
@@ -78,8 +82,6 @@ def match_affixes(tok: str, prefix_map: Dict[str, str], suffix_map: Dict[str, st
         for p in pref_list:
             if root.startswith(p) and len(root) > len(p):
                 residual = root[len(p):]
-                if valid_stems is not None and residual not in valid_stems:
-                    continue
                 fams.append(prefix_map[p])
                 root = residual
                 changed = True
@@ -92,11 +94,14 @@ def match_affixes(tok: str, prefix_map: Dict[str, str], suffix_map: Dict[str, st
         for s in suf_list:
             if root.endswith(s) and len(root) > len(s):
                 residual = root[:-len(s)]
-                if valid_stems is not None and residual not in valid_stems:
-                    continue
                 fams.append(suffix_map[s])
                 root = residual
                 changed = True
                 break
 
+    # Validate final root against valid_stems if provided.
+    # If validation fails, return no affixes (treat as unmatched).
+    if valid_stems is not None and fams and root not in valid_stems:
+        return [], t
+
     return fams, root

tests/test_affix_residual_validation.py

@@ -19,13 +19,29 @@ def test_backend_parser_affix_does_not_fire_on_invalid_residuals():
     assert all((not hasattr(x, "bone_type") or x.bone_type != "affix") for x in out)
 
 
-def test_backend_parser_affix_positive_cases_still_emit():
+def test_backend_parser_affix_positive_cases_still_emit_for_canon_valid_stems():
     canon = CanonLoader()
     c = _BoneClassifier(canon)
+
+    # Guaranteed by current canon word inventory: "redo" -> residual "do" exists.
     out = c.classify_sequence(["redo"])
     assert [getattr(x, "bone_type", None) for x in out] == ["affix"]
 
 
+def test_backend_parser_examples_unhappy_and_linking_are_canon_dependent():
+    canon = CanonLoader()
+    c = _BoneClassifier(canon)
+
+    # These examples are morphologically valid in English, but backend affix emission
+    # depends on whether residual stems are present in the canon word index.
+    # This test documents that parser behavior remains canon-driven, not heuristic.
+    for tok, residual in (("unhappy", "happy"), ("linking", "link")):
+        out = c.classify_sequence([tok])[0]
+        emits_affix = getattr(out, "bone_type", None) == "affix"
+        residual_in_canon = canon.lookup_word(residual) is not None
+        assert emits_affix == residual_in_canon
+
+
 def test_core_matcher_affix_respects_valid_stems_negative_and_positive():
     prefix_map = {"un": "P", "re": "K"}
     suffix_map = {"ing": "K"}
@@ -36,3 +52,29 @@ def test_core_matcher_affix_respects_valid_stems_negative_and_positive():
 
     fams2, root2 = match_affixes("uncle", prefix_map, suffix_map, valid_stems=valid_stems)
     assert fams2 == [] and root2 == "uncle"
+
+
+def test_core_matcher_multi_affix_words_validate_final_root_only():
+    """
+    Regression test for issue where intermediate residuals were validated too early.
+    For "redoing" (re+do+ing), the intermediate "doing" is not a valid stem, but
+    the final root "do" is. Validation should be deferred until all affixes are stripped.
+    """
+    prefix_map = {"re": "K", "un": "P"}
+    suffix_map = {"ing": "K", "ed": "K"}
+    valid_stems = {"do", "happy"}
+
+    # "redoing" -> strip "re" -> "doing" -> strip "ing" -> "do" (valid!)
+    fams, root = match_affixes("redoing", prefix_map, suffix_map, valid_stems=valid_stems)
+    assert fams == ["K", "K"], f"Expected ['K', 'K'] but got {fams}"
+    assert root == "do", f"Expected 'do' but got {root}"
+
+    # "unhappying" -> strip "un" -> "happying" -> strip "ing" -> "happy" (valid!)
+    fams2, root2 = match_affixes("unhappying", prefix_map, suffix_map, valid_stems=valid_stems)
+    assert fams2 == ["P", "K"], f"Expected ['P', 'K'] but got {fams2}"
+    assert root2 == "happy", f"Expected 'happy' but got {root2}"
+
+    # "rethinking" -> strip "re" -> "thinking" -> strip "ing" -> "think" (NOT valid!)
+    fams3, root3 = match_affixes("rethinking", prefix_map, suffix_map, valid_stems=valid_stems)
+    assert fams3 == [], f"Expected [] but got {fams3}"
+    assert root3 == "rethinking", f"Expected 'rethinking' but got {root3}"

tests/test_apostrophe_normalization_and_tokenization.py

@@ -5,7 +5,9 @@
 
 
 def _parser_word_re():
-    src = Path("backend/src/edcmbone/parser/turns_rounds.py").read_text()
+    repo_root = Path(__file__).resolve().parents[1]
+    src_path = repo_root / "backend" / "src" / "edcmbone" / "parser" / "turns_rounds.py"
+    src = src_path.read_text()
     m = re.search(r"_WORD_RE\s*=\s*re\.compile\(r\"([^\"]+)\"\)", src)
     assert m, "Could not locate _WORD_RE in parser/turns_rounds.py"
     return re.compile(m.group(1))
@@ -32,3 +34,8 @@ def test_parser_word_re_keeps_smart_contractions_whole_after_normalization():
     normalized = normalize_text_for_matching("don’t can’t it’s")
     tokens = _parser_raw_tokens_like_impl(normalized)
     assert tokens == ["don't", "can't", "it's"]
+
+
+def test_parser_word_re_keeps_hyphenated_compounds_as_one_surface_token():
+    tokens = _parser_raw_tokens_like_impl("state-of-the-art")
+    assert tokens == ["state-of-the-art"]