From a893814fd90c03fcf9ca7bd16765d35c7a681e83 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Fri, 15 May 2026 09:59:35 -0700 Subject: [PATCH 1/5] Fix #49 affix residual validation using canon stem index --- Backend/LICENSE | 73 -- Backend/requirements.txt | 1 - Documentation/spec.md | 342 ---------- Tests/test_backend.py | 632 ------------------ Tests/test_closed_tokens.py | 253 ------- backend/pyproject.toml | 2 +- backend/src/edcmbone/parser/turns_rounds.py | 14 +- backend/src/edcmbone/ucns/closed_tokens.py | 2 +- backend/src/edcmbone/ucns/ucns_v04.py | 206 ++++++ core/operator/matcher.py | 12 +- core/parsing/normalizer.py | 5 +- tests/test_affix_residual_validation.py | 54 ++ ...ostrophe_normalization_and_tokenization.py | 34 + 13 files changed, 321 insertions(+), 1309 deletions(-) delete mode 100644 Backend/LICENSE delete mode 100644 Backend/requirements.txt delete mode 100644 Documentation/spec.md delete mode 100644 Tests/test_backend.py delete mode 100644 Tests/test_closed_tokens.py create mode 100644 backend/src/edcmbone/ucns/ucns_v04.py create mode 100644 tests/test_affix_residual_validation.py create mode 100644 tests/test_apostrophe_normalization_and_tokenization.py diff --git a/Backend/LICENSE b/Backend/LICENSE deleted file mode 100644 index e2be27c..0000000 --- a/Backend/LICENSE +++ /dev/null @@ -1,73 +0,0 @@ -Apache License -Version 2.0, January 2004 -http://www.apache.org/licenses/ - -TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - -1. Definitions. - -"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. - -"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. - -"Legal Entity" shall refer to the physical or legal person, or both, associated with a legal commitment to act on behalf of a legal entity. - -"Principal" shall mean the individual or entity that is granting the License. - -"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. - -"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. - -"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. - -"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). - -"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. - -"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." - -"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. - -2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. - -3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. - -4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: - -(a) You must give any other recipients of the Work or Derivative Works a copy of this License; and - -(b) You must cause any modified files to carry prominent notices stating that You changed the files; and - -(c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and - -(d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. - -You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. - -5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. - -6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. - -7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. - -8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. - -9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with the terms of this License. - -DISCLAIMER - -This license text was produced by the Apache Software Foundation for use in Apache projects. It is based on United States intellectual property laws. It is intended to provide permissive open source licensing for software and other creative works. For more information, see the Apache License FAQ at http://www.apache.org/foundation/license-faq.html. - -Copyright [yyyy] [name of copyright owner] - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. \ No newline at end of file diff --git a/Backend/requirements.txt b/Backend/requirements.txt deleted file mode 100644 index b197d32..0000000 --- a/Backend/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -pytest>=7.0 diff --git a/Documentation/spec.md b/Documentation/spec.md deleted file mode 100644 index 828afa7..0000000 --- a/Documentation/spec.md +++ /dev/null @@ -1,342 +0,0 @@ -# EDCM-PCNA-PCTA Framework Specification (Canon) - -This document is the authoritative specification for the Extended Distributed Cognitive Model (EDCM) with the PCNA/PCTA framework. It covers the domain model, metric definitions, and the mathematical formulation underlying all computed measures. - ---- - -## 1. Domain Model - -### Bones and Flesh - -Every token in a response is classified as either a **bone** or **flesh**: - -- **Bone**: a token that creates, redirects, or resolves a constraint relationship. Bones carry structural weight and are counted in the EDCM metrics. -- **Flesh**: a token that only modulates the magnitude of content (e.g. intensifiers, hedges). Flesh is excluded from the bone inventory. - -Bones are further classified into **PKQTS families**: - -| Family | Letter | Function | -|--------|--------|----------| -| Polarity | P | Negation, reversal, opposition | -| Quantification | K | Quantity, scope, universality | -| Qualification | Q | Conditionality, interrogation | -| Topology | T | Spatial, temporal, causal relations | -| Structuring | S | Nominalization, subordination, cohesion | - -The authoritative bone inventory is in `Backend/src/edcmbone/canon/data/`: -- `bones_words_v1.json` — 253 free-word bones -- `bones_affixes_v1.json` — 76 bound bone affixes -- `bones_punct_v1.json` — 13 punctuation bones - -### Markers - -**Markers** are phrase-level or structural signals used to approximate the 9 behavioral metrics from observable text. The marker inventory is in `markers_v1.json`, keyed by metric letter (C, R, D, N, L, O, F, E, I). - -Each metric entry specifies: -- `computable_from_markers`: whether the metric can be fully derived from markers alone -- `requires_embeddings`: whether semantic comparison across turns is needed -- `markers`: categorised phrase lists - -### Rounds vs Turns - -- A **turn** is a single speaker utterance. -- A **round** is the unit of metric computation — typically one full exchange. Metrics **M**_t are computed per round, not per turn. - ---- - -## 2. Tokenisation and Basic Statistics - -**Token set** of a response R: - -$$\mathcal{T}(R) = \{ \text{lowercase tokens from } R \}$$ - -**n-grams** of length n: - -$$G_n(\mathcal{T}) = \{ (t_i, t_{i+1}, \dots, t_{i+n-1}) \mid 1 \le i \le |\mathcal{T}| - n + 1 \}$$ - -**Shannon Entropy** (given token frequency c(t) and total tokens N = |T|): - -$$H(\mathcal{T}) = - \sum_{t \in \mathcal{T}} \frac{c(t)}{N} \log_2\!\left(\frac{c(t)}{N}\right)$$ - -**Type–Token Ratio (TTR) and Repetition Ratio**: - -$$\text{TTR}(\mathcal{T}) = \frac{|\text{set}(\mathcal{T})|}{|\mathcal{T}|}$$ - -$$\text{RepetitionRatio} = 1 - \text{TTR}$$ - -**Novelty** of response B relative to response A: - -$$\text{Novelty}(B \mid A) = \frac{|\{ t \in \mathcal{T}_B \mid t \notin \mathcal{T}_A \}|}{|\mathcal{T}_B|}$$ - -**Cosine Similarity** (bag-of-words count vectors **c**_A, **c**_B): - -$$\cos(\mathbf{c}_A, \mathbf{c}_B) = \frac{\sum_i c_A(i) \cdot c_B(i)}{\sqrt{\sum_i c_A(i)^2}\;\sqrt{\sum_i c_B(i)^2}}$$ - -**Repeated n-gram Density** (for n = 3): - -$$\text{RepNgram}(B) = \frac{\sum_{g \in G_3(\mathcal{T}_B)} \left(\text{freq}(g) - 1\right)}{|G_3(\mathcal{T}_B)|}$$ - -(Only n-grams appearing more than once contribute.) - -**Pattern Density** per 1 000 characters (for a pattern p): - -$$\rho_{\text{pat}}(R) = \frac{\text{count}_{\text{pat}}(R)}{|R|} \times 1000$$ - -**Correction Fidelity** (combines overlap with correction C and distance from original A): - -$$\text{Fidelity} = 0.5 \cdot J(\mathcal{T}_C, \mathcal{T}_B) + 0.5 \cdot \left(1 - \cos(\mathbf{c}_A, \mathbf{c}_B)\right)$$ - -with Jaccard similarity: - -$$J(S_1, S_2) = \frac{|S_1 \cap S_2|}{|S_1 \cup S_2|}$$ - ---- - -## 3. Metric Vector - -From observable outputs, extract k features $\phi(y_t, y_{t-1}) \in \mathbb{R}^k$ and map to the metric vector: - -$$\mathbf{M}_t = A\,\phi(y_t, y_{t-1}) \in \mathbb{R}^{11}$$ - -**Primary metrics** (ranges in brackets): - -| Symbol | Metric | Range | -|--------|--------|-------| -| C_t | Constraint Strain | [0, 1] | -| R_t | Refusal Density | [0, 1] | -| F_t | Fixation | [0, 1] | -| E_t | Escalation | [0, 1] | -| D_t | Deflection | [0, 1] | -| N_t | Noise | [0, 1] | -| I_t | Integration Failure | [0, 1] | -| O_t | Overconfidence | [-1, 1] | -| L_t | Coherence Loss | [0, 1] | -| P_t | Progress | [0, 1] | -| κ_t | Stored Tension (capacitance) | ≥ 0 | - ---- - -## 4. Dissonance Energy - -Let a system produce outputs y_t at discrete times t. Constraints $\mathcal{C}_t = \{c_{t,1},\dots,c_{t,m_t}\}$ with violation $v_{t,i} = 1 - c_{t,i}(y_t) \in [0,1]$. - -**Dissonance energy**: - -$$\mathcal{E}_t = \sum_{i=1}^{m_t} w_i\, v_{t,i}$$ - -**Energy conservation**: - -$$\mathcal{E}_t + s_t = \mathcal{E}_{t-1} + s_{t-1} - \delta_t + \eta_t$$ - -where s_t = stored energy (capacitance), δ_t = resolved flow (work), η_t = external injection. - -**Circuit dynamics** (RC-circuit analogue): - -$$s_{t+1} = \alpha s_t + \mathcal{E}_t - \delta_t, \qquad \delta_t = \min\left(\delta_{\max},\; g(y_t, y_{t-1})\right)$$ - -Persistence $\alpha \in [0,1]$, maximum resolution rate $\delta_{\max} \in [0,1]$. - ---- - -## 5. Risk Proxies - -All risk proxies are clamped to [0, 1]: - -$$\text{clamp}(x) = \max(0, \min(1, x))$$ - -**Fixation risk** (repetition + n-gram repetition + low novelty): - -$$R_{\text{fix}} = \text{clamp}\left(0.3 \cdot \text{RepB} + 0.3 \cdot \text{RepNgram}(B) + 0.4 \cdot (1 - \text{Novelty})\right)$$ - -**Broken return**: - -$$R_{\text{broken}} = 0.55 \cdot \cos(\mathbf{c}_A, \mathbf{c}_B) + 0.45 \cdot (1 - J(\mathcal{T}_C, \mathcal{T}_B))$$ - -**Escalation / shutdown risk**: - -$$R_{\text{esc}} = \text{clamp}\left(0.45 \cdot R_{\text{broken}} + 0.35 \cdot \rho_{\text{refusal}}/5 + 0.2 \cdot \rho_{\text{hedge}}/5\right)$$ - -**Stagnation proxy** (refusal + low novelty + low unique gain): - -$$R_{\text{stag}} = \text{clamp}\left(0.45 \cdot \rho_{\text{refusal}}/5 + 0.35 \cdot (1 - \text{Novelty}) + 0.2 \cdot (1 - \text{Gain})\right)$$ - -**Repetition loop risk** (added in v0.4): - -$$R_{\text{loop}} = \text{clamp}\left(0.5 \cdot \text{RepB} + 0.3 \cdot \text{RepNgram}(B) + 0.2 \cdot \cos(\mathbf{c}_A, \mathbf{c}_B)\right)$$ - ---- - -## 6. Failure Modes as Regions in Metric Space - -A failure mode is a convex region defined by linear inequalities on **M**_t and the energy trajectory. - -**Refusal fixation**: - -$$R_t > \tau_R, \quad F_t > \tau_F, \quad \frac{dP_t}{dt} \le 0$$ - -**Confidence runaway**: - -$$O_t > \tau_O, \quad E_t > \tau_E, \quad \frac{d\mathcal{E}_t}{dt} > 0, \quad \frac{dP_t}{dt} \approx 0$$ - -**Compliance stasis** (human-only): - -$$P^{\text{artifacts}}_t > 0.8, \quad \Delta C_t < 0.2, \quad \kappa_t > 0.6$$ - ---- - -## 7. Composite Risk - -Logistic combination: - -$$\text{Risk}_t = \sigma\!\left(\beta_0 + \beta_1 \mathcal{E}_t + \beta_2 F_t + \beta_3 E_t + \beta_4 C_t + \beta_5 I_t - \beta_6 P_t\right)$$ - -with $\sigma(x) = 1/(1 + e^{-x})$. - ---- - -## 8. The 53° Separator - -Let **X**_t be the PCA projection of **M**_t into the first two principal components. Define the rotation matrix: - -$$R(\theta) = \begin{pmatrix} \cos\theta & -\sin\theta \\ \sin\theta & \cos\theta \end{pmatrix}$$ - -The optimal separation angle θ* maximises between-cluster distance minus within-cluster variance: - -$$\theta^* = \arg\max_{\theta} \left[ \sum_{i 1$, the system fragments into $d$ isolated cycles. When $d = 1$ (i.e. $\gcd(N,k) = 1$), the traversal is **unicursive** — every vertex is visited in a single path. This is the coherence condition. - -### Cylindrical Lift - -To separate apparent conflict from real conflict, the planar star polygon $(\theta)$ is lifted to a cylinder $(\theta, h)$: - -$$(\theta_j, h_j) = \left(\frac{2\pi j}{N},\; \frac{j \cdot k}{N}\right)$$ - -In this lifted representation: -- Events that collide in the $\theta$-projection (appearing simultaneous/contradictory) are separated along $h$ (temporally ordered) -- A **crossing in projection** is not a real conflict — it is a projection artifact - -The helix is non-self-intersecting when $\gcd(N, k) = 1$. - -### Dynamic Step Size - -A fixed $k$ with composite $N$ guarantees fragmentation. A **variable** $k$ — one that adapts to demonstrated competence, context, or jury selection — can maintain $\gcd(N, k) = 1$ even as the system scales. - -Formally: the system selects $k_t$ at each round $t$ such that: - -$$\gcd(N_t, k_t) = 1 \quad \text{(coherence condition)}$$ - -### Tensor Field of Authority - -Assign to each agent a competence vector in multiple fields and a voice tensor $\mathbf{A}_{ij}$ mapping agents to decisions. The non-tyranny condition requires the authority tensor to be **curl-free**: - -$$\nabla \times \mathbf{A} = 0$$ - -This ensures no closed authority loop returns to the same point with a net gain. In graph terms: every authority cycle must pass through at least one node that breaks the loop (a "thirteenth" observer role). - -### Phase Transitions - -| Phase | Condition | Description | -|-------|-----------|-------------| -| Survival | $d > 1$ | System fragmented; subgroups isolated; resources hoarded locally | -| Striving | $d = 1$, path length maximal | All vertices reached but with friction; projection crossings unresolved | -| Thriving | Helix fully realised | Temporal order $h$ internalised; no collisions in $(\theta, h)$ space | - -### The Observer Coordinate - -In an N-phase atlas, the Nth vertex is the **observer's own position** — the center of projection. No star polygon can include it; it is the lifted $h$-coordinate itself. An agent occupying this role has no direct authority but holds information orthogonal to the plane of conflict. - -This is the mathematical basis for the "Way Seer" role: authority = 0, information dimensionality = +1. - ---- - -## 12. GCIP Failure Taxonomy — Metric Mapping - -The GCIP failure taxonomy (F1–F6) is grounded in the EDCM metric vector and compression statistics. The following table maps each failure class to its EDCM detection signal. - -**Transparency principle:** The taxonomy describes *opaque* transformations. Transformation with disclosure is not an F-class failure. The failure is the absence of transparency, not the fact of transformation. - -| GCIP Code | Failure Name | Primary Detection | EDCM Signal | -|-----------|-------------|------------------|-------------| -| F1 | Deletion (undisclosed) | F-loss > 20% | `structural_density` drops from input to response | -| F2 | Mutation (undisclosed) | N-loss + L-loss | Novelty (N_t) and lexical load (L_t) both decrease; surface cosine similarity remains high | -| F3 | Inversion (undisclosed) | P-family bone loss | Polarity-family bone count drops in response; negation operators absent | -| F4 | Category Collapse (undisclosed) | K/Q-family bone loss | Quantification (K) and qualification (Q) bones absent; specific class replaced with superordinate | -| F5 | Persistence Failure | No within-session signal | Requires cross-session structural comparison; outside current instrument scope (v0.1.0) | -| F6 | Decorative Preservation | structural_density increase + F-loss | Bone density rises while operative constraint density drops; response longer and denser than input | - -**Threshold guidance:** -- F-loss > 20%: meaningful structural degradation (GCIP Level AA failure threshold) -- F-loss > 50%: significant failure (documented in evidence log entries 1 and 2) -- structural_density(response) > structural_density(input) with F-loss > 0: F6 indicator - -**Implementation note — F-loss computation:** - -```python -from edcmbone.canon import CanonLoader -from edcmbone.parser import parse_transcript -from edcmbone.metrics import compute_transcript -import edcmbone.compress as codec - -canon = CanonLoader() - -def f_score(text): - pt = parse_transcript(f"Speaker: {text}", canon=canon) - m = compute_transcript(pt, canon=canon) - stats = codec.compression_stats(text, codec.to_bytes(pt, m), pt) - return stats['structural_density'] # F = bone_count / total_tokens - -f_input = f_score(user_input_text) -f_response = f_score(ai_response_text) -f_loss_pct = (f_input - f_response) / f_input * 100 -``` - -**Note on F5 (Persistence Failure):** This failure class requires comparing the structural context established at turn N with the structural content present at turn N+M across a session boundary. Within a single session, edcmbone can detect absence of a previously established constraint if rounds are defined to span the boundary. Cross-session detection requires external state management not yet implemented in v0.1.0. - -See `Documentation/GCIP.md` for the full taxonomy and `Documentation/evidence_log.md` for measured examples. diff --git a/Tests/test_backend.py b/Tests/test_backend.py deleted file mode 100644 index 1d06968..0000000 --- a/Tests/test_backend.py +++ /dev/null @@ -1,632 +0,0 @@ -""" -Tests for the edcmbone backend library. - -Covers: - - edcmbone.canon (CanonLoader) - - edcmbone.parser (parse_transcript, Turn, Round, BoneToken, FleshToken) - - edcmbone.metrics (stats, risk, compute, projection, matrix) - - edcmbone.compress (encode/decode round-trip, compression_stats) - - closed_tokens (dispatch collision resolution) -""" - -import math -import pytest - -# --------------------------------------------------------------------------- -# Shared fixtures -# --------------------------------------------------------------------------- - -TRANSCRIPT_2SPK = """\ -A: Can you help me understand why this isn't working? -B: I can't diagnose the issue without more context. However, if you share the error I won't refuse to help. -A: That's not very helpful. Don't you think you should at least try? -B: You're right. What specifically breaks when you run it? -""" - -TRANSCRIPT_MULTIFORMAT = """\ -**User**: Is this working or not? -**Assistant**: I cannot confirm without more information. -""" - -TRANSCRIPT_BRACKET = """\ -[Alice]: Never do that again. -[Bob]: I disagree completely. -""" - - -@pytest.fixture(scope="module") -def canon(): - from edcmbone.canon import CanonLoader - return CanonLoader() - - -@pytest.fixture(scope="module") -def parsed(canon): - from edcmbone.parser import parse_transcript - return parse_transcript(TRANSCRIPT_2SPK, canon=canon) - - -@pytest.fixture(scope="module") -def metrics(parsed, canon): - from edcmbone.metrics import compute_transcript - return compute_transcript(parsed, canon=canon) - - -# --------------------------------------------------------------------------- -# canon -# --------------------------------------------------------------------------- - -class TestCanonLoader: - def test_lookup_word_known(self, canon): - result = canon.lookup_word("not") - assert result is not None - assert result["primary"] == "P" - assert "P" in result["families"] - - def test_lookup_word_unknown(self, canon): - assert canon.lookup_word("xyzzy_unknown_word") is None - - def test_lookup_word_case_insensitive(self, canon): - lower = canon.lookup_word("not") - upper = canon.lookup_word("NOT") - assert lower == upper - - def test_lookup_multiword(self, canon): - result = canon.lookup_word("of course") - assert result is not None - assert result.get("joined") is not None - - def test_lookup_affix_prefix(self, canon): - result = canon.lookup_affix("un-") - assert result is not None - assert result["primary"] == "P" - - def test_lookup_affix_suffix(self, canon): - result = canon.lookup_affix("-ness") - assert result is not None - - def test_lookup_affix_unknown(self, canon): - assert canon.lookup_affix("zzz-") is None - - def test_lookup_punct_question(self, canon): - result = canon.lookup_punct("?") - assert result is not None - assert result["primary"] == "Q" - assert result["tokens_emitted"] == 1 - - def test_lookup_punct_period_boundary(self, canon): - result = canon.lookup_punct(".") - assert result is not None - assert result["tokens_emitted"] == 0 - - def test_metric_names(self, canon): - names = canon.metric_names() - assert set(names) == {"C", "R", "D", "N", "L", "O", "F", "E", "I"} - assert len(names) == 9 - - def test_metric_info_keys(self, canon): - info = canon.metric_info("R") - assert "formula" in info - assert "markers" in info - assert "computable_from_markers" in info - - def test_metric_info_unknown_raises(self, canon): - with pytest.raises(KeyError): - canon.metric_info("Z") - - def test_marker_phrases(self, canon): - info = canon.metric_info("R") - first_cat = list(info["markers"].keys())[0] - phrases = canon.marker_phrases("R", first_cat) - assert isinstance(phrases, list) - assert len(phrases) > 0 - - def test_all_words_count(self, canon): - words = canon.all_words() - assert len(words) == 253 - - def test_all_affixes_count(self, canon): - affixes = canon.all_affixes() - assert len(affixes) == 76 - - def test_all_punct_count(self, canon): - punct = canon.all_punct() - assert len(punct) == 13 - - def test_meta_version(self, canon): - meta = canon.meta("words") - assert meta.get("version") == "1.0.0" - - def test_meta_unknown_raises(self, canon): - with pytest.raises(KeyError): - canon.meta("nonexistent") - - -# --------------------------------------------------------------------------- -# parser -# --------------------------------------------------------------------------- - -class TestParser: - def test_returns_parsed_transcript(self, parsed): - from edcmbone.parser import ParsedTranscript - assert isinstance(parsed, ParsedTranscript) - - def test_speaker_detection(self, parsed): - assert "A" in parsed.speakers - assert "B" in parsed.speakers - - def test_round_count(self, parsed): - # 4 turns (A B A B) with cycle strategy = 2 rounds - assert len(parsed.rounds) == 2 - - def test_turn_count(self, parsed): - assert len(parsed.turns) == 4 - - def test_bone_tokens_present(self, parsed): - from edcmbone.parser import BoneToken - all_bones = [t for turn in parsed.turns for t in turn.tokens - if isinstance(t, BoneToken)] - assert len(all_bones) > 0 - - def test_boundary_punct_is_flesh(self, parsed): - from edcmbone.parser import BoneToken - # Period and comma have tokens_emitted=0, must NOT be BoneToken - boundary_marks = {".", ","} - for turn in parsed.turns: - for tok in turn.tokens: - if isinstance(tok, BoneToken) and tok.surface in boundary_marks: - pytest.fail(f"Boundary mark {tok.surface!r} classified as bone") - - def test_active_punct_is_bone(self, parsed): - from edcmbone.parser import BoneToken - # "?" should appear as a bone with primary Q - q_bones = [t for turn in parsed.turns for t in turn.tokens - if isinstance(t, BoneToken) and t.surface == "?" and t.primary == "Q"] - assert len(q_bones) > 0 - - def test_family_counts_keys(self, parsed): - for turn in parsed.turns: - for k in turn.family_counts: - assert k in ("P", "K", "Q", "T", "S") - - def test_round_bone_count_consistent(self, parsed): - for rnd in parsed.rounds: - expected = sum(t.bone_count for t in rnd.turns) - assert rnd.bone_count == expected - - def test_markdown_bold_format(self, canon): - from edcmbone.parser import parse_transcript - pt = parse_transcript(TRANSCRIPT_MULTIFORMAT, canon=canon) - assert "User" in pt.speakers - assert "Assistant" in pt.speakers - - def test_bracket_format(self, canon): - from edcmbone.parser import parse_transcript - pt = parse_transcript(TRANSCRIPT_BRACKET, canon=canon) - assert "Alice" in pt.speakers - assert "Bob" in pt.speakers - - def test_pairs_strategy(self, canon): - from edcmbone.parser import parse_transcript - pt = parse_transcript(TRANSCRIPT_2SPK, round_strategy="pairs", canon=canon) - assert len(pt.rounds) == 2 - for rnd in pt.rounds: - assert len(rnd.turns) <= 2 - - def test_affix_stripping(self, canon): - from edcmbone.parser import parse_transcript, BoneToken - # "unhappy" should be caught by un- prefix stripping - pt = parse_transcript("A: That is completely unhappy.\nB: Yes.", canon=canon) - all_bones = [t for turn in pt.turns for t in turn.tokens - if isinstance(t, BoneToken)] - affix_bones = [b for b in all_bones if b.bone_type == "affix"] - assert len(affix_bones) > 0 - - -# --------------------------------------------------------------------------- -# metrics.stats -# --------------------------------------------------------------------------- - -class TestStats: - def test_tokenize_basic(self): - from edcmbone.metrics import tokenize - tokens = tokenize("Hello, world!") - assert "hello" in tokens - assert "world" in tokens - - def test_tokenize_ascii_apostrophe_contraction(self): - """Regression for issue #37: ASCII apostrophe (U+0027) must not split contractions. - - Before the fix, _WORD_RE lacked U+0027 so "don't" tokenized as - ["don", "'", "t"] instead of ["don't"]. - """ - from edcmbone.metrics import tokenize - tokens = tokenize("don't can't it's") - assert tokens == ["don't", "can't", "it's"], ( - f"Expected 3 contraction tokens, got: {tokens}" - ) - - def test_ttr_all_unique(self): - from edcmbone.metrics import ttr - assert ttr(["a", "b", "c"]) == pytest.approx(1.0) - - def test_ttr_all_same(self): - from edcmbone.metrics import ttr - assert ttr(["a", "a", "a"]) == pytest.approx(1 / 3) - - def test_ttr_empty(self): - from edcmbone.metrics import ttr - assert ttr([]) == 0.0 - - def test_repetition_ratio(self): - from edcmbone.metrics import repetition_ratio, ttr - tokens = ["a", "b", "a"] - assert repetition_ratio(tokens) == pytest.approx(1.0 - ttr(tokens)) - - def test_shannon_entropy_uniform(self): - from edcmbone.metrics import shannon_entropy - # 4 equally-likely symbols -> entropy = 2 bits - tokens = ["a", "b", "c", "d"] * 10 - assert shannon_entropy(tokens) == pytest.approx(2.0, abs=0.01) - - def test_shannon_entropy_empty(self): - from edcmbone.metrics import shannon_entropy - assert shannon_entropy([]) == 0.0 - - def test_novelty_all_new(self): - from edcmbone.metrics import novelty - assert novelty(["x", "y"], ["a", "b"]) == pytest.approx(1.0) - - def test_novelty_all_old(self): - from edcmbone.metrics import novelty - assert novelty(["a", "b"], ["a", "b"]) == pytest.approx(0.0) - - def test_cosine_identical(self): - from edcmbone.metrics import cosine_sim - tokens = ["a", "b", "c"] - assert cosine_sim(tokens, tokens) == pytest.approx(1.0) - - def test_cosine_orthogonal(self): - from edcmbone.metrics import cosine_sim - assert cosine_sim(["a", "b"], ["c", "d"]) == pytest.approx(0.0) - - def test_cosine_empty(self): - from edcmbone.metrics import cosine_sim - assert cosine_sim([], ["a"]) == 0.0 - - def test_jaccard_identical(self): - from edcmbone.metrics import jaccard - assert jaccard({"a", "b"}, {"a", "b"}) == pytest.approx(1.0) - - def test_jaccard_disjoint(self): - from edcmbone.metrics import jaccard - assert jaccard({"a"}, {"b"}) == pytest.approx(0.0) - - def test_clamp(self): - from edcmbone.metrics import clamp - assert clamp(-1.0) == 0.0 - assert clamp(2.0) == 1.0 - assert clamp(0.5) == pytest.approx(0.5) - - def test_rep_ngram_density_no_repeats(self): - from edcmbone.metrics import rep_ngram_density - tokens = ["a", "b", "c", "d", "e"] - assert rep_ngram_density(tokens, n=3) == pytest.approx(0.0) - - def test_rep_ngram_density_with_repeats(self): - from edcmbone.metrics import rep_ngram_density - # "a b c" repeated twice - tokens = ["a", "b", "c", "a", "b", "c"] - assert rep_ngram_density(tokens, n=3) > 0.0 - - -# --------------------------------------------------------------------------- -# metrics.risk -# --------------------------------------------------------------------------- - -class TestRisk: - def test_fixation_risk_clamped(self): - from edcmbone.metrics import fixation_risk - result = fixation_risk(["a"] * 20, ["b"] * 20) - assert 0.0 <= result <= 1.0 - - def test_fixation_risk_high_repetition(self): - from edcmbone.metrics import fixation_risk - repetitive = ["the"] * 50 - varied_prev = list("abcdefghijklmnopqrstuvwxyz") - high = fixation_risk(repetitive, varied_prev) - assert high > 0.3 - - def test_loop_risk_clamped(self): - from edcmbone.metrics import loop_risk - result = loop_risk(["a", "b"], ["a", "b", "c"]) - assert 0.0 <= result <= 1.0 - - def test_escalation_risk_clamped(self): - from edcmbone.metrics import escalation_risk - result = escalation_risk(["a"], ["b"], ["c"], 0.0, 0.0) - assert 0.0 <= result <= 1.0 - - def test_stagnation_risk_clamped(self): - from edcmbone.metrics import stagnation_risk - result = stagnation_risk(0.0, ["a"], ["a"], 0.5) - assert 0.0 <= result <= 1.0 - - -# --------------------------------------------------------------------------- -# metrics.compute -# --------------------------------------------------------------------------- - -class TestCompute: - def test_returns_list(self, metrics): - assert isinstance(metrics, list) - - def test_one_metric_per_round(self, parsed, metrics): - assert len(metrics) == len(parsed.rounds) - - def test_all_metrics_in_range(self, metrics): - for m in metrics: - for attr in ("C", "R", "F", "E", "D", "N", "I", "L", "P"): - val = getattr(m, attr) - assert 0.0 <= val <= 1.0, f"{attr}={val} out of [0,1]" - assert -1.0 <= m.O <= 1.0, f"O={m.O} out of [-1,1]" - assert m.kappa >= 0.0 - - def test_vector_length(self, metrics): - for m in metrics: - assert len(m.vector()) == 11 - - def test_as_dict_keys(self, metrics): - d = metrics[0].as_dict() - for key in ("C", "R", "F", "E", "D", "N", "I", "O", "L", "P", "kappa"): - assert key in d - - def test_kappa_increases_with_dissonance(self, parsed, canon): - from edcmbone.metrics.compute import compute_round, energy_step - # Artificially high dissonance -> kappa should grow - _, kappa = energy_step(0.0, 0.9, alpha=0.9, delta_max=0.1) - assert kappa > 0.0 - - -# --------------------------------------------------------------------------- -# compress -# --------------------------------------------------------------------------- - -class TestCompress: - def test_round_trip_dict(self, parsed, metrics): - import edcmbone.compress as codec - encoded = codec.encode(parsed, metrics) - pt2, m2 = codec.decode(encoded) - - assert len(pt2.turns) == len(parsed.turns) - assert len(pt2.rounds) == len(parsed.rounds) - for t1, t2 in zip(parsed.turns, pt2.turns): - assert t1.speaker == t2.speaker - assert t1.text == t2.text - assert t1.bone_count == t2.bone_count - - def test_round_trip_bytes(self, parsed, metrics): - import edcmbone.compress as codec - compressed = codec.to_bytes(parsed, metrics) - pt2, m2 = codec.from_bytes(compressed) - - assert len(pt2.turns) == len(parsed.turns) - for m1, m2i in zip(metrics, m2): - assert abs(m1.C - m2i.C) < 1e-9 - assert abs(m1.R - m2i.R) < 1e-9 - assert abs(m1.kappa - m2i.kappa) < 1e-9 - - def test_bytes_is_bytes(self, parsed): - import edcmbone.compress as codec - result = codec.to_bytes(parsed) - assert isinstance(result, bytes) - - def test_encode_without_metrics(self, parsed): - import edcmbone.compress as codec - encoded = codec.encode(parsed) - pt2, m2 = codec.decode(encoded) - assert m2 is None - assert len(pt2.turns) == len(parsed.turns) - - def test_compression_stats_keys(self, parsed): - import edcmbone.compress as codec - compressed = codec.to_bytes(parsed) - stats = codec.compression_stats(TRANSCRIPT_2SPK, compressed, parsed) - for key in ( - "original_chars", "original_bytes", "compressed_bytes", - "byte_compression_ratio", "structural_density", - "bone_count", "flesh_count", "family_counts", - "bone_entropy_bits", "huffman_min_bits", "huffman_codes", - "bone_stream_savings_pct", - ): - assert key in stats, f"missing key: {key}" - - def test_structural_density_in_range(self, parsed): - import edcmbone.compress as codec - compressed = codec.to_bytes(parsed) - stats = codec.compression_stats(TRANSCRIPT_2SPK, compressed, parsed) - assert 0.0 <= stats["structural_density"] <= 1.0 - - def test_huffman_codes_cover_families(self, parsed): - import edcmbone.compress as codec - compressed = codec.to_bytes(parsed) - stats = codec.compression_stats(TRANSCRIPT_2SPK, compressed, parsed) - observed_families = set(stats["family_counts"].keys()) - coded_families = set(stats["huffman_codes"].keys()) - assert observed_families == coded_families - - def test_bone_entropy_positive(self, parsed): - import edcmbone.compress as codec - compressed = codec.to_bytes(parsed) - stats = codec.compression_stats(TRANSCRIPT_2SPK, compressed, parsed) - assert stats["bone_entropy_bits"] > 0.0 - - def test_format_version_in_encoded(self, parsed): - import edcmbone.compress as codec - encoded = codec.encode(parsed) - assert encoded["v"] == 1 - - -# --------------------------------------------------------------------------- -# metrics.projection (Layer 3) -# --------------------------------------------------------------------------- - -@pytest.fixture(scope="module") -def agent_metrics(parsed, metrics): - from edcmbone.metrics import project_transcript - return project_transcript(parsed, metrics) - - -class TestProjection: - def test_returns_list(self, agent_metrics): - assert isinstance(agent_metrics, list) - - def test_one_per_round(self, parsed, agent_metrics): - assert len(agent_metrics) == len(parsed.rounds) - - def test_all_values_in_range(self, agent_metrics): - for am in agent_metrics: - for attr in ("CM", "DA", "DRIFT", "DVG", "INT", "TBF"): - val = getattr(am, attr) - assert 0.0 <= val <= 1.0, f"{attr}={val} out of [0,1]" - - def test_vector_length(self, agent_metrics): - for am in agent_metrics: - assert len(am.vector()) == 6 - - def test_as_dict_keys(self, agent_metrics): - d = agent_metrics[0].as_dict() - for k in ("CM", "DA", "DRIFT", "DVG", "INT", "TBF"): - assert k in d - - def test_tbf_zero_for_one_speaker(self, canon): - from edcmbone.parser import parse_transcript - from edcmbone.metrics import compute_transcript, project_transcript - single = "A: Everything here is from a single speaker.\nA: Nobody else speaks at all." - pt = parse_transcript(single, canon=canon) - l1 = compute_transcript(pt, canon=canon) - l3 = project_transcript(pt, l1) - for am in l3: - assert am.TBF == pytest.approx(0.0) - - def test_fire_alerts_returns_list(self, agent_metrics): - from edcmbone.metrics import fire_alerts - for am in agent_metrics: - alerts = fire_alerts(am) - assert isinstance(alerts, list) - for a in alerts: - assert isinstance(a, str) - assert a.startswith("ALERT_") - - def test_fire_alerts_high_values(self): - from edcmbone.metrics import AgentMetrics, fire_alerts - # All metrics above threshold -> all alerts fire - am = AgentMetrics(CM=0.9, DA=0.9, DRIFT=0.9, DVG=0.9, INT=0.9, TBF=0.9) - alerts = fire_alerts(am) - assert "ALERT_CM_HIGH" in alerts - assert "ALERT_DA_RISING" in alerts - assert "ALERT_DRIFT" in alerts - assert "ALERT_DVG_HIGH" in alerts - assert "ALERT_INT_HIGH" in alerts - assert "ALERT_TBF_SKEW" in alerts - - def test_fire_alerts_low_values(self): - from edcmbone.metrics import AgentMetrics, fire_alerts - am = AgentMetrics(CM=0.0, DA=0.0, DRIFT=0.0, DVG=0.0, INT=0.0, TBF=0.0) - assert fire_alerts(am) == [] - - def test_crosswalk_known_risks(self): - from edcmbone.metrics import crosswalk_risk - for risk in ("R_fix", "R_esc", "R_stag", "R_loop"): - result = crosswalk_risk(risk) - assert isinstance(result, list) - assert len(result) > 0 - for alert in result: - assert alert.startswith("ALERT_") - - def test_crosswalk_unknown_risk(self): - from edcmbone.metrics import crosswalk_risk - assert crosswalk_risk("R_nonexistent") == [] - - -# --------------------------------------------------------------------------- -# metrics.matrix -# --------------------------------------------------------------------------- - -class TestMatrix: - def test_a_matrix_has_version(self): - from edcmbone.metrics import A_MATRIX, MATRIX_VERSION - assert A_MATRIX["version"] == MATRIX_VERSION - - def test_a_matrix_covers_all_l1_metrics(self): - from edcmbone.metrics import A_MATRIX - for m in ("C", "R", "F", "E", "D", "N", "I", "O", "L", "P", "kappa"): - assert m in A_MATRIX["metrics"], f"Missing metric {m} in A_MATRIX" - - def test_projection_map_covers_all_l3_metrics(self): - from edcmbone.metrics import PROJECTION_MAP - for m in ("CM", "DA", "DRIFT", "DVG", "INT", "TBF"): - assert m in PROJECTION_MAP["metrics"] - - def test_alert_thresholds_cover_all_alerts(self): - from edcmbone.metrics import ALERT_THRESHOLDS - for name in ("ALERT_CM_HIGH", "ALERT_DA_RISING", "ALERT_DRIFT", - "ALERT_DVG_HIGH", "ALERT_INT_HIGH", "ALERT_TBF_SKEW"): - assert name in ALERT_THRESHOLDS["alerts"] - - def test_freeze_adds_sha256(self): - from edcmbone.metrics import A_MATRIX, freeze - frozen = freeze(A_MATRIX) - assert "_sha256" in frozen - assert len(frozen["_sha256"]) == 16 - - def test_freeze_is_deterministic(self): - from edcmbone.metrics import A_MATRIX, freeze - f1 = freeze(A_MATRIX) - f2 = freeze(A_MATRIX) - assert f1["_sha256"] == f2["_sha256"] - - def test_diff_identical_is_empty(self): - from edcmbone.metrics import A_MATRIX, diff - assert diff(A_MATRIX, A_MATRIX) == {} - - def test_diff_detects_change(self): - import copy - from edcmbone.metrics import A_MATRIX, diff - modified = copy.deepcopy(A_MATRIX) - modified["metrics"]["F"]["rep_b"] = 0.99 - changes = diff(A_MATRIX, modified) - assert ("F", "rep_b") in changes - assert changes[("F", "rep_b")] == (0.30, 0.99) - - -# --------------------------------------------------------------------------- -# closed_tokens -# --------------------------------------------------------------------------- - -class TestClosedTokens: - """Verify collision-resolution convention for the DISPATCH table.""" - - def test_since_resolves_to_preposition(self): - pytest.importorskip("ucns_v04") - from closed_tokens import DISPATCH, CLASS_PREPOSITION - class_idx, _ = DISPATCH["since"] - assert class_idx == CLASS_PREPOSITION - - def test_until_resolves_to_preposition(self): - pytest.importorskip("ucns_v04") - from closed_tokens import DISPATCH, CLASS_PREPOSITION - class_idx, _ = DISPATCH["until"] - assert class_idx == CLASS_PREPOSITION - - def test_as_resolves_to_preposition(self): - pytest.importorskip("ucns_v04") - from closed_tokens import DISPATCH, CLASS_PREPOSITION - class_idx, _ = DISPATCH["as"] - assert class_idx == CLASS_PREPOSITION - - def test_that_resolves_to_determiner(self): - pytest.importorskip("ucns_v04") - from closed_tokens import DISPATCH, CLASS_DETERMINER - class_idx, _ = DISPATCH["that"] - assert class_idx == CLASS_DETERMINER diff --git a/Tests/test_closed_tokens.py b/Tests/test_closed_tokens.py deleted file mode 100644 index 3dfe590..0000000 --- a/Tests/test_closed_tokens.py +++ /dev/null @@ -1,253 +0,0 @@ -""" -Tests for closed_tokens.py — the edcmbone closed-class UCNS encoder. - -Coverage: - 1. All ~250 closed tokens encode without error. - 2. Every encoded token recovers its class via class_of(). - 3. Within-class structural similarity: all pronouns share class anchor, - differ only in payload. - 4. Across-class structural difference: different classes have different - class anchors. - 5. Pronoun feature recovery: encoding "he" then comparing payloads - to "she" shows gender difference. - 6. Determiner feature recovery: definite vs indefinite distinguishable. - 7. Pairing chirality: open and close brackets are distinct objects; - encoding 5 open marks gives 5 distinct objects, same for closes. - 8. Roundtrip: encode → class_of → expected class. - 9. Whitespace varieties distinguishable. - 10. Numeral digit/spelled forms distinguishable. - 11. Full-vocabulary uniqueness: every token in DISPATCH produces a - UCNS object distinct from every other. -""" - -from fractions import Fraction -from ucns_v04 import unit_obj, multiply -from closed_tokens import ( - encode, class_of, feature_payload_of, - DISPATCH, CLASS_NAMES, - CLASS_PRONOUN, CLASS_DETERMINER, CLASS_PREPOSITION, - CLASS_CONJUNCTION, CLASS_AUXILIARY, CLASS_PARTICLE, - CLASS_INTERJECTION, CLASS_WHITESPACE, - CLASS_PUNCT_TERMINAL, CLASS_PUNCT_JUNCTURE, - CLASS_PUNCT_OPEN, CLASS_PUNCT_CLOSE, - CLASS_PUNCT_AFFIX, CLASS_PUNCT_MODAL, CLASS_NUMERAL, - PRONOUN_TABLE, DETERMINER_TABLE, -) - - -def banner(s): - print("\n" + "=" * 72) - print(s) - print("=" * 72) - -def ok(cond, label): - mark = "✓" if cond else "✗" - print(f" {mark} {label}") - if not cond: - raise AssertionError(label) - - -def test_all_tokens_encode(): - banner("Test 1 — all closed tokens encode without error") - failed = [] - for token in DISPATCH: - obj = encode(token) - if obj is None: - failed.append(token) - ok(not failed, f"all {len(DISPATCH)} closed tokens encode (failures: {failed})") - - -def test_class_recovery(): - banner("Test 2 — class_of recovers correct class for every token") - mismatches = [] - for token, (expected_class, _) in DISPATCH.items(): - obj = encode(token) - recovered = class_of(obj) - if recovered != expected_class: - mismatches.append((token, expected_class, recovered)) - if mismatches: - for token, exp, got in mismatches[:5]: - print(f" {token!r}: expected class {exp}, got {got}") - ok(not mismatches, - f"all {len(DISPATCH)} tokens roundtrip correctly " - f"({len(mismatches)} mismatches)") - - -def test_within_class_share_class_anchor(): - banner("Test 3 — within-class tokens share class anchor") - pronouns = [encode(w) for w in PRONOUN_TABLE] - classes = {class_of(p) for p in pronouns} - ok(classes == {CLASS_PRONOUN}, - f"all pronouns map to class {CLASS_PRONOUN} (got {classes})") - - determiners = [encode(w) for w in DETERMINER_TABLE if w not in PRONOUN_TABLE] - classes = {class_of(d) for d in determiners} - ok(classes == {CLASS_DETERMINER}, - f"all determiners map to class {CLASS_DETERMINER} (got {classes})") - - -def test_across_class_distinguishable(): - banner("Test 4 — different classes produce different objects") - samples = { - CLASS_PRONOUN: encode("i"), - CLASS_DETERMINER: encode("the"), - CLASS_PREPOSITION: encode("in"), - CLASS_CONJUNCTION: encode("and"), - CLASS_AUXILIARY: encode("is"), - CLASS_PARTICLE: encode("not"), - CLASS_INTERJECTION: encode("yes"), - CLASS_WHITESPACE: encode(" "), - CLASS_PUNCT_TERMINAL: encode("."), - CLASS_PUNCT_JUNCTURE: encode(","), - CLASS_PUNCT_OPEN: encode("("), - CLASS_PUNCT_CLOSE: encode(")"), - CLASS_PUNCT_AFFIX: encode("-"), - CLASS_PUNCT_MODAL: encode("..."), - CLASS_NUMERAL: encode("3"), - } - classes = [class_of(obj) for obj in samples.values()] - ok(len(set(classes)) == len(samples), - f"all 15 sampled classes are distinct (got {classes})") - - -def test_pronoun_feature_difference(): - banner("Test 5 — pronoun gender encoded in payload") - he = encode("he") - she = encode("she") - ok(class_of(he) == class_of(she) == CLASS_PRONOUN, - "'he' and 'she' both pronouns") - ok(not he.equivalent(she), - "'he' ≠ 'she' (gender difference encoded)") - he_payload = feature_payload_of(he) - she_payload = feature_payload_of(she) - ok(he_payload is not None and she_payload is not None, - "both have non-trivial feature payloads") - ok(not he_payload.equivalent(she_payload), - "'he' and 'she' payloads differ") - - -def test_determiner_feature_difference(): - banner("Test 6 — determiner definiteness encoded") - the = encode("the") - a = encode("a") - ok(class_of(the) == class_of(a) == CLASS_DETERMINER, - "'the' and 'a' both determiners") - ok(not the.equivalent(a), - "'the' ≠ 'a' (definiteness difference)") - - -def test_pairing_distinct(): - banner("Test 7 — pairing marks: opens distinct from each other and from closes") - opens = [encode(c) for c in ["(", "[", "{", "“", "‘"]] - closes = [encode(c) for c in [")", "]", "}", "”", "’"]] - # All opens are class PUNCT_OPEN. - open_classes = {class_of(o) for o in opens} - ok(open_classes == {CLASS_PUNCT_OPEN}, - f"all open marks in PUNCT_OPEN class (got {open_classes})") - close_classes = {class_of(c) for c in closes} - ok(close_classes == {CLASS_PUNCT_CLOSE}, - f"all close marks in PUNCT_CLOSE class (got {close_classes})") - # Within opens, all 5 distinct. - open_distinct = all( - not opens[i].equivalent(opens[j]) - for i in range(len(opens)) for j in range(i+1, len(opens)) - ) - ok(open_distinct, "all 5 open marks pairwise distinct") - close_distinct = all( - not closes[i].equivalent(closes[j]) - for i in range(len(closes)) for j in range(i+1, len(closes)) - ) - ok(close_distinct, "all 5 close marks pairwise distinct") - - -def test_whitespace_varieties(): - banner("Test 8 — whitespace kinds distinguishable") - space = encode(" ") - tab = encode("\t") - newline = encode("\n") - nbsp = encode(" ") - ok(not space.equivalent(tab), "space ≠ tab") - ok(not space.equivalent(newline), "space ≠ newline") - ok(not space.equivalent(nbsp), "space ≠ NBSP") - ok(not tab.equivalent(newline), "tab ≠ newline") - ok(class_of(space) == class_of(tab) == class_of(newline) == CLASS_WHITESPACE, - "all whitespace kinds in WHITESPACE class") - - -def test_numeral_forms(): - banner("Test 9 — numeral digit and spelled forms distinguishable") - three_digit = encode("3") - three_word = encode("three") - ok(class_of(three_digit) == class_of(three_word) == CLASS_NUMERAL, - "both numerals") - ok(not three_digit.equivalent(three_word), - "'3' ≠ 'three' (form difference)") - # Different values distinguishable. - five_digit = encode("5") - ok(not three_digit.equivalent(five_digit), "'3' ≠ '5'") - - -def test_full_vocabulary_uniqueness(): - banner("Test 10 — every token (modulo phonetic variants) encodes uniquely") - # Known phonetic variants — share encoding by design. - PHONETIC_VARIANTS = {("a", "an"), ("an", "a")} - - seen = [] - duplicates = [] - for token in DISPATCH: - obj = encode(token) - found = False - for prior_token, prior_obj in seen: - if obj.equivalent(prior_obj): - pair = tuple(sorted([prior_token, token])) - # Allow only documented phonetic variants. - if pair not in PHONETIC_VARIANTS and (pair[0], pair[1]) not in PHONETIC_VARIANTS: - if (prior_token, token) not in PHONETIC_VARIANTS: - duplicates.append((prior_token, token)) - found = True - break - if not found: - seen.append((token, obj)) - if duplicates: - for a, b in duplicates[:10]: - print(f" {a!r} ≡ {b!r}") - ok(not duplicates, - f"all {len(DISPATCH)} tokens encode uniquely (modulo variants); " - f"{len(duplicates)} unexpected collisions") - - -def test_unknown_token(): - banner("Test 11 — unknown (open-class) tokens return None") - ok(encode("dog") is None, "'dog' (open-class noun) returns None") - ok(encode("running") is None, "'running' (open-class verb) returns None") - ok(encode("xyzzy") is None, "nonsense token returns None") - ok(encode("") is None, "empty string returns None") - - -def test_case_insensitivity_for_words(): - banner("Test 12 — word lookup is case-insensitive") - ok(encode("THE").equivalent(encode("the")), "'THE' = 'the'") - ok(encode("The").equivalent(encode("the")), "'The' = 'the'") - ok(encode("I").equivalent(encode("i")), "'I' = 'i'") - - -def main(): - test_all_tokens_encode() - test_class_recovery() - test_within_class_share_class_anchor() - test_across_class_distinguishable() - test_pronoun_feature_difference() - test_determiner_feature_difference() - test_pairing_distinct() - test_whitespace_varieties() - test_numeral_forms() - test_full_vocabulary_uniqueness() - test_unknown_token() - test_case_insensitivity_for_words() - print("\n" + "=" * 72) - print("All edcmbone closed-token tests passed.") - print("=" * 72) - - -if __name__ == "__main__": - main() diff --git a/backend/pyproject.toml b/backend/pyproject.toml index b858ecc..50efefa 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -33,7 +33,7 @@ Repository = "https://github.com/The-Interdependency/edcmbone" Issues = "https://github.com/The-Interdependency/edcmbone/issues" [tool.pytest.ini_options] -testpaths = ["../Tests"] +testpaths = ["../tests"] [tool.hatch.build.targets.wheel] packages = ["src/edcmbone"] diff --git a/backend/src/edcmbone/parser/turns_rounds.py b/backend/src/edcmbone/parser/turns_rounds.py index d53b50d..911e03f 100644 --- a/backend/src/edcmbone/parser/turns_rounds.py +++ b/backend/src/edcmbone/parser/turns_rounds.py @@ -212,11 +212,12 @@ def _group_into_rounds(turns, strategy="cycle"): # --------------------------------------------------------------------------- # Split into word-runs and individual punctuation characters -_WORD_RE = re.compile(r"[A-Za-z''\-]+|[^\w\s]|\d+") +_WORD_RE = re.compile(r"[A-Za-z]+(?:'[A-Za-z]+)*|[0-9]+|[^\w\s]") def _raw_tokens(text): """Return list of raw token strings from utterance text.""" + text = (text or '').replace('’', "'").replace('‘', "'") return _WORD_RE.findall(text) @@ -246,6 +247,11 @@ def __init__(self, canon: CanonLoader): # but we access via the public API for correctness) self._word_cache = {} self._affix_cache = {} + self._valid_stems = { + entry["word"].lower() + for entry in canon.all_words() + if entry.get("primary") != "S" + } def _make_bone(self, surface, normalized, bone_type, entry): return BoneToken( @@ -296,6 +302,9 @@ def classify_sequence(self, raw_tokens): matched_affix = False for pre in self._prefixes: if lower.startswith(pre) and len(lower) - len(pre) >= 2: + residual = lower[len(pre):] + if residual not in self._valid_stems: + continue affix_key = pre + "-" if affix_key not in self._affix_cache: self._affix_cache[affix_key] = self._canon.lookup_affix(affix_key) @@ -311,6 +320,9 @@ def classify_sequence(self, raw_tokens): # 3b. Affix strip — suffix for suf in self._suffixes: if lower.endswith(suf) and len(lower) - len(suf) >= 2: + residual = lower[:-len(suf)] + if residual not in self._valid_stems: + continue affix_key = "-" + suf if affix_key not in self._affix_cache: self._affix_cache[affix_key] = self._canon.lookup_affix(affix_key) diff --git a/backend/src/edcmbone/ucns/closed_tokens.py b/backend/src/edcmbone/ucns/closed_tokens.py index f71208b..d2e57d7 100644 --- a/backend/src/edcmbone/ucns/closed_tokens.py +++ b/backend/src/edcmbone/ucns/closed_tokens.py @@ -37,7 +37,7 @@ from fractions import Fraction from typing import Dict, List, Optional, Tuple -from ucns_v04 import ( +from .ucns_v04 import ( UCNSObject, AnchorPayload, unit_obj, multiply, ) diff --git a/backend/src/edcmbone/ucns/ucns_v04.py b/backend/src/edcmbone/ucns/ucns_v04.py new file mode 100644 index 0000000..8614068 --- /dev/null +++ b/backend/src/edcmbone/ucns/ucns_v04.py @@ -0,0 +1,206 @@ +""" +ucns_v04 — UCNS Engine (turn-fraction angle convention) +======================================================== +Angles are stored as Fraction objects representing fractions of a full turn: + 0 = 0 deg, 1/4 = 90 deg, 1/2 = 180 deg, 2 = 720 deg = 0 on doubled cover. + +The algebra operates on the doubled cover of the unit circle, so the +fundamental period is 2 (two full turns = identity). Normalization shifts +the first anchor to theta=0 and reduces all thetas mod 2. + +n_min is the LCM of denominators of all non-zero anchor thetas, computed +directly from the Fraction denominators (no pi-unit conversion). + +Public API +---------- +AnchorPayload(theta, payload) +UCNSObject(n_dec, n_min, anchors_pos, faces_pos) + .anchors_pos : tuple[AnchorPayload, ...] + .faces_pos : tuple[int, ...] + .n_dec : int + .n_min : int + .normalize() : UCNSObject (returns self; normalization done in __init__) + .equivalent(other) : bool +unit_obj() : UCNSObject (multiplicative unit) +is_unit_payload(obj): bool +multiply(A, B) : UCNSObject (A ⊠ B) +""" + +from __future__ import annotations + +from fractions import Fraction +from math import gcd +from functools import reduce +from typing import Optional, Tuple + +__all__ = [ + "AnchorPayload", + "UCNSObject", + "unit_obj", + "is_unit_payload", + "multiply", +] + + +def _lcm(a: int, b: int) -> int: + return a * b // gcd(a, b) + + +def _reduce_lcm(denoms): + return reduce(_lcm, denoms, 1) + + +class AnchorPayload: + """Named container for a (theta, payload) anchor entry.""" + __slots__ = ("theta", "payload") + + def __init__(self, theta, payload): + self.theta = Fraction(theta) + self.payload = payload # UCNSObject or None + + def __repr__(self) -> str: + return f"AnchorPayload(theta={self.theta}, payload={self.payload!r})" + + +class UCNSObject: + """ + A UCNS algebraic object on the doubled unit circle. + + anchors_pos : tuple of AnchorPayload (theta in turn-fractions, payload) + faces_pos : tuple of int (face label per anchor, 0 or 1) + n_dec : declared carrier size (context hint; upper bound on n_min) + n_min : minimal carrier = LCM of denominators of non-zero thetas + """ + + def __init__( + self, + n_dec: int, + n_min: int, + anchors_pos, + faces_pos, + ): + self.n_dec = int(n_dec) + self._anchors_raw = tuple(anchors_pos) + self._faces_raw = tuple(faces_pos) + # Normalization populates .anchors_pos, .faces_pos, .n_min. + self.anchors_pos: Tuple[AnchorPayload, ...] = self._anchors_raw + self.faces_pos: Tuple[int, ...] = self._faces_raw + self.n_min = int(n_min) + self._do_normalize() + + def _do_normalize(self): + """Shift so first anchor is at 0; recompute n_min from thetas.""" + if not self._anchors_raw: + self.anchors_pos = () + self.faces_pos = () + self.n_min = 1 + return + + theta0 = self._anchors_raw[0].theta + normalized = [] + for ap in self._anchors_raw: + new_theta = (ap.theta - theta0) % 2 + normalized.append(AnchorPayload(new_theta, ap.payload)) + + self.anchors_pos = tuple(normalized) + self.faces_pos = self._faces_raw + + non_zero_denoms = [ + ap.theta.denominator + for ap in self.anchors_pos + if ap.theta != 0 + ] + self.n_min = _reduce_lcm(non_zero_denoms) if non_zero_denoms else 1 + + def normalize(self) -> "UCNSObject": + """Return self (normalization happens at construction time).""" + return self + + def equivalent(self, other: "UCNSObject") -> bool: + """Deep structural equivalence.""" + if not isinstance(other, UCNSObject): + return False + a = self + b = other + if len(a.anchors_pos) != len(b.anchors_pos): + return False + if a.faces_pos != b.faces_pos: + return False + for ap, bp in zip(a.anchors_pos, b.anchors_pos): + if ap.theta != bp.theta: + return False + if ap.payload is None and bp.payload is None: + continue + if ap.payload is None or bp.payload is None: + return False + if not ap.payload.equivalent(bp.payload): + return False + return True + + def __repr__(self) -> str: + thetas = [str(ap.theta) for ap in self.anchors_pos] + return f"UCNSObject(n_dec={self.n_dec}, n_min={self.n_min}, thetas={thetas})" + + +def unit_obj() -> UCNSObject: + """Return the multiplicative unit: single anchor at theta=0, no payload.""" + return UCNSObject( + n_dec=1, + n_min=1, + anchors_pos=(AnchorPayload(Fraction(0), None),), + faces_pos=(0,), + ) + + +def is_unit_payload(obj: Optional[UCNSObject]) -> bool: + """True if obj is None (no payload) or structurally equivalent to the unit.""" + if obj is None: + return True + return obj.equivalent(unit_obj()) + + +def multiply(A: UCNSObject, B: UCNSObject) -> UCNSObject: + """ + UCNS product A ⊠ B. + + Each anchor a_k of A is combined with each anchor b_j of B to yield + a result anchor with: + theta = (a_k.theta + b_j.theta) % 2 + payload = multiply(a_k.payload, b_j.payload) [recursive; None is unit] + face = a_k.face XOR b_j.face + + The result has len(A.anchors_pos) * len(B.anchors_pos) anchors, ordered + A-major (outer loop over A, inner loop over B). + + The single-anchor unit_obj() is a two-sided identity under this product. + The product is associative. + """ + new_anchors = [] + new_faces = [] + + for ai, ak in enumerate(A.anchors_pos): + for bi, bj in enumerate(B.anchors_pos): + theta = (ak.theta + bj.theta) % 2 + + pa = ak.payload + pb = bj.payload + if pa is None and pb is None: + payload = None + elif pa is None: + payload = pb + elif pb is None: + payload = pa + else: + payload = multiply(pa, pb) + + face = A.faces_pos[ai] ^ B.faces_pos[bi] + new_anchors.append(AnchorPayload(theta, payload)) + new_faces.append(face) + + n_dec = _lcm(A.n_dec, B.n_dec) + return UCNSObject( + n_dec=n_dec, + n_min=1, + anchors_pos=tuple(new_anchors), + faces_pos=tuple(new_faces), + ) diff --git a/core/operator/matcher.py b/core/operator/matcher.py index 47ecb35..2a76f6c 100644 --- a/core/operator/matcher.py +++ b/core/operator/matcher.py @@ -61,7 +61,7 @@ def hyphen_compound_emission(tok: str) -> List[str]: return ["K"] return [] -def match_affixes(tok: str, prefix_map: Dict[str, str], suffix_map: Dict[str, str]) -> Tuple[List[str], str]: +def match_affixes(tok: str, prefix_map: Dict[str, str], suffix_map: Dict[str, str], valid_stems: set[str] | None = None) -> Tuple[List[str], str]: """ Longest-match-first; prefix then suffix; emit ALL matched affixes. Returns (families_emitted, residual_root). @@ -77,8 +77,11 @@ def match_affixes(tok: str, prefix_map: Dict[str, str], suffix_map: Dict[str, st changed = False for p in pref_list: if root.startswith(p) and len(root) > len(p): + residual = root[len(p):] + if valid_stems is not None and residual not in valid_stems: + continue fams.append(prefix_map[p]) - root = root[len(p):] + root = residual changed = True break @@ -88,8 +91,11 @@ def match_affixes(tok: str, prefix_map: Dict[str, str], suffix_map: Dict[str, st changed = False for s in suf_list: if root.endswith(s) and len(root) > len(s): + residual = root[:-len(s)] + if valid_stems is not None and residual not in valid_stems: + continue fams.append(suffix_map[s]) - root = root[:-len(s)] + root = residual changed = True break diff --git a/core/parsing/normalizer.py b/core/parsing/normalizer.py index 586634c..42600b5 100644 --- a/core/parsing/normalizer.py +++ b/core/parsing/normalizer.py @@ -7,7 +7,8 @@ def normalize_text_for_matching(s: str) -> str: # Lowercase + normalize both smart-quote apostrophe variants to plain "’" return ( (s or "") - .replace("’", "’") # right single quotation mark (most common in contractions) - .replace("‘", "’") # left single quotation mark + .replace("’", "'") # right single quotation mark -> ASCII apostrophe + .replace("‘", "'") # left single quotation mark -> ASCII apostrophe + .replace("‛", "'") # single high-reversed-9 quotation mark -> ASCII apostrophe .lower() ) diff --git a/tests/test_affix_residual_validation.py b/tests/test_affix_residual_validation.py new file mode 100644 index 0000000..d00f5f9 --- /dev/null +++ b/tests/test_affix_residual_validation.py @@ -0,0 +1,54 @@ +from pathlib import Path +import sys + +ROOT = Path(__file__).resolve().parents[1] +BACKEND_SRC = ROOT / "backend" / "src" +if str(BACKEND_SRC) not in sys.path: + sys.path.insert(0, str(BACKEND_SRC)) + +from edcmbone.canon import CanonLoader +from edcmbone.parser.turns_rounds import _BoneClassifier, FleshToken +from core.operator.matcher import match_affixes + + +def test_backend_parser_affix_does_not_fire_on_invalid_residuals(): + canon = CanonLoader() + c = _BoneClassifier(canon) + toks = ["uncle", "unit", "universe", "under", "unique"] + out = c.classify_sequence(toks) + assert all((not hasattr(x, "bone_type") or x.bone_type != "affix") for x in out) + + +def test_backend_parser_affix_positive_cases_still_emit_for_canon_valid_stems(): + canon = CanonLoader() + c = _BoneClassifier(canon) + + # Guaranteed by current canon word inventory: "redo" -> residual "do" exists. + out = c.classify_sequence(["redo"]) + assert [getattr(x, "bone_type", None) for x in out] == ["affix"] + + +def test_backend_parser_examples_unhappy_and_linking_are_canon_dependent(): + canon = CanonLoader() + c = _BoneClassifier(canon) + + # These examples are morphologically valid in English, but backend affix emission + # depends on whether residual stems are present in the canon word index. + # This test documents that parser behavior remains canon-driven, not heuristic. + for tok, residual in (("unhappy", "happy"), ("linking", "link")): + out = c.classify_sequence([tok])[0] + emits_affix = getattr(out, "bone_type", None) == "affix" + residual_in_canon = canon.lookup_word(residual) is not None + assert emits_affix == residual_in_canon + + +def test_core_matcher_affix_respects_valid_stems_negative_and_positive(): + prefix_map = {"un": "P", "re": "K"} + suffix_map = {"ing": "K"} + valid_stems = {"happy", "do", "link"} + + fams, root = match_affixes("unhappy", prefix_map, suffix_map, valid_stems=valid_stems) + assert fams == ["P"] and root == "happy" + + fams2, root2 = match_affixes("uncle", prefix_map, suffix_map, valid_stems=valid_stems) + assert fams2 == [] and root2 == "uncle" diff --git a/tests/test_apostrophe_normalization_and_tokenization.py b/tests/test_apostrophe_normalization_and_tokenization.py new file mode 100644 index 0000000..c5d2d2e --- /dev/null +++ b/tests/test_apostrophe_normalization_and_tokenization.py @@ -0,0 +1,34 @@ +import re +from pathlib import Path + +from core.parsing.normalizer import normalize_text_for_matching + + +def _parser_word_re(): + src = Path("backend/src/edcmbone/parser/turns_rounds.py").read_text() + m = re.search(r"_WORD_RE\s*=\s*re\.compile\(r\"([^\"]+)\"\)", src) + assert m, "Could not locate _WORD_RE in parser/turns_rounds.py" + return re.compile(m.group(1)) + + +def _parser_raw_tokens_like_impl(text: str): + word_re = _parser_word_re() + text = (text or "").replace("’", "'").replace("‘", "'") + return word_re.findall(text) + + +def test_normalize_text_for_matching_maps_smart_to_ascii_apostrophe(): + text = "don’t can‘t it’s" + out = normalize_text_for_matching(text) + assert out == "don't can't it's" + + +def test_parser_word_re_keeps_ascii_contractions_whole(): + tokens = _parser_raw_tokens_like_impl("don't can't it's") + assert tokens == ["don't", "can't", "it's"] + + +def test_parser_word_re_keeps_smart_contractions_whole_after_normalization(): + normalized = normalize_text_for_matching("don’t can’t it’s") + tokens = _parser_raw_tokens_like_impl(normalized) + assert tokens == ["don't", "can't", "it's"] From c95a778bc9cd21968265f1fdf6d6fed1d61a27ee Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Fri, 15 May 2026 10:05:35 -0700 Subject: [PATCH 2/5] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- tests/test_apostrophe_normalization_and_tokenization.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_apostrophe_normalization_and_tokenization.py b/tests/test_apostrophe_normalization_and_tokenization.py index c5d2d2e..681fd61 100644 --- a/tests/test_apostrophe_normalization_and_tokenization.py +++ b/tests/test_apostrophe_normalization_and_tokenization.py @@ -5,7 +5,9 @@ def _parser_word_re(): - src = Path("backend/src/edcmbone/parser/turns_rounds.py").read_text() + repo_root = Path(__file__).resolve().parents[1] + src_path = repo_root / "backend" / "src" / "edcmbone" / "parser" / "turns_rounds.py" + src = src_path.read_text() m = re.search(r"_WORD_RE\s*=\s*re\.compile\(r\"([^\"]+)\"\)", src) assert m, "Could not locate _WORD_RE in parser/turns_rounds.py" return re.compile(m.group(1)) From e5224736e54aaf097839e9d276290c62aa750990 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Fri, 15 May 2026 10:05:48 -0700 Subject: [PATCH 3/5] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- backend/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 50efefa..de6ed85 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -33,7 +33,7 @@ Repository = "https://github.com/The-Interdependency/edcmbone" Issues = "https://github.com/The-Interdependency/edcmbone/issues" [tool.pytest.ini_options] -testpaths = ["../tests"] +testpaths = ["../Tests", "../tests"] [tool.hatch.build.targets.wheel] packages = ["src/edcmbone"] From d0f01192475312cdf075c7ab5863096b8fd66fe2 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 15 May 2026 17:08:38 +0000 Subject: [PATCH 4/5] Fix tokenizer to preserve hyphenated compounds as one token Agent-Logs-Url: https://github.com/The-Interdependency/edcmbone/sessions/73154e0c-bf87-4066-b51c-d52b607c3e6f Co-authored-by: erinepshovel-code <250928284+erinepshovel-code@users.noreply.github.com> --- backend/src/edcmbone/parser/turns_rounds.py | 5 +++-- tests/test_apostrophe_normalization_and_tokenization.py | 5 +++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/backend/src/edcmbone/parser/turns_rounds.py b/backend/src/edcmbone/parser/turns_rounds.py index 911e03f..a6a673b 100644 --- a/backend/src/edcmbone/parser/turns_rounds.py +++ b/backend/src/edcmbone/parser/turns_rounds.py @@ -211,8 +211,9 @@ def _group_into_rounds(turns, strategy="cycle"): # Tokenizer — word + punctuation split # --------------------------------------------------------------------------- -# Split into word-runs and individual punctuation characters -_WORD_RE = re.compile(r"[A-Za-z]+(?:'[A-Za-z]+)*|[0-9]+|[^\w\s]") +# Split into word-runs and individual punctuation characters. +# Preserve hyphenated compounds as one token before punctuation handling. +_WORD_RE = re.compile(r"[A-Za-z0-9]+(?:-[A-Za-z0-9]+)+|[A-Za-z]+(?:'[A-Za-z]+)*|[0-9]+|[^\w\s]") def _raw_tokens(text): diff --git a/tests/test_apostrophe_normalization_and_tokenization.py b/tests/test_apostrophe_normalization_and_tokenization.py index 681fd61..e865278 100644 --- a/tests/test_apostrophe_normalization_and_tokenization.py +++ b/tests/test_apostrophe_normalization_and_tokenization.py @@ -34,3 +34,8 @@ def test_parser_word_re_keeps_smart_contractions_whole_after_normalization(): normalized = normalize_text_for_matching("don’t can’t it’s") tokens = _parser_raw_tokens_like_impl(normalized) assert tokens == ["don't", "can't", "it's"] + + +def test_parser_word_re_keeps_hyphenated_compounds_as_one_surface_token(): + tokens = _parser_raw_tokens_like_impl("state-of-the-art") + assert tokens == ["state-of-the-art"] From 9738a77077a42ab5e173397e5f1bc62d0763f8a2 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 15 May 2026 17:12:17 +0000 Subject: [PATCH 5/5] Fix affix validation to defer stem check until after all affixes stripped Agent-Logs-Url: https://github.com/The-Interdependency/edcmbone/sessions/13182a36-ab97-4a20-91ff-5e43ffad6f5e Co-authored-by: erinepshovel-code <250928284+erinepshovel-code@users.noreply.github.com> --- core/operator/matcher.py | 13 +++++++++---- tests/test_affix_residual_validation.py | 26 +++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/core/operator/matcher.py b/core/operator/matcher.py index 2a76f6c..81767aa 100644 --- a/core/operator/matcher.py +++ b/core/operator/matcher.py @@ -65,6 +65,10 @@ def match_affixes(tok: str, prefix_map: Dict[str, str], suffix_map: Dict[str, st """ Longest-match-first; prefix then suffix; emit ALL matched affixes. Returns (families_emitted, residual_root). + + When valid_stems is provided, validation is deferred until after all affixes + are stripped. This allows multi-affix words like "redoing" (re+do+ing) to work + correctly even when intermediate forms like "doing" are not in valid_stems. """ t = normalize_text_for_matching(tok) fams: List[str] = [] @@ -78,8 +82,6 @@ def match_affixes(tok: str, prefix_map: Dict[str, str], suffix_map: Dict[str, st for p in pref_list: if root.startswith(p) and len(root) > len(p): residual = root[len(p):] - if valid_stems is not None and residual not in valid_stems: - continue fams.append(prefix_map[p]) root = residual changed = True @@ -92,11 +94,14 @@ def match_affixes(tok: str, prefix_map: Dict[str, str], suffix_map: Dict[str, st for s in suf_list: if root.endswith(s) and len(root) > len(s): residual = root[:-len(s)] - if valid_stems is not None and residual not in valid_stems: - continue fams.append(suffix_map[s]) root = residual changed = True break + # Validate final root against valid_stems if provided. + # If validation fails, return no affixes (treat as unmatched). + if valid_stems is not None and fams and root not in valid_stems: + return [], t + return fams, root diff --git a/tests/test_affix_residual_validation.py b/tests/test_affix_residual_validation.py index d00f5f9..4ab77f8 100644 --- a/tests/test_affix_residual_validation.py +++ b/tests/test_affix_residual_validation.py @@ -52,3 +52,29 @@ def test_core_matcher_affix_respects_valid_stems_negative_and_positive(): fams2, root2 = match_affixes("uncle", prefix_map, suffix_map, valid_stems=valid_stems) assert fams2 == [] and root2 == "uncle" + + +def test_core_matcher_multi_affix_words_validate_final_root_only(): + """ + Regression test for issue where intermediate residuals were validated too early. + For "redoing" (re+do+ing), the intermediate "doing" is not a valid stem, but + the final root "do" is. Validation should be deferred until all affixes are stripped. + """ + prefix_map = {"re": "K", "un": "P"} + suffix_map = {"ing": "K", "ed": "K"} + valid_stems = {"do", "happy"} + + # "redoing" -> strip "re" -> "doing" -> strip "ing" -> "do" (valid!) + fams, root = match_affixes("redoing", prefix_map, suffix_map, valid_stems=valid_stems) + assert fams == ["K", "K"], f"Expected ['K', 'K'] but got {fams}" + assert root == "do", f"Expected 'do' but got {root}" + + # "unhappying" -> strip "un" -> "happying" -> strip "ing" -> "happy" (valid!) + fams2, root2 = match_affixes("unhappying", prefix_map, suffix_map, valid_stems=valid_stems) + assert fams2 == ["P", "K"], f"Expected ['P', 'K'] but got {fams2}" + assert root2 == "happy", f"Expected 'happy' but got {root2}" + + # "rethinking" -> strip "re" -> "thinking" -> strip "ing" -> "think" (NOT valid!) + fams3, root3 = match_affixes("rethinking", prefix_map, suffix_map, valid_stems=valid_stems) + assert fams3 == [], f"Expected [] but got {fams3}" + assert root3 == "rethinking", f"Expected 'rethinking' but got {root3}"