From 38298b33c3e218ee79cff20d7590b24b74715a03 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Tue, 12 May 2026 12:34:59 -0700 Subject: [PATCH 1/4] fix: dispatch collisions, dead code, NBSP key, docstring (issues #31 #32 #33 #36) --- CLAUDE.md | 9 ++++--- closed_tokens.py | 64 ++++++++++++++++++++++++++++-------------------- 2 files changed, 42 insertions(+), 31 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index e9172b6..daff57f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -71,7 +71,7 @@ edcmbone/ │ ├── parse/ # parser refactor (scaffolded) │ └── routing/ # routing layer (scaffolded) │ -├── core/ # Core framework package (IN PROGRESS — stubs) +├── core/ # Core framework package (IN PROGRESS — partially implemented) │ ├── __init__.py │ ├── behavioral/ │ ├── bridge/ @@ -114,9 +114,9 @@ The repo currently has **two parallel package structures**: | Layout | Path | Status | When to Use | |--------|------|--------|-------------| | **Stable** | `Backend/src/edcmbone/` | 87 tests passing, production-ready | Installing and consuming the library | -| **Refactor** | `edcmbone/` (root) and `core/` | Scaffolded stubs, 0-byte files | Adding new architecture — fill in stubs here | +| **Refactor** | `edcmbone/` (root) and `core/` | Partially implemented refactor-in-progress; see individual module docstrings for what is and isn't complete | Adding new architecture — fill in stubs here | -Do not assume files in `edcmbone/` (root) or `core/` contain working implementations — most are 0-byte stubs. The **stable** code lives under `Backend/src/edcmbone/`. +Do not assume all files in `edcmbone/` (root) or `core/` contain working implementations. `core/parsing/`, `core/operator/`, and `core/bridge/` now contain substantive implementations; other subdirectories may still be stubs. The **stable** code lives under `Backend/src/edcmbone/`. --- @@ -237,7 +237,8 @@ npm run build - No CI/CD pipeline - No linting configs (prefer `ruff` when adding) - No pre-commit hooks -- `edcmbone/` (root) and `core/` modules are stubs — not yet implemented +- `edcmbone/` (root) stubs — not yet fully implemented (see individual module docstrings) +- `core/` — partially implemented refactor-in-progress; `core/parsing/`, `core/operator/`, and `core/bridge/` contain substantive implementations; other subdirectories may still be stubs - `aimmh-lib/backend/server.py` is a stub - `tailwind.config.js` needs `content` globs before use - Frontend tests diff --git a/closed_tokens.py b/closed_tokens.py index c73fee4..7fd1c4d 100644 --- a/closed_tokens.py +++ b/closed_tokens.py @@ -29,17 +29,23 @@ - face bit unused at the host level (set to 0) The pairing-mark rule: smart open quote and curly open brackets go in class 10; -their close counterparts in class 11. The disk-flip of an open-mark UCNS -object equals the close-mark object (verified by test). +their close counterparts in class 11. The disk-flip property (open-mark ↔ +close-mark symmetry) is stated as a design invariant; implementation is in +ucns_v04.py (TODO: add explicit test). + +Dispatch collision convention: when a token appears in multiple class tables, +the first table wins by ordering. Specifically: for 'since' and 'until' the +preposition sense wins; for 'as' the preposition sense wins; for 'that' the +determiner sense wins. """ from __future__ import annotations +import warnings from fractions import Fraction from typing import Dict, List, Optional, Tuple from ucns_v04 import ( UCNSObject, AnchorPayload, - unit_obj, multiply, ) @@ -57,7 +63,7 @@ CLASS_PUNCT_JUNCTURE = 9 CLASS_PUNCT_OPEN = 10 CLASS_PUNCT_CLOSE = 11 -CLASS_PUNCT_QUOTE = 12 # reserved; quote glyphs are routed to OPEN/CLOSE +CLASS_PUNCT_QUOTE = 12 # reserved but unpopulated; quote glyphs are routed to OPEN/CLOSE CLASS_PUNCT_AFFIX = 13 CLASS_PUNCT_MODAL = 14 CLASS_NUMERAL = 15 @@ -75,25 +81,20 @@ # ---------- helpers ---------- -def _class_anchor(class_idx: int) -> Fraction: - """Anchor position on the 16-gonal lattice (in turns).""" - return Fraction(class_idx, HOST_CARRIER) - - def _wrap_with_class(class_idx: int, payload: Optional[UCNSObject]) -> UCNSObject: """ Encode class and feature payload as a 2-anchor object on a 32-gonal lattice. Anchor 0: at position 0 (structural marker, no payload). - Anchor 1: at position (class_idx + 1) / 32, carrying the feature payload. + Anchor 1: at position (class_idx + 1) / (2 * HOST_CARRIER), carrying the feature payload. After normalize, the first anchor stays at 0 (it's already there), and - the second stays at (class_idx + 1) / 32 — a relative offset that + the second stays at (class_idx + 1) / (2 * HOST_CARRIER) — a relative offset that encodes the class index uniquely for class_idx ∈ {0..15}. """ - class_offset = Fraction(class_idx + 1, 32) + class_offset = Fraction(class_idx + 1, 2 * HOST_CARRIER) return UCNSObject( - n_dec=32, + n_dec=2 * HOST_CARRIER, n_min=1, # normalize will recompute anchors_pos=( AnchorPayload(Fraction(0), None), @@ -352,11 +353,11 @@ def _lcm(a, b): # Whitespace: kind 0=space, 1=tab, 2=newline, 3=double-newline, 4=non-breaking WHITESPACE_TABLE = { - " ": {"kind": (0, 5), "breaking": (0, 2)}, - "\t": {"kind": (1, 5), "breaking": (0, 2)}, - "\n": {"kind": (2, 5), "breaking": (0, 2)}, + " ": {"kind": (0, 5), "breaking": (0, 2)}, + "\t": {"kind": (1, 5), "breaking": (0, 2)}, + "\n": {"kind": (2, 5), "breaking": (0, 2)}, "\n\n":{"kind": (3, 5), "breaking": (0, 2)}, - " ": {"kind": (4, 5), "breaking": (1, 2)}, # NBSP + "\xa0": {"kind": (4, 5), "breaking": (1, 2)}, # U+00A0 NON-BREAKING SPACE } # Sentence terminals (modality) @@ -379,15 +380,15 @@ def _lcm(a, b): "(": {"shape": (0, 5)}, "[": {"shape": (1, 5)}, "{": {"shape": (2, 5)}, - "“": {"shape": (3, 5)}, # “ - "‘": {"shape": (4, 5)}, # ‘ + "“": {"shape": (3, 5)}, # “ LEFT DOUBLE QUOTATION MARK + "‘": {"shape": (4, 5)}, # ‘ LEFT SINGLE QUOTATION MARK } PAIR_CLOSE_TABLE = { ")": {"shape": (0, 5)}, "]": {"shape": (1, 5)}, "}": {"shape": (2, 5)}, - "”": {"shape": (3, 5)}, # ” - "’": {"shape": (4, 5)}, # ’ + "”": {"shape": (3, 5)}, # ” RIGHT DOUBLE QUOTATION MARK + "’": {"shape": (4, 5)}, # ’ RIGHT SINGLE QUOTATION MARK } # Affix and connective marks @@ -424,8 +425,14 @@ def _build_dispatch_table(): Phonetic variants ('a' / 'an') are canonically merged: both produce the same UCNS object since they differ only in surface phonology. + + Collision convention: when a token appears in multiple class tables the + first entry wins and a warning is emitted. By table-ordering convention: + - 'since' and 'until' resolve to the preposition sense + - 'as' resolves to the preposition sense + - 'that' resolves to the determiner sense """ - PHONETIC_VARIANTS = {"an": "a"} # an → a as canonical encoding + PHONETIC_VARIANTS = {"an": "a"} # an -> a as canonical encoding out: Dict[str, Tuple[int, Dict[str, Tuple[int, int]]]] = {} @@ -434,7 +441,11 @@ def add(token, class_idx, feats): # Skip — variant will be aliased after the canonical is in place. return if token in out: - return # earlier class entry wins + warnings.warn( + f"closed_tokens: dispatch collision on {token!r}; keeping first entry", + stacklevel=2, + ) + return out[token] = (class_idx, feats) for w, feats in PRONOUN_TABLE.items(): @@ -530,11 +541,10 @@ def class_of(obj: UCNSObject) -> Optional[int]: return None if obj.anchors_pos[0].payload is not None: return None - # Second anchor's theta is (class_idx + 1) / 32 by construction. + # Second anchor's theta is (class_idx + 1) / (2 * HOST_CARRIER) by construction. second_theta = obj.anchors_pos[1].theta - expected_denominator_factor = 32 - # Reconstruct class_idx: theta * 32 - 1. - # Use exact rational arithmetic: theta = (k+1)/32 → k = theta*32 - 1. + expected_denominator_factor = 2 * HOST_CARRIER + # Reconstruct class_idx: theta * (2 * HOST_CARRIER) - 1. candidate = second_theta * expected_denominator_factor - 1 if candidate.denominator != 1: return None From c1d6491f80a81663b4ca68ec5d532685cd24c21a Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Tue, 12 May 2026 18:38:33 -0700 Subject: [PATCH 2/4] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- closed_tokens.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/closed_tokens.py b/closed_tokens.py index 7fd1c4d..9c7ca33 100644 --- a/closed_tokens.py +++ b/closed_tokens.py @@ -30,8 +30,9 @@ The pairing-mark rule: smart open quote and curly open brackets go in class 10; their close counterparts in class 11. The disk-flip property (open-mark ↔ -close-mark symmetry) is stated as a design invariant; implementation is in -ucns_v04.py (TODO: add explicit test). +close-mark symmetry) is currently stated as a design invariant only; there is +no explicit disk-flip operation documented here yet (TODO: add the op + explicit +test). Dispatch collision convention: when a token appears in multiple class tables, the first table wins by ordering. Specifically: for 'since' and 'until' the From 0573490c42e17d709d32dbc7a93029403ae0d5a4 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Tue, 12 May 2026 18:38:50 -0700 Subject: [PATCH 3/4] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- closed_tokens.py | 1 + 1 file changed, 1 insertion(+) diff --git a/closed_tokens.py b/closed_tokens.py index 9c7ca33..19b8c36 100644 --- a/closed_tokens.py +++ b/closed_tokens.py @@ -41,6 +41,7 @@ """ from __future__ import annotations + import warnings from fractions import Fraction from typing import Dict, List, Optional, Tuple From 0be2cd680c013a3e7b763311c985116bf74085e5 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Tue, 12 May 2026 18:42:11 -0700 Subject: [PATCH 4/4] fix(closed_tokens): remove import-time warn; fix docstring; add collision tests - Remove warnings.warn() from _build_dispatch_table: the known collisions (since/until/as/that) are intentional and documented in the module docstring; firing a warning unconditionally at import time breaks any -W error CI environment merely by importing the module - Remove 'import warnings' (now unused) - Add blank line after 'from __future__ import annotations' (PEP 8) - Module docstring: disk-flip sentence now says 'design invariant with no implementation yet' instead of 'implementation is in ucns_v04.py' (there is no disk-flip op in ucns_v04.py) - Tests/test_backend.py: add TestClosedTokens covering collision resolution convention for since/until/as/that --- Tests/test_backend.py | 33 +++++++++++++++++++++++++++++++++ closed_tokens.py | 13 ++++--------- 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/Tests/test_backend.py b/Tests/test_backend.py index 6babbae..dcefd74 100644 --- a/Tests/test_backend.py +++ b/Tests/test_backend.py @@ -6,6 +6,7 @@ - edcmbone.parser (parse_transcript, Turn, Round, BoneToken, FleshToken) - edcmbone.metrics (stats, risk, compute, projection, matrix) - edcmbone.compress (encode/decode round-trip, compression_stats) + - closed_tokens (dispatch collision resolution) """ import math @@ -585,3 +586,35 @@ def test_diff_detects_change(self): changes = diff(A_MATRIX, modified) assert ("F", "rep_b") in changes assert changes[("F", "rep_b")] == (0.30, 0.99) + + +# --------------------------------------------------------------------------- +# closed_tokens +# --------------------------------------------------------------------------- + +class TestClosedTokens: + """Verify collision-resolution convention for the DISPATCH table.""" + + def test_since_resolves_to_preposition(self): + pytest.importorskip("ucns_v04") + from closed_tokens import DISPATCH, CLASS_PREPOSITION + class_idx, _ = DISPATCH["since"] + assert class_idx == CLASS_PREPOSITION + + def test_until_resolves_to_preposition(self): + pytest.importorskip("ucns_v04") + from closed_tokens import DISPATCH, CLASS_PREPOSITION + class_idx, _ = DISPATCH["until"] + assert class_idx == CLASS_PREPOSITION + + def test_as_resolves_to_preposition(self): + pytest.importorskip("ucns_v04") + from closed_tokens import DISPATCH, CLASS_PREPOSITION + class_idx, _ = DISPATCH["as"] + assert class_idx == CLASS_PREPOSITION + + def test_that_resolves_to_determiner(self): + pytest.importorskip("ucns_v04") + from closed_tokens import DISPATCH, CLASS_DETERMINER + class_idx, _ = DISPATCH["that"] + assert class_idx == CLASS_DETERMINER diff --git a/closed_tokens.py b/closed_tokens.py index 19b8c36..b84a62f 100644 --- a/closed_tokens.py +++ b/closed_tokens.py @@ -30,9 +30,8 @@ The pairing-mark rule: smart open quote and curly open brackets go in class 10; their close counterparts in class 11. The disk-flip property (open-mark ↔ -close-mark symmetry) is currently stated as a design invariant only; there is -no explicit disk-flip operation documented here yet (TODO: add the op + explicit -test). +close-mark symmetry) is a design invariant with no implementation yet +(TODO: add disk-flip op + explicit test). Dispatch collision convention: when a token appears in multiple class tables, the first table wins by ordering. Specifically: for 'since' and 'until' the @@ -42,7 +41,6 @@ from __future__ import annotations -import warnings from fractions import Fraction from typing import Dict, List, Optional, Tuple @@ -429,7 +427,8 @@ def _build_dispatch_table(): same UCNS object since they differ only in surface phonology. Collision convention: when a token appears in multiple class tables the - first entry wins and a warning is emitted. By table-ordering convention: + first entry wins (silent; collisions are documented in the module docstring). + By table-ordering convention: - 'since' and 'until' resolve to the preposition sense - 'as' resolves to the preposition sense - 'that' resolves to the determiner sense @@ -443,10 +442,6 @@ def add(token, class_idx, feats): # Skip — variant will be aliased after the canonical is in place. return if token in out: - warnings.warn( - f"closed_tokens: dispatch collision on {token!r}; keeping first entry", - stacklevel=2, - ) return out[token] = (class_idx, feats)