From c97b2d152f4c8f6e963b3ebd1c9da09cc3b49f5b Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 30 Jul 2026 20:56:23 +0000 Subject: [PATCH 1/2] My Words: tolerate typography when matching the writer's words MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Edits were failing for reasons that had nothing to do with the edit. A phrase has to be found in text a host stores, and the sources disagree constantly: Word autocorrects a hyphen into an en dash and an apostrophe into a curly one, transcription emits straight ASCII, paste brings non-breaking spaces, and hyphenation is simply unstable ("well-being" / "well being"). Exact `indexOf` read every one of those as "not found", so the model burned its turn re-viewing an edit it had got right. Add a fold ladder (`utilities/textMatching`) — exact, then typographic, then loose (case, collapsed spaces, hyphen-as-space) — exhausting each rung across the whole search space before loosening, so an exact hit always beats a folded one elsewhere. Folds carry an index map, and spans are sliced from the source, so what gets replaced or selected is always the writer's real characters. `ops.findSpan` and the Lexical `selectPhrase` both use it. Closed vs. hyphenated compounds ("email" / "e-mail") stay a miss on purpose: matching across a deleted separator risks silently rewriting the wrong span. The word-bank gate had the same problem from the other side — it tokenized "well-being" as one atomic word, so a hyphen disagreement rejected the writer's own words as not theirs, in both directions. Treat an intra-word hyphen as a separator instead. The phrase rule is unchanged: "well being" must still appear contiguously in the corpus for "well-being" to pass. Two things this surfaced: - `move` re-inserted `op.phrase`, so a lenient match would have let it restyle the text it carried. It now re-places the characters it cut. - Failure notices outlived their cause. The strip cleared only on replacement or dismissal, so one word-bank rejection sat there through many successful turns, and a stale "that was blocked" is indistinguishable from a live one. Retract it when the writer takes the floor or an edit lands (session.ts invariant 6); the log keeps history. Also adds frontend/src/pages/my-words/CLAUDE.md — a call-path index for the feature, including the four places text gets matched, which were the expensive thing to establish here. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01GDgXb9TvnpCgoPtTWjqe9p --- docs/my-words-interaction-design.md | 39 +++- frontend/src/editor/editor.tsx | 46 +++-- frontend/src/pages/my-words/CLAUDE.md | 70 +++++++ .../pages/my-words/__tests__/corpus.test.ts | 29 +++ .../src/pages/my-words/__tests__/ops.test.ts | 91 +++++++++ .../my-words/__tests__/voiceControl.test.ts | 86 ++++++++- frontend/src/pages/my-words/corpus.ts | Bin 5967 -> 7205 bytes .../src/pages/my-words/interaction/ops.ts | 159 ++++++++++------ .../src/pages/my-words/voice/VoiceSession.tsx | 12 +- frontend/src/pages/my-words/voice/session.ts | 26 ++- .../utilities/__tests__/textMatching.test.ts | 120 ++++++++++++ frontend/src/utilities/textMatching.ts | 174 ++++++++++++++++++ 12 files changed, 772 insertions(+), 80 deletions(-) create mode 100644 frontend/src/pages/my-words/CLAUDE.md create mode 100644 frontend/src/utilities/__tests__/textMatching.test.ts create mode 100644 frontend/src/utilities/textMatching.ts diff --git a/docs/my-words-interaction-design.md b/docs/my-words-interaction-design.md index 05020b4b..6357dc2e 100644 --- a/docs/my-words-interaction-design.md +++ b/docs/my-words-interaction-design.md @@ -161,7 +161,44 @@ in the prompt — they crowd out the room the model needs to *be* a partner. Tel the **stance** ("one small move, then listen; name what you'd do next") and let `validateText` be the cop. -### 4.5 Responses API — a separate experiment +### 4.5 A failed move is news for one turn, not for the session + +Every path where an edit doesn't land raises one writer-visible notice — a +refusal has to be *legible*, or the partner just going quiet reads as broken. +But the strip that carries it is about the present attempt only. It is retracted +as soon as the writer takes the floor again or the partner lands an edit +(`session.ts` invariant 6); the debug log keeps the history. Before that, a +notice cleared only on replacement or manual dismissal, so a single word-bank +rejection sat on screen through many successful turns — and a stale "that was +blocked" is indistinguishable from a live one, which turns a principled refusal +back into the impression of a broken session. + +### 4.6 Tolerate the writer's typography, not the AI's vocabulary + +The word-bank constraint is about *which words*, and it should not be enforcing a +particular spelling of them. Three sources have to agree on a phrase — what the +writer typed, what the word processor autocorrected it into, and what the model +heard or read back — and they routinely disagree over hyphenation ("well-being" / +"well being"), curly vs. straight apostrophes, and spacing. Treating those as a +mismatch failed edits the model had got right, and rejected the writer's own +words as not theirs. + +So both gates are tolerant of typography and strict about vocabulary: + +- **Locating** a phrase runs a leniency ladder (`frontend/src/utilities/ + textMatching.ts`), exhausting each rung document-wide before loosening, so an + exact hit always beats a folded one elsewhere. Spans are then sliced from the + *source* text, never the needle. +- **Validating** treats an intra-word hyphen as a word separator (`corpus.ts`). + This doesn't loosen the phrase rule: "well being" must still appear + contiguously in the corpus for "well-being" to pass. The AI gains only the + freedom to hyphenate the writer's own adjacent words. + +Deliberately still a miss: closed vs. hyphenated compounds ("email" / "e-mail"). +Matching across a deleted separator risks silently rewriting the wrong span, and +a clean miss the model can retry is the cheaper failure. + +### 4.7 Responses API — a separate experiment Worth doing for persisted reasoning, but change the loop *first*. Doing both at once means we won't know which fixed it, and the loop is the bigger lever. diff --git a/frontend/src/editor/editor.tsx b/frontend/src/editor/editor.tsx index 6ff2991c..06a91a53 100644 --- a/frontend/src/editor/editor.tsx +++ b/frontend/src/editor/editor.tsx @@ -28,6 +28,8 @@ import { } from 'lexical'; import { useEffect } from 'react'; +import { MATCH_FOLDS, srcIndex } from '@/utilities/textMatching'; + import classes from './editor.module.css'; /** @@ -198,23 +200,33 @@ function ControlsPlugin({ }; collect($getRoot()); - const needle = phrase.toLowerCase(); - for (const node of textNodes) { - const idx = node - .getTextContent() - .toLowerCase() - .indexOf(needle); - if (idx === -1) continue; - const selection = $createRangeSelection(); - selection.anchor.set(node.getKey(), idx, 'text'); - selection.focus.set( - node.getKey(), - idx + phrase.length, - 'text', - ); - $setSelection(selection); - found = true; - return; + // Exhaust each rung of the leniency ladder across every node + // before loosening, so an exact hit later in the document + // still beats a folded hit in the first node. Offsets come + // back through the fold's index map, because a fold can drop + // or merge characters — the selection has to cover the real + // source text, not the needle's length. + for (const fold of MATCH_FOLDS) { + const needle = fold(phrase).text; + for (const node of textNodes) { + const haystack = fold(node.getTextContent()); + const at = haystack.text.indexOf(needle); + if (at === -1) continue; + const selection = $createRangeSelection(); + selection.anchor.set( + node.getKey(), + srcIndex(haystack, at), + 'text', + ); + selection.focus.set( + node.getKey(), + srcIndex(haystack, at + needle.length), + 'text', + ); + $setSelection(selection); + found = true; + return; + } } }); return found; diff --git a/frontend/src/pages/my-words/CLAUDE.md b/frontend/src/pages/my-words/CLAUDE.md new file mode 100644 index 00000000..90520142 --- /dev/null +++ b/frontend/src/pages/my-words/CLAUDE.md @@ -0,0 +1,70 @@ +# My Words + +A map of this page, for finding the right file before reading any of them. Each +module's own header comment is the real documentation — this is only an index to +them. + +**The concept:** the AI may edit the writer's document but never *originate* +words. Everything it places is lifted from the writer's own corpus. See +`docs/my-words-interaction-design.md` for the why. + +## Two turn-loops, one edit path + +The page has three tabs (`index.tsx`). Walkthrough and Propose share a +text-model loop; Voice runs its own, because the realtime model drives turns +itself. Only the *turn-taking* differs — an edit is an edit in both. + +``` +text tabs voice tab +───────── ───────── +liveResponder.ts (model, 1 step) voice/realtime.ts (OpenAI Realtime) + → strategies/{walkthrough,propose} → voice/session.ts (tools + invariants) + → interaction/shared.ts ───┐ ┌───┘ + ↓ ↓ + interaction/ops.ts lowerOp: EditOp → ParagraphSplice[] + interaction/editor.ts reveal beat, veto, splice → host + ↓ + EditorAPI (Word / Google Docs / Lexical / MockEditor) +``` + +- `interaction/types.ts` — `EditOp`, `Responder`, `AssistantMove`. Start here. +- `interaction/ops.ts` — **every** op lowers to a `ParagraphSplice`. Pure, so + preview and apply can't drift. Paragraphs are the coordinate system `view` + numbers and inserts index into. +- `interaction/editor.ts` — reveal-then-apply, and the adapter for hosts without + a native `applySplice`. +- `voice/session.ts` — the six tools' dispatcher, and the numbered **control + invariants**. Change one and `__tests__/voiceControl.test.ts` should be how you + find out. +- `voice/tools.ts` — tool schemas; the descriptions are the model's only guidance + on paragraph targeting. +- `corpus.ts` — the word-bank rule. The riskiest file here, unit-tested directly. + +## Where text gets matched against the document + +Four places, and they should agree. A phrase the model utters has to be found in +text some host stores, and they disagree constantly over hyphens, curly quotes, +and spacing — see `@/utilities/textMatching`, which owns the fold ladder all of +these use or should use. + +| What | Where | +| --- | --- | +| Locating an op's target span | `interaction/ops.ts` → `findSpan` | +| Word-bank validation (tokenizer) | `corpus.ts` → `tokenize` | +| `highlight` / reveal, standalone editor | `@/editor/editor.tsx` → `selectPhrase` | +| `highlight` / reveal, Word | `@/api/wordEditorAPI.ts` → `selectPhrase` | + +Google Docs (`@/api/googleDocsEditorAPI.ts`) delegates to Apps Script and is not +yet folded; `demo/mockEditor.ts` always succeeds, so a miss there is invisible in +tests. + +## Tests + +`__tests__/` splits by class, and the split is deliberate: + +- `ops.test.ts`, `corpus.test.ts`, `spliceAdapter.test.ts` — pure functions. +- `voiceControl.test.ts` — *sequencing*: a whole model turn through the real + session, against `MockEditor` and a fake transport. + +`demo/` is a standalone harness (`DemoApp.tsx`, scripted responder, mock editor) +for driving the interaction without a model or an Office host. diff --git a/frontend/src/pages/my-words/__tests__/corpus.test.ts b/frontend/src/pages/my-words/__tests__/corpus.test.ts index 5a565371..bb3b3663 100644 --- a/frontend/src/pages/my-words/__tests__/corpus.test.ts +++ b/frontend/src/pages/my-words/__tests__/corpus.test.ts @@ -93,6 +93,35 @@ describe('validateText — phrase-level rule', () => { expect(validateText('don’t', corpus).ok).toBe(true); }); + it('treats hyphenation as spelling, not as a different word', () => { + // The writer typed it hyphenated; the model (or the transcriber) didn't. + const hyphenated = corpusOf('I care about well-being here'); + expect(validateText('well being', hyphenated).ok).toBe(true); + expect(validateText('well-being', hyphenated).ok).toBe(true); + + // And the other direction, which is the one voice hits constantly. + const spaced = corpusOf('I care about well being here'); + expect(validateText('well-being', spaced).ok).toBe(true); + }); + + it('splits every hyphen in a chain', () => { + const corpus = corpusOf('a state-of-the-art result'); + expect(validateText('state of the art', corpus).ok).toBe(true); + expect(validateText('state-of-the-art', corpus).ok).toBe(true); + }); + + it('still requires the hyphenated words to be adjacent in the corpus', () => { + // Splitting hyphens must not become a way to invent an adjacency: a + // hyphen is spelling, not a bridge the AI may introduce. + const corpus = corpusOf('the big cat and the small dog'); + expect(validateText('big-dog', corpus).ok).toBe(false); + }); + + it('keeps a standalone dash as a bridge, not a word joiner', () => { + const corpus = corpusOf('the big cat and the small dog'); + expect(validateText('big — dog', corpus).ok).toBe(true); + }); + it('reports a segmentation that labels lifted / glue / punct parts', () => { const corpus = corpusOf('honesty hard work'); const result = validateText('honesty and hard work, please', corpus); diff --git a/frontend/src/pages/my-words/__tests__/ops.test.ts b/frontend/src/pages/my-words/__tests__/ops.test.ts index 27d1fa77..e03f85ab 100644 --- a/frontend/src/pages/my-words/__tests__/ops.test.ts +++ b/frontend/src/pages/my-words/__tests__/ops.test.ts @@ -91,6 +91,97 @@ describe('applyOp', () => { }); }); +describe('tolerant targeting', () => { + // The model utters the writer's phrase back with its own typography — a + // straight apostrophe where Word autocorrected a curly one, a hyphen where + // the transcriber heard two words. Those are the same phrase, and treating + // them as a miss was costing edits that were otherwise right. + it('finds a hyphenated phrase the model spelled with a space', () => { + expect( + applyOp(['their well-being matters most'], { + kind: 'str_replace', + oldStr: 'well being', + newStr: 'well-being', + }), + ).toEqual(['their well-being matters most']); + }); + + it('finds a spaced phrase the model spelled with a hyphen', () => { + expect( + applyOp(['a state of the art result'], { + kind: 'str_replace', + oldStr: 'state-of-the-art', + newStr: 'state of the art', + }), + ).toEqual(['a state of the art result']); + }); + + it('finds a curly apostrophe behind a straight one', () => { + expect( + applyOp(['I don’t think so'], { + kind: 'str_replace', + oldStr: "don't think", + newStr: 'think', + }), + ).toEqual(['I think so']); + }); + + it('replaces the writer’s real characters, not the needle’s', () => { + // The span is located leniently but sliced from the source, so the en + // dash goes away with the rest of the matched text. + expect( + applyOp(['the well–being section'], { + kind: 'str_replace', + oldStr: 'well-being', + newStr: 'wellness', + }), + ).toEqual(['the wellness section']); + }); + + it('still prefers an exact match elsewhere over a folded one', () => { + expect( + applyOp(['a well-being note', 'a well being note'], { + kind: 'str_replace', + oldStr: 'well being', + newStr: 'health', + }), + ).toEqual(['a well-being note', 'a health note']); + }); + + it('move carries the writer’s characters, not the needle’s', () => { + // A move adds no words; it must not restyle the ones it relocates. + expect( + applyOp(['keep this', 'their well-being'], { + kind: 'move', + phrase: 'well being', + paragraph: 1, + position: 'before', + }), + ).toEqual(['well-being', 'keep this', 'their ']); + }); + + it('keeps a genuine miss a miss', () => { + expect(() => + applyOp(['the cat sat'], { + kind: 'str_replace', + oldStr: 'the dog sat', + newStr: 'x', + }), + ).toThrow(/not found in the document/); + }); + + it('honors paragraph scoping when matching leniently', () => { + expect(() => + applyOp(['well-being', 'nothing here'], { + kind: 'str_replace', + oldStr: 'well being', + newStr: 'health', + paragraph: 2, + }), + ).toThrow(/paragraph 2/); + }); +}); + describe('newline lowering (splits and merges)', () => { it('a newline in newStr splits the paragraph', () => { expect( diff --git a/frontend/src/pages/my-words/__tests__/voiceControl.test.ts b/frontend/src/pages/my-words/__tests__/voiceControl.test.ts index a339c869..1a4d1ead 100644 --- a/frontend/src/pages/my-words/__tests__/voiceControl.test.ts +++ b/frontend/src/pages/my-words/__tests__/voiceControl.test.ts @@ -71,7 +71,10 @@ function fakeTransport() { async function harness(seed: string[] = SEED, initialScratchpad = '') { const editor = new MockEditor(seed); let scratchpad = initialScratchpad; + // Every notice ever raised, and separately the one currently on screen — + // invariant 6 is about the difference between the two. const notices: string[] = []; + let standingNotice: string | null = null; const toolLog: string[] = []; let reveal: { anchor?: string; cancel: () => void } | null = null; let undoState: { depth: number; description?: string } = { depth: 0 }; @@ -88,7 +91,10 @@ async function harness(seed: string[] = SEED, initialScratchpad = '') { }, }, audioEl: {} as HTMLAudioElement, - onNotice: (t) => notices.push(t), + onNotice: (t) => { + standingNotice = t; + if (t !== null) notices.push(t); + }, onTool: (t) => toolLog.push(t), onReveal: (info) => { reveal = info; @@ -104,6 +110,8 @@ async function harness(seed: string[] = SEED, initialScratchpad = '') { session, fake, notices, + /** What the writer is looking at right now, if anything. */ + notice: (): string | null => standingNotice, toolLog, paragraphs: () => editor.snapshot().paragraphs, scratchpad: () => scratchpad, @@ -416,6 +424,82 @@ describe('invariant 5: highlighting is not an edit', () => { }); }); +describe('invariant 6: a notice describes one attempt, not the session', () => { + /** Raise a notice the honest way: an edit the word-bank refuses. */ + const provokeNotice = (h: Awaited>) => + h.fake.call('str_replace', { + old_str: 'the words come out stiff', + new_str: 'the words come out clunky', + }); + + it('retracts the notice when the writer takes the floor again', async () => { + const h = await harness(); + + await provokeNotice(h); + expect(h.notice()).toMatch(/clunky/); + + h.fake.speak(); + expect(h.notice()).toBeNull(); + }); + + it('retracts the notice when an edit lands', async () => { + const h = await harness(); + + await provokeNotice(h); + expect(h.notice()).not.toBeNull(); + + // Same turn, no speech: the model corrects itself and the edit lands. + const applied = await callThroughBeat( + h.fake.call('str_replace', TIGHTEN), + ); + expect(applied).toMatch(/^Applied/); + expect(h.notice()).toBeNull(); + }); + + it('retracts it for a scratchpad edit too', async () => { + const h = await harness(SEED, 'stiff words I keep deleting'); + + await provokeNotice(h); + expect(h.notice()).not.toBeNull(); + + const applied = await callThroughBeat( + h.fake.call('str_replace', { + old_str: 'stiff words', + new_str: 'stiff', + target: 'scratchpad', + }), + ); + expect(applied).toMatch(/^Applied/); + expect(h.notice()).toBeNull(); + }); + + it('keeps every notice in the log after retracting it from the strip', async () => { + const h = await harness(); + + await provokeNotice(h); + h.fake.speak(); + + // The strip is clear, but the history the writer can scroll back to is not + // — retraction is a UI state change, not an erasure. + expect(h.notice()).toBeNull(); + expect(h.notices).toHaveLength(1); + }); + + it('leaves a fresh notice standing until something supersedes it', async () => { + const h = await harness(); + + // A failure *after* the writer's turn started must survive the rest of + // that turn — retraction keys on new events, not on elapsed time. + h.fake.speak(); + await provokeNotice(h); + await h.fake.call('view', {}); + await callThroughBeat( + h.fake.call('highlight', { phrase: 'I sit down' }), + ); + expect(h.notice()).toMatch(/clunky/); + }); +}); + describe('session wiring', () => { it('exposes all six tools and stops the transport', async () => { const h = await harness(); diff --git a/frontend/src/pages/my-words/corpus.ts b/frontend/src/pages/my-words/corpus.ts index ab1429e23c347855c23fb1c1227e110fa77b6b24..db03869668be85bf7f95a89cdb9083a74db3d31c 100644 GIT binary patch delta 1344 zcmZ8hO^Xys5QW7>?74gK5;Q`8(ADF5atA~P6h;<5P(gGUG^;AQDxK_%A~ULc+e;02 z@u;>3{Q=$uul@oL?q87pB}P>Bu%oxCip&=;-g{C1O+Q`x{OVlOgE8DOoR!uWU?XHE zVq$$nA1&Jjg7G=bC=_;~CrnB;Lskd30#i}sKvjuRPbpHlk~eSIl80eLgy&4h=F11k9zV5G}{8s!304q@gnTs1`4*Y%3246rC@z&&VBgv(h#S` z6#6;&0rw}p-CgK+6bEtrPj9fUdwYL;`?~(zyWH%;{&sj6i+VQ4@fJ!&?LZ-7WrAS} z6QRQ(FD#V0(WU90K4}+6bZlhI92}a3sH+mp$T?anMTe|1)jlI1>pA8S8-lDf(in#Q z$c8bYSt7R*g_Ab@5yZ+_22mOpDC&1kP2u2-W5QHBNz=$6FsW0GT2-<@v8^hgNL$r$ z5Ft_>L~__Pt+l^67)2~}E=f@U`jZyiW~^@_cXr8DB~g_@_6R~P3g*;)l2{#sI^cpFUA1V0aj%nU zJHM!Ior6+{>#S8JS%yF189JxAUOHXhHicR{jXTiNgUMf*Ow?&NLAu^n)NNUoE3r*}Fk*NxiSDD?D5l8HW;IWH z8hnP^cOO02fA;v^gPYH;9z3~!aO*Byhuz`fe)7~LFRvtjAB_%^&B>e1$=l8AcSpk; z*Uy&L-+Q0VA8t-Qe*1c~UhLqgci{r#zdQ|CIhk{K2EId|z~J&dZjplMA`67 /** Split op text into paragraph parts (`''` stays a single empty part). */ const splitParas = (s: string) => normalizeBreaks(s).split('\n'); -/** - * Locate `needle` in the paragraphs, possibly spanning paragraph boundaries - * when it contains `\n` (paragraphs match as if joined by single newlines). - * Returns the span as (paragraph, offset) endpoints. Throws with the same - * messages `applyOp` has always used. - */ -function findSpan( - paragraphs: string[], - rawNeedle: string, - paragraph?: number, -): { +interface Span { startPara: number; startOffset: number; endPara: number; endOffset: number; -} { - const needle = normalizeBreaks(rawNeedle); - const scopeMiss = () => - new Error(`"${rawNeedle}" not found in paragraph ${paragraph}.`); - const docMiss = () => - new Error(`"${rawNeedle}" not found in the document.`); +} + +/** One tier of `findSpan`'s search: locate `needle` under a single fold. */ +function findSpanWith( + paragraphs: string[], + needle: string, + paragraph: number | undefined, + fold: (typeof MATCH_FOLDS)[number], +): Span | null { + const foldedNeedle = fold(needle).text; if (!needle.includes('\n')) { // Single-paragraph needle: first paragraph containing it (or the scoped one). - if (paragraph !== undefined) { - const i = paragraph - 1; - const at = paragraphs[i]?.indexOf(needle) ?? -1; - if (at === -1) throw scopeMiss(); + const search = (i: number): Span | null => { + if (i < 0 || i >= paragraphs.length) return null; + const folded = fold(paragraphs[i]); + const at = folded.text.indexOf(foldedNeedle); + if (at === -1) return null; return { startPara: i, - startOffset: at, + startOffset: srcIndex(folded, at), endPara: i, - endOffset: at + needle.length, + endOffset: srcIndex(folded, at + foldedNeedle.length), }; - } + }; + if (paragraph !== undefined) return search(paragraph - 1); for (let i = 0; i < paragraphs.length; i++) { - const at = paragraphs[i].indexOf(needle); - if (at !== -1) - return { - startPara: i, - startOffset: at, - endPara: i, - endOffset: at + needle.length, - }; + const hit = search(i); + if (hit) return hit; } - throw docMiss(); + return null; } // Cross-paragraph needle: match against the single-newline join and map the - // hit back to (paragraph, offset) coordinates. - const joined = paragraphs.join('\n'); + // hit back to (paragraph, offset) coordinates. Folds never touch newlines, + // so the join stays the coordinate system on both sides. const starts: number[] = []; let acc = 0; for (const p of paragraphs) { @@ -94,26 +90,77 @@ function findSpan( return i; }; - let from = 0; - if (paragraph !== undefined) { - const i = paragraph - 1; - if (i < 0 || i >= paragraphs.length) throw scopeMiss(); - from = starts[i]; + if (paragraph !== undefined && (paragraph < 1 || paragraph > starts.length)) + return null; + const folded = fold(paragraphs.join('\n')); + + // Walk the occurrences: unscoped takes the first, scoped takes the first + // that *starts* in the requested paragraph. + for ( + let at = folded.text.indexOf(foldedNeedle); + at !== -1; + at = folded.text.indexOf(foldedNeedle, at + 1) + ) { + const start = srcIndex(folded, at); + const startPara = paraAt(start); + if (paragraph !== undefined && startPara !== paragraph - 1) continue; + const end = srcIndex(folded, at + foldedNeedle.length); + const endPara = paraAt(end); + return { + startPara, + startOffset: start - starts[startPara], + endPara, + endOffset: end - starts[endPara], + }; + } + return null; +} + +/** + * Locate `needle` in the paragraphs, possibly spanning paragraph boundaries + * when it contains `\n` (paragraphs match as if joined by single newlines). + * Returns the span as (paragraph, offset) endpoints. Throws with the same + * messages `applyOp` has always used. + * + * The search runs the whole document at each rung of the `MATCH_FOLDS` ladder + * before loosening, so an exact hit always wins over a typographically-folded + * one elsewhere. That rung is what lets the model's "well-being" find the + * writer's "well being", or its straight apostrophe find Word's curly one, + * instead of reporting a miss for an edit it got right. + */ +function findSpan( + paragraphs: string[], + rawNeedle: string, + paragraph?: number, +): Span { + const needle = normalizeBreaks(rawNeedle); + for (const fold of MATCH_FOLDS) { + const hit = findSpanWith(paragraphs, needle, paragraph, fold); + if (hit) return hit; } - const at = joined.indexOf(needle, from); - if (at === -1) throw paragraph !== undefined ? scopeMiss() : docMiss(); - if (paragraph !== undefined && paraAt(at) !== paragraph - 1) - throw scopeMiss(); + throw paragraph !== undefined + ? new Error(`"${rawNeedle}" not found in paragraph ${paragraph}.`) + : new Error(`"${rawNeedle}" not found in the document.`); +} - const end = at + needle.length; - const startPara = paraAt(at); - const endPara = paraAt(end); - return { - startPara, - startOffset: at - starts[startPara], - endPara, - endOffset: end - starts[endPara], - }; +/** + * The source text a span covers — the writer's real characters, which are not + * always the ones the needle was spelled with (see `findSpan`). Paragraph + * boundaries inside the span come back as newlines, the same coordinate system + * op text uses. + */ +function sliceSpan(paragraphs: string[], span: Span): string { + if (span.startPara === span.endPara) { + return paragraphs[span.startPara].slice( + span.startOffset, + span.endOffset, + ); + } + return [ + paragraphs[span.startPara].slice(span.startOffset), + ...paragraphs.slice(span.startPara + 1, span.endPara), + paragraphs[span.endPara].slice(0, span.endOffset), + ].join('\n'); } /** Replace a located span with (possibly multi-paragraph) text, as a splice. */ @@ -192,6 +239,10 @@ export function lowerOp(paragraphs: string[], op: EditOp): ParagraphSplice[] { // the target (numbered against the post-removal array, as `view` // would show it after the cut). const span = findSpan(paragraphs, op.phrase); + // Re-place the text that was actually cut, not `op.phrase`: a move + // adds no words, and it must not quietly restyle the ones it carries + // (the model's hyphen for the writer's space, say). + const moved = sliceSpan(paragraphs, span); const cut = spliceForSpan(paragraphs, span, ''); const emptied = cut.insert.every((p) => p.length === 0); const removal: ParagraphSplice = emptied @@ -205,7 +256,7 @@ export function lowerOp(paragraphs: string[], op: EditOp): ParagraphSplice[] { ); return [ removal, - { index: at, remove: [], insert: splitParas(op.phrase) }, + { index: at, remove: [], insert: splitParas(moved) }, ]; } } diff --git a/frontend/src/pages/my-words/voice/VoiceSession.tsx b/frontend/src/pages/my-words/voice/VoiceSession.tsx index 97d86fcc..0af32f2b 100644 --- a/frontend/src/pages/my-words/voice/VoiceSession.tsx +++ b/frontend/src/pages/my-words/voice/VoiceSession.tsx @@ -80,8 +80,10 @@ export function VoiceSession(props: { anchor?: string; cancel: () => void; } | null>(null); - // The most recent thing that didn't happen, shown until it's replaced or - // dismissed. Every notice also lands in the log. + // The most recent thing that didn't happen, shown until the session retracts + // it (the writer speaks again, or an edit lands — invariant 6), it's + // replaced, or it's dismissed. Every notice also lands in the log, which is + // where the history lives; this strip is only about the current attempt. const [notice, setNotice] = useState(null); // Depth drives the button's presence; description tells the writer what // pressing it would revert. @@ -143,10 +145,12 @@ export function VoiceSession(props: { }, onTool: (text) => pushLog({ kind: 'tool', text }), onStatus: (text) => pushLog({ kind: 'system', text }), - // Writer-visible: an attempt that didn't land (see invariant 3). + // Writer-visible: an attempt that didn't land (invariant 3), or + // `null` retracting one that's no longer current (invariant 6). + // Only real notices are logged — a retraction isn't an event. onNotice: (text) => { setNotice(text); - pushLog({ kind: 'notice', text }); + if (text !== null) pushLog({ kind: 'notice', text }); }, onScratchpadHighlight: setAgentHighlight, onReveal: setReveal, diff --git a/frontend/src/pages/my-words/voice/session.ts b/frontend/src/pages/my-words/voice/session.ts index 4d928380..318c6eba 100644 --- a/frontend/src/pages/my-words/voice/session.ts +++ b/frontend/src/pages/my-words/voice/session.ts @@ -17,7 +17,7 @@ * * ## Control invariants * - * These five are enforced *here, in code*, not asked for in the prompt — the + * These six are enforced *here, in code*, not asked for in the prompt — the * prompt spends its words on stance (`prompt.ts`). Each exists because the * writer's sense of being in control fails in a specific way without it. They * are covered by `../__tests__/voiceControl.test.ts`; if you change one, that @@ -53,6 +53,14 @@ * but that beat carries no veto. There's nothing to undo about a selection, * so unlike an edit's reveal window, speech during it is not treated as an * objection and does not touch `pendingVeto`. + * 6. **A notice describes one attempt, not the session.** The counterpart to 3: + * whatever raised the last notice is over once the writer takes the floor + * again or the partner lands an edit, so both retract it (`onNotice(null)`). + * A strip that cleared only on replacement or manual dismissal outlived its + * cause by many successful turns, and a stale "that was blocked" is + * indistinguishable from a live one — it reads as the session still being + * broken. The debug log keeps the whole history; the strip is only ever + * about *now*. */ import type { Corpus } from '../corpus'; @@ -113,8 +121,10 @@ export interface VoiceSessionOptions { /** * Something the *writer* needs to know: an edit was refused, cancelled, or * couldn't be applied. Invariant 3 — one per failed attempt, never silent. + * `null` retracts the standing notice: the thing it described is over (see + * invariant 6). */ - onNotice?: (text: string) => void; + onNotice?: (text: string | null) => void; /** The partner points at a scratchpad phrase (null clears the highlight). */ onScratchpadHighlight?: (phrase: string | null) => void; /** A veto window opened (info) or closed (null); wire `cancel` to a ✕ chip. */ @@ -204,6 +214,12 @@ export async function startVoiceSession( opts.onTool?.(logText); }; + /** + * Retract the standing notice (invariant 6). Not logged — the notice's own + * log line stays; this only takes it off the writer's screen. + */ + const clearNotice = () => opts.onNotice?.(null); + // Inverses of applied edits, most recent last. Popped by `undo`; each entry // is freshness-checked against the live surface before it restores anything. // The shape ({target, splices, description}) is also the substrate a future @@ -234,8 +250,10 @@ export async function startVoiceSession( let pendingVeto: (() => void) | null = null; const onSpeechStart = () => { - // The writer took the floor: a new move is allowed. + // The writer took the floor: a new move is allowed, and whatever the + // last notice was about belongs to the exchange they just ended. editBudget = 1; + clearNotice(); if (pendingVeto) { const cancel = pendingVeto; pendingVeto = null; @@ -405,6 +423,7 @@ export async function startVoiceSession( description: describeOp(op), }); editBudget = 0; + clearNotice(); opts.onTool?.(`${name} applied to scratchpad`); return appliedReport(after, op, 'scratchpad'); } catch (e) { @@ -447,6 +466,7 @@ export async function startVoiceSession( ); return `Could not apply that: ${(e as Error).message} Re-\`view\` and check the paragraph numbers, then try again.`; } + clearNotice(); opts.onTool?.(`${name} applied`); // Hand back a window around the change so the model tracks any shifts. return appliedReport(await editor.getParagraphs(), op); diff --git a/frontend/src/utilities/__tests__/textMatching.test.ts b/frontend/src/utilities/__tests__/textMatching.test.ts new file mode 100644 index 00000000..0e4f0df8 --- /dev/null +++ b/frontend/src/utilities/__tests__/textMatching.test.ts @@ -0,0 +1,120 @@ +/** + * The fold ladder that lets the model's spelling of a phrase find the writer's. + * + * Two properties matter here and neither is "it matches more things": that the + * ladder never *reorders* which occurrence wins (exact beats folded, everywhere), + * and that a folded hit still maps back to real source offsets — a fold can drop + * or merge characters, so an off-by-one here would have an edit eat a neighboring + * character of the writer's prose. + */ + +import { describe, expect, it } from 'vitest'; + +import { findPhrase, MATCH_FOLDS, srcIndex } from '../textMatching'; + +/** The source text a hit covers — what a caller would replace or select. */ +const matched = (hay: string, needle: string) => { + const hit = findPhrase(hay, needle); + return hit ? hay.slice(hit.start, hit.end) : null; +}; + +describe('findPhrase', () => { + it('matches exactly when it can', () => { + expect(findPhrase('the cat sat', 'cat')).toEqual({ start: 4, end: 7 }); + }); + + it('reports a real miss as a miss', () => { + expect(findPhrase('the cat sat', 'dog')).toBeNull(); + }); + + it('bridges hyphenation in both directions', () => { + expect(matched('their well-being matters', 'well being')).toBe( + 'well-being', + ); + expect(matched('their well being matters', 'well-being')).toBe( + 'well being', + ); + }); + + it('splits a hyphen chain the same way', () => { + expect(matched('a state-of-the-art result', 'state of the art')).toBe( + 'state-of-the-art', + ); + }); + + it('matches across the dash family, not just ASCII hyphens', () => { + // Word autocorrects a typed hyphen into an en dash. + expect(matched('a well–being note', 'well-being')).toBe('well–being'); + }); + + it('matches a straight apostrophe against a curly one', () => { + expect(matched('I don’t think so', "don't")).toBe('don’t'); + expect(matched("I don't think so", 'don’t')).toBe("don't"); + }); + + it('matches across non-breaking and exotic spaces', () => { + expect(matched('the\u00A0cat sat', 'the cat')).toBe('the\u00A0cat'); + }); + + it('ignores invisible characters in the source', () => { + // A soft hyphen from a paste; the writer never sees it. + expect(matched('well\u00ADbeing matters', 'wellbeing')).toBe( + 'well\u00ADbeing', + ); + }); + + it('collapses runs of spaces, so double-spacing is not a miss', () => { + expect(matched('stop. Then go', 'stop. Then')).toBe('stop. Then'); + }); + + it('falls back to case-insensitivity last', () => { + expect(matched('Reproducible research', 'reproducible')).toBe( + 'Reproducible', + ); + }); + + it('prefers an exact hit later in the text over a folded one earlier', () => { + // The loose tier would match "Cat" at 0; the exact tier must win. + const hay = 'Cat and cat'; + expect(findPhrase(hay, 'cat')).toEqual({ start: 8, end: 11 }); + }); + + it('does not fold a newline into a space', () => { + // Paragraph structure is the callers' coordinate system; a fold that + // blurred it would let a same-paragraph needle match across a break. + expect(findPhrase('one\ntwo', 'one two')).toBeNull(); + }); +}); + +describe('fold index maps', () => { + it('map back to source offsets even when characters are dropped', () => { + const loose = MATCH_FOLDS[MATCH_FOLDS.length - 1]; + const source = 'a\u00AD B-c'; + const folded = loose(source); + expect(folded.text).toBe('a b c'); + // Every folded index points at the character it came from. + for (let i = 0; i < folded.text.length; i++) { + const at = srcIndex(folded, i); + expect(at).toBeGreaterThanOrEqual(0); + expect(at).toBeLessThan(source.length); + } + expect(srcIndex(folded, folded.text.length)).toBe(source.length); + }); + + it('are monotonic, so a span never inverts', () => { + const loose = MATCH_FOLDS[MATCH_FOLDS.length - 1]; + const folded = loose('The well-being of “quiet” work'); + for (let i = 1; i <= folded.text.length; i++) { + expect(srcIndex(folded, i)).toBeGreaterThan( + srcIndex(folded, i - 1), + ); + } + }); + + it('leave the exact tier as a plain identity', () => { + const exact = MATCH_FOLDS[0]; + const source = 'well–being'; + expect(exact(source).text).toBe(source); + expect(srcIndex(exact(source), 4)).toBe(4); + }); +}); diff --git a/frontend/src/utilities/textMatching.ts b/frontend/src/utilities/textMatching.ts new file mode 100644 index 00000000..dc3cf1a0 --- /dev/null +++ b/frontend/src/utilities/textMatching.ts @@ -0,0 +1,174 @@ +/** + * Tolerant text matching for locating a phrase in the writer's document. + * + * Every tool that acts on the writer's words — `str_replace`, `insert … after`, + * `move`, `highlight` — has to find a phrase the *model* uttered inside text the + * *host* stores, and those two strings disagree far more often than they look + * like they should. Word autocorrects `-` into an en dash and `'` into a curly + * apostrophe as the writer types; speech transcription emits straight ASCII; + * copy-paste brings non-breaking spaces; a writer types two spaces after a + * period and the model repeats the sentence with one; and hyphenation is simply + * unstable across sources ("well-being" / "well being"). A plain `indexOf` reads + * every one of those as "not found", so an edit the model got *right* fails and + * it burns the turn re-`view`ing. + * + * So matching runs a ladder, most literal first, and stops at the first tier + * that hits document-wide. Exact text always wins over a folded match anywhere + * else, so leniency only ever rescues a search that would otherwise have failed + * outright: + * + * 1. **exact** — byte-for-byte. + * 2. **typographic** — dashes, quotes and exotic spaces canonicalized; + * invisible characters (soft hyphen, zero-width, BOM) dropped. + * 3. **loose** — the above, plus case-folding, runs of spaces collapsed to + * one, and a hyphen treated as a space, which is what makes "well-being" + * find "well being" and vice versa. + * + * Deliberately *not* folded: newlines (they're paragraph boundaries, and the + * callers rely on that coordinate system), and the gap between a hyphenated and + * a closed compound ("e-mail" / "email"). The latter would mean matching across + * deleted separators, and a wrong match silently rewrites the writer's prose — + * worse than a clean miss the model can retry. + * + * A fold may drop or merge characters, so it carries an index map back to the + * source: a hit at folded `[a, b)` is the source range `[srcIndex(f, a), + * srcIndex(f, b))`. Callers slice the *original* text with those, so what gets + * replaced or selected is always the writer's real characters. + */ + +/** Folded text plus, when it isn't 1:1, the source index of each folded char. */ +export interface Folded { + text: string; + /** + * `map[i]` is the source index of folded character `i`, and `map[text.length]` + * is the source length. Absent for the identity fold, where they're equal. + */ + map?: number[]; +} + +/** Where folded index `i` sits in the source text. */ +export function srcIndex(folded: Folded, i: number): number { + return folded.map ? folded.map[i] : i; +} + +/** Characters that carry no text, only layout hints from some other editor. */ +const DROPPED = new Set([ + '­', // soft hyphen + '​', // zero-width space + '‌', // zero-width non-joiner + '‍', // zero-width joiner + '', // BOM / zero-width no-break space +]); + +/** One-for-one canonicalization: typographic variants → their ASCII form. */ +const CANON: Record = { + // dashes and hyphens + '‐': '-', // hyphen + '‑': '-', // non-breaking hyphen + '‒': '-', // figure dash + '–': '-', // en dash + '—': '-', // em dash + '―': '-', // horizontal bar + '−': '-', // minus sign + '－': '-', // fullwidth hyphen-minus + // apostrophes and single quotes + '‘': "'", + '’': "'", + '‚': "'", + '‛': "'", + ʼ: "'", + '´': "'", + '`': "'", + // double quotes + '“': '"', + '”': '"', + '„': '"', + '‟': '"', + // spaces (never \n or \r — those are paragraph structure) + '\t': ' ', + ' ': ' ', + ' ': ' ', + ' ': ' ', + ' ': ' ', + ' ': ' ', + ' ': ' ', + ' ': ' ', + ' ': ' ', + ' ': ' ', + ' ': ' ', + ' ': ' ', + ' ': ' ', + ' ': ' ', + ' ': ' ', + ' ': ' ', + '　': ' ', +}; + +/** Lowercase a single character, unless casing would change its length. */ +function lowerChar(ch: string): string { + const lower = ch.toLowerCase(); + return lower.length === 1 ? lower : ch; +} + +/** + * Build a fold. `loose` adds case-folding, hyphen-as-space, and whitespace + * collapsing on top of the typographic canonicalization. + */ +function makeFold(loose: boolean): (source: string) => Folded { + return (source) => { + let text = ''; + const map: number[] = []; + let prevWasSpace = false; + for (let i = 0; i < source.length; i++) { + const ch = source[i]; + if (DROPPED.has(ch)) continue; + let out = CANON[ch] ?? ch; + if (loose) { + // A hyphen is a word separator like any other here, so a + // compound matches however either side spelled it. + out = out === '-' ? ' ' : lowerChar(out); + // Runs of spaces collapse to one; the run maps to its first + // character, so the source span still covers all of it. + if (out === ' ' && prevWasSpace) continue; + } + prevWasSpace = out === ' '; + map.push(i); + text += out; + } + map.push(source.length); + return { text, map }; + }; +} + +/** + * The leniency ladder, most literal first. Apply one tier across the *whole* + * search space before moving to the next, so an exact match anywhere beats a + * folded match somewhere else. + */ +export const MATCH_FOLDS: readonly ((source: string) => Folded)[] = [ + (source) => ({ text: source }), + makeFold(false), + makeFold(true), +]; + +/** + * Locate `needle` in `hay` under the ladder, as a source-index range. For + * single-haystack callers; anything searching several strings (paragraphs, + * text nodes) should loop the tiers itself so a whole tier is exhausted first. + */ +export function findPhrase( + hay: string, + needle: string, +): { start: number; end: number } | null { + for (const fold of MATCH_FOLDS) { + const folded = fold(hay); + const foldedNeedle = fold(needle).text; + const at = folded.text.indexOf(foldedNeedle); + if (at === -1) continue; + return { + start: srcIndex(folded, at), + end: srcIndex(folded, at + foldedNeedle.length), + }; + } + return null; +} From 6ad776167bdabb46fe851bff399075be2f7bba7d Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 30 Jul 2026 21:20:24 +0000 Subject: [PATCH 2/2] My Words: match closed compounds, and derive folds from Unicode categories MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two gaps in the previous commit, both from review. "email" / "e-mail" is the same ASR ambiguity as "well-being" / "well being" — one utterance, and the transcriber picks a spelling that need not be the document's. Excluding it was inconsistent. What it actually needs is care about *how* it matches, since dropping the separator lets letters re-segment: - In the locator, add a run-together tier that drops separators, anchored to word boundaries in the source. Without the anchor "email" also matches inside "the ache mail" and the span rewrites the wrong text; the anchor is on that rung alone, so literal substring matches behave as they always have. - In the word bank, re-segmentation invents meaning rather than just picking a bad span — "notable" to "not able" is a claim the writer never made. So the corpus carries both readings of a *hyphenated* compound and no others: a closed-up spelling is legal exactly where a hyphen appears on one side or the other, never off a plain space. The same rule makes a hyphen bind tighter than the glue rule, so "in-depth" is lifted whole instead of read as "in" plus a stray "depth". Span ends now resolve to one past the last matched character rather than the next character the fold kept, which is what makes "e-mail" come back as a span of "e-mail" and not "e-mail ". Replace the hand-listed character table with Unicode general categories (\p{Pd} dashes, \p{Zs} spaces, \p{Cf} invisibles) so a variant nobody listed can't go missing; quotes stay enumerated because Pi/Pf don't tell single from double. Normalizing by dropping non-alphanumerics instead would break two things, now written down in the module header: punctuation is editable content here, so a locator blind to it can't tell whether a trailing period is inside the span; and it collapses the ladder, leaving nothing between byte-exact and maximally lenient. Also replaces the two literal NUL bytes in corpus.ts with named escapes, which is why git stops treating that file as binary. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01GDgXb9TvnpCgoPtTWjqe9p --- docs/my-words-interaction-design.md | 32 ++- frontend/src/editor/editor.tsx | 30 +-- frontend/src/pages/my-words/CLAUDE.md | 5 +- .../pages/my-words/__tests__/corpus.test.ts | 36 +++ frontend/src/pages/my-words/corpus.ts | Bin 7205 -> 10648 bytes .../src/pages/my-words/interaction/ops.ts | 43 ++-- .../utilities/__tests__/textMatching.test.ts | 107 +++++++- frontend/src/utilities/textMatching.ts | 241 +++++++++++------- 8 files changed, 347 insertions(+), 147 deletions(-) diff --git a/docs/my-words-interaction-design.md b/docs/my-words-interaction-design.md index 6357dc2e..d690cd44 100644 --- a/docs/my-words-interaction-design.md +++ b/docs/my-words-interaction-design.md @@ -189,14 +189,30 @@ So both gates are tolerant of typography and strict about vocabulary: textMatching.ts`), exhausting each rung document-wide before loosening, so an exact hit always beats a folded one elsewhere. Spans are then sliced from the *source* text, never the needle. -- **Validating** treats an intra-word hyphen as a word separator (`corpus.ts`). - This doesn't loosen the phrase rule: "well being" must still appear - contiguously in the corpus for "well-being" to pass. The AI gains only the - freedom to hyphenate the writer's own adjacent words. - -Deliberately still a miss: closed vs. hyphenated compounds ("email" / "e-mail"). -Matching across a deleted separator risks silently rewriting the wrong span, and -a clean miss the model can retry is the cheaper failure. +- **Validating** treats an intra-word hyphen as a word separator, and offers both + readings of a hyphenated compound (`corpus.ts`). This doesn't loosen the phrase + rule: "well being" must still appear contiguously in the corpus for + "well-being" to pass. The AI gains only the freedom to hyphenate — or unhyphenate + — the writer's own adjacent words. + +**Where the leniency stops, and why it isn't where you'd guess.** "email" / +"e-mail" is the same ASR ambiguity as "well-being" / "well being" — one utterance, +and the transcriber picks a spelling — so it gets the same treatment. What makes +it need more care is *how* it's matched, not whether it should be: dropping the +separator means letters re-segment, and that can find text no reader would call +the same phrase. + +- In the **locator**, "email" would otherwise match inside "the ache mail" and + splice over the wrong words. So that rung — and only that rung — requires the + hit to begin and end on a word boundary in the source. +- In the **word bank**, re-segmentation is worse than a bad span: it invents + meaning. "notable" → "not able" is a claim the writer never made. So the closed + form is offered only where a hyphen actually appears, on one side or the other; + a plain space never binds. A hyphen is a mark the sources disagree about, a + space is not. + +The same rule makes a hyphen bind tighter than the glue-word rule: "in-depth" is +one compound to be lifted whole, not "in" (glue) plus a stray "depth". ### 4.7 Responses API — a separate experiment diff --git a/frontend/src/editor/editor.tsx b/frontend/src/editor/editor.tsx index 06a91a53..89809952 100644 --- a/frontend/src/editor/editor.tsx +++ b/frontend/src/editor/editor.tsx @@ -28,7 +28,7 @@ import { } from 'lexical'; import { useEffect } from 'react'; -import { MATCH_FOLDS, srcIndex } from '@/utilities/textMatching'; +import { firstMatch, MATCH_TIERS } from '@/utilities/textMatching'; import classes from './editor.module.css'; @@ -202,27 +202,25 @@ function ControlsPlugin({ // Exhaust each rung of the leniency ladder across every node // before loosening, so an exact hit later in the document - // still beats a folded hit in the first node. Offsets come - // back through the fold's index map, because a fold can drop - // or merge characters — the selection has to cover the real - // source text, not the needle's length. - for (const fold of MATCH_FOLDS) { - const needle = fold(phrase).text; + // still beats a folded hit in the first node. The offsets + // are source offsets, not needle lengths — a fold can drop + // or merge characters, and the selection has to land on the + // writer's real text. + for (const tier of MATCH_TIERS) { for (const node of textNodes) { - const haystack = fold(node.getTextContent()); - const at = haystack.text.indexOf(needle); - if (at === -1) continue; + const hit = firstMatch( + tier, + node.getTextContent(), + phrase, + ); + if (!hit) continue; const selection = $createRangeSelection(); selection.anchor.set( node.getKey(), - srcIndex(haystack, at), - 'text', - ); - selection.focus.set( - node.getKey(), - srcIndex(haystack, at + needle.length), + hit.start, 'text', ); + selection.focus.set(node.getKey(), hit.end, 'text'); $setSelection(selection); found = true; return; diff --git a/frontend/src/pages/my-words/CLAUDE.md b/frontend/src/pages/my-words/CLAUDE.md index 90520142..ba662ac6 100644 --- a/frontend/src/pages/my-words/CLAUDE.md +++ b/frontend/src/pages/my-words/CLAUDE.md @@ -45,7 +45,10 @@ liveResponder.ts (model, 1 step) voice/realtime.ts (OpenAI Realtime) Four places, and they should agree. A phrase the model utters has to be found in text some host stores, and they disagree constantly over hyphens, curly quotes, and spacing — see `@/utilities/textMatching`, which owns the fold ladder all of -these use or should use. +these use or should use. Its last rung drops separators entirely ("email" finds +"e-mail") and is the only one anchored to word boundaries; loosening it further, +or dropping the anchor, is how a match starts landing on text no reader would +call the same phrase. | What | Where | | --- | --- | diff --git a/frontend/src/pages/my-words/__tests__/corpus.test.ts b/frontend/src/pages/my-words/__tests__/corpus.test.ts index bb3b3663..c68b2320 100644 --- a/frontend/src/pages/my-words/__tests__/corpus.test.ts +++ b/frontend/src/pages/my-words/__tests__/corpus.test.ts @@ -122,6 +122,42 @@ describe('validateText — phrase-level rule', () => { expect(validateText('big — dog', corpus).ok).toBe(true); }); + it('reads a hyphenated compound closed up, and vice versa', () => { + // Same ASR ambiguity as "well-being"/"well being", with the separator + // gone entirely: the transcriber picks a spelling and it need not be the + // document's. + const hyphenated = corpusOf('I sent an e-mail yesterday'); + expect(validateText('email', hyphenated).ok).toBe(true); + expect(validateText('email yesterday', hyphenated).ok).toBe(true); + + const closed = corpusOf('I sent an email yesterday'); + expect(validateText('e-mail', closed).ok).toBe(true); + expect(validateText('e-mail yesterday', closed).ok).toBe(true); + }); + + it('will not re-segment across a plain space', () => { + // The line this leniency stops at. A hyphen is a mark the sources + // disagree about; a space is not, and joining across one invents a word + // the writer never wrote. + const corpus = corpusOf('she was not able to finish'); + expect(validateText('notable', corpus).ok).toBe(false); + + // Nor the reverse: splitting one of the writer's words into two. + const closed = corpusOf('a notable essay'); + expect(validateText('not able', closed).ok).toBe(false); + }); + + it('lets a hyphen bind tighter than the glue rule', () => { + // "in" is a glue word, but "in-depth" is one compound — reading it as + // glue + "depth" would let the AI assemble it from any stray "depth". + const corpus = corpusOf('we need more depth here'); + expect(validateText('in-depth', corpus).ok).toBe(false); + expect(validateText('in depth', corpus).ok).toBe(true); + + const wrote = corpusOf('we need an in-depth look'); + expect(validateText('in-depth', wrote).ok).toBe(true); + }); + it('reports a segmentation that labels lifted / glue / punct parts', () => { const corpus = corpusOf('honesty hard work'); const result = validateText('honesty and hard work, please', corpus); diff --git a/frontend/src/pages/my-words/corpus.ts b/frontend/src/pages/my-words/corpus.ts index db03869668be85bf7f95a89cdb9083a74db3d31c..7da2330ad674bfac1406999fa5290259263f217a 100644 GIT binary patch delta 4206 zcmaJ^O>7&-6^0PxXBk_LVL7#}7JVWFla$Dnlta_XrUk_jYFCI|NJC4?Pq;6xjmJDFq4?2#`w)U5eg&?x8@B1?p3Q06i8x^w96kEX9?gC29U6Ltlp85h)#w6bZdff(A#UfecHuL_={z z`+nG^D5fMd|Egu#7z#fq(+ZX4^dI(Ou~w&j>4`*1@-_Lei1qk@&tiKjN|V@=in_iM z{a8v8`%$9rVB07Ven@I0JZX*%BR#bh%Z8Hup$xSj_TjYFGFEKR4-c`ZXdJ~|Wg;5K zK3-BN5Ai#WBQ2>~hB2`p$TB^j{rN}4k2o05wUghx_{$g4k^@tWixNZp=vay&jeR{J zDZD|BhtRT1UNjs<2>|heNZ~md!37;r1U%UXV`&P}b3V<21Y}giamLXoQX+t}LL*)o z1S!LGYMIj#;RoJ}BV$i#KL`Md_6Ab9v#6fqUJJb(|9gQiLa;mtLC5{`2bMWeTF`Kn$el zmR44>%B9V#BG%;^FbcpWp?UR#OBmMe)L4E`9e^9E&K*b$r@^CZmj2D&@fTR0mGXu%vN8w?;F+}A+CL6}vM;?AOizm#@5$uB`B3d(12o1Tb7;J;?47#lj>RGyv^= z2(iHT3pB>Lgot3ykkk(qDD4B8Bg<6wP05jfSrDSngp?`t63gHSxdmkn;dtE>AuGTH zhjbo-$9fP)Nq+z`gF2y%MX)}R8BN7!x!(1AJsHDat2$$x{VJ#|7g=gmme&C!o7KyX zeo>?@ZA&?tyl0HqB$RGdZBtGWpKKWlUU#t_neIV7OQ*R2vZqo?IqVSXWTX}I!)nlq zw*m1Q{qd=QH;{eEQob)d4JD0H0*k&*58%pH1;%+gHXovzb?XdW=THQG7lhc8@9R#D z=AZxc*Lfv^=Vyyt_U4s`DJyGN!%M}VvyxE=5}h;aPEqDw6sly1(#ds|HPxwB18s{ANfU0q$P+$ovEkT&QqK=Soi?jS^~P-UtUpc)Ngf#S`D zAGH_=i~(PdV;vTb*nRx&&i$>YZ#{nZ(c7ClPfm|nt(^Si$}2N3GsCf%{pvMJJBw?J z&k8;rcWQS^4l+w48FC7_B4Be*1WMu!qfoVx#A3{cTgAAh09N-NZzH&!tr~4?P^(>| zXC=q+dsOAk&1SP0KEJ%~IL_IFbezcS>HPKV->-eN=@e012MvD6HSYH9qH}eeaLi7h zGh{qI`PtQf%s}Y02c(LmWS$rL$@;WhOxuW>)P=l=`fylR0g6#eYrT9_^XrwT7tYETPPB4Owq+Z|j~*Fn{E7&naK zp$XlwmI48WHLdb(7ikrYyC`FQw2ua@wT(Sa4BK_AwH)l!^Z=l2UKx|@5&Ej^ad-H_ zn{#~Q&urpzkH6Xz_-oEs20rMTZulmGGVUFd1nCTg&RDt_f#d9d*M5+m^TmKugusAz z!KUL<$%&;-;?SIDK4E4RzMVYoO3Y!Q9HZ5117FK^UHbvuo&2>ix80r}VBo1%$2Dr+ zqcJg(=Z|N@HT-0s%-zlYIXCwmU)|@l_me1555ualJsAgD?K;ItIDM?0Hfwrz^4ay( zne5ZK@&jz)qQx|~)wymG7?Z@{geskD0RlFI8VY%#gicDd+9+lJn)|dpL58C}=bJS? z2_L7{=)(`cdYgsDL{Td_wUfWT{>Pao)WNr0YgEf`cZ|+r3FBsN#3;rXP{;3FIN;$0vg4H1qh}_$9T;w=RAmYP8l3Y86jyF6yNM_7+A*)yzE`aZ&#R*)#zQnLLzrjS zZ|mn29j<@aI8pOGTCMSOtFXojr{$(GWuzp7B`Aq^7y(S{M$UP>;eE$-F520jZ@kSl za|a1P%Zd|^$5nY+(kf`{W4;YwT;uM{J%QibxuMY$9+2+MbqwNB8guf%k zljx0rY}{xOdD!?QMcAfctkMRCu`%K5<9W1C0l`M zI$#Lwn_QBF_M_06Gt*JwZdJ!%mRV=_hKFfY_ zv*7~QtsxKoz`qjXed`vz!Q?kW%+=={vjR=wS1!KV!i}zK zn_JF>?C{n>YFTO+I>~82NbIGqm|b+(F<;zzQ4mC9Ok>J)Yda%heNxz1WbobE+K>Hi zU*h`VhmEcr=|OqYt-+7L?`ioI_o@NzL-sR68MJfkFN1Vt`^_{9d=XbWet*Dhx1qIs zY+P$^xW##fMx-vS8j{I|a1Z6fZo5(}+1K J`}_Rk{{t7WWvKuF delta 782 zcmXw1O;6NN6t#e8%P=3Jjt*e1gG}3G9MFYPhH-)kAp`;m8SF`1*xMVTP=b+f7o_Ev+T)6V! z5DtRje%2V2Jp7a$tAj_I#47`5ARGz{%g`iJ4}2yZIS@8GA}QDbr50<+c5rsv4)uwk zu|_IaKr=U+=-tar;7;y3ejZzjzMmadG_mDS4mC|x@#i_H>Q}CTZdO`E;>O+emBqCu z8;d5?>+s-FRa0#ifU(8o&p<%Bz-&mEl$+4Q!}G5PLNj`q|DoWYi<5Y(@Gw=Y)j*H! z)lI;JkbW@6jnx+O+bOIlG5Oa7rc{Qk*1D|$5ei#W}v0YElz%aTd)NVr; zF2}M`MT&l#m{0eUs3hE{b^Lef?YO4)*E~{&5Nbz(RU>P1R*5ojpkTTvw3pX(%5#IW`0H#$|kLteifs;|If)63gcL Ee}dHbMF0Q* diff --git a/frontend/src/pages/my-words/interaction/ops.ts b/frontend/src/pages/my-words/interaction/ops.ts index 0d358bcd..e8f4bf69 100644 --- a/frontend/src/pages/my-words/interaction/ops.ts +++ b/frontend/src/pages/my-words/interaction/ops.ts @@ -22,7 +22,12 @@ * `findSpan`. */ -import { MATCH_FOLDS, srcIndex } from '@/utilities/textMatching'; +import { + firstMatch, + MATCH_TIERS, + tierMatches, + type MatchTier, +} from '@/utilities/textMatching'; import type { EditOp } from './types'; @@ -44,27 +49,24 @@ interface Span { endOffset: number; } -/** One tier of `findSpan`'s search: locate `needle` under a single fold. */ +/** One tier of `findSpan`'s search: locate `needle` at a single leniency. */ function findSpanWith( paragraphs: string[], needle: string, paragraph: number | undefined, - fold: (typeof MATCH_FOLDS)[number], + tier: MatchTier, ): Span | null { - const foldedNeedle = fold(needle).text; - if (!needle.includes('\n')) { // Single-paragraph needle: first paragraph containing it (or the scoped one). const search = (i: number): Span | null => { if (i < 0 || i >= paragraphs.length) return null; - const folded = fold(paragraphs[i]); - const at = folded.text.indexOf(foldedNeedle); - if (at === -1) return null; + const hit = firstMatch(tier, paragraphs[i], needle); + if (!hit) return null; return { startPara: i, - startOffset: srcIndex(folded, at), + startOffset: hit.start, endPara: i, - endOffset: srcIndex(folded, at + foldedNeedle.length), + endOffset: hit.end, }; }; if (paragraph !== undefined) return search(paragraph - 1); @@ -92,19 +94,16 @@ function findSpanWith( if (paragraph !== undefined && (paragraph < 1 || paragraph > starts.length)) return null; - const folded = fold(paragraphs.join('\n')); // Walk the occurrences: unscoped takes the first, scoped takes the first // that *starts* in the requested paragraph. - for ( - let at = folded.text.indexOf(foldedNeedle); - at !== -1; - at = folded.text.indexOf(foldedNeedle, at + 1) - ) { - const start = srcIndex(folded, at); + for (const { start, end } of tierMatches( + tier, + paragraphs.join('\n'), + needle, + )) { const startPara = paraAt(start); if (paragraph !== undefined && startPara !== paragraph - 1) continue; - const end = srcIndex(folded, at + foldedNeedle.length); const endPara = paraAt(end); return { startPara, @@ -122,9 +121,9 @@ function findSpanWith( * Returns the span as (paragraph, offset) endpoints. Throws with the same * messages `applyOp` has always used. * - * The search runs the whole document at each rung of the `MATCH_FOLDS` ladder + * The search runs the whole document at each rung of the `MATCH_TIERS` ladder * before loosening, so an exact hit always wins over a typographically-folded - * one elsewhere. That rung is what lets the model's "well-being" find the + * one elsewhere. Those rungs are what let the model's "well-being" find the * writer's "well being", or its straight apostrophe find Word's curly one, * instead of reporting a miss for an edit it got right. */ @@ -134,8 +133,8 @@ function findSpan( paragraph?: number, ): Span { const needle = normalizeBreaks(rawNeedle); - for (const fold of MATCH_FOLDS) { - const hit = findSpanWith(paragraphs, needle, paragraph, fold); + for (const tier of MATCH_TIERS) { + const hit = findSpanWith(paragraphs, needle, paragraph, tier); if (hit) return hit; } throw paragraph !== undefined diff --git a/frontend/src/utilities/__tests__/textMatching.test.ts b/frontend/src/utilities/__tests__/textMatching.test.ts index 0e4f0df8..045bb3e7 100644 --- a/frontend/src/utilities/__tests__/textMatching.test.ts +++ b/frontend/src/utilities/__tests__/textMatching.test.ts @@ -10,7 +10,18 @@ import { describe, expect, it } from 'vitest'; -import { findPhrase, MATCH_FOLDS, srcIndex } from '../textMatching'; +import { + endIndex, + findPhrase, + firstMatch, + MATCH_TIERS, + srcIndex, + tierMatches, +} from '../textMatching'; + +/** The loosest tier: separators dropped, hits anchored to word boundaries. */ +const RUN_TOGETHER = MATCH_TIERS[MATCH_TIERS.length - 1]; +const LOOSE = MATCH_TIERS[MATCH_TIERS.length - 2]; /** The source text a hit covers — what a caller would replace or select. */ const matched = (hay: string, needle: string) => { @@ -86,11 +97,64 @@ describe('findPhrase', () => { }); }); +describe('the run-together tier (closed vs. hyphenated compounds)', () => { + // Transcription is as unsure about "email" / "e-mail" as it is about + // "well-being" / "well being": same utterance, and the transcriber picks a + // spelling that need not be the document's. + it('matches a closed compound against a hyphenated one', () => { + expect(matched('send an e-mail now', 'email')).toBe('e-mail'); + expect(matched('send an email now', 'e-mail')).toBe('email'); + }); + + it('matches across a space too', () => { + expect(matched('send an e mail now', 'email')).toBe('e mail'); + }); + + it('does not glue across a word gap', () => { + // The reason this tier is anchored: "the ache mail" contains the letters + // of "email" across a boundary no reader would call the same phrase, and + // matching it would splice over the wrong text. + expect(findPhrase('nursing the ache mail arrived', 'email')).toBeNull(); + }); + + it('rejects a hit that starts or ends mid-word', () => { + // Both of these only match once separators are dropped, and both would + // splice into the middle of a word. + expect(firstMatch(RUN_TOGETHER, 'none-mail', 'email')).toBeNull(); + expect(firstMatch(RUN_TOGETHER, 'e-mails', 'email')).toBeNull(); + // …but the same needle is fine where the boundaries are real. + expect(findPhrase('an e-mail.', 'email')).toEqual({ start: 3, end: 9 }); + }); + + it('does not retroactively anchor the exact tier', () => { + // A needle that appears verbatim inside a longer word still matches, as + // it always has — the anchor is about what *separator-dropping* may glue + // together, not a new rule for literal substrings. + expect(findPhrase('nonemail', 'email')).toEqual({ start: 3, end: 8 }); + }); + + it('only anchors the tier that needs it', () => { + // The looser rungs still allow a partial-word match, exactly as exact + // `indexOf` always has ("cat" inside "concatenate"). + expect(RUN_TOGETHER.wordBounded).toBe(true); + expect(LOOSE.wordBounded).toBeFalsy(); + expect(matched('concatenate', 'cat')).toBe('cat'); + }); + + it('is reached only after the looser tiers miss', () => { + // "e-mail" is present verbatim; the run-together tier must not get to + // re-point the match at the other occurrence. + expect(findPhrase('an email and an e-mail', 'e-mail')).toEqual({ + start: 16, + end: 22, + }); + }); +}); + describe('fold index maps', () => { it('map back to source offsets even when characters are dropped', () => { - const loose = MATCH_FOLDS[MATCH_FOLDS.length - 1]; const source = 'a\u00AD B-c'; - const folded = loose(source); + const folded = LOOSE.fold(source); expect(folded.text).toBe('a b c'); // Every folded index points at the character it came from. for (let i = 0; i < folded.text.length; i++) { @@ -102,8 +166,7 @@ describe('fold index maps', () => { }); it('are monotonic, so a span never inverts', () => { - const loose = MATCH_FOLDS[MATCH_FOLDS.length - 1]; - const folded = loose('The well-being of “quiet” work'); + const folded = LOOSE.fold('The well-being of “quiet” work'); for (let i = 1; i <= folded.text.length; i++) { expect(srcIndex(folded, i)).toBeGreaterThan( srcIndex(folded, i - 1), @@ -112,9 +175,37 @@ describe('fold index maps', () => { }); it('leave the exact tier as a plain identity', () => { - const exact = MATCH_FOLDS[0]; const source = 'well–being'; - expect(exact(source).text).toBe(source); - expect(srcIndex(exact(source), 4)).toBe(4); + expect(MATCH_TIERS[0].fold(source).text).toBe(source); + expect(srcIndex(MATCH_TIERS[0].fold(source), 4)).toBe(4); + }); + + it('end one past the last matched character, not at the next kept one', () => { + // `srcIndex` of the end would point past any dropped characters — for + // "e-mail now" that is the space, and the span would carry it along. + const source = 'send an e-mail now'; + const folded = RUN_TOGETHER.fold(source); + const at = folded.text.indexOf('email'); + expect( + source.slice(srcIndex(folded, at), endIndex(folded, at + 5)), + ).toBe('e-mail'); + }); +}); + +describe('tierMatches', () => { + it('yields every hit in order, so callers can pick by position', () => { + const hits = [...tierMatches(MATCH_TIERS[0], 'cat and cat', 'cat')]; + expect(hits).toEqual([ + { start: 0, end: 3 }, + { start: 8, end: 11 }, + ]); + }); + + it('skips boundary-violating hits rather than stopping at them', () => { + // The first candidate is mid-word; the tier must keep looking. + const hits = [ + ...tierMatches(RUN_TOGETHER, 'nonemail and e-mail', 'email'), + ]; + expect(hits).toEqual([{ start: 13, end: 19 }]); }); }); diff --git a/frontend/src/utilities/textMatching.ts b/frontend/src/utilities/textMatching.ts index dc3cf1a0..fb05ef8b 100644 --- a/frontend/src/utilities/textMatching.ts +++ b/frontend/src/utilities/textMatching.ts @@ -8,31 +8,53 @@ * apostrophe as the writer types; speech transcription emits straight ASCII; * copy-paste brings non-breaking spaces; a writer types two spaces after a * period and the model repeats the sentence with one; and hyphenation is simply - * unstable across sources ("well-being" / "well being"). A plain `indexOf` reads - * every one of those as "not found", so an edit the model got *right* fails and - * it burns the turn re-`view`ing. + * unstable across sources ("well-being" / "well being" / "wellbeing"). A plain + * `indexOf` reads every one of those as "not found", so an edit the model got + * *right* fails and it burns the turn re-`view`ing. * * So matching runs a ladder, most literal first, and stops at the first tier - * that hits document-wide. Exact text always wins over a folded match anywhere - * else, so leniency only ever rescues a search that would otherwise have failed - * outright: + * that hits across the whole search space. Exact text always wins over a folded + * match anywhere else, so leniency only ever rescues a search that would + * otherwise have failed outright: * * 1. **exact** — byte-for-byte. * 2. **typographic** — dashes, quotes and exotic spaces canonicalized; * invisible characters (soft hyphen, zero-width, BOM) dropped. * 3. **loose** — the above, plus case-folding, runs of spaces collapsed to - * one, and a hyphen treated as a space, which is what makes "well-being" - * find "well being" and vice versa. + * one, and a hyphen treated as a space: "well-being" finds "well being". + * 4. **run-together** — separators dropped entirely, so "email" finds + * "e-mail". This tier alone is *word-bounded*: the hit must begin and end + * on a word boundary in the source. Without that, "email" also matches + * inside "the ache mail", and the span silently rewrites the wrong text. + * Gluing across a word gap is the one fold that can find a match no reader + * would call the same phrase, so it's the one that needs the anchor. + * + * ## Why substitute by category rather than strip punctuation + * + * Tempting shortcut: normalize by dropping everything non-alphanumeric. It + * breaks two things. Punctuation is *editable content* here — `str_replace` is + * routinely used to change a comma to a period — so a locator that can't see + * punctuation can't tell whether a trailing `.` is inside the match or after it, + * and the span it returns eats or drops a character of the writer's prose. And + * it collapses the ladder: if the second rung already ignored all punctuation, + * there is nothing between "byte-exact" and "maximally lenient", so a curly-quote + * mismatch would be resolved at the same leniency as ignoring a sentence + * boundary. What the sources actually disagree about is *which glyph* they used + * for one character, so the fix is a same-length substitution, not an erasure. + * + * The substitutions come from Unicode general categories (`\p{Pd}` dashes, + * `\p{Zs}` spaces, `\p{Cf}` invisibles) rather than a hand-listed table, so a + * dash variant nobody thought of can't go missing. Quotes stay explicit: their + * categories (`Pi`/`Pf`) don't distinguish single from double, and mapping both + * to one character would make `'` match `"`. * * Deliberately *not* folded: newlines (they're paragraph boundaries, and the - * callers rely on that coordinate system), and the gap between a hyphenated and - * a closed compound ("e-mail" / "email"). The latter would mean matching across - * deleted separators, and a wrong match silently rewrites the writer's prose — - * worse than a clean miss the model can retry. + * callers rely on that coordinate system) and accents (é vs e is a different + * word, not a different rendering of one). * * A fold may drop or merge characters, so it carries an index map back to the * source: a hit at folded `[a, b)` is the source range `[srcIndex(f, a), - * srcIndex(f, b))`. Callers slice the *original* text with those, so what gets + * endIndex(f, b))`. Callers slice the *original* text with those, so what gets * replaced or selected is always the writer's real characters. */ @@ -46,63 +68,35 @@ export interface Folded { map?: number[]; } -/** Where folded index `i` sits in the source text. */ +/** Where folded index `i` starts in the source text. */ export function srcIndex(folded: Folded, i: number): number { - return folded.map ? folded.map[i] : i; + if (!folded.map) return i; + return folded.map[i]; } -/** Characters that carry no text, only layout hints from some other editor. */ -const DROPPED = new Set([ - '­', // soft hyphen - '​', // zero-width space - '‌', // zero-width non-joiner - '‍', // zero-width joiner - '', // BOM / zero-width no-break space -]); +/** + * One past the last source character of a match ending at folded index `i` — + * *not* `srcIndex(f, i)`, which points at the next character the fold kept and + * would swallow any dropped characters sitting in between. For "send an e-mail + * now" that's the difference between a span of `e-mail` and one of `e-mail `. + */ +export function endIndex(folded: Folded, i: number): number { + if (!folded.map) return i; + return i === 0 ? folded.map[0] : folded.map[i - 1] + 1; +} + +/** Invisible formatting characters: soft hyphen, zero-widths, BOM, bidi marks. */ +const INVISIBLE = /\p{Cf}/u; +/** Dash punctuation, plus the minus sign (which is math, not punctuation). */ +const DASH = /[\p{Pd}−]/u; +/** Space separators, plus the tab. */ +const SPACE = /[\p{Zs}\t]/u; -/** One-for-one canonicalization: typographic variants → their ASCII form. */ -const CANON: Record = { - // dashes and hyphens - '‐': '-', // hyphen - '‑': '-', // non-breaking hyphen - '‒': '-', // figure dash - '–': '-', // en dash - '—': '-', // em dash - '―': '-', // horizontal bar - '−': '-', // minus sign - '－': '-', // fullwidth hyphen-minus - // apostrophes and single quotes - '‘': "'", - '’': "'", - '‚': "'", - '‛': "'", - ʼ: "'", - '´': "'", - '`': "'", - // double quotes - '“': '"', - '”': '"', - '„': '"', - '‟': '"', - // spaces (never \n or \r — those are paragraph structure) - '\t': ' ', - ' ': ' ', - ' ': ' ', - ' ': ' ', - ' ': ' ', - ' ': ' ', - ' ': ' ', - ' ': ' ', - ' ': ' ', - ' ': ' ', - ' ': ' ', - ' ': ' ', - ' ': ' ', - ' ': ' ', - ' ': ' ', - ' ': ' ', - '　': ' ', -}; +// Quote folding stays enumerated: `\p{Pi}`/`\p{Pf}` cover both single and double +// curly quotes without telling them apart, so a category-wide rule would fold +// `'` and `"` together. +const SINGLE_QUOTE = new Set(['‘', '’', '‚', '‛', '′', 'ʼ', '´', '`']); +const DOUBLE_QUOTE = new Set(['“', '”', '„', '‟', '″']); /** Lowercase a single character, unless casing would change its length. */ function lowerChar(ch: string): string { @@ -110,26 +104,41 @@ function lowerChar(ch: string): string { return lower.length === 1 ? lower : ch; } -/** - * Build a fold. `loose` adds case-folding, hyphen-as-space, and whitespace - * collapsing on top of the typographic canonicalization. - */ -function makeFold(loose: boolean): (source: string) => Folded { +/** The typographic canonical form of one character (always 1:1). */ +function canon(ch: string): string { + if (DASH.test(ch)) return '-'; + if (SPACE.test(ch)) return ' '; + if (SINGLE_QUOTE.has(ch)) return "'"; + if (DOUBLE_QUOTE.has(ch)) return '"'; + return ch; +} + +interface FoldOptions { + /** Case-fold, treat a hyphen as a space, and collapse runs of spaces. */ + loose?: boolean; + /** Drop separators (spaces and hyphens) outright: "e-mail" → "email". */ + runTogether?: boolean; +} + +function makeFold(opts: FoldOptions): (source: string) => Folded { return (source) => { let text = ''; const map: number[] = []; let prevWasSpace = false; for (let i = 0; i < source.length; i++) { const ch = source[i]; - if (DROPPED.has(ch)) continue; - let out = CANON[ch] ?? ch; - if (loose) { + if (INVISIBLE.test(ch)) continue; + let out = canon(ch); + if (opts.loose || opts.runTogether) { // A hyphen is a word separator like any other here, so a // compound matches however either side spelled it. out = out === '-' ? ' ' : lowerChar(out); + } + if (out === ' ') { + if (opts.runTogether) continue; // Runs of spaces collapse to one; the run maps to its first // character, so the source span still covers all of it. - if (out === ' ' && prevWasSpace) continue; + if (opts.loose && prevWasSpace) continue; } prevWasSpace = out === ' '; map.push(i); @@ -140,35 +149,83 @@ function makeFold(loose: boolean): (source: string) => Folded { }; } +/** A rung of the ladder: how to fold, and whether hits must sit on word edges. */ +export interface MatchTier { + fold: (source: string) => Folded; + /** + * Require the hit to begin and end on a word boundary in the source. Only + * the separator-dropping tier needs this — it's the only fold that can match + * across a gap between words. + */ + wordBounded?: boolean; +} + /** - * The leniency ladder, most literal first. Apply one tier across the *whole* + * The leniency ladder, most literal first. Exhaust one tier across the *whole* * search space before moving to the next, so an exact match anywhere beats a * folded match somewhere else. */ -export const MATCH_FOLDS: readonly ((source: string) => Folded)[] = [ - (source) => ({ text: source }), - makeFold(false), - makeFold(true), +export const MATCH_TIERS: readonly MatchTier[] = [ + { fold: (source) => ({ text: source }) }, + { fold: makeFold({}) }, + { fold: makeFold({ loose: true }) }, + { fold: makeFold({ runTogether: true }), wordBounded: true }, ]; +const isWordChar = (ch: string | undefined) => + ch !== undefined && /[A-Za-z0-9]/u.test(ch); + +/** + * Every hit of `needle` in `hay` under one tier, in order, as source-index + * ranges. A generator because callers stop at different points: the first hit + * per paragraph, or the first that starts inside a scoped paragraph. + */ +export function* tierMatches( + tier: MatchTier, + hay: string, + needle: string, +): Generator<{ start: number; end: number }> { + const folded = tier.fold(hay); + const foldedNeedle = tier.fold(needle).text; + for ( + let at = folded.text.indexOf(foldedNeedle); + at !== -1; + at = folded.text.indexOf(foldedNeedle, at + 1) + ) { + const start = srcIndex(folded, at); + const end = endIndex(folded, at + foldedNeedle.length); + if ( + tier.wordBounded && + (isWordChar(hay[start - 1]) || isWordChar(hay[end])) + ) { + continue; + } + yield { start, end }; + } +} + +/** The first hit of `needle` in `hay` under one tier. */ +export function firstMatch( + tier: MatchTier, + hay: string, + needle: string, +): { start: number; end: number } | null { + for (const hit of tierMatches(tier, hay, needle)) return hit; + return null; +} + /** - * Locate `needle` in `hay` under the ladder, as a source-index range. For - * single-haystack callers; anything searching several strings (paragraphs, - * text nodes) should loop the tiers itself so a whole tier is exhausted first. + * Locate `needle` in `hay` under the whole ladder. For single-haystack callers; + * anything searching several strings (paragraphs, text nodes) should loop the + * tiers itself so a whole tier is exhausted before loosening. */ export function findPhrase( hay: string, needle: string, ): { start: number; end: number } | null { - for (const fold of MATCH_FOLDS) { - const folded = fold(hay); - const foldedNeedle = fold(needle).text; - const at = folded.text.indexOf(foldedNeedle); - if (at === -1) continue; - return { - start: srcIndex(folded, at), - end: srcIndex(folded, at + foldedNeedle.length), - }; + for (const tier of MATCH_TIERS) { + const hit = firstMatch(tier, hay, needle); + if (hit) return hit; } return null; }