From 9c14f40a1ca4bfe2ecb7dfe74db389576ec5a8b6 Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Thu, 1 Oct 2026 17:02:02 +0200 Subject: [PATCH 1/2] fix(stt): keep a sentence's last word in the stretch it closes DTW can anchor a sentence's last word in the pause after it, past the kept tail. The next stretch then owned it, put it on its onset after the words that follow it, and the aligner looked for it in the wrong audio (issue #948). Ownership is now one function, read by both passes; a sentence-final word anchored before the next stretch's speech closes the stretch before it. Word starts also come out monotonic and never inverted. --- electron/stt/ctcAlign.test.ts | 16 ++++ electron/stt/ctcAlign.ts | 14 ++-- electron/stt/snapWordBoundaries.test.ts | 81 ++++++++++++++++++- electron/stt/snapWordBoundaries.ts | 77 ++++++++++++++++-- .../transcription-and-captions.md | 12 ++- 5 files changed, 184 insertions(+), 16 deletions(-) diff --git a/electron/stt/ctcAlign.test.ts b/electron/stt/ctcAlign.test.ts index 91cc15d4c..579f97a37 100644 --- a/electron/stt/ctcAlign.test.ts +++ b/electron/stt/ctcAlign.test.ts @@ -185,6 +185,22 @@ describe("alignWordsOnEmissions", () => { expect(r(out[0].startSec)).toBe(r(30 * STRIDE + PAUSE_START_OFFSET_SEC)); }); + it("aligns a sentence's last word in the stretch it closes, even anchored in the pause", () => { + // "b." is said on frame 25 (0.5 s), but DTW anchored it at 0.75 s, past the + // first stretch's tail (issue #948). It is aligned there, not in "c"'s audio. + const tokens = [...gap(15), "a", "|", ...gap(8), "b", ...gap(49), "c", ...gap(24)]; + const out = alignWordsOnEmissions( + [word("a", 0.25, 0.35, 0.3), word("b.", 0.35, 0.75, 0.75), word("c", 0.75, 1.6, 1.55)], + [ + { startSec: 0.2, endSec: 0.6 }, + { startSec: 1.4, endSec: 1.8 }, + ], + emissions(tokens), + ); + expect(r(out[1].startSec)).toBe(r(25 * STRIDE + PAUSE_START_OFFSET_SEC)); + expect(r(out[2].startSec)).toBe(r(75 * STRIDE + PAUSE_START_OFFSET_SEC)); + }); + it("keeps the helper's times where it cannot align", () => { const words = [word("ab", 0.3, 0.5, 0.35), word("c", 1.5, 1.7, 1.6)]; // The second word is outside every stretch; the first one's letters do not fit. diff --git a/electron/stt/ctcAlign.ts b/electron/stt/ctcAlign.ts index 335b6c196..4e142671e 100644 --- a/electron/stt/ctcAlign.ts +++ b/electron/stt/ctcAlign.ts @@ -18,6 +18,7 @@ // Self-contained on purpose: tools/stt-eval/word-timing imports this file // straight into Node, which resolves no extensionless import. +import { ownWords } from "./snapWordBoundaries.ts"; import type { SttVadSegment } from "./transcriptionContract"; /** What `/emissions` answers, decoded. */ @@ -58,7 +59,7 @@ export const PAUSE_END_OFFSET_SEC = 0.15; /** Audio kept on each side of a stretch of speech, so its first and last letters have context. */ export const REGION_MARGIN_SEC = 0.3; -/** As in snapWordBoundaries.ts: a stretch owns the words anchored before its end plus this tail. */ +/** As in snapWordBoundaries.ts: the audio the helper keeps past each speech offset. */ const TAIL_SEC = 0.1; const WILDCARD = -1; @@ -175,8 +176,8 @@ export function ctcViterbi( } /** - * Re-time `words` on `emissions`. Each speech stretch owns the words anchored - * in it (the rule of snapWordBoundaries.ts); they are aligned on the frames + * Re-time `words` on `emissions`. Each speech stretch owns the words + * `ownWords` (snapWordBoundaries.ts) gives it; they are aligned on the frames * between the neighbouring stretches, with the vocabulary's word delimiter * between words when it has one. Words outside every stretch, stretches the * model cannot fit, and a vocabulary with no word in it keep the helper's @@ -192,12 +193,11 @@ export function alignWordsOnEmissions( const delimiter = index.get("|"); const V = emissions.vocab.length; const { strideSec } = emissions; - let k = 0; + const ranges = ownWords(words, speech); for (let i = 0; i < speech.length; i++) { const { startSec: onset, endSec: offset } = speech[i]; - const until = Math.min(offset + TAIL_SEC, speech[i + 1]?.startSec ?? Number.POSITIVE_INFINITY); - const owned: number[] = []; - for (; k < out.length && words[k].anchorSec < until; k++) owned.push(k); + const [from, to] = ranges[i]; + const owned = Array.from({ length: to - from }, (_, n) => from + n); // A region's frames stop up to one receptive field (25 ms) short of the audio // it was cut from, so a stretch that runs to the end of the upload ends // past them; two frames of slack keep it. `hi` below stays on the frames. diff --git a/electron/stt/snapWordBoundaries.test.ts b/electron/stt/snapWordBoundaries.test.ts index dcd260f92..62d48cf12 100644 --- a/electron/stt/snapWordBoundaries.test.ts +++ b/electron/stt/snapWordBoundaries.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { anchorWordsOnSpeech, type HelperWord } from "./snapWordBoundaries"; +import { anchorWordsOnSpeech, type HelperWord, ownWords } from "./snapWordBoundaries"; import type { SttWordSegment } from "./transcriptionContract"; /** A helper word; its anchor defaults to its start, as for a request's first word. */ @@ -95,4 +95,83 @@ describe("anchorWordsOnSpeech", () => { ["d", 3.5, 4], ]); }); + + // Issue #948, measured on a Zira TTS clip: "…as if nothing happened. The + // transcript…" with speech stopping at 30.59 and resuming at 31.33. DTW + // anchored "happened." at 30.69, past the first stretch's tail. + const pause = [ + { startSec: 28.13, endSec: 30.59 }, + { startSec: 31.33, endSec: 35.23 }, + ]; + + it("keeps a sentence's last word in the stretch it closes when DTW anchors it in the pause", () => { + const words = [ + word("and", 27.97, 28.35, 28.35), + word("nothing", 29.65, 30.03, 30.03), + word("happened.", 30.03, 30.69, 30.69), + word("The", 30.69, 31.47, 31.47), + word("transcript", 31.47, 32.13, 32.13), + ]; + expect(times(anchorWordsOnSpeech(words, pause))).toEqual([ + ["and", 28.13, 28.35], + ["nothing", 29.65, 30.03], + ["happened.", 30.03, 30.59], + ["The", 31.33, 31.47], + ["transcript", 31.47, 32.13], + ]); + expect(ownWords(words, pause)).toEqual([ + [0, 3], + [3, 5], + ]); + }); + + it("keeps the punctuation whisper split off a sentence's last word with it", () => { + const words = [ + word("et", 28.2, 28.4, 28.4), + word("vraiment", 28.4, 30.69, 30.69), + word("?", 30.69, 30.7, 30.7), + word("Oui", 30.7, 31.6, 31.5), + ]; + expect(ownWords(words, pause)).toEqual([ + [0, 3], + [3, 4], + ]); + }); + + it("lets a one-word sentence open a stretch when it is anchored in its speech", () => { + const words = [ + word("Agreed?", 28.13, 30.5, 30.4), + word("Yes.", 30.5, 31.6, 31.5), + word("Good", 31.6, 32, 32), + ]; + expect(ownWords(words, pause)).toEqual([ + [0, 1], + [1, 3], + ]); + }); + + it("gives a sentence's last word to the next stretch when the one before has no words", () => { + const words = [word("Done.", 31, 31.6, 31.2), word("Next", 31.6, 32, 32)]; + expect(ownWords(words, pause)).toEqual([ + [0, 0], + [0, 2], + ]); + }); + + it("puts the words in order, none inverted, whatever stretch owns them", () => { + // The aligner-on shape of the report, as the aligner left it when the next + // stretch owned "happened": a 20 ms word after the pause and "The" earlier. + const aligned = [ + word("nothing", 29.68, 30.59, 30.03), + word("happened", 31.33, 31.35, 30.69), + word("The", 30.69, 31.48, 31.47), + word("transcript", 31.48, 32.12, 32.13), + ]; + const out = anchorWordsOnSpeech(aligned, pause); + for (const [j, w] of out.entries()) { + expect(w.endSec).toBeGreaterThanOrEqual(w.startSec); + if (j > 0) expect(w.startSec).toBeGreaterThanOrEqual(out[j - 1].startSec); + } + expect(out.map((w) => w.word)).toEqual(["nothing", "happened", "The", "transcript"]); + }); }); diff --git a/electron/stt/snapWordBoundaries.ts b/electron/stt/snapWordBoundaries.ts index 247eac6ad..7998a0f8e 100644 --- a/electron/stt/snapWordBoundaries.ts +++ b/electron/stt/snapWordBoundaries.ts @@ -49,6 +49,61 @@ const TAIL_SEC = 0.1; /** French puts a space before `!`, `?`, `:` and `;`, so whisper emits them as words. */ const isPunctuation = (word: string) => /^[\p{P}\p{S}]+$/u.test(word); +/** `.`, `!`, `?`, `…` or a full-width form, maybe inside a closing quote or bracket. */ +const endsSentence = (text: string) => /[.!?…。！？．｡][\p{Pe}\p{Pf}"']*$/u.test(text); + +/** End of the audio the helper kept for stretch `i`: its speech, plus a tail short of the next. */ +const keptUntil = (speech: SttVadSegment[], i: number) => + Math.min(speech[i].endSec + TAIL_SEC, speech[i + 1]?.startSec ?? Number.POSITIVE_INFINITY); + +/** + * The words each speech stretch owns, as `[from, to)` ranges of `words`: those + * anchored before the end of the audio the helper kept for it. Both the aligner + * (ctcAlign.ts) and the edge anchoring below read this, so a word is aligned in + * the stretch whose edges it is put on. + * + * One exception. DTW can anchor a sentence's last word in the pause after it, + * past the kept tail: the next stretch then put it on its onset, after the + * words that follow it, and the aligner looked for it in the wrong audio (issue + * #948). So a word that ends a sentence and is anchored in the pause, before the + * next stretch's speech, closes the stretch before it, with its punctuation, + * when that stretch has words. One anchored inside the next speech still opens + * it: on LibriSpeech, nine one-word sentences ("Heaven!", "What?") open a + * stretch that way, and giving them to the stretch before cost phrase deletes. + */ +export function ownWords( + words: ReadonlyArray<{ word: string; anchorSec: number }>, + speech: SttVadSegment[], +): Array<[number, number]> { + const out: Array<[number, number]> = []; + let k = 0; + for (let i = 0; i < speech.length; i++) { + const from = k; + const until = keptUntil(speech, i); + while (k < words.length && words[k].anchorSec < until) k++; + // The word that would open the next stretch, and the punctuation after it. + let first = k; + while (first < words.length && isPunctuation(words[first].word)) first++; + let end = first + 1; + while (end < words.length && isPunctuation(words[end].word)) end++; + if ( + i + 1 < speech.length && + first < words.length && + words[first].anchorSec < speech[i + 1].startSec && + words.slice(from, k).some((w) => !isPunctuation(w.word)) && + endsSentence( + words + .slice(first, end) + .map((w) => w.word) + .join(""), + ) + ) + k = end; + out.push([from, k]); + } + return out; +} + /** * Put the first and last word of every speech stretch on the stretch's edges. * The onset already carries the VAD's 30 ms pad, so a cut there lands just @@ -65,16 +120,25 @@ export function anchorWordsOnSpeech( ...w, endSec: Math.max(w.endSec, w.startSec + MIN_WORD_SEC), })); - if (!speech) return out; - let k = 0; + if (speech) anchorEdges(out, words, speech); + // Whatever the stretches did, words come out in order and never inverted. A + // start past the next word's comes back to it rather than pushing the next + // one: the onset clamp is what overshoots (a word whisper invented, put on + // the onset after the real one the aligner found earlier). + for (let j = out.length - 1; j >= 0; j--) { + if (j + 1 < out.length) out[j].startSec = Math.min(out[j].startSec, out[j + 1].startSec); + out[j].endSec = Math.max(out[j].endSec, out[j].startSec); + } + return out; +} + +function anchorEdges(out: SttWordSegment[], words: HelperWord[], speech: SttVadSegment[]) { + const owned = ownWords(words, speech); for (let i = 0; i < speech.length; i++) { const { startSec: onset, endSec: offset } = speech[i]; - // A stretch owns the words anchored before the end of the audio the helper - // kept for it: its speech, plus a tail that stops short of the next one. - const tail = Math.min(offset + TAIL_SEC, speech[i + 1]?.startSec ?? Number.POSITIVE_INFINITY); let first = -1; let last = -1; - for (; k < out.length && words[k].anchorSec < tail; k++) { + for (let k = owned[i][0]; k < owned[i][1]; k++) { if (isPunctuation(out[k].word)) continue; if (first < 0) first = k; last = k; @@ -95,5 +159,4 @@ export function anchorWordsOnSpeech( out[j].endSec = out[last].endSec; } } - return out; } diff --git a/technical-documentation/architecture/transcription-and-captions.md b/technical-documentation/architecture/transcription-and-captions.md index d3b1dfcaf..0f16828cb 100644 --- a/technical-documentation/architecture/transcription-and-captions.md +++ b/technical-documentation/architecture/transcription-and-captions.md @@ -309,7 +309,17 @@ it verbatim in the response. - its last real word ends on the offset, and the punctuation closing the phrase collapses to a point there; - a stretch owns the words whose `anchor` falls in it or in the 0.1 s tail - the helper keeps past its offset. + the helper keeps past its offset (`ownWords`, which step 5 reads too); + - except a word that ends a sentence (`.`, `!`, `?`, `…`, full-width forms + included) anchored in the pause, before the next stretch's speech: it + closes the stretch before it, with its punctuation. DTW put "happened." + of "…as if nothing happened. The transcript…" there on an rc.6 take, so + the next stretch put it on its onset, after "The", as a 20 ms word, and + the aligner looked for it in "The"'s audio (issue #948). A one-word + sentence anchored inside the next speech ("Heaven!") still opens it; + moving those too cost phrase deletes on LibriSpeech; + - last, starts are made monotonic and no word ends before it starts: a + start past the next word's comes back to it. Words own the speech, never the pause: deleting a phrase's last word keeps the pause after it. `MAX_ANCHOR_SEC` bounds the stretching only: a word more From d50bfabce3f3fcd689b75e005d4ccad5c9b3b88f Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Thu, 1 Oct 2026 17:10:02 +0200 Subject: [PATCH 2/2] fix(stt): stop a sentence end's punctuation scan at an opening bracket A separate "(" after a pause-anchored "b." made the scan read "b.(", no sentence end, so "b." opened the next stretch again. --- electron/stt/snapWordBoundaries.test.ts | 15 +++++++++++++++ electron/stt/snapWordBoundaries.ts | 5 ++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/electron/stt/snapWordBoundaries.test.ts b/electron/stt/snapWordBoundaries.test.ts index 62d48cf12..f3272b38d 100644 --- a/electron/stt/snapWordBoundaries.test.ts +++ b/electron/stt/snapWordBoundaries.test.ts @@ -138,6 +138,21 @@ describe("anchorWordsOnSpeech", () => { ]); }); + it("does not read an opening bracket after a sentence's last word as part of its end", () => { + // "b." is anchored in the pause, then whisper split "(" off the next word: + // "b.(" is no sentence end, so "b." would have opened the next stretch. + const words = [ + word("a", 28.2, 30.03, 30.03), + word("b.", 30.03, 30.69, 30.69), + word("(", 30.69, 31.4, 31.4), + word("c)", 31.4, 32, 32), + ]; + expect(ownWords(words, pause)).toEqual([ + [0, 2], + [2, 4], + ]); + }); + it("lets a one-word sentence open a stretch when it is anchored in its speech", () => { const words = [ word("Agreed?", 28.13, 30.5, 30.4), diff --git a/electron/stt/snapWordBoundaries.ts b/electron/stt/snapWordBoundaries.ts index 7998a0f8e..0cbcb6991 100644 --- a/electron/stt/snapWordBoundaries.ts +++ b/electron/stt/snapWordBoundaries.ts @@ -49,6 +49,9 @@ const TAIL_SEC = 0.1; /** French puts a space before `!`, `?`, `:` and `;`, so whisper emits them as words. */ const isPunctuation = (word: string) => /^[\p{P}\p{S}]+$/u.test(word); +/** An opening bracket or quote starts what follows, so it never closes a sentence. */ +const opens = (word: string) => /^[\p{Ps}\p{Pi}]+$/u.test(word); + /** `.`, `!`, `?`, `…` or a full-width form, maybe inside a closing quote or bracket. */ const endsSentence = (text: string) => /[.!?…。！？．｡][\p{Pe}\p{Pf}"']*$/u.test(text); @@ -85,7 +88,7 @@ export function ownWords( let first = k; while (first < words.length && isPunctuation(words[first].word)) first++; let end = first + 1; - while (end < words.length && isPunctuation(words[end].word)) end++; + while (end < words.length && isPunctuation(words[end].word) && !opens(words[end].word)) end++; if ( i + 1 < speech.length && first < words.length &&