diff --git a/electron/stt/ctcAlign.test.ts b/electron/stt/ctcAlign.test.ts index 91cc15d4c..579f97a37 100644 --- a/electron/stt/ctcAlign.test.ts +++ b/electron/stt/ctcAlign.test.ts @@ -185,6 +185,22 @@ describe("alignWordsOnEmissions", () => { expect(r(out[0].startSec)).toBe(r(30 * STRIDE + PAUSE_START_OFFSET_SEC)); }); + it("aligns a sentence's last word in the stretch it closes, even anchored in the pause", () => { + // "b." is said on frame 25 (0.5 s), but DTW anchored it at 0.75 s, past the + // first stretch's tail (issue #948). It is aligned there, not in "c"'s audio. + const tokens = [...gap(15), "a", "|", ...gap(8), "b", ...gap(49), "c", ...gap(24)]; + const out = alignWordsOnEmissions( + [word("a", 0.25, 0.35, 0.3), word("b.", 0.35, 0.75, 0.75), word("c", 0.75, 1.6, 1.55)], + [ + { startSec: 0.2, endSec: 0.6 }, + { startSec: 1.4, endSec: 1.8 }, + ], + emissions(tokens), + ); + expect(r(out[1].startSec)).toBe(r(25 * STRIDE + PAUSE_START_OFFSET_SEC)); + expect(r(out[2].startSec)).toBe(r(75 * STRIDE + PAUSE_START_OFFSET_SEC)); + }); + it("keeps the helper's times where it cannot align", () => { const words = [word("ab", 0.3, 0.5, 0.35), word("c", 1.5, 1.7, 1.6)]; // The second word is outside every stretch; the first one's letters do not fit. diff --git a/electron/stt/ctcAlign.ts b/electron/stt/ctcAlign.ts index 335b6c196..4e142671e 100644 --- a/electron/stt/ctcAlign.ts +++ b/electron/stt/ctcAlign.ts @@ -18,6 +18,7 @@ // Self-contained on purpose: tools/stt-eval/word-timing imports this file // straight into Node, which resolves no extensionless import. +import { ownWords } from "./snapWordBoundaries.ts"; import type { SttVadSegment } from "./transcriptionContract"; /** What `/emissions` answers, decoded. */ @@ -58,7 +59,7 @@ export const PAUSE_END_OFFSET_SEC = 0.15; /** Audio kept on each side of a stretch of speech, so its first and last letters have context. */ export const REGION_MARGIN_SEC = 0.3; -/** As in snapWordBoundaries.ts: a stretch owns the words anchored before its end plus this tail. */ +/** As in snapWordBoundaries.ts: the audio the helper keeps past each speech offset. */ const TAIL_SEC = 0.1; const WILDCARD = -1; @@ -175,8 +176,8 @@ export function ctcViterbi( } /** - * Re-time `words` on `emissions`. Each speech stretch owns the words anchored - * in it (the rule of snapWordBoundaries.ts); they are aligned on the frames + * Re-time `words` on `emissions`. Each speech stretch owns the words + * `ownWords` (snapWordBoundaries.ts) gives it; they are aligned on the frames * between the neighbouring stretches, with the vocabulary's word delimiter * between words when it has one. Words outside every stretch, stretches the * model cannot fit, and a vocabulary with no word in it keep the helper's @@ -192,12 +193,11 @@ export function alignWordsOnEmissions( const delimiter = index.get("|"); const V = emissions.vocab.length; const { strideSec } = emissions; - let k = 0; + const ranges = ownWords(words, speech); for (let i = 0; i < speech.length; i++) { const { startSec: onset, endSec: offset } = speech[i]; - const until = Math.min(offset + TAIL_SEC, speech[i + 1]?.startSec ?? Number.POSITIVE_INFINITY); - const owned: number[] = []; - for (; k < out.length && words[k].anchorSec < until; k++) owned.push(k); + const [from, to] = ranges[i]; + const owned = Array.from({ length: to - from }, (_, n) => from + n); // A region's frames stop up to one receptive field (25 ms) short of the audio // it was cut from, so a stretch that runs to the end of the upload ends // past them; two frames of slack keep it. `hi` below stays on the frames. diff --git a/electron/stt/snapWordBoundaries.test.ts b/electron/stt/snapWordBoundaries.test.ts index dcd260f92..f3272b38d 100644 --- a/electron/stt/snapWordBoundaries.test.ts +++ b/electron/stt/snapWordBoundaries.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { anchorWordsOnSpeech, type HelperWord } from "./snapWordBoundaries"; +import { anchorWordsOnSpeech, type HelperWord, ownWords } from "./snapWordBoundaries"; import type { SttWordSegment } from "./transcriptionContract"; /** A helper word; its anchor defaults to its start, as for a request's first word. */ @@ -95,4 +95,98 @@ describe("anchorWordsOnSpeech", () => { ["d", 3.5, 4], ]); }); + + // Issue #948, measured on a Zira TTS clip: "…as if nothing happened. The + // transcript…" with speech stopping at 30.59 and resuming at 31.33. DTW + // anchored "happened." at 30.69, past the first stretch's tail. + const pause = [ + { startSec: 28.13, endSec: 30.59 }, + { startSec: 31.33, endSec: 35.23 }, + ]; + + it("keeps a sentence's last word in the stretch it closes when DTW anchors it in the pause", () => { + const words = [ + word("and", 27.97, 28.35, 28.35), + word("nothing", 29.65, 30.03, 30.03), + word("happened.", 30.03, 30.69, 30.69), + word("The", 30.69, 31.47, 31.47), + word("transcript", 31.47, 32.13, 32.13), + ]; + expect(times(anchorWordsOnSpeech(words, pause))).toEqual([ + ["and", 28.13, 28.35], + ["nothing", 29.65, 30.03], + ["happened.", 30.03, 30.59], + ["The", 31.33, 31.47], + ["transcript", 31.47, 32.13], + ]); + expect(ownWords(words, pause)).toEqual([ + [0, 3], + [3, 5], + ]); + }); + + it("keeps the punctuation whisper split off a sentence's last word with it", () => { + const words = [ + word("et", 28.2, 28.4, 28.4), + word("vraiment", 28.4, 30.69, 30.69), + word("?", 30.69, 30.7, 30.7), + word("Oui", 30.7, 31.6, 31.5), + ]; + expect(ownWords(words, pause)).toEqual([ + [0, 3], + [3, 4], + ]); + }); + + it("does not read an opening bracket after a sentence's last word as part of its end", () => { + // "b." is anchored in the pause, then whisper split "(" off the next word: + // "b.(" is no sentence end, so "b." would have opened the next stretch. + const words = [ + word("a", 28.2, 30.03, 30.03), + word("b.", 30.03, 30.69, 30.69), + word("(", 30.69, 31.4, 31.4), + word("c)", 31.4, 32, 32), + ]; + expect(ownWords(words, pause)).toEqual([ + [0, 2], + [2, 4], + ]); + }); + + it("lets a one-word sentence open a stretch when it is anchored in its speech", () => { + const words = [ + word("Agreed?", 28.13, 30.5, 30.4), + word("Yes.", 30.5, 31.6, 31.5), + word("Good", 31.6, 32, 32), + ]; + expect(ownWords(words, pause)).toEqual([ + [0, 1], + [1, 3], + ]); + }); + + it("gives a sentence's last word to the next stretch when the one before has no words", () => { + const words = [word("Done.", 31, 31.6, 31.2), word("Next", 31.6, 32, 32)]; + expect(ownWords(words, pause)).toEqual([ + [0, 0], + [0, 2], + ]); + }); + + it("puts the words in order, none inverted, whatever stretch owns them", () => { + // The aligner-on shape of the report, as the aligner left it when the next + // stretch owned "happened": a 20 ms word after the pause and "The" earlier. + const aligned = [ + word("nothing", 29.68, 30.59, 30.03), + word("happened", 31.33, 31.35, 30.69), + word("The", 30.69, 31.48, 31.47), + word("transcript", 31.48, 32.12, 32.13), + ]; + const out = anchorWordsOnSpeech(aligned, pause); + for (const [j, w] of out.entries()) { + expect(w.endSec).toBeGreaterThanOrEqual(w.startSec); + if (j > 0) expect(w.startSec).toBeGreaterThanOrEqual(out[j - 1].startSec); + } + expect(out.map((w) => w.word)).toEqual(["nothing", "happened", "The", "transcript"]); + }); }); diff --git a/electron/stt/snapWordBoundaries.ts b/electron/stt/snapWordBoundaries.ts index 247eac6ad..0cbcb6991 100644 --- a/electron/stt/snapWordBoundaries.ts +++ b/electron/stt/snapWordBoundaries.ts @@ -49,6 +49,64 @@ const TAIL_SEC = 0.1; /** French puts a space before `!`, `?`, `:` and `;`, so whisper emits them as words. */ const isPunctuation = (word: string) => /^[\p{P}\p{S}]+$/u.test(word); +/** An opening bracket or quote starts what follows, so it never closes a sentence. */ +const opens = (word: string) => /^[\p{Ps}\p{Pi}]+$/u.test(word); + +/** `.`, `!`, `?`, `…` or a full-width form, maybe inside a closing quote or bracket. */ +const endsSentence = (text: string) => /[.!?…。！？．｡][\p{Pe}\p{Pf}"']*$/u.test(text); + +/** End of the audio the helper kept for stretch `i`: its speech, plus a tail short of the next. */ +const keptUntil = (speech: SttVadSegment[], i: number) => + Math.min(speech[i].endSec + TAIL_SEC, speech[i + 1]?.startSec ?? Number.POSITIVE_INFINITY); + +/** + * The words each speech stretch owns, as `[from, to)` ranges of `words`: those + * anchored before the end of the audio the helper kept for it. Both the aligner + * (ctcAlign.ts) and the edge anchoring below read this, so a word is aligned in + * the stretch whose edges it is put on. + * + * One exception. DTW can anchor a sentence's last word in the pause after it, + * past the kept tail: the next stretch then put it on its onset, after the + * words that follow it, and the aligner looked for it in the wrong audio (issue + * #948). So a word that ends a sentence and is anchored in the pause, before the + * next stretch's speech, closes the stretch before it, with its punctuation, + * when that stretch has words. One anchored inside the next speech still opens + * it: on LibriSpeech, nine one-word sentences ("Heaven!", "What?") open a + * stretch that way, and giving them to the stretch before cost phrase deletes. + */ +export function ownWords( + words: ReadonlyArray<{ word: string; anchorSec: number }>, + speech: SttVadSegment[], +): Array<[number, number]> { + const out: Array<[number, number]> = []; + let k = 0; + for (let i = 0; i < speech.length; i++) { + const from = k; + const until = keptUntil(speech, i); + while (k < words.length && words[k].anchorSec < until) k++; + // The word that would open the next stretch, and the punctuation after it. + let first = k; + while (first < words.length && isPunctuation(words[first].word)) first++; + let end = first + 1; + while (end < words.length && isPunctuation(words[end].word) && !opens(words[end].word)) end++; + if ( + i + 1 < speech.length && + first < words.length && + words[first].anchorSec < speech[i + 1].startSec && + words.slice(from, k).some((w) => !isPunctuation(w.word)) && + endsSentence( + words + .slice(first, end) + .map((w) => w.word) + .join(""), + ) + ) + k = end; + out.push([from, k]); + } + return out; +} + /** * Put the first and last word of every speech stretch on the stretch's edges. * The onset already carries the VAD's 30 ms pad, so a cut there lands just @@ -65,16 +123,25 @@ export function anchorWordsOnSpeech( ...w, endSec: Math.max(w.endSec, w.startSec + MIN_WORD_SEC), })); - if (!speech) return out; - let k = 0; + if (speech) anchorEdges(out, words, speech); + // Whatever the stretches did, words come out in order and never inverted. A + // start past the next word's comes back to it rather than pushing the next + // one: the onset clamp is what overshoots (a word whisper invented, put on + // the onset after the real one the aligner found earlier). + for (let j = out.length - 1; j >= 0; j--) { + if (j + 1 < out.length) out[j].startSec = Math.min(out[j].startSec, out[j + 1].startSec); + out[j].endSec = Math.max(out[j].endSec, out[j].startSec); + } + return out; +} + +function anchorEdges(out: SttWordSegment[], words: HelperWord[], speech: SttVadSegment[]) { + const owned = ownWords(words, speech); for (let i = 0; i < speech.length; i++) { const { startSec: onset, endSec: offset } = speech[i]; - // A stretch owns the words anchored before the end of the audio the helper - // kept for it: its speech, plus a tail that stops short of the next one. - const tail = Math.min(offset + TAIL_SEC, speech[i + 1]?.startSec ?? Number.POSITIVE_INFINITY); let first = -1; let last = -1; - for (; k < out.length && words[k].anchorSec < tail; k++) { + for (let k = owned[i][0]; k < owned[i][1]; k++) { if (isPunctuation(out[k].word)) continue; if (first < 0) first = k; last = k; @@ -95,5 +162,4 @@ export function anchorWordsOnSpeech( out[j].endSec = out[last].endSec; } } - return out; } diff --git a/technical-documentation/architecture/transcription-and-captions.md b/technical-documentation/architecture/transcription-and-captions.md index d3b1dfcaf..0f16828cb 100644 --- a/technical-documentation/architecture/transcription-and-captions.md +++ b/technical-documentation/architecture/transcription-and-captions.md @@ -309,7 +309,17 @@ it verbatim in the response. - its last real word ends on the offset, and the punctuation closing the phrase collapses to a point there; - a stretch owns the words whose `anchor` falls in it or in the 0.1 s tail - the helper keeps past its offset. + the helper keeps past its offset (`ownWords`, which step 5 reads too); + - except a word that ends a sentence (`.`, `!`, `?`, `…`, full-width forms + included) anchored in the pause, before the next stretch's speech: it + closes the stretch before it, with its punctuation. DTW put "happened." + of "…as if nothing happened. The transcript…" there on an rc.6 take, so + the next stretch put it on its onset, after "The", as a 20 ms word, and + the aligner looked for it in "The"'s audio (issue #948). A one-word + sentence anchored inside the next speech ("Heaven!") still opens it; + moving those too cost phrase deletes on LibriSpeech; + - last, starts are made monotonic and no word ends before it starts: a + start past the next word's comes back to it. Words own the speech, never the pause: deleting a phrase's last word keeps the pause after it. `MAX_ANCHOR_SEC` bounds the stretching only: a word more