fix(stt): keep a sentence's last word in the stretch it closes

electron/stt/ctcAlign.test.ts

@@ -185,6 +185,22 @@ describe("alignWordsOnEmissions", () => {
 		expect(r(out[0].startSec)).toBe(r(30 * STRIDE + PAUSE_START_OFFSET_SEC));
 	});
 
+	it("aligns a sentence's last word in the stretch it closes, even anchored in the pause", () => {
+		// "b." is said on frame 25 (0.5 s), but DTW anchored it at 0.75 s, past the
+		// first stretch's tail (issue #948). It is aligned there, not in "c"'s audio.
+		const tokens = [...gap(15), "a", "|", ...gap(8), "b", ...gap(49), "c", ...gap(24)];
+		const out = alignWordsOnEmissions(
+			[word("a", 0.25, 0.35, 0.3), word("b.", 0.35, 0.75, 0.75), word("c", 0.75, 1.6, 1.55)],
+			[
+				{ startSec: 0.2, endSec: 0.6 },
+				{ startSec: 1.4, endSec: 1.8 },
+			],
+			emissions(tokens),
+		);
+		expect(r(out[1].startSec)).toBe(r(25 * STRIDE + PAUSE_START_OFFSET_SEC));
+		expect(r(out[2].startSec)).toBe(r(75 * STRIDE + PAUSE_START_OFFSET_SEC));
+	});
+
 	it("keeps the helper's times where it cannot align", () => {
 		const words = [word("ab", 0.3, 0.5, 0.35), word("c", 1.5, 1.7, 1.6)];
 		// The second word is outside every stretch; the first one's letters do not fit.

electron/stt/ctcAlign.ts

@@ -18,6 +18,7 @@
 // Self-contained on purpose: tools/stt-eval/word-timing imports this file
 // straight into Node, which resolves no extensionless import.
 
+import { ownWords } from "./snapWordBoundaries.ts";
 import type { SttVadSegment } from "./transcriptionContract";
 
 /** What `/emissions` answers, decoded. */
@@ -58,7 +59,7 @@ export const PAUSE_END_OFFSET_SEC = 0.15;
 /** Audio kept on each side of a stretch of speech, so its first and last letters have context. */
 export const REGION_MARGIN_SEC = 0.3;
 
-/** As in snapWordBoundaries.ts: a stretch owns the words anchored before its end plus this tail. */
+/** As in snapWordBoundaries.ts: the audio the helper keeps past each speech offset. */
 const TAIL_SEC = 0.1;
 
 const WILDCARD = -1;
@@ -175,8 +176,8 @@ export function ctcViterbi(
 }
 
 /**
- * Re-time `words` on `emissions`. Each speech stretch owns the words anchored
- * in it (the rule of snapWordBoundaries.ts); they are aligned on the frames
+ * Re-time `words` on `emissions`. Each speech stretch owns the words
+ * `ownWords` (snapWordBoundaries.ts) gives it; they are aligned on the frames
  * between the neighbouring stretches, with the vocabulary's word delimiter
  * between words when it has one. Words outside every stretch, stretches the
  * model cannot fit, and a vocabulary with no word in it keep the helper's
@@ -192,12 +193,11 @@ export function alignWordsOnEmissions<W extends AlignableWord>(
 	const delimiter = index.get("|");
 	const V = emissions.vocab.length;
 	const { strideSec } = emissions;
-	let k = 0;
+	const ranges = ownWords(words, speech);
 	for (let i = 0; i < speech.length; i++) {
 		const { startSec: onset, endSec: offset } = speech[i];
-		const until = Math.min(offset + TAIL_SEC, speech[i + 1]?.startSec ?? Number.POSITIVE_INFINITY);
-		const owned: number[] = [];
-		for (; k < out.length && words[k].anchorSec < until; k++) owned.push(k);
+		const [from, to] = ranges[i];
+		const owned = Array.from({ length: to - from }, (_, n) => from + n);
 		// A region's frames stop up to one receptive field (25 ms) short of the audio
 		// it was cut from, so a stretch that runs to the end of the upload ends
 		// past them; two frames of slack keep it. `hi` below stays on the frames.

electron/stt/snapWordBoundaries.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, it } from "vitest";
-import { anchorWordsOnSpeech, type HelperWord } from "./snapWordBoundaries";
+import { anchorWordsOnSpeech, type HelperWord, ownWords } from "./snapWordBoundaries";
 import type { SttWordSegment } from "./transcriptionContract";
 
 /** A helper word; its anchor defaults to its start, as for a request's first word. */
@@ -95,4 +95,98 @@ describe("anchorWordsOnSpeech", () => {
 			["d", 3.5, 4],
 		]);
 	});
+
+	// Issue #948, measured on a Zira TTS clip: "…as if nothing happened. The
+	// transcript…" with speech stopping at 30.59 and resuming at 31.33. DTW
+	// anchored "happened." at 30.69, past the first stretch's tail.
+	const pause = [
+		{ startSec: 28.13, endSec: 30.59 },
+		{ startSec: 31.33, endSec: 35.23 },
+	];
+
+	it("keeps a sentence's last word in the stretch it closes when DTW anchors it in the pause", () => {
+		const words = [
+			word("and", 27.97, 28.35, 28.35),
+			word("nothing", 29.65, 30.03, 30.03),
+			word("happened.", 30.03, 30.69, 30.69),
+			word("The", 30.69, 31.47, 31.47),
+			word("transcript", 31.47, 32.13, 32.13),
+		];
+		expect(times(anchorWordsOnSpeech(words, pause))).toEqual([
+			["and", 28.13, 28.35],
+			["nothing", 29.65, 30.03],
+			["happened.", 30.03, 30.59],
+			["The", 31.33, 31.47],
+			["transcript", 31.47, 32.13],
+		]);
+		expect(ownWords(words, pause)).toEqual([
+			[0, 3],
+			[3, 5],
+		]);
+	});
+
+	it("keeps the punctuation whisper split off a sentence's last word with it", () => {
+		const words = [
+			word("et", 28.2, 28.4, 28.4),
+			word("vraiment", 28.4, 30.69, 30.69),
+			word("?", 30.69, 30.7, 30.7),
+			word("Oui", 30.7, 31.6, 31.5),
+		];
+		expect(ownWords(words, pause)).toEqual([
+			[0, 3],
+			[3, 4],
+		]);
+	});
+
+	it("does not read an opening bracket after a sentence's last word as part of its end", () => {
+		// "b." is anchored in the pause, then whisper split "(" off the next word:
+		// "b.(" is no sentence end, so "b." would have opened the next stretch.
+		const words = [
+			word("a", 28.2, 30.03, 30.03),
+			word("b.", 30.03, 30.69, 30.69),
+			word("(", 30.69, 31.4, 31.4),
+			word("c)", 31.4, 32, 32),
+		];
+		expect(ownWords(words, pause)).toEqual([
+			[0, 2],
+			[2, 4],
+		]);
+	});
+
+	it("lets a one-word sentence open a stretch when it is anchored in its speech", () => {
+		const words = [
+			word("Agreed?", 28.13, 30.5, 30.4),
+			word("Yes.", 30.5, 31.6, 31.5),
+			word("Good", 31.6, 32, 32),
+		];
+		expect(ownWords(words, pause)).toEqual([
+			[0, 1],
+			[1, 3],
+		]);
+	});
+
+	it("gives a sentence's last word to the next stretch when the one before has no words", () => {
+		const words = [word("Done.", 31, 31.6, 31.2), word("Next", 31.6, 32, 32)];
+		expect(ownWords(words, pause)).toEqual([
+			[0, 0],
+			[0, 2],
+		]);
+	});
+
+	it("puts the words in order, none inverted, whatever stretch owns them", () => {
+		// The aligner-on shape of the report, as the aligner left it when the next
+		// stretch owned "happened": a 20 ms word after the pause and "The" earlier.
+		const aligned = [
+			word("nothing", 29.68, 30.59, 30.03),
+			word("happened", 31.33, 31.35, 30.69),
+			word("The", 30.69, 31.48, 31.47),
+			word("transcript", 31.48, 32.12, 32.13),
+		];
+		const out = anchorWordsOnSpeech(aligned, pause);
+		for (const [j, w] of out.entries()) {
+			expect(w.endSec).toBeGreaterThanOrEqual(w.startSec);
+			if (j > 0) expect(w.startSec).toBeGreaterThanOrEqual(out[j - 1].startSec);
+		}
+		expect(out.map((w) => w.word)).toEqual(["nothing", "happened", "The", "transcript"]);
+	});
 });

electron/stt/snapWordBoundaries.ts

@@ -49,6 +49,64 @@ const TAIL_SEC = 0.1;
 /** French puts a space before `!`, `?`, `:` and `;`, so whisper emits them as words. */
 const isPunctuation = (word: string) => /^[\p{P}\p{S}]+$/u.test(word);
 
+/** An opening bracket or quote starts what follows, so it never closes a sentence. */
+const opens = (word: string) => /^[\p{Ps}\p{Pi}]+$/u.test(word);
+
+/** `.`, `!`, `?`, `…` or a full-width form, maybe inside a closing quote or bracket. */
+const endsSentence = (text: string) => /[.!?…。！？．｡][\p{Pe}\p{Pf}"']*$/u.test(text);
+
+/** End of the audio the helper kept for stretch `i`: its speech, plus a tail short of the next. */
+const keptUntil = (speech: SttVadSegment[], i: number) =>
+	Math.min(speech[i].endSec + TAIL_SEC, speech[i + 1]?.startSec ?? Number.POSITIVE_INFINITY);
+
+/**
+ * The words each speech stretch owns, as `[from, to)` ranges of `words`: those
+ * anchored before the end of the audio the helper kept for it. Both the aligner
+ * (ctcAlign.ts) and the edge anchoring below read this, so a word is aligned in
+ * the stretch whose edges it is put on.
+ *
+ * One exception. DTW can anchor a sentence's last word in the pause after it,
+ * past the kept tail: the next stretch then put it on its onset, after the
+ * words that follow it, and the aligner looked for it in the wrong audio (issue
+ * #948). So a word that ends a sentence and is anchored in the pause, before the
+ * next stretch's speech, closes the stretch before it, with its punctuation,
+ * when that stretch has words. One anchored inside the next speech still opens
+ * it: on LibriSpeech, nine one-word sentences ("Heaven!", "What?") open a
+ * stretch that way, and giving them to the stretch before cost phrase deletes.
+ */
+export function ownWords(
+	words: ReadonlyArray<{ word: string; anchorSec: number }>,
+	speech: SttVadSegment[],
+): Array<[number, number]> {
+	const out: Array<[number, number]> = [];
+	let k = 0;
+	for (let i = 0; i < speech.length; i++) {
+		const from = k;
+		const until = keptUntil(speech, i);
+		while (k < words.length && words[k].anchorSec < until) k++;
+		// The word that would open the next stretch, and the punctuation after it.
+		let first = k;
+		while (first < words.length && isPunctuation(words[first].word)) first++;
+		let end = first + 1;
+		while (end < words.length && isPunctuation(words[end].word) && !opens(words[end].word)) end++;
+		if (
+			i + 1 < speech.length &&
+			first < words.length &&
+			words[first].anchorSec < speech[i + 1].startSec &&
+			words.slice(from, k).some((w) => !isPunctuation(w.word)) &&
+			endsSentence(
+				words
+					.slice(first, end)
+					.map((w) => w.word)
+					.join(""),
+			)
+		)
+			k = end;
+		out.push([from, k]);
+	}
+	return out;
+}
+
 /**
  * Put the first and last word of every speech stretch on the stretch's edges.
  * The onset already carries the VAD's 30 ms pad, so a cut there lands just
@@ -65,16 +123,25 @@ export function anchorWordsOnSpeech(
 		...w,
 		endSec: Math.max(w.endSec, w.startSec + MIN_WORD_SEC),
 	}));
-	if (!speech) return out;
-	let k = 0;
+	if (speech) anchorEdges(out, words, speech);
+	// Whatever the stretches did, words come out in order and never inverted. A
+	// start past the next word's comes back to it rather than pushing the next
+	// one: the onset clamp is what overshoots (a word whisper invented, put on
+	// the onset after the real one the aligner found earlier).
+	for (let j = out.length - 1; j >= 0; j--) {
+		if (j + 1 < out.length) out[j].startSec = Math.min(out[j].startSec, out[j + 1].startSec);
+		out[j].endSec = Math.max(out[j].endSec, out[j].startSec);
+	}
+	return out;
+}
+
+function anchorEdges(out: SttWordSegment[], words: HelperWord[], speech: SttVadSegment[]) {
+	const owned = ownWords(words, speech);
 	for (let i = 0; i < speech.length; i++) {
 		const { startSec: onset, endSec: offset } = speech[i];
-		// A stretch owns the words anchored before the end of the audio the helper
-		// kept for it: its speech, plus a tail that stops short of the next one.
-		const tail = Math.min(offset + TAIL_SEC, speech[i + 1]?.startSec ?? Number.POSITIVE_INFINITY);
 		let first = -1;
 		let last = -1;
-		for (; k < out.length && words[k].anchorSec < tail; k++) {
+		for (let k = owned[i][0]; k < owned[i][1]; k++) {
 			if (isPunctuation(out[k].word)) continue;
 			if (first < 0) first = k;
 			last = k;
@@ -95,5 +162,4 @@ export function anchorWordsOnSpeech(
 			out[j].endSec = out[last].endSec;
 		}
 	}
-	return out;
 }

technical-documentation/architecture/transcription-and-captions.md

@@ -309,7 +309,17 @@ it verbatim in the response.
    - its last real word ends on the offset, and the punctuation closing the
      phrase collapses to a point there;
    - a stretch owns the words whose `anchor` falls in it or in the 0.1 s tail
-     the helper keeps past its offset.
+     the helper keeps past its offset (`ownWords`, which step 5 reads too);
+   - except a word that ends a sentence (`.`, `!`, `?`, `…`, full-width forms
+     included) anchored in the pause, before the next stretch's speech: it
+     closes the stretch before it, with its punctuation. DTW put "happened."
+     of "…as if nothing happened. The transcript…" there on an rc.6 take, so
+     the next stretch put it on its onset, after "The", as a 20 ms word, and
+     the aligner looked for it in "The"'s audio (issue #948). A one-word
+     sentence anchored inside the next speech ("Heaven!") still opens it;
+     moving those too cost phrase deletes on LibriSpeech;
+   - last, starts are made monotonic and no word ends before it starts: a
+     start past the next word's comes back to it.
 
    Words own the speech, never the pause: deleting a phrase's last word keeps
    the pause after it. `MAX_ANCHOR_SEC` bounds the stretching only: a word more