Detect untranscribed filled pauses as ... filler placeholders

components/Timeline.tsx

@@ -33,6 +33,7 @@ import {
   trimEdgeBounds,
 } from "@/lib/edits";
 import type { ClipSegment, Word } from "@/lib/types";
+import { isDisfluencyPlaceholder } from "@/lib/disfluencies";
 import { VAD_SAMPLE_RATE } from "@/lib/vad";
 import { useCutRanges } from "@/hooks/useCutRanges";
 import { useIsDark } from "@/hooks/useIsDark";
@@ -726,6 +727,7 @@ export default function Timeline() {
               const hovered = hoveredWordId === w.id;
               const cutOut = isWordCutOut(w, cuts);
               const wordSelected = selectedWordIds.includes(w.id);
+              const placeholder = isDisfluencyPlaceholder(w.text);
               const showWordHandles = showHandles && (hovered || wWidth > 28);
               return (
                 <div
@@ -736,9 +738,13 @@ export default function Timeline() {
                       ? "border-red-200/90 bg-red-50/95 text-red-400 line-through dark:border-red-900/90 dark:bg-red-950/60 dark:text-red-400"
                       : wordSelected
                         ? "border-indigo-300 bg-indigo-100/70 text-zinc-800 dark:border-indigo-500/60 dark:bg-indigo-950/50 dark:text-zinc-100"
-                        : hovered
-                          ? "border-neutral-300 bg-white text-zinc-700 shadow-sm shadow-neutral-500/10 dark:border-neutral-600 dark:bg-zinc-800 dark:text-zinc-200 dark:shadow-black/20"
-                          : "border-zinc-200/90 bg-white/95 text-zinc-600 dark:border-zinc-700/90 dark:bg-zinc-800/95 dark:text-zinc-300"
+                        : placeholder
+                          ? hovered
+                            ? "border-amber-300/90 bg-amber-50 text-amber-800 shadow-sm shadow-amber-500/10 dark:border-amber-700/80 dark:bg-amber-950/50 dark:text-amber-300"
+                            : "border-amber-200/90 bg-amber-50/90 text-amber-700/90 dark:border-amber-800/80 dark:bg-amber-950/40 dark:text-amber-400/90"
+                          : hovered
+                            ? "border-neutral-300 bg-white text-zinc-700 shadow-sm shadow-neutral-500/10 dark:border-neutral-600 dark:bg-zinc-800 dark:text-zinc-200 dark:shadow-black/20"
+                            : "border-zinc-200/90 bg-white/95 text-zinc-600 dark:border-zinc-700/90 dark:bg-zinc-800/95 dark:text-zinc-300"
                   } ${wordSelected ? "ring-1 ring-indigo-400/80" : ""}`}
                   style={{
                     left: w.start * pps,
@@ -747,9 +753,13 @@ export default function Timeline() {
                     height: WORDBAR_H - 10,
                   }}
                   title={
-                    showHandles
-                      ? `${w.text} — drag edges to adjust timing`
-                      : w.text
+                    placeholder
+                      ? showHandles
+                        ? "… detected hesitation — drag edges to adjust, or Remove filler words to cut"
+                        : "… detected hesitation — cut with Remove filler words"
+                      : showHandles
+                        ? `${w.text} — drag edges to adjust timing`
+                        : w.text
                   }
                   onPointerEnter={() => setHoveredWordId(w.id)}
                   onPointerLeave={() =>

components/TranscriptPanel.tsx

@@ -16,6 +16,7 @@ import {
   X,
 } from "lucide-react";
 import { useEditorStore } from "@/lib/store";
+import { isDisfluencyPlaceholder } from "@/lib/disfluencies";
 import { findFillerWordIds } from "@/lib/fillers";
 import { findSilenceRanges } from "@/lib/silences";
 import {
@@ -60,18 +61,23 @@ const WordSpan = memo(function WordSpan({
   active: boolean;
   onClick: (word: Word, el: HTMLElement) => void;
 }) {
+  const placeholder = isDisfluencyPlaceholder(word.text);
   // The trailing space lives inside the span so that selection and deletion
   // highlights are continuous across words instead of breaking at each gap.
   return (
     <span
       data-wid={word.id}
       data-cut={cutOut ? "" : undefined}
+      data-placeholder={placeholder ? "" : undefined}
+      title={placeholder ? "Detected hesitation (not transcribed) — cut with Remove filler words" : undefined}
       onClick={(e) => onClick(word, e.currentTarget)}
       className={`py-0.5 cursor-pointer transition-colors duration-75 ${cutOut
         ? "word-deleted bg-red-50 text-red-600 line-through decoration-red-300 dark:bg-red-950/40 dark:text-red-400 dark:decoration-red-800"
         : active
           ? "bg-neutral-200/80 text-zinc-900 dark:bg-neutral-700/80 dark:text-zinc-50"
-          : "text-zinc-800 hover:bg-neutral-50 dark:text-zinc-200 dark:hover:bg-neutral-800/60"
+          : placeholder
+            ? "font-medium text-amber-700/90 hover:bg-amber-50 dark:text-amber-400/90 dark:hover:bg-amber-950/40"
+            : "text-zinc-800 hover:bg-neutral-50 dark:text-zinc-200 dark:hover:bg-neutral-800/60"
         }`}
     >
       {word.text}{" "}
@@ -315,7 +321,7 @@ export default function TranscriptPanel() {
           {status === "ready" && fillerIds.length > 0 && (
             <button
               onClick={removeFillers}
-              title='Cut filler words ("um", "uh", …) from the video'
+              title='Cut filler words ("um", "uh", "...", …) from the video'
               className="flex cursor-pointer h-7 items-center gap-1.5 rounded-lg px-2 text-xs text-zinc-500 transition hover:bg-zinc-100 line-clamp-1 dark:text-zinc-400 dark:hover:bg-zinc-800"
             >
               <WandSparkles size={14} />

lib/disfluencies.ts

@@ -0,0 +1,130 @@
+/**
+ * Detect filled pauses / hesitations that ASR dropped from the transcript.
+ *
+ * Whisper (and similar models) often omit "um" / "uh". The audio is still
+ * speech according to VAD, so those spans show up as speech frames with no
+ * covering word. We insert `...` placeholders there so they appear in the
+ * transcript / timeline and can be cut via Remove fillers.
+ *
+ * Pure helpers — takes VAD frame flags directly so tests need no model.
+ */
+import type { Word } from "./types";
+import { VAD_FRAME_SIZE, VAD_SAMPLE_RATE } from "./vad";
+
+/** Marker text for a detected, untranscribed disfluency. */
+export const DISFLUENCY_PLACEHOLDER = "...";
+
+/**
+ * Minimum uncovered speech (seconds) to emit a placeholder. Must sit above
+ * Silero's ~150 ms post-speech hangover so word tails aren't mistaken for fillers.
+ */
+export const MIN_DISFLUENCY_DURATION = 0.3;
+
+/**
+ * Trim this much from the start of an uncovered run that abuts a prior word.
+ * Absorbs Silero hangover so the placeholder tracks the filled pause, not the
+ * bleed after the previous token.
+ */
+const HANGOVER_TRIM_S = 0.12;
+
+export interface DisfluencyOptions {
+  frameSize?: number;
+  sampleRate?: number;
+  minDurationS?: number;
+  /** Media duration; placeholder ends are clamped to it when > 0. */
+  duration?: number;
+}
+
+/** Whether `text` is a disfluency placeholder token. */
+export function isDisfluencyPlaceholder(text: string): boolean {
+  return text.trim() === DISFLUENCY_PLACEHOLDER;
+}
+
+/** Contiguous [start, end) runs of uncovered speech, in seconds. */
+function uncoveredSpeechRuns(
+  words: Word[],
+  speechFrames: boolean[],
+  frameS: number
+): Array<{ start: number; end: number }> {
+  if (speechFrames.length === 0) return [];
+
+  const covered = new Uint8Array(speechFrames.length);
+  for (const w of words) {
+    if (w.deleted) continue;
+    const a = Math.max(0, Math.floor(w.start / frameS));
+    const b = Math.min(speechFrames.length, Math.ceil(w.end / frameS));
+    for (let i = a; i < b; i++) covered[i] = 1;
+  }
+
+  const runs: Array<{ start: number; end: number }> = [];
+  for (let i = 0; i < speechFrames.length; ) {
+    if (!speechFrames[i] || covered[i]) {
+      i++;
+      continue;
+    }
+    let j = i + 1;
+    while (j < speechFrames.length && speechFrames[j] && !covered[j]) j++;
+    runs.push({ start: i * frameS, end: j * frameS });
+    i = j;
+  }
+  return runs;
+}
+
+/**
+ * Insert `...` words over VAD speech that no transcript token covers.
+ * Returns a new array (re-indexed); input is untouched. No-op when there are
+ * no speech frames or nothing qualifies.
+ */
+export function insertDisfluencyPlaceholders(
+  words: Word[],
+  speechFrames: boolean[],
+  {
+    frameSize = VAD_FRAME_SIZE,
+    sampleRate = VAD_SAMPLE_RATE,
+    minDurationS = MIN_DISFLUENCY_DURATION,
+    duration = 0,
+  }: DisfluencyOptions = {}
+): Word[] {
+  if (speechFrames.length === 0) return words;
+
+  const frameS = frameSize / sampleRate;
+  const maxT = duration > 0 ? duration : Infinity;
+  const sorted = words.slice().sort((a, b) => a.start - b.start || a.end - b.end);
+  const runs = uncoveredSpeechRuns(sorted, speechFrames, frameS);
+  if (runs.length === 0) return words;
+
+  const placeholders: Word[] = [];
+  for (const run of runs) {
+    let start = run.start;
+    let end = Math.min(run.end, maxT);
+
+    // Hangover after the preceding word is not a filled pause.
+    const prev = sorted.filter((w) => !w.deleted && w.end <= start + frameS).pop();
+    if (prev && start - prev.end <= frameS + 1e-4) {
+      start = Math.min(end, start + HANGOVER_TRIM_S);
+    }
+
+    // Don't spill into the next word (alignment / frame rounding).
+    const next = sorted.find((w) => !w.deleted && w.start >= start - frameS);
+    if (next) end = Math.min(end, next.start);
+
+    if (end - start < minDurationS - 1e-4) continue;
+    if (start >= maxT - 1e-4) continue;
+
+    placeholders.push({
+      id: -1,
+      text: DISFLUENCY_PLACEHOLDER,
+      start,
+      end,
+      speaker: 0,
+      deleted: false,
+    });
+  }
+
+  if (placeholders.length === 0) return words;
+
+  const merged = [...sorted, ...placeholders].sort(
+    (a, b) => a.start - b.start || a.end - b.end
+  );
+  return merged.map((w, i) => (w.id === i ? w : { ...w, id: i }));
+}

lib/fillers.ts

@@ -1,9 +1,11 @@
+import { isDisfluencyPlaceholder } from "./disfluencies";
 import type { Word } from "./types";
 
 /**
  * Filler sounds that carry no meaning and can safely be cut. Deliberately
  * conservative: ambiguous words like "like", "so" or "right" are excluded
- * because they are usually legitimate.
+ * because they are usually legitimate. Also includes `...` placeholders for
+ * filled pauses ASR dropped (see lib/disfluencies.ts).
  */
 const FILLER_WORDS = new Set([
   // English
@@ -66,6 +68,7 @@ function normalize(text: string): string {
 }
 
 export function isFillerWord(text: string): boolean {
+  if (isDisfluencyPlaceholder(text)) return true;
   return FILLER_WORDS.has(normalize(text));
 }
 

tests/disfluencies-test.ts

@@ -0,0 +1,114 @@
+/**
+ * Run: npx tsx tests/disfluencies-test.ts
+ */
+import {
+  DISFLUENCY_PLACEHOLDER,
+  MIN_DISFLUENCY_DURATION,
+  insertDisfluencyPlaceholders,
+  isDisfluencyPlaceholder,
+} from "../lib/disfluencies";
+import { findFillerWordIds, isFillerWord } from "../lib/fillers";
+import { VAD_FRAME_SIZE, VAD_SAMPLE_RATE } from "../lib/vad";
+import type { Word } from "../lib/types";
+
+function assert(cond: boolean, msg: string) {
+  if (!cond) throw new Error(msg);
+}
+
+const FRAME_S = VAD_FRAME_SIZE / VAD_SAMPLE_RATE;
+
+function framesFor(ranges: Array<[number, number]>, totalS: number): boolean[] {
+  const n = Math.ceil(totalS / FRAME_S);
+  const frames = new Array<boolean>(n).fill(false);
+  for (const [a, b] of ranges) {
+    for (let i = Math.round(a / FRAME_S); i < Math.round(b / FRAME_S); i++) {
+      if (i >= 0 && i < n) frames[i] = true;
+    }
+  }
+  return frames;
+}
+
+function word(id: number, text: string, start: number, end: number): Word {
+  return { id, text, start, end, speaker: 0, deleted: false };
+}
+
+{
+  assert(isDisfluencyPlaceholder("..."), "exact placeholder");
+  assert(isDisfluencyPlaceholder(" ... "), "trimmed placeholder");
+  assert(!isDisfluencyPlaceholder("um"), "um is not a placeholder token");
+  assert(!isDisfluencyPlaceholder("[*]"), "legacy [*] marker is not used");
+  assert(isFillerWord(DISFLUENCY_PLACEHOLDER), "... counts as a filler");
+  console.log("placeholder identity: ok");
+}
+
+{
+  // Continuous speech with a mid gap Whisper skipped → one "..." filler.
+  // Words cover 1.0–1.4 and 2.0–2.4; speech runs 1.0–2.4 (includes filled pause).
+  const words = [word(0, "Hello", 1.0, 1.4), word(1, "there", 2.0, 2.4)];
+  const frames = framesFor([[1.0, 2.4]], 3);
+  const out = insertDisfluencyPlaceholders(words, frames, { duration: 3 });
+  const placeholders = out.filter((w) => isDisfluencyPlaceholder(w.text));
+  assert(placeholders.length === 1, `expected 1 placeholder, got ${placeholders.length}`);
+  const p = placeholders[0]!;
+  assert(p.text === "...", "placeholder text is ...");
+  assert(p.end - p.start >= MIN_DISFLUENCY_DURATION - 1e-3, "placeholder long enough");
+  assert(p.start >= 1.4 - 1e-3, "placeholder after Hello");
+  assert(p.end <= 2.0 + 1e-3, "placeholder before there");
+  assert(out.length === 3, "merged length");
+  assert(
+    out.map((w) => w.id).join(",") === "0,1,2",
+    `ids reindexed: ${out.map((w) => w.id).join(",")}`
+  );
+  const fillerIds = findFillerWordIds(out);
+  assert(fillerIds.length === 1 && fillerIds[0] === p.id, "Remove fillers sees ...");
+  console.log("mid-phrase filled pause: ok");
+}
+
+{
+  // Short uncovered hangover after a word must not become "...".
+  const words = [word(0, "Hi", 1.0, 1.5), word(1, "there", 1.7, 2.2)];
+  // Speech covers words + 0.15s hangover bridge — under the min duration after trim.
+  const frames = framesFor([[1.0, 1.65], [1.7, 2.2]], 3);
+  const out = insertDisfluencyPlaceholders(words, frames, { duration: 3 });
+  assert(
+    out.filter((w) => isDisfluencyPlaceholder(w.text)).length === 0,
+    "hangover-sized gaps ignored"
+  );
+  console.log("hangover ignored: ok");
+}
+
+{
+  // Pure silence between words is not a disfluency (Remove silences owns that).
+  const words = [word(0, "Hi", 1.0, 1.4), word(1, "there", 2.2, 2.6)];
+  const frames = framesFor(
+    [
+      [1.0, 1.4],
+      [2.2, 2.6],
+    ],
+    3
+  );
+  const out = insertDisfluencyPlaceholders(words, frames, { duration: 3 });
+  assert(out.length === 2, "silence gaps produce no placeholders");
+  console.log("silence not placeholder: ok");
+}
+
+{
+  // Empty frames → no-op (full-audio fallback path).
+  const words = [word(0, "Hi", 0, 0.5)];
+  const out = insertDisfluencyPlaceholders(words, [], { duration: 1 });
+  assert(out === words, "empty frames returns same array");
+  console.log("empty frames no-op: ok");
+}
+
+{
+  // Leading uncovered speech before the first word.
+  const words = [word(0, "Hello", 1.5, 2.0)];
+  const frames = framesFor([[0.8, 2.0]], 2.5);
+  const out = insertDisfluencyPlaceholders(words, frames, { duration: 2.5 });
+  const placeholders = out.filter((w) => isDisfluencyPlaceholder(w.text));
+  assert(placeholders.length === 1, "leading filled pause");
+  assert(placeholders[0]!.end <= 1.5 + 1e-3, "leading ends before first word");
+  console.log("leading filled pause: ok");
+}
+
+console.log("ALL DISFLUENCY TESTS PASSED");

tests/fillers-test.ts

@@ -46,4 +46,13 @@ function w(text: string, id: number, deleted = false): Word {
   if (ids.join(",") !== "2,4,5,6") throw new Error(`unexpected filler ids: ${ids.join(",")}`);
 }
 
+{
+  if (!isFillerWord("...")) throw new Error('expected "..." to be a filler');
+  const words = [w("Hello", 1), w("...", 2), w("um", 3), w("...", 4, true)];
+  const ids = findFillerWordIds(words);
+  if (ids.join(",") !== "2,3") {
+    throw new Error(`unexpected filler ids with placeholders: ${ids.join(",")}`);
+  }
+}
+
 console.log("ALL FILLER TESTS PASSED");