From 58b777e445971640cbd7db3df6111197eccdeb0e Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Wed, 14 Oct 2026 19:52:10 +0200 Subject: [PATCH 1/2] fix(stt): drop the renderer audio-decode fallback and say ffmpeg is missing When no ffmpeg resolved, transcribeAsset fell back to decoding the media in the renderer. In the CLI's hidden captions runner that decode reads the file through read-binary-file, which refuses any path the runner never approved, so `captions` failed with "File path is not approved or is not a supported video file". MCP's transcribe runs in the main process and had no fallback at all. The fallback is removed rather than widened: every package now ships ffmpeg (before-pack refuses a macOS or Linux payload without it, Windows stages it with the compositor DLLs), the renderer decode is what froze the editor at open, and approving project media for the hidden runner would add a grant path for code nobody reaches. FfmpegUnavailableError now says that ffmpeg is missing and how to restore it, which the editor, the CLI and MCP all show. The renderer decode it relied on goes with it: extractMono16k.ts, the web-demuxer extraction and transcribeMono16kToSegments. The waveform's audioDataFrameToMono moves into streamingAudioPeaks.ts, its only user. --- electron/stt/extractAudio.test.ts | 13 +- electron/stt/extractAudio.ts | 15 +- electron/stt/index.ts | 2 +- electron/stt/transcriptionContract.ts | 24 +-- scripts/before-pack.cjs | 7 +- scripts/before-pack.test.mjs | 2 +- src/hooks/streamingAudioPeaks.ts | 34 ++- .../ai-edition/document/transcribe.test.ts | 76 ++----- src/lib/ai-edition/document/transcribe.ts | 45 +--- src/lib/ai-edition/transcription/status.ts | 7 +- src/lib/captioning/captionConstants.ts | 2 - src/lib/captioning/extractMono16k.ts | 197 ----------------- .../captioning/extractMono16kWebDemuxer.ts | 198 ------------------ src/lib/captioning/index.ts | 3 +- src/lib/captioning/transcribe.test.ts | 19 +- src/lib/captioning/transcribe.ts | 60 ++---- .../transcription-and-captions.md | 24 +-- 17 files changed, 132 insertions(+), 596 deletions(-) delete mode 100644 src/lib/captioning/captionConstants.ts delete mode 100644 src/lib/captioning/extractMono16k.ts delete mode 100644 src/lib/captioning/extractMono16kWebDemuxer.ts diff --git a/electron/stt/extractAudio.test.ts b/electron/stt/extractAudio.test.ts index d87217668..4bfe6a8b3 100644 --- a/electron/stt/extractAudio.test.ts +++ b/electron/stt/extractAudio.test.ts @@ -14,9 +14,7 @@ vi.mock("../media/audioPeaks", () => ({ const { extractMono16kPcm, FfmpegUnavailableError, MediaUnreadableError, NoAudioTrackError } = await import("./extractAudio"); -const { STT_MEDIA_UNREADABLE, STT_NATIVE_EXTRACTION_UNAVAILABLE } = await import( - "./transcriptionContract" -); +const { STT_MEDIA_UNREADABLE } = await import("./transcriptionContract"); /** A stand-in for the ffmpeg child: two pipes and a close event, nothing more. */ function fakeChild() { @@ -179,13 +177,12 @@ describe("extractMono16kPcm", () => { expect(Array.from(await promise)).toEqual([expect.closeTo(0.1, 6), expect.closeTo(0.2, 6)]); }); - it("refuses with the marker the renderer falls back on when ffmpeg is missing", async () => { - // The string is the contract across the IPC boundary, which drops the class. + it("says ffmpeg is missing when none resolves", async () => { + // The message is all that crosses the IPC boundary, and nothing falls back on it: + // it is what the editor, the CLI and MCP show. resolveFfmpegMock.mockReturnValue(null); await expect(extractMono16kPcm("/tmp/a.mp3")).rejects.toBeInstanceOf(FfmpegUnavailableError); - await expect(extractMono16kPcm("/tmp/a.mp3")).rejects.toThrow( - STT_NATIVE_EXTRACTION_UNAVAILABLE, - ); + await expect(extractMono16kPcm("/tmp/a.mp3")).rejects.toThrow(/^ffmpeg is missing/); expect(spawnMock).not.toHaveBeenCalled(); }); diff --git a/electron/stt/extractAudio.ts b/electron/stt/extractAudio.ts index 2ffebf559..f4308c2e9 100644 --- a/electron/stt/extractAudio.ts +++ b/electron/stt/extractAudio.ts @@ -21,7 +21,7 @@ import { spawn } from "node:child_process"; import { mediaClockAudioArgs, resolveFfmpeg } from "../media/audioPeaks"; -import { STT_MEDIA_UNREADABLE, STT_NATIVE_EXTRACTION_UNAVAILABLE } from "./transcriptionContract"; +import { STT_MEDIA_UNREADABLE } from "./transcriptionContract"; /** What whisper.cpp wants, and what `decodePeaks` already asks ffmpeg for. */ const SAMPLE_RATE = 16_000; @@ -30,14 +30,17 @@ const SAMPLE_RATE = 16_000; const EXTRACT_TIMEOUT_MS = 60_000; /** - * Thrown when no ffmpeg can be resolved, so the caller can fall back to the - * renderer pipeline rather than failing the transcription outright. A distinct type - * because "there is no ffmpeg here" and "this file has no audio" want opposite - * responses: fall back, versus report a permanent failure for this asset. + * Thrown when no ffmpeg can be resolved. There is no fallback behind it: the renderer + * decode that used to catch this reads the whole file on the UI thread, and the hidden + * CLI runner could not read the file at all. Every package ships ffmpeg, so the message + * names what is missing and how to put it back, for the editor, the CLI and MCP alike. */ export class FfmpegUnavailableError extends Error { constructor() { - super(`${STT_NATIVE_EXTRACTION_UNAVAILABLE}: no ffmpeg binary for native audio extraction`); + super( + "ffmpeg is missing: transcription needs it to read the audio, and this install has none. " + + "Reinstall OpenScreen, or set OPENSCREEN_FFMPEG_PATH to an ffmpeg executable.", + ); this.name = "FfmpegUnavailableError"; } } diff --git a/electron/stt/index.ts b/electron/stt/index.ts index 7a536409b..1272b99fa 100644 --- a/electron/stt/index.ts +++ b/electron/stt/index.ts @@ -53,7 +53,7 @@ import { WhisperServerManager } from "./whisperServer"; * worth revisiting only if a much smaller model ever becomes the default. */ -/** The renderer always sends mono 16 kHz (see `extractMono16kFromVideoUrl`). */ +/** What `extractMono16kPcm` decodes to (see `extractAudio.ts`). */ const SAMPLE_RATE = 16_000; /** Attempts per chunk before the whole transcription fails. */ diff --git a/electron/stt/transcriptionContract.ts b/electron/stt/transcriptionContract.ts index d30181487..ce82aff5d 100644 --- a/electron/stt/transcriptionContract.ts +++ b/electron/stt/transcriptionContract.ts @@ -107,10 +107,9 @@ export interface SttStatusEvent { /** IPC request: renderer → main. */ export interface SttTranscribeRequest { /** - * Mono-16k samples the CALLER decoded. Optional since native extraction landed: - * pass `sourcePath` instead and the main process decodes with ffmpeg, off the UI - * thread and without the renderer ever holding the audio. Kept for the caption - * path, which already has samples in hand and has no file to point at. + * Mono-16k samples the CALLER decoded. No renderer sends these any more: it passes + * `sourcePath` and the main process decodes with ffmpeg, off the UI thread and + * without the renderer ever holding the audio. Kept as the manager's own seam. * * Exactly one of `samples` / `sourcePath` is required. */ @@ -120,8 +119,7 @@ export interface SttTranscribeRequest { * Preferred: the renderer's own pipeline read the whole file, copied it twice and * resampled it on the UI thread, which is what froze the editor at open. * - * The caller falls back to its own decode when this cannot be honoured — see - * `FfmpegUnavailableError`. + * Fails with `FfmpegUnavailableError` when no ffmpeg can be resolved. */ sourcePath?: string; /** @@ -136,17 +134,6 @@ export interface SttTranscribeRequest { prompt?: string; } -/** - * Marker carried in the error message when the main process cannot decode a - * `sourcePath` because no ffmpeg is resolvable on this install. - * - * A string rather than an error class because this crosses `ipcRenderer.invoke`, - * which reconstructs a plain `Error` from the message and drops the prototype and - * the `name`. Exported so neither side spells it out by hand — a fallback keyed on - * a literal typed twice is a fallback that silently stops working. - */ -export const STT_NATIVE_EXTRACTION_UNAVAILABLE = "stt:native-extraction-unavailable"; - /** * Marker carried in the error message when audio extraction could not read the * `sourcePath` itself: the OS refused the read, or ffmpeg hung opening the file @@ -154,7 +141,8 @@ export const STT_NATIVE_EXTRACTION_UNAVAILABLE = "stt:native-extraction-unavaila * * Only `extractAudio.ts` sets it, so the renderer can tell "this file's folder is * not readable" from an engine failure that happens to mention a permission (model - * loading, the whisper helper). Same IPC reason as above for being a string. + * loading, the whisper helper). A string rather than an error class because this + * crosses `ipcRenderer.invoke`, which rebuilds a plain `Error` from the message. */ export const STT_MEDIA_UNREADABLE = "stt:media-unreadable"; diff --git a/scripts/before-pack.cjs b/scripts/before-pack.cjs index d4ecb5d48..cdb8b05c6 100644 --- a/scripts/before-pack.cjs +++ b/scripts/before-pack.cjs @@ -134,8 +134,7 @@ const MAC_REQUIRED = [ match: (name) => name === "ffmpeg", what: "the LGPL ffmpeg CLI (spawned for waveform peaks and STT audio extraction)", breaks: - "transcription falls back to the renderer decode or fails outright on machines with no\n" + - 'system ffmpeg, shown to the user only as "Failed to fetch" (#616)', + 'transcription fails in the editor, over MCP and in the CLI with "ffmpeg is missing" (#616)', fix: "Build it with:\n\n npm run build:native:compositor:mac\n\nwhich stages the SDK's ffmpeg beside the vendored dylibs.", }, ]; @@ -194,9 +193,7 @@ const LINUX_REQUIRED = [ { match: (name) => name === "ffmpeg", what: "the LGPL ffmpeg CLI (spawned for waveform peaks and STT audio extraction)", - breaks: - "transcription fails over MCP and the CLI with stt:native-extraction-unavailable,\n" + - "and the editor falls back to a slow renderer decode", + breaks: 'transcription fails in the editor, over MCP and in the CLI with "ffmpeg is missing"', fix: `${FIX_LINUX_HELPER}\n\nwhich stages the SDK's shared ffmpeg beside helper-ffmpeg/.`, }, { diff --git a/scripts/before-pack.test.mjs b/scripts/before-pack.test.mjs index b7df9a3bd..027f847b6 100644 --- a/scripts/before-pack.test.mjs +++ b/scripts/before-pack.test.mjs @@ -359,7 +359,7 @@ describe("MAC_REQUIRED", () => { }); // The Linux twin of #616: packages shipped the libav shared objects but no ffmpeg CLI, so -// transcription over MCP and the CLI failed with stt:native-extraction-unavailable. +// transcription over MCP and the CLI failed for want of ffmpeg. describe("LINUX_REQUIRED", () => { it("refuses a payload that has everything except the ffmpeg binary", () => { const { LINUX_REQUIRED, checkNativePayload } = testing(); diff --git a/src/hooks/streamingAudioPeaks.ts b/src/hooks/streamingAudioPeaks.ts index 8e906ad21..ed9f314af 100644 --- a/src/hooks/streamingAudioPeaks.ts +++ b/src/hooks/streamingAudioPeaks.ts @@ -1,5 +1,4 @@ import { WebDemuxer } from "web-demuxer"; -import { audioDataFrameToMono } from "@/lib/captioning/extractMono16kWebDemuxer"; /** * Streaming trim-waveform peaks for recordings too large to load into memory. @@ -17,6 +16,39 @@ import { audioDataFrameToMono } from "@/lib/captioning/extractMono16kWebDemuxer" * min/max starting from 0 (silence baseline) and channels averaged per sample. */ +/** Mixes one WebCodecs AudioData frame down to mono (averaged across channels). */ +function audioDataFrameToMono(frame: AudioData): Float32Array { + const frames = frame.numberOfFrames; + const ch = frame.numberOfChannels; + const out = new Float32Array(frames); + const fmt = frame.format || ""; + const planar = fmt.includes("planar"); + + if (planar) { + const plane = new Float32Array(frames); + for (let c = 0; c < ch; c++) { + frame.copyTo(plane, { planeIndex: c }); + for (let i = 0; i < frames; i++) { + out[i] += plane[i]; + } + } + for (let i = 0; i < frames; i++) { + out[i] /= ch; + } + } else { + const interleaved = new Float32Array(frames * ch); + frame.copyTo(interleaved, { planeIndex: 0 }); + for (let i = 0; i < frames; i++) { + let sum = 0; + for (let c = 0; c < ch; c++) { + sum += interleaved[i * ch + c]; + } + out[i] = sum / ch; + } + } + return out; +} + const DECODE_QUEUE_BACKPRESSURE = 20; const LOAD_TIMEOUT_MS = 60_000; const READ_END_PADDING_SEC = 0.5; diff --git a/src/lib/ai-edition/document/transcribe.test.ts b/src/lib/ai-edition/document/transcribe.test.ts index a7c2d194b..b94ddf454 100644 --- a/src/lib/ai-edition/document/transcribe.test.ts +++ b/src/lib/ai-edition/document/transcribe.test.ts @@ -1,29 +1,15 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; -import { STT_NATIVE_EXTRACTION_UNAVAILABLE } from "../../../../electron/stt/transcriptionContract"; import { type AxcutDocument, axcutSchemaVersion, transcriptSchema } from "../schema"; +import { classifyTranscriptionError } from "../transcription/status"; import { transcribeAsset } from "./transcribe"; -vi.mock("@/components/video-editor/projectPersistence", () => ({ - toFileUrl: (path: string) => `file://${path}`, -})); - vi.mock("@/lib/captioning", () => ({ - extractMono16kFromVideoUrl: vi.fn(async () => ({ - samples: new Float32Array([0, 0, 0]), - sampleRate: 16_000, - })), - transcribeMono16kToSegments: vi.fn(), transcribeSourceFileToSegments: vi.fn(), })); -const { extractMono16kFromVideoUrl, transcribeMono16kToSegments, transcribeSourceFileToSegments } = - await import("@/lib/captioning"); -// `transcribeAsset` sends the PATH now and lets the main process decode; the samples -// entry point is only reached when no ffmpeg can be resolved. The assertions below -// therefore target the native call, and the fallback has tests of its own at the end. +const { transcribeSourceFileToSegments } = await import("@/lib/captioning"); +// `transcribeAsset` sends the PATH and lets the main process decode. const transcribeMock = vi.mocked(transcribeSourceFileToSegments); -const rendererMock = vi.mocked(transcribeMono16kToSegments); -const extractMock = vi.mocked(extractMono16kFromVideoUrl); function makeDoc(): AxcutDocument { return { @@ -183,10 +169,7 @@ describe("transcribeAsset native extraction", () => { vi.clearAllMocks(); }); - it("hands the main process a path instead of decoding in the renderer", async () => { - // The point of the change: the renderer must not touch the audio at all on the - // happy path. `extractMono16kFromVideoUrl` reads the whole file, copies it twice - // and resamples on the UI thread — that is the freeze this avoids. + it("hands the main process a path", async () => { transcribeMock.mockResolvedValueOnce({ segments: [{ startSec: 0, endSec: 1, text: "hi" }], granularity: "word", @@ -194,51 +177,38 @@ describe("transcribeAsset native extraction", () => { }); await transcribeAsset(makeDoc(), "asset_1"); expect(transcribeMock).toHaveBeenCalledWith("/tmp/demo.mp4", expect.anything()); - expect(extractMock).not.toHaveBeenCalled(); - expect(rendererMock).not.toHaveBeenCalled(); }); - it("falls back to the renderer decode when the install has no ffmpeg", async () => { - // A dev checkout that never fetched ffmpeg, or a platform build missing it, must - // still transcribe rather than lose the feature. + it("says ffmpeg is missing instead of falling back to a renderer decode", async () => { + // The fallback used to catch this and decode in the renderer, which the CLI's + // hidden runner was not allowed to read: "File path is not approved or is not a + // supported video file". Now the reason reaches the caller as the IPC delivers it. + const missing = + "ffmpeg is missing: transcription needs it to read the audio, and this install has none."; transcribeMock.mockRejectedValueOnce( - new Error(`${STT_NATIVE_EXTRACTION_UNAVAILABLE}: no ffmpeg binary`), + new Error( + `Error invoking remote method 'stt:transcribe': FfmpegUnavailableError: ${missing}`, + ), ); - rendererMock.mockResolvedValueOnce({ - segments: [{ startSec: 0, endSec: 1, text: "hi" }], - granularity: "word", - detectedLanguage: "en", - }); - const transcript = await transcribeAsset(makeDoc(), "asset_1"); - expect(extractMock).toHaveBeenCalled(); - expect(rendererMock).toHaveBeenCalled(); - expect(transcript.segments.length).toBeGreaterThan(0); + const error = await transcribeAsset(makeDoc(), "asset_1").catch((e: unknown) => e); + expect(transcribeMock).toHaveBeenCalledTimes(1); + // What the CLI prints after "Transcription failed: ", and the editor shows. + expect(classifyTranscriptionError(error)).toEqual({ kind: "error", message: missing }); }); - it("passes the project's names and terms on both paths", async () => { + it("passes the project's names and terms", async () => { const doc = makeDoc(); doc.project.vocabulary = "OpenScreen, Kubernetes"; - transcribeMock.mockRejectedValueOnce( - new Error(`${STT_NATIVE_EXTRACTION_UNAVAILABLE}: no ffmpeg binary`), - ); - rendererMock.mockResolvedValueOnce({ + transcribeMock.mockResolvedValueOnce({ segments: [], granularity: "word", detectedLanguage: "en", }); await transcribeAsset(doc, "asset_1"); - const withPrompt = expect.objectContaining({ prompt: "OpenScreen, Kubernetes" }); - expect(transcribeMock).toHaveBeenCalledWith("/tmp/demo.mp4", withPrompt); - expect(rendererMock).toHaveBeenCalledWith(expect.anything(), withPrompt); - }); - - it("does NOT fall back on any other failure", async () => { - // "This file has no audio" is a verdict. Re-deriving it in the renderer would buy - // the same answer for the price of the decode this change exists to avoid. - transcribeMock.mockRejectedValueOnce(new Error("No decodable audio in /tmp/demo.mp4")); - await expect(transcribeAsset(makeDoc(), "asset_1")).rejects.toThrow("No decodable audio"); - expect(extractMock).not.toHaveBeenCalled(); - expect(rendererMock).not.toHaveBeenCalled(); + expect(transcribeMock).toHaveBeenCalledWith( + "/tmp/demo.mp4", + expect.objectContaining({ prompt: "OpenScreen, Kubernetes" }), + ); }); }); diff --git a/src/lib/ai-edition/document/transcribe.ts b/src/lib/ai-edition/document/transcribe.ts index 4f3f1c701..4253eaf22 100644 --- a/src/lib/ai-edition/document/transcribe.ts +++ b/src/lib/ai-edition/document/transcribe.ts @@ -2,17 +2,11 @@ // src/lib/captioning/) as a transcribeAsset function that returns an // AxcutTranscript and persists it into the document. // -// ponytail: reuses extractMono16kFromVideoUrl + transcribeMono16kToSegments -// verbatim. No Python, no faster-whisper, no network calls. Privacy-safe. +// ponytail: reuses transcribeSourceFileToSegments verbatim. No Python, no +// faster-whisper, no network calls. Privacy-safe. -import { toFileUrl } from "@/components/video-editor/projectPersistence"; -import { - extractMono16kFromVideoUrl, - transcribeMono16kToSegments, - transcribeSourceFileToSegments, -} from "@/lib/captioning"; +import { transcribeSourceFileToSegments } from "@/lib/captioning"; import type { SttRendererStatus } from "@/lib/captioning/transcribe"; -import { STT_NATIVE_EXTRACTION_UNAVAILABLE } from "../../../../electron/stt/transcriptionContract"; import type { AxcutDocument, AxcutTranscript } from "../schema"; import { transcriptFromSegments } from "../transcription/fromStt"; @@ -51,8 +45,6 @@ export async function transcribeAsset( throw new Error(`Asset ${assetId} not found in document.`); } - const videoUrl = toFileUrl(asset.originalPath); - options.onStatus?.({ phase: "extracting-audio" }); // Stay on loading-model until the main process reports inference chunks. @@ -86,38 +78,17 @@ export async function transcribeAsset( ...(status.totalBytes !== undefined ? { totalBytes: status.totalBytes } : {}), }); - // Native first. `extractMono16kFromVideoUrl` runs in the RENDERER: it reads the - // whole media into memory, copies it twice and resamples on the UI thread, which - // is what froze the editor at project open on a long import (measured on a - // four-minute bed: ~86 MB of decoded float32 there against 15.7 MB in the main - // process). Handing the path over keeps every byte on the other side of the IPC, - // and the whisper helper it feeds was already a separate process. - // - // The fallback is not decoration: an install with no resolvable ffmpeg — a dev - // checkout that never fetched it, a platform build missing the binary — must still - // transcribe rather than lose the feature. Only THAT case falls back. "This file - // has no audio" is a verdict, and re-deriving it in the renderer would buy the same - // answer for the price of the decode this exists to avoid. + // The main process decodes the file with ffmpeg; the renderer never holds the audio. + // No renderer decode behind it: that path read the whole file on the UI thread, + // and the CLI's hidden runner was not even allowed to read it. Every package ships + // ffmpeg, so a missing one surfaces as `FfmpegUnavailableError`'s message, which + // says so, in the editor, the CLI and MCP alike. const result = await transcribeSourceFileToSegments(asset.originalPath, { trimRegions: [], signal: options.signal, language: forcedLanguage, prompt, onStatus: forwardStatus, - }).catch(async (error: unknown) => { - const message = error instanceof Error ? error.message : String(error); - if (!message.includes(STT_NATIVE_EXTRACTION_UNAVAILABLE)) throw error; - const audioResult = await extractMono16kFromVideoUrl(videoUrl, { - signal: options.signal, - }); - options.onStatus?.({ phase: "transcribing" }); - return transcribeMono16kToSegments(audioResult.samples, { - trimRegions: [], - signal: options.signal, - language: forcedLanguage, - prompt, - onStatus: forwardStatus, - }); }); // Prefer the model-reported language (covers both forced picks and auto-detect), diff --git a/src/lib/ai-edition/transcription/status.ts b/src/lib/ai-edition/transcription/status.ts index ea317e60e..081766695 100644 --- a/src/lib/ai-edition/transcription/status.ts +++ b/src/lib/ai-edition/transcription/status.ts @@ -149,10 +149,9 @@ export function withTranscriptionFailure( /** * Map an exception out of `transcribeAsset` onto a failure the UI can explain. - * The deterministic cases come from whichever layer decoded the audio, and there - * are two of them: `extractMono16kWebDemuxer` in the renderer, and — since - * native extraction landed — `extractAudio.ts`'s `NoAudioTrackError` in the main - * process, whose wording ("No decodable audio in …") is its own. + * The deterministic cases come from `extractAudio.ts`'s `NoAudioTrackError` in the + * main process ("No decodable audio in …"). The other phrasings are the renderer + * decode's, gone since the main process decodes every file. * * Matched on the message rather than on the class because `ipcRenderer.invoke` * rebuilds a plain `Error` and drops both the prototype and the `name`. Missing diff --git a/src/lib/captioning/captionConstants.ts b/src/lib/captioning/captionConstants.ts deleted file mode 100644 index 1bacb7cc7..000000000 --- a/src/lib/captioning/captionConstants.ts +++ /dev/null @@ -1,2 +0,0 @@ -/** Max audio length for auto-captions (decode + transcribe); keep demuxer read aligned with this. */ -export const MAX_CAPTION_AUDIO_SEC = 4 * 60 * 60; diff --git a/src/lib/captioning/extractMono16k.ts b/src/lib/captioning/extractMono16k.ts deleted file mode 100644 index 1116bb9a2..000000000 --- a/src/lib/captioning/extractMono16k.ts +++ /dev/null @@ -1,197 +0,0 @@ -import { materializeLocalSourceFile, releaseLocalSourceFile } from "@/lib/exporter/localSourceFile"; -import { MAX_IN_MEMORY_SOURCE_BYTES } from "@/lib/exporter/sourceFileLimits"; -import { MAX_CAPTION_AUDIO_SEC } from "./captionConstants"; -import { extractMonoPcmViaWebDemuxer } from "./extractMono16kWebDemuxer"; - -export { MAX_CAPTION_AUDIO_SEC }; - -const FETCH_TIMEOUT_MS = 120_000; -// The demuxer caption path holds every decoded AudioData frame plus full-rate -// merge buffers in memory (~50 MB per minute of 48 kHz audio all-in), so very -// long recordings would exhaust the renderer heap well before the 4 h caption -// ceiling. For sources too large to load in memory anyway, cap the decoded -// audio; captions come back truncated instead of crashing the renderer. -const LARGE_FILE_CAPTION_SEC = 30 * 60; - -async function fetchWithTimeout(url: string, signal?: AbortSignal): Promise { - const ctrl = new AbortController(); - const timer = window.setTimeout(() => ctrl.abort(), FETCH_TIMEOUT_MS); - const onAbort = () => ctrl.abort(); - if (signal) { - if (signal.aborted) ctrl.abort(); - else signal.addEventListener("abort", onAbort, { once: true }); - } - try { - return await fetch(url, { signal: ctrl.signal }); - } finally { - window.clearTimeout(timer); - if (signal) signal.removeEventListener("abort", onAbort); - } -} - -/** - * Load the editor video like `StreamingVideoDecoder`: Electron `readBinaryFile` - * for local paths (fetch(file://) is unreliable in the renderer), otherwise - * HTTP/blob/data URLs via fetch. - */ -async function loadSourceVideoFile(videoUrl: string, signal?: AbortSignal): Promise { - const isRemoteUrl = /^(https?:|blob:|data:)/i.test(videoUrl); - - if (!isRemoteUrl && window.electronAPI) { - // Streams large recordings through OPFS instead of reading them whole, so - // captions work for multi-GB files just like export does. The signal also - // aborts the copy itself when the caption pass is cancelled. - const filename = (videoUrl.split(/[\\/]/).pop() || "video").replace(/^file:/, ""); - return materializeLocalSourceFile(videoUrl, filename, { signal }); - } - - const response = await fetchWithTimeout(videoUrl, signal); - if (!response.ok) { - throw new Error(`Failed to load video for captions: ${response.status} ${response.statusText}`); - } - const blob = await response.blob(); - if (signal?.aborted) throw new DOMException("Aborted", "AbortError"); - const filename = videoUrl.split("/").pop() || "video"; - return new File([blob], filename, { type: blob.type || "video/webm" }); -} - -function mixToMono(audioBuffer: AudioBuffer): Float32Array { - const { length, numberOfChannels } = audioBuffer; - if (numberOfChannels === 0) return new Float32Array(length); - // `getChannelData` is a WebIDL call, so calling it INSIDE the sample loop cost - // one call per sample per channel — ~57 M of them for a ten-minute stereo - // recording, seconds of blocked main thread. That was survivable while - // transcription only ran when the user asked for it; it now runs by itself when - // a project opens, and a frozen window (spinners included) is exactly what the - // automatic pass must not look like. Hoisting the channel arrays out of the loop - // leaves plain typed-array indexing. - const channels: Float32Array[] = []; - for (let c = 0; c < numberOfChannels; c++) channels.push(audioBuffer.getChannelData(c)); - // Mono source: the mixdown is a copy. `slice` keeps the caller's contract of - // owning its buffer (the AudioBuffer's own array is reused by the context). - if (numberOfChannels === 1) return channels[0].slice(); - const out = new Float32Array(length); - for (let i = 0; i < length; i++) { - let sum = 0; - for (let c = 0; c < numberOfChannels; c++) sum += channels[c][i]; - out[i] = sum / numberOfChannels; - } - return out; -} - -async function resampleMono( - mono: Float32Array, - fromRate: number, - toRate: number, - signal?: AbortSignal, -): Promise { - if (signal?.aborted) throw new DOMException("Aborted", "AbortError"); - if (fromRate === toRate) return mono; - const durationSec = mono.length / fromRate; - const outLength = Math.max(1, Math.ceil(durationSec * toRate)); - const offline = new OfflineAudioContext(1, outLength, toRate); - const buf = offline.createBuffer(1, mono.length, fromRate); - buf.copyToChannel(Float32Array.from(mono), 0); - const src = offline.createBufferSource(); - src.buffer = buf; - src.connect(offline.destination); - src.start(0); - const rendered = await offline.startRendering(); - if (signal?.aborted) throw new DOMException("Aborted", "AbortError"); - return rendered.getChannelData(0).slice(); -} - -async function truncateAndResampleTo16k( - mono: Float32Array, - fromRate: number, - durationSec: number, - signal?: AbortSignal, -): Promise<{ samples: Float32Array; truncated: boolean; durationSec: number }> { - let truncated = false; - let work = mono; - if (durationSec > MAX_CAPTION_AUDIO_SEC) { - const maxSamples = Math.floor(MAX_CAPTION_AUDIO_SEC * fromRate); - work = mono.subarray(0, Math.min(mono.length, maxSamples)); - truncated = true; - } - - const samples = await resampleMono(work, fromRate, 16_000, signal); - return { samples, truncated, durationSec: samples.length / 16_000 }; -} - -/** - * Decode the video's audio track to mono 16 kHz float samples (Whisper input). - * Prefers `decodeAudioData` when the container is supported, else the same - * web-demuxer + AudioDecoder path as export. - */ -export async function extractMono16kFromVideoUrl( - videoUrl: string, - options?: { signal?: AbortSignal }, -): Promise<{ samples: Float32Array; truncated: boolean; durationSec: number }> { - const file = await loadSourceVideoFile(videoUrl, options?.signal); - - /** When this returns null, use web-demuxer + AudioDecoder (same as export). */ - const tryDecodeAudioDataPath = async (): Promise<{ - samples: Float32Array; - truncated: boolean; - durationSec: number; - } | null> => { - const audioContext = new AudioContext(); - try { - const ab = await file.arrayBuffer(); - if (options?.signal?.aborted) throw new DOMException("Aborted", "AbortError"); - const audioBuffer = await audioContext.decodeAudioData(ab.slice(0)); - if ( - audioBuffer.numberOfChannels === 0 || - audioBuffer.length === 0 || - !Number.isFinite(audioBuffer.duration) || - audioBuffer.duration <= 0 - ) { - return null; - } - const durationSec = audioBuffer.duration; - const mono = mixToMono(audioBuffer); - const fromRate = audioBuffer.sampleRate; - const out = await truncateAndResampleTo16k(mono, fromRate, durationSec, options?.signal); - // decodeAudioData can resolve for some WebM/Matroska inputs yet yield almost no usable - // PCM, and captions only fall back to the demuxer path on throw, so return null to recover. - if (out.samples.length < 800) { - return null; - } - return out; - } catch { - return null; - } finally { - await audioContext.close().catch(() => undefined); - } - }; - - try { - // Large recordings skip the in-memory decodeAudioData path (it would load - // the whole file) and go straight to the streaming web-demuxer path below. - const isLargeFile = file.size > MAX_IN_MEMORY_SOURCE_BYTES; - const primary = isLargeFile ? null : await tryDecodeAudioDataPath(); - if (primary) { - return primary; - } - - // For oversized sources, also cap how much audio the demuxer path decodes - // — its frame/merge buffers are in-memory and scale with duration. - const pcm = await extractMonoPcmViaWebDemuxer( - file, - options?.signal, - isLargeFile ? LARGE_FILE_CAPTION_SEC : undefined, - ); - const out = await truncateAndResampleTo16k( - pcm.mono, - pcm.sampleRate, - pcm.durationSec, - options?.signal, - ); - return { ...out, truncated: out.truncated || pcm.capped }; - } finally { - // Release the OPFS cache reference taken when streaming a large source. - // The File name is the cache-entry key (no-op for small/remote sources). - releaseLocalSourceFile(file.name); - } -} diff --git a/src/lib/captioning/extractMono16kWebDemuxer.ts b/src/lib/captioning/extractMono16kWebDemuxer.ts deleted file mode 100644 index 0d1a2ba78..000000000 --- a/src/lib/captioning/extractMono16kWebDemuxer.ts +++ /dev/null @@ -1,198 +0,0 @@ -import { WebDemuxer } from "web-demuxer"; - -import { MAX_CAPTION_AUDIO_SEC } from "./captionConstants"; - -const DECODE_QUEUE_BACKPRESSURE = 20; -const SOURCE_LOAD_TIMEOUT_MS = 60_000; -const READ_END_PADDING_SEC = 0.5; - -function webDemuxerWasmUrl(): string { - return new URL("./wasm/web-demuxer.wasm", window.location.href).href; -} - -/** Mixes one WebCodecs AudioData frame down to mono (averaged across channels). */ -export function audioDataFrameToMono(frame: AudioData): Float32Array { - const frames = frame.numberOfFrames; - const ch = frame.numberOfChannels; - const out = new Float32Array(frames); - const fmt = frame.format || ""; - const planar = fmt.includes("planar"); - - if (planar) { - const plane = new Float32Array(frames); - for (let c = 0; c < ch; c++) { - frame.copyTo(plane, { planeIndex: c }); - for (let i = 0; i < frames; i++) { - out[i] += plane[i]; - } - } - for (let i = 0; i < frames; i++) { - out[i] /= ch; - } - } else { - const interleaved = new Float32Array(frames * ch); - frame.copyTo(interleaved, { planeIndex: 0 }); - for (let i = 0; i < frames; i++) { - let sum = 0; - for (let c = 0; c < ch; c++) { - sum += interleaved[i * ch + c]; - } - out[i] = sum / ch; - } - } - return out; -} - -function mergeAndConsumeDecodedAudioToMonoLinear( - frames: AudioData[], - sampleRate: number, - durationSec: number, -): Float32Array { - const sorted = [...frames].sort((a, b) => a.timestamp - b.timestamp); - const totalSamples = Math.max(1, Math.ceil(durationSec * sampleRate)); - const acc = new Float32Array(totalSamples); - const weight = new Float32Array(totalSamples); - - for (const frame of sorted) { - const startSample = Math.round((frame.timestamp / 1e6) * sampleRate); - const slice = audioDataFrameToMono(frame); - for (let i = 0; i < slice.length; i++) { - const pos = startSample + i; - if (pos >= 0 && pos < totalSamples) { - acc[pos] += slice[i]; - weight[pos] += 1; - } - } - frame.close(); - } - - for (let i = 0; i < totalSamples; i++) { - if (weight[i] > 0) { - acc[i] /= weight[i]; - } - } - return acc; -} - -function withTimeout(promise: Promise, ms: number, message: string): Promise { - return new Promise((resolve, reject) => { - const id = window.setTimeout(() => reject(new Error(message)), ms); - promise - .then((v) => { - window.clearTimeout(id); - resolve(v); - }) - .catch((e) => { - window.clearTimeout(id); - reject(e instanceof Error ? e : new Error(String(e))); - }); - }); -} - -/** - * Demux + WebCodecs audio decode (same stack as export). Use when `decodeAudioData` - * can't handle the container (e.g. WebM with video). - * - * @param maxReadSec Optional cap on how much audio to demux/decode. The decoded - * frames and merge buffers are held in memory, so very long recordings must be - * capped below MAX_CAPTION_AUDIO_SEC to avoid exhausting the renderer heap. - * `capped` reports whether the cap actually cut the track short. - */ -export async function extractMonoPcmViaWebDemuxer( - file: File, - signal?: AbortSignal, - maxReadSec?: number, -): Promise<{ mono: Float32Array; sampleRate: number; durationSec: number; capped: boolean }> { - const demuxer = new WebDemuxer({ wasmFilePath: webDemuxerWasmUrl() }); - await withTimeout( - demuxer.load(file), - SOURCE_LOAD_TIMEOUT_MS, - "Timed out while parsing the source video for captions.", - ); - - if (signal?.aborted) throw new DOMException("Aborted", "AbortError"); - - const mediaInfo = await withTimeout( - demuxer.getMediaInfo(), - SOURCE_LOAD_TIMEOUT_MS, - "Timed out while reading media info for captions.", - ); - - const reportedDurationSec = - Number.isFinite(mediaInfo.duration) && mediaInfo.duration > 0 ? mediaInfo.duration : 0; - - let audioConfig: AudioDecoderConfig; - try { - audioConfig = await demuxer.getDecoderConfig("audio"); - } catch { - throw new Error("No audio track found in this video."); - } - - const codecCheck = await AudioDecoder.isConfigSupported(audioConfig); - if (!codecCheck.supported) { - throw new Error(`Audio codec not supported for captions: ${audioConfig.codec}`); - } - - const sampleRate = audioConfig.sampleRate || 48_000; - - // Many WebM/Matroska files report a too-short duration, so capping read at reported time stops - // demux early and clips everything past that. Read to the caption-decode ceiling instead; the - // demuxer stops when the track ends. - const readCapSec = Math.min(maxReadSec ?? MAX_CAPTION_AUDIO_SEC, MAX_CAPTION_AUDIO_SEC); - const readEndSec = readCapSec + READ_END_PADDING_SEC; - const decodedFrames: AudioData[] = []; - - const decoder = new AudioDecoder({ - output: (data: AudioData) => decodedFrames.push(data), - error: (e: DOMException) => console.error("[captioning] AudioDecoder error:", e), - }); - decoder.configure(audioConfig); - - const reader = demuxer.read("audio", 0, readEndSec).getReader(); - try { - while (!signal?.aborted) { - const { done, value: chunk } = await reader.read(); - if (done || !chunk) break; - decoder.decode(chunk); - while (decoder.decodeQueueSize > DECODE_QUEUE_BACKPRESSURE && !signal?.aborted) { - await new Promise((r) => setTimeout(r, 1)); - } - } - } finally { - try { - await reader.cancel(); - } catch { - /* already closed */ - } - } - - if (decoder.state === "configured") { - await decoder.flush(); - decoder.close(); - } - - if (signal?.aborted) { - for (const f of decodedFrames) f.close(); - throw new DOMException("Aborted", "AbortError"); - } - - if (decodedFrames.length === 0) { - throw new Error("Decoded zero audio frames from this video."); - } - - let maxEndUs = 0; - for (const f of decodedFrames) { - const end = f.timestamp + (f.duration ?? 0); - if (end > maxEndUs) maxEndUs = end; - } - const inferredDurationSec = maxEndUs / 1e6; - // Prefer extent implied by decoded frames (fixes bad container duration); fall back to reported - // metadata when frames lack duration. - const durationSec = inferredDurationSec > 0.02 ? inferredDurationSec : reportedDurationSec; - - // The cap cut the track short when the reported extent exceeds what we read. - const capped = reportedDurationSec > readCapSec + READ_END_PADDING_SEC; - - const mono = mergeAndConsumeDecodedAudioToMonoLinear(decodedFrames, sampleRate, durationSec); - return { mono, sampleRate, durationSec, capped }; -} diff --git a/src/lib/captioning/index.ts b/src/lib/captioning/index.ts index a5ad5dabd..339782578 100644 --- a/src/lib/captioning/index.ts +++ b/src/lib/captioning/index.ts @@ -6,10 +6,9 @@ export { mergeAdjacentCaptionSegments, splitMergedCaptionsByWordBounds, } from "./annotationsFromCaptions"; -export { extractMono16kFromVideoUrl, MAX_CAPTION_AUDIO_SEC } from "./extractMono16k"; export type { CaptionSegment, CaptionTimestampGranularity, TranscribeMono16kResult, } from "./transcribe"; -export { transcribeMono16kToSegments, transcribeSourceFileToSegments } from "./transcribe"; +export { transcribeSourceFileToSegments } from "./transcribe"; diff --git a/src/lib/captioning/transcribe.test.ts b/src/lib/captioning/transcribe.test.ts index 505fc59b0..9ef1d94b3 100644 --- a/src/lib/captioning/transcribe.test.ts +++ b/src/lib/captioning/transcribe.test.ts @@ -1,10 +1,10 @@ // @vitest-environment jsdom import { afterEach, beforeEach, describe, expect, it, type Mock, vi } from "vitest"; -import { transcribeMono16kToSegments } from "./transcribe"; +import { transcribeSourceFileToSegments } from "./transcribe"; /** - * The renderer-side `transcribeMono16kToSegments` is a thin adapter on top of + * The renderer-side `transcribeSourceFileToSegments` is a thin adapter on top of * `electronAPI.stt.transcribe`. It must: * - emit each word from `wordSegments` as a single-token `CaptionSegment`, * - tag the run `granularity: "word"`, @@ -75,7 +75,7 @@ const removeMockApi = () => { lastStatusCb = null; }; -describe("transcribeMono16kToSegments", () => { +describe("transcribeSourceFileToSegments", () => { beforeEach(() => { installMockApi(); }); @@ -95,8 +95,7 @@ describe("transcribeMono16kToSegments", () => { backend: "whisper-cpu", }); - const samples = new Float32Array(1600); - const result = await transcribeMono16kToSegments(samples); + const result = await transcribeSourceFileToSegments("/tmp/a.mp4"); expect(result.granularity).toBe("word"); expect(result.segments).toEqual([ @@ -104,7 +103,7 @@ describe("transcribeMono16kToSegments", () => { { text: "world", startSec: 0.31, endSec: 0.65 }, ]); expect(mockApi.transcribe).toHaveBeenCalledWith({ - samples, + sourcePath: "/tmp/a.mp4", language: undefined, }); }); @@ -117,7 +116,7 @@ describe("transcribeMono16kToSegments", () => { backend: "whisper-cpu", }); - const result = await transcribeMono16kToSegments(new Float32Array(1600)); + const result = await transcribeSourceFileToSegments("/tmp/a.mp4"); expect(result.granularity).toBe("phrase"); expect(result.segments).toEqual([{ text: "hello world", startSec: 0, endSec: 0.65 }]); }); @@ -136,7 +135,7 @@ describe("transcribeMono16kToSegments", () => { }; }); - await transcribeMono16kToSegments(new Float32Array(1600), { onStatus }); + await transcribeSourceFileToSegments("/tmp/a.mp4", { onStatus }); expect(onStatus).toHaveBeenCalledWith({ phase: "model" }); // The chunk progress must survive the hop, not just the phase — it is what // drives the progress bar. @@ -151,7 +150,7 @@ describe("transcribeMono16kToSegments", () => { it("returns empty results when running outside Electron (browser tests)", async () => { removeMockApi(); - const result = await transcribeMono16kToSegments(new Float32Array(1600)); + const result = await transcribeSourceFileToSegments("/tmp/a.mp4"); expect(result.granularity).toBe("word"); expect(result.segments).toEqual([]); }); @@ -160,7 +159,7 @@ describe("transcribeMono16kToSegments", () => { const controller = new AbortController(); controller.abort(); await expect( - transcribeMono16kToSegments(new Float32Array(1600), { signal: controller.signal }), + transcribeSourceFileToSegments("/tmp/a.mp4", { signal: controller.signal }), ).rejects.toMatchObject({ name: "AbortError" }); expect(mockApi.transcribe).not.toHaveBeenCalled(); }); diff --git a/src/lib/captioning/transcribe.ts b/src/lib/captioning/transcribe.ts index 806b66914..0d4cf997e 100644 --- a/src/lib/captioning/transcribe.ts +++ b/src/lib/captioning/transcribe.ts @@ -10,7 +10,7 @@ export interface CaptionSegment { /** * How caption layout should interpret `CaptionSegment` times from - * `transcribeMono16kToSegments`. The native pipeline gets per-word timestamps + * `transcribeSourceFileToSegments`. The native pipeline gets per-word timestamps * from whisper.cpp's own output; `"phrase"` is a fallback for the rare case * where whisper reports zero words for a segment. */ @@ -53,7 +53,7 @@ export interface SttRendererStatus { } interface RendererSttApi { - transcribe: (request: { samples: Float32Array; language?: string; prompt?: string }) => Promise<{ + transcribe: (request: { sourcePath: string; language?: string; prompt?: string }) => Promise<{ segments: CaptionSegment[]; wordSegments: SttWordSegment[]; detectedLanguage: string; @@ -63,42 +63,6 @@ interface RendererSttApi { onStatus?: (callback: (event: SttRendererStatus) => void) => () => void; } -/** - * Transcribes mono 16 kHz audio into per-word timed caption segments. The - * renderer is a thin IPC adapter: it forwards the audio to the Electron main - * process where `whisper-server` runs recognition and emits its own - * per-word timestamps in the same pass. - * - * The previous in-Web-Worker implementation (Transformers.js + ORT-WASM) was - * 0.5× realtime on tiny models and had no word-level accuracy under 50 ms; - * see `technical-documentation/architecture/transcription-and-captions.md` § History for context. - */ -export function transcribeMono16kToSegments( - samples: Float32Array, - options?: TranscribeOptions, -): Promise { - return runTranscription({ samples }, options); -} - -/** - * Same recognition, from a FILE the main process decodes itself. - * - * Preferred over `transcribeMono16kToSegments` wherever there is a path to point at. - * The samples entry point decodes in the renderer — whole file into memory, an - * `arrayBuffer()` copy, a `slice(0)` copy, then a resample loop on the UI thread — - * which is what froze the editor at open on a long import. Here the renderer sends a - * string and gets segments back. - * - * Rejects with a message carrying `STT_NATIVE_EXTRACTION_UNAVAILABLE` when the - * install has no ffmpeg, so the caller can fall back rather than lose the transcript. - */ -export function transcribeSourceFileToSegments( - sourcePath: string, - options?: TranscribeOptions, -): Promise { - return runTranscription({ sourcePath }, options); -} - export interface TranscribeOptions { trimRegions?: TrimRegion[]; onStatus?: (status: SttRendererStatus) => void; @@ -108,8 +72,22 @@ export interface TranscribeOptions { prompt?: string; } -function runTranscription( - payload: { samples: Float32Array } | { sourcePath: string }, +/** + * Transcribes a media FILE into per-word timed caption segments. The renderer is a + * thin IPC adapter: it sends the path, the main process decodes it with ffmpeg and + * `whisper-server` runs recognition, emitting per-word timestamps in the same pass. + * The renderer never holds the audio: decoding it here read the whole file, copied it + * twice and resampled on the UI thread, which is what froze the editor at open. + * + * Rejects with `FfmpegUnavailableError`'s message when the install has no ffmpeg. + * There is no renderer decode to fall back on any more. + * + * The previous in-Web-Worker implementation (Transformers.js + ORT-WASM) was + * 0.5× realtime on tiny models and had no word-level accuracy under 50 ms; + * see `technical-documentation/architecture/transcription-and-captions.md` § History for context. + */ +export function transcribeSourceFileToSegments( + sourcePath: string, options?: TranscribeOptions, ): Promise { if (options?.signal?.aborted) { @@ -138,7 +116,7 @@ function runTranscription( // trimmed leading silence with a peak detector and got false positives on // quiet music intros / room tone. VAD or nothing. return api - .transcribe({ ...payload, language: forcedLanguage, prompt: options?.prompt }) + .transcribe({ sourcePath, language: forcedLanguage, prompt: options?.prompt }) .then((result) => ({ ...segmentsFromStt(result), detectedLanguage: result.detectedLanguage })) .catch((error: unknown) => { // A run the caller cancelled surfaces as an abort, not as an engine diff --git a/technical-documentation/architecture/transcription-and-captions.md b/technical-documentation/architecture/transcription-and-captions.md index a39171c6d..f71ff8b33 100644 --- a/technical-documentation/architecture/transcription-and-captions.md +++ b/technical-documentation/architecture/transcription-and-captions.md @@ -15,8 +15,8 @@ the transcript). flowchart LR Z["Asset added
(import or recording)"] -- "sync()" --> Y["transcriptionStore queue
(src/lib/ai-edition/store/transcriptionStore.ts)"] Y -- "one job at a time" --> A["Recorded audio"] - A -- "extract mono 16 kHz" --> B["transcribeAsset
(src/lib/ai-edition/document/transcribe.ts)"] - B -- "IPC: Float32Array + language" --> C["SttManager
(electron/stt/index.ts)"] + A -- "file path" --> B["transcribeAsset
(src/lib/ai-edition/document/transcribe.ts)"] + B -- "IPC: source path + language" --> C["SttManager
(electron/stt/index.ts)
ffmpeg: mono 16 kHz"] C -- "POST /inference (WAV)" --> D["whisper-stt-server
(electron/native/whisper-stt/)"] D -- "VAD cut + whisper_full() + DTW" --> E["SttTranscribeResponse
(segments + wordSegments)"] E --> C @@ -27,10 +27,11 @@ flowchart LR H -- "annotation path
(scene description)" --> I["Native compositor
(preview AND export)"] ``` -The renderer pieces — `transcribeMono16kToSegments` +The renderer pieces — `transcribeSourceFileToSegments` (`src/lib/captioning/transcribe.ts`) called from `transcribeAsset` (`src/lib/ai-edition/document/transcribe.ts`) — are a thin IPC adapter: the -audio crosses to the main process, the helper does the recognition, and the +file path crosses to the main process, which decodes it with ffmpeg +(`electron/stt/extractAudio.ts`), the helper does the recognition, and the result is mapped back onto the `AxcutTranscript` shape the rest of the editor reads. From the moment that transcript is persisted, captions and the words they show cannot drift — captions are a derived view, not a parallel store. @@ -71,8 +72,7 @@ What the store owns and what it does not: the panes' one button: every timeline asset that still lacks a transcript, skipping the ones already known to be silent and waiting on (rather than duplicating) a run the background pass already has in flight. -- **One run at a time.** whisper-server is a single process and the audio - extraction path holds decoded frames in renderer memory, so the pump is a +- **One run at a time.** whisper-server is a single process, so the pump is a sequential loop, not a fan-out. - **The auto pass never loops.** `sync` enqueues an asset only when it has no transcript, no job entry (queued / running / failed alike) and no persisted @@ -132,12 +132,12 @@ Three edges make that promise hold, and each is load-bearing: clip's word stream swallow Backspace and hover-bin clicks in silence for the whole background pass. -Still synchronous, and still on the main thread: `mixToMono` -([`src/lib/captioning/extractMono16k.ts`](../../src/lib/captioning/extractMono16k.ts)) -now hoists its channel arrays out of the sample loop (it was making one WebIDL -call per sample per channel), but a long recording will still block the renderer -for a moment during "extracting-audio". Moving the mixdown to a worker is the -next step if it shows up in practice. +Audio extraction never runs in the renderer. There is no renderer decode to fall +back on: it read the whole file on the UI thread, froze the editor at open, and +the CLI's hidden runner was not even allowed to read the file. An install with +no ffmpeg fails with "ffmpeg is missing", in the editor, the CLI and MCP alike; +every package ships one, and `before-pack` refuses a macOS or Linux payload +without it. ## The STT engine From 864a268549ec29acf59d9d66668fea766fcec4d5 Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Wed, 14 Oct 2026 20:02:14 +0200 Subject: [PATCH 2/2] docs(stt): describe the transcription request as path-first --- .../architecture/transcription-and-captions.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/technical-documentation/architecture/transcription-and-captions.md b/technical-documentation/architecture/transcription-and-captions.md index f71ff8b33..00c425d65 100644 --- a/technical-documentation/architecture/transcription-and-captions.md +++ b/technical-documentation/architecture/transcription-and-captions.md @@ -606,8 +606,10 @@ For anything user-facing, though, read them off the STATUS events rather than this response: the response lands only once the whole recording is done, which is far too late to explain a wait that is already happening. -The request takes a raw `Float32Array` of mono 16 kHz PCM and an optional -ISO 639-1 `language` code; `"auto"` or absent leaves detection to Whisper. +The renderer sends a `sourcePath`, which the main process decodes with ffmpeg +into mono 16 kHz PCM; `samples`, a raw `Float32Array`, stays as `SttManager`'s +own seam. The request takes an optional ISO 639-1 `language` code; `"auto"` or +absent leaves detection to Whisper. An optional `prompt` carries the project's names and terms (`project.vocabulary`, the media stage's "Names and terms" field, issue #1023). `whisperServer.ts` sends it framed as `Glossary: .`, and the helper sets