diff --git a/electron/stt/extractAudio.test.ts b/electron/stt/extractAudio.test.ts index d872176686..4bfe6a8b30 100644 --- a/electron/stt/extractAudio.test.ts +++ b/electron/stt/extractAudio.test.ts @@ -14,9 +14,7 @@ vi.mock("../media/audioPeaks", () => ({ const { extractMono16kPcm, FfmpegUnavailableError, MediaUnreadableError, NoAudioTrackError } = await import("./extractAudio"); -const { STT_MEDIA_UNREADABLE, STT_NATIVE_EXTRACTION_UNAVAILABLE } = await import( - "./transcriptionContract" -); +const { STT_MEDIA_UNREADABLE } = await import("./transcriptionContract"); /** A stand-in for the ffmpeg child: two pipes and a close event, nothing more. */ function fakeChild() { @@ -179,13 +177,12 @@ describe("extractMono16kPcm", () => { expect(Array.from(await promise)).toEqual([expect.closeTo(0.1, 6), expect.closeTo(0.2, 6)]); }); - it("refuses with the marker the renderer falls back on when ffmpeg is missing", async () => { - // The string is the contract across the IPC boundary, which drops the class. + it("says ffmpeg is missing when none resolves", async () => { + // The message is all that crosses the IPC boundary, and nothing falls back on it: + // it is what the editor, the CLI and MCP show. resolveFfmpegMock.mockReturnValue(null); await expect(extractMono16kPcm("/tmp/a.mp3")).rejects.toBeInstanceOf(FfmpegUnavailableError); - await expect(extractMono16kPcm("/tmp/a.mp3")).rejects.toThrow( - STT_NATIVE_EXTRACTION_UNAVAILABLE, - ); + await expect(extractMono16kPcm("/tmp/a.mp3")).rejects.toThrow(/^ffmpeg is missing/); expect(spawnMock).not.toHaveBeenCalled(); }); diff --git a/electron/stt/extractAudio.ts b/electron/stt/extractAudio.ts index 2ffebf559c..f4308c2e98 100644 --- a/electron/stt/extractAudio.ts +++ b/electron/stt/extractAudio.ts @@ -21,7 +21,7 @@ import { spawn } from "node:child_process"; import { mediaClockAudioArgs, resolveFfmpeg } from "../media/audioPeaks"; -import { STT_MEDIA_UNREADABLE, STT_NATIVE_EXTRACTION_UNAVAILABLE } from "./transcriptionContract"; +import { STT_MEDIA_UNREADABLE } from "./transcriptionContract"; /** What whisper.cpp wants, and what `decodePeaks` already asks ffmpeg for. */ const SAMPLE_RATE = 16_000; @@ -30,14 +30,17 @@ const SAMPLE_RATE = 16_000; const EXTRACT_TIMEOUT_MS = 60_000; /** - * Thrown when no ffmpeg can be resolved, so the caller can fall back to the - * renderer pipeline rather than failing the transcription outright. A distinct type - * because "there is no ffmpeg here" and "this file has no audio" want opposite - * responses: fall back, versus report a permanent failure for this asset. + * Thrown when no ffmpeg can be resolved. There is no fallback behind it: the renderer + * decode that used to catch this reads the whole file on the UI thread, and the hidden + * CLI runner could not read the file at all. Every package ships ffmpeg, so the message + * names what is missing and how to put it back, for the editor, the CLI and MCP alike. */ export class FfmpegUnavailableError extends Error { constructor() { - super(`${STT_NATIVE_EXTRACTION_UNAVAILABLE}: no ffmpeg binary for native audio extraction`); + super( + "ffmpeg is missing: transcription needs it to read the audio, and this install has none. " + + "Reinstall OpenScreen, or set OPENSCREEN_FFMPEG_PATH to an ffmpeg executable.", + ); this.name = "FfmpegUnavailableError"; } } diff --git a/electron/stt/index.ts b/electron/stt/index.ts index 7a536409bf..1272b99faf 100644 --- a/electron/stt/index.ts +++ b/electron/stt/index.ts @@ -53,7 +53,7 @@ import { WhisperServerManager } from "./whisperServer"; * worth revisiting only if a much smaller model ever becomes the default. */ -/** The renderer always sends mono 16 kHz (see `extractMono16kFromVideoUrl`). */ +/** What `extractMono16kPcm` decodes to (see `extractAudio.ts`). */ const SAMPLE_RATE = 16_000; /** Attempts per chunk before the whole transcription fails. */ diff --git a/electron/stt/transcriptionContract.ts b/electron/stt/transcriptionContract.ts index d30181487d..ce82aff5d2 100644 --- a/electron/stt/transcriptionContract.ts +++ b/electron/stt/transcriptionContract.ts @@ -107,10 +107,9 @@ export interface SttStatusEvent { /** IPC request: renderer → main. */ export interface SttTranscribeRequest { /** - * Mono-16k samples the CALLER decoded. Optional since native extraction landed: - * pass `sourcePath` instead and the main process decodes with ffmpeg, off the UI - * thread and without the renderer ever holding the audio. Kept for the caption - * path, which already has samples in hand and has no file to point at. + * Mono-16k samples the CALLER decoded. No renderer sends these any more: it passes + * `sourcePath` and the main process decodes with ffmpeg, off the UI thread and + * without the renderer ever holding the audio. Kept as the manager's own seam. * * Exactly one of `samples` / `sourcePath` is required. */ @@ -120,8 +119,7 @@ export interface SttTranscribeRequest { * Preferred: the renderer's own pipeline read the whole file, copied it twice and * resampled it on the UI thread, which is what froze the editor at open. * - * The caller falls back to its own decode when this cannot be honoured — see - * `FfmpegUnavailableError`. + * Fails with `FfmpegUnavailableError` when no ffmpeg can be resolved. */ sourcePath?: string; /** @@ -136,17 +134,6 @@ export interface SttTranscribeRequest { prompt?: string; } -/** - * Marker carried in the error message when the main process cannot decode a - * `sourcePath` because no ffmpeg is resolvable on this install. - * - * A string rather than an error class because this crosses `ipcRenderer.invoke`, - * which reconstructs a plain `Error` from the message and drops the prototype and - * the `name`. Exported so neither side spells it out by hand — a fallback keyed on - * a literal typed twice is a fallback that silently stops working. - */ -export const STT_NATIVE_EXTRACTION_UNAVAILABLE = "stt:native-extraction-unavailable"; - /** * Marker carried in the error message when audio extraction could not read the * `sourcePath` itself: the OS refused the read, or ffmpeg hung opening the file @@ -154,7 +141,8 @@ export const STT_NATIVE_EXTRACTION_UNAVAILABLE = "stt:native-extraction-unavaila * * Only `extractAudio.ts` sets it, so the renderer can tell "this file's folder is * not readable" from an engine failure that happens to mention a permission (model - * loading, the whisper helper). Same IPC reason as above for being a string. + * loading, the whisper helper). A string rather than an error class because this + * crosses `ipcRenderer.invoke`, which rebuilds a plain `Error` from the message. */ export const STT_MEDIA_UNREADABLE = "stt:media-unreadable"; diff --git a/scripts/before-pack.cjs b/scripts/before-pack.cjs index d4ecb5d486..cdb8b05c63 100644 --- a/scripts/before-pack.cjs +++ b/scripts/before-pack.cjs @@ -134,8 +134,7 @@ const MAC_REQUIRED = [ match: (name) => name === "ffmpeg", what: "the LGPL ffmpeg CLI (spawned for waveform peaks and STT audio extraction)", breaks: - "transcription falls back to the renderer decode or fails outright on machines with no\n" + - 'system ffmpeg, shown to the user only as "Failed to fetch" (#616)', + 'transcription fails in the editor, over MCP and in the CLI with "ffmpeg is missing" (#616)', fix: "Build it with:\n\n npm run build:native:compositor:mac\n\nwhich stages the SDK's ffmpeg beside the vendored dylibs.", }, ]; @@ -194,9 +193,7 @@ const LINUX_REQUIRED = [ { match: (name) => name === "ffmpeg", what: "the LGPL ffmpeg CLI (spawned for waveform peaks and STT audio extraction)", - breaks: - "transcription fails over MCP and the CLI with stt:native-extraction-unavailable,\n" + - "and the editor falls back to a slow renderer decode", + breaks: 'transcription fails in the editor, over MCP and in the CLI with "ffmpeg is missing"', fix: `${FIX_LINUX_HELPER}\n\nwhich stages the SDK's shared ffmpeg beside helper-ffmpeg/.`, }, { diff --git a/scripts/before-pack.test.mjs b/scripts/before-pack.test.mjs index b7df9a3bd4..027f847b61 100644 --- a/scripts/before-pack.test.mjs +++ b/scripts/before-pack.test.mjs @@ -359,7 +359,7 @@ describe("MAC_REQUIRED", () => { }); // The Linux twin of #616: packages shipped the libav shared objects but no ffmpeg CLI, so -// transcription over MCP and the CLI failed with stt:native-extraction-unavailable. +// transcription over MCP and the CLI failed for want of ffmpeg. describe("LINUX_REQUIRED", () => { it("refuses a payload that has everything except the ffmpeg binary", () => { const { LINUX_REQUIRED, checkNativePayload } = testing(); diff --git a/src/hooks/streamingAudioPeaks.ts b/src/hooks/streamingAudioPeaks.ts index 8e906ad210..ed9f314af8 100644 --- a/src/hooks/streamingAudioPeaks.ts +++ b/src/hooks/streamingAudioPeaks.ts @@ -1,5 +1,4 @@ import { WebDemuxer } from "web-demuxer"; -import { audioDataFrameToMono } from "@/lib/captioning/extractMono16kWebDemuxer"; /** * Streaming trim-waveform peaks for recordings too large to load into memory. @@ -17,6 +16,39 @@ import { audioDataFrameToMono } from "@/lib/captioning/extractMono16kWebDemuxer" * min/max starting from 0 (silence baseline) and channels averaged per sample. */ +/** Mixes one WebCodecs AudioData frame down to mono (averaged across channels). */ +function audioDataFrameToMono(frame: AudioData): Float32Array { + const frames = frame.numberOfFrames; + const ch = frame.numberOfChannels; + const out = new Float32Array(frames); + const fmt = frame.format || ""; + const planar = fmt.includes("planar"); + + if (planar) { + const plane = new Float32Array(frames); + for (let c = 0; c < ch; c++) { + frame.copyTo(plane, { planeIndex: c }); + for (let i = 0; i < frames; i++) { + out[i] += plane[i]; + } + } + for (let i = 0; i < frames; i++) { + out[i] /= ch; + } + } else { + const interleaved = new Float32Array(frames * ch); + frame.copyTo(interleaved, { planeIndex: 0 }); + for (let i = 0; i < frames; i++) { + let sum = 0; + for (let c = 0; c < ch; c++) { + sum += interleaved[i * ch + c]; + } + out[i] = sum / ch; + } + } + return out; +} + const DECODE_QUEUE_BACKPRESSURE = 20; const LOAD_TIMEOUT_MS = 60_000; const READ_END_PADDING_SEC = 0.5; diff --git a/src/lib/ai-edition/document/transcribe.test.ts b/src/lib/ai-edition/document/transcribe.test.ts index a7c2d194bb..b94ddf4542 100644 --- a/src/lib/ai-edition/document/transcribe.test.ts +++ b/src/lib/ai-edition/document/transcribe.test.ts @@ -1,29 +1,15 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; -import { STT_NATIVE_EXTRACTION_UNAVAILABLE } from "../../../../electron/stt/transcriptionContract"; import { type AxcutDocument, axcutSchemaVersion, transcriptSchema } from "../schema"; +import { classifyTranscriptionError } from "../transcription/status"; import { transcribeAsset } from "./transcribe"; -vi.mock("@/components/video-editor/projectPersistence", () => ({ - toFileUrl: (path: string) => `file://${path}`, -})); - vi.mock("@/lib/captioning", () => ({ - extractMono16kFromVideoUrl: vi.fn(async () => ({ - samples: new Float32Array([0, 0, 0]), - sampleRate: 16_000, - })), - transcribeMono16kToSegments: vi.fn(), transcribeSourceFileToSegments: vi.fn(), })); -const { extractMono16kFromVideoUrl, transcribeMono16kToSegments, transcribeSourceFileToSegments } = - await import("@/lib/captioning"); -// `transcribeAsset` sends the PATH now and lets the main process decode; the samples -// entry point is only reached when no ffmpeg can be resolved. The assertions below -// therefore target the native call, and the fallback has tests of its own at the end. +const { transcribeSourceFileToSegments } = await import("@/lib/captioning"); +// `transcribeAsset` sends the PATH and lets the main process decode. const transcribeMock = vi.mocked(transcribeSourceFileToSegments); -const rendererMock = vi.mocked(transcribeMono16kToSegments); -const extractMock = vi.mocked(extractMono16kFromVideoUrl); function makeDoc(): AxcutDocument { return { @@ -183,10 +169,7 @@ describe("transcribeAsset native extraction", () => { vi.clearAllMocks(); }); - it("hands the main process a path instead of decoding in the renderer", async () => { - // The point of the change: the renderer must not touch the audio at all on the - // happy path. `extractMono16kFromVideoUrl` reads the whole file, copies it twice - // and resamples on the UI thread — that is the freeze this avoids. + it("hands the main process a path", async () => { transcribeMock.mockResolvedValueOnce({ segments: [{ startSec: 0, endSec: 1, text: "hi" }], granularity: "word", @@ -194,51 +177,38 @@ describe("transcribeAsset native extraction", () => { }); await transcribeAsset(makeDoc(), "asset_1"); expect(transcribeMock).toHaveBeenCalledWith("/tmp/demo.mp4", expect.anything()); - expect(extractMock).not.toHaveBeenCalled(); - expect(rendererMock).not.toHaveBeenCalled(); }); - it("falls back to the renderer decode when the install has no ffmpeg", async () => { - // A dev checkout that never fetched ffmpeg, or a platform build missing it, must - // still transcribe rather than lose the feature. + it("says ffmpeg is missing instead of falling back to a renderer decode", async () => { + // The fallback used to catch this and decode in the renderer, which the CLI's + // hidden runner was not allowed to read: "File path is not approved or is not a + // supported video file". Now the reason reaches the caller as the IPC delivers it. + const missing = + "ffmpeg is missing: transcription needs it to read the audio, and this install has none."; transcribeMock.mockRejectedValueOnce( - new Error(`${STT_NATIVE_EXTRACTION_UNAVAILABLE}: no ffmpeg binary`), + new Error( + `Error invoking remote method 'stt:transcribe': FfmpegUnavailableError: ${missing}`, + ), ); - rendererMock.mockResolvedValueOnce({ - segments: [{ startSec: 0, endSec: 1, text: "hi" }], - granularity: "word", - detectedLanguage: "en", - }); - const transcript = await transcribeAsset(makeDoc(), "asset_1"); - expect(extractMock).toHaveBeenCalled(); - expect(rendererMock).toHaveBeenCalled(); - expect(transcript.segments.length).toBeGreaterThan(0); + const error = await transcribeAsset(makeDoc(), "asset_1").catch((e: unknown) => e); + expect(transcribeMock).toHaveBeenCalledTimes(1); + // What the CLI prints after "Transcription failed: ", and the editor shows. + expect(classifyTranscriptionError(error)).toEqual({ kind: "error", message: missing }); }); - it("passes the project's names and terms on both paths", async () => { + it("passes the project's names and terms", async () => { const doc = makeDoc(); doc.project.vocabulary = "OpenScreen, Kubernetes"; - transcribeMock.mockRejectedValueOnce( - new Error(`${STT_NATIVE_EXTRACTION_UNAVAILABLE}: no ffmpeg binary`), - ); - rendererMock.mockResolvedValueOnce({ + transcribeMock.mockResolvedValueOnce({ segments: [], granularity: "word", detectedLanguage: "en", }); await transcribeAsset(doc, "asset_1"); - const withPrompt = expect.objectContaining({ prompt: "OpenScreen, Kubernetes" }); - expect(transcribeMock).toHaveBeenCalledWith("/tmp/demo.mp4", withPrompt); - expect(rendererMock).toHaveBeenCalledWith(expect.anything(), withPrompt); - }); - - it("does NOT fall back on any other failure", async () => { - // "This file has no audio" is a verdict. Re-deriving it in the renderer would buy - // the same answer for the price of the decode this change exists to avoid. - transcribeMock.mockRejectedValueOnce(new Error("No decodable audio in /tmp/demo.mp4")); - await expect(transcribeAsset(makeDoc(), "asset_1")).rejects.toThrow("No decodable audio"); - expect(extractMock).not.toHaveBeenCalled(); - expect(rendererMock).not.toHaveBeenCalled(); + expect(transcribeMock).toHaveBeenCalledWith( + "/tmp/demo.mp4", + expect.objectContaining({ prompt: "OpenScreen, Kubernetes" }), + ); }); }); diff --git a/src/lib/ai-edition/document/transcribe.ts b/src/lib/ai-edition/document/transcribe.ts index 4f3f1c7016..4253eaf22b 100644 --- a/src/lib/ai-edition/document/transcribe.ts +++ b/src/lib/ai-edition/document/transcribe.ts @@ -2,17 +2,11 @@ // src/lib/captioning/) as a transcribeAsset function that returns an // AxcutTranscript and persists it into the document. // -// ponytail: reuses extractMono16kFromVideoUrl + transcribeMono16kToSegments -// verbatim. No Python, no faster-whisper, no network calls. Privacy-safe. +// ponytail: reuses transcribeSourceFileToSegments verbatim. No Python, no +// faster-whisper, no network calls. Privacy-safe. -import { toFileUrl } from "@/components/video-editor/projectPersistence"; -import { - extractMono16kFromVideoUrl, - transcribeMono16kToSegments, - transcribeSourceFileToSegments, -} from "@/lib/captioning"; +import { transcribeSourceFileToSegments } from "@/lib/captioning"; import type { SttRendererStatus } from "@/lib/captioning/transcribe"; -import { STT_NATIVE_EXTRACTION_UNAVAILABLE } from "../../../../electron/stt/transcriptionContract"; import type { AxcutDocument, AxcutTranscript } from "../schema"; import { transcriptFromSegments } from "../transcription/fromStt"; @@ -51,8 +45,6 @@ export async function transcribeAsset( throw new Error(`Asset ${assetId} not found in document.`); } - const videoUrl = toFileUrl(asset.originalPath); - options.onStatus?.({ phase: "extracting-audio" }); // Stay on loading-model until the main process reports inference chunks. @@ -86,38 +78,17 @@ export async function transcribeAsset( ...(status.totalBytes !== undefined ? { totalBytes: status.totalBytes } : {}), }); - // Native first. `extractMono16kFromVideoUrl` runs in the RENDERER: it reads the - // whole media into memory, copies it twice and resamples on the UI thread, which - // is what froze the editor at project open on a long import (measured on a - // four-minute bed: ~86 MB of decoded float32 there against 15.7 MB in the main - // process). Handing the path over keeps every byte on the other side of the IPC, - // and the whisper helper it feeds was already a separate process. - // - // The fallback is not decoration: an install with no resolvable ffmpeg — a dev - // checkout that never fetched it, a platform build missing the binary — must still - // transcribe rather than lose the feature. Only THAT case falls back. "This file - // has no audio" is a verdict, and re-deriving it in the renderer would buy the same - // answer for the price of the decode this exists to avoid. + // The main process decodes the file with ffmpeg; the renderer never holds the audio. + // No renderer decode behind it: that path read the whole file on the UI thread, + // and the CLI's hidden runner was not even allowed to read it. Every package ships + // ffmpeg, so a missing one surfaces as `FfmpegUnavailableError`'s message, which + // says so, in the editor, the CLI and MCP alike. const result = await transcribeSourceFileToSegments(asset.originalPath, { trimRegions: [], signal: options.signal, language: forcedLanguage, prompt, onStatus: forwardStatus, - }).catch(async (error: unknown) => { - const message = error instanceof Error ? error.message : String(error); - if (!message.includes(STT_NATIVE_EXTRACTION_UNAVAILABLE)) throw error; - const audioResult = await extractMono16kFromVideoUrl(videoUrl, { - signal: options.signal, - }); - options.onStatus?.({ phase: "transcribing" }); - return transcribeMono16kToSegments(audioResult.samples, { - trimRegions: [], - signal: options.signal, - language: forcedLanguage, - prompt, - onStatus: forwardStatus, - }); }); // Prefer the model-reported language (covers both forced picks and auto-detect), diff --git a/src/lib/ai-edition/transcription/status.ts b/src/lib/ai-edition/transcription/status.ts index ea317e60e8..081766695d 100644 --- a/src/lib/ai-edition/transcription/status.ts +++ b/src/lib/ai-edition/transcription/status.ts @@ -149,10 +149,9 @@ export function withTranscriptionFailure( /** * Map an exception out of `transcribeAsset` onto a failure the UI can explain. - * The deterministic cases come from whichever layer decoded the audio, and there - * are two of them: `extractMono16kWebDemuxer` in the renderer, and — since - * native extraction landed — `extractAudio.ts`'s `NoAudioTrackError` in the main - * process, whose wording ("No decodable audio in …") is its own. + * The deterministic cases come from `extractAudio.ts`'s `NoAudioTrackError` in the + * main process ("No decodable audio in …"). The other phrasings are the renderer + * decode's, gone since the main process decodes every file. * * Matched on the message rather than on the class because `ipcRenderer.invoke` * rebuilds a plain `Error` and drops both the prototype and the `name`. Missing diff --git a/src/lib/captioning/captionConstants.ts b/src/lib/captioning/captionConstants.ts deleted file mode 100644 index 1bacb7cc7a..0000000000 --- a/src/lib/captioning/captionConstants.ts +++ /dev/null @@ -1,2 +0,0 @@ -/** Max audio length for auto-captions (decode + transcribe); keep demuxer read aligned with this. */ -export const MAX_CAPTION_AUDIO_SEC = 4 * 60 * 60; diff --git a/src/lib/captioning/extractMono16k.ts b/src/lib/captioning/extractMono16k.ts deleted file mode 100644 index 1116bb9a2a..0000000000 --- a/src/lib/captioning/extractMono16k.ts +++ /dev/null @@ -1,197 +0,0 @@ -import { materializeLocalSourceFile, releaseLocalSourceFile } from "@/lib/exporter/localSourceFile"; -import { MAX_IN_MEMORY_SOURCE_BYTES } from "@/lib/exporter/sourceFileLimits"; -import { MAX_CAPTION_AUDIO_SEC } from "./captionConstants"; -import { extractMonoPcmViaWebDemuxer } from "./extractMono16kWebDemuxer"; - -export { MAX_CAPTION_AUDIO_SEC }; - -const FETCH_TIMEOUT_MS = 120_000; -// The demuxer caption path holds every decoded AudioData frame plus full-rate -// merge buffers in memory (~50 MB per minute of 48 kHz audio all-in), so very -// long recordings would exhaust the renderer heap well before the 4 h caption -// ceiling. For sources too large to load in memory anyway, cap the decoded -// audio; captions come back truncated instead of crashing the renderer. -const LARGE_FILE_CAPTION_SEC = 30 * 60; - -async function fetchWithTimeout(url: string, signal?: AbortSignal): Promise { - const ctrl = new AbortController(); - const timer = window.setTimeout(() => ctrl.abort(), FETCH_TIMEOUT_MS); - const onAbort = () => ctrl.abort(); - if (signal) { - if (signal.aborted) ctrl.abort(); - else signal.addEventListener("abort", onAbort, { once: true }); - } - try { - return await fetch(url, { signal: ctrl.signal }); - } finally { - window.clearTimeout(timer); - if (signal) signal.removeEventListener("abort", onAbort); - } -} - -/** - * Load the editor video like `StreamingVideoDecoder`: Electron `readBinaryFile` - * for local paths (fetch(file://) is unreliable in the renderer), otherwise - * HTTP/blob/data URLs via fetch. - */ -async function loadSourceVideoFile(videoUrl: string, signal?: AbortSignal): Promise { - const isRemoteUrl = /^(https?:|blob:|data:)/i.test(videoUrl); - - if (!isRemoteUrl && window.electronAPI) { - // Streams large recordings through OPFS instead of reading them whole, so - // captions work for multi-GB files just like export does. The signal also - // aborts the copy itself when the caption pass is cancelled. - const filename = (videoUrl.split(/[\\/]/).pop() || "video").replace(/^file:/, ""); - return materializeLocalSourceFile(videoUrl, filename, { signal }); - } - - const response = await fetchWithTimeout(videoUrl, signal); - if (!response.ok) { - throw new Error(`Failed to load video for captions: ${response.status} ${response.statusText}`); - } - const blob = await response.blob(); - if (signal?.aborted) throw new DOMException("Aborted", "AbortError"); - const filename = videoUrl.split("/").pop() || "video"; - return new File([blob], filename, { type: blob.type || "video/webm" }); -} - -function mixToMono(audioBuffer: AudioBuffer): Float32Array { - const { length, numberOfChannels } = audioBuffer; - if (numberOfChannels === 0) return new Float32Array(length); - // `getChannelData` is a WebIDL call, so calling it INSIDE the sample loop cost - // one call per sample per channel — ~57 M of them for a ten-minute stereo - // recording, seconds of blocked main thread. That was survivable while - // transcription only ran when the user asked for it; it now runs by itself when - // a project opens, and a frozen window (spinners included) is exactly what the - // automatic pass must not look like. Hoisting the channel arrays out of the loop - // leaves plain typed-array indexing. - const channels: Float32Array[] = []; - for (let c = 0; c < numberOfChannels; c++) channels.push(audioBuffer.getChannelData(c)); - // Mono source: the mixdown is a copy. `slice` keeps the caller's contract of - // owning its buffer (the AudioBuffer's own array is reused by the context). - if (numberOfChannels === 1) return channels[0].slice(); - const out = new Float32Array(length); - for (let i = 0; i < length; i++) { - let sum = 0; - for (let c = 0; c < numberOfChannels; c++) sum += channels[c][i]; - out[i] = sum / numberOfChannels; - } - return out; -} - -async function resampleMono( - mono: Float32Array, - fromRate: number, - toRate: number, - signal?: AbortSignal, -): Promise { - if (signal?.aborted) throw new DOMException("Aborted", "AbortError"); - if (fromRate === toRate) return mono; - const durationSec = mono.length / fromRate; - const outLength = Math.max(1, Math.ceil(durationSec * toRate)); - const offline = new OfflineAudioContext(1, outLength, toRate); - const buf = offline.createBuffer(1, mono.length, fromRate); - buf.copyToChannel(Float32Array.from(mono), 0); - const src = offline.createBufferSource(); - src.buffer = buf; - src.connect(offline.destination); - src.start(0); - const rendered = await offline.startRendering(); - if (signal?.aborted) throw new DOMException("Aborted", "AbortError"); - return rendered.getChannelData(0).slice(); -} - -async function truncateAndResampleTo16k( - mono: Float32Array, - fromRate: number, - durationSec: number, - signal?: AbortSignal, -): Promise<{ samples: Float32Array; truncated: boolean; durationSec: number }> { - let truncated = false; - let work = mono; - if (durationSec > MAX_CAPTION_AUDIO_SEC) { - const maxSamples = Math.floor(MAX_CAPTION_AUDIO_SEC * fromRate); - work = mono.subarray(0, Math.min(mono.length, maxSamples)); - truncated = true; - } - - const samples = await resampleMono(work, fromRate, 16_000, signal); - return { samples, truncated, durationSec: samples.length / 16_000 }; -} - -/** - * Decode the video's audio track to mono 16 kHz float samples (Whisper input). - * Prefers `decodeAudioData` when the container is supported, else the same - * web-demuxer + AudioDecoder path as export. - */ -export async function extractMono16kFromVideoUrl( - videoUrl: string, - options?: { signal?: AbortSignal }, -): Promise<{ samples: Float32Array; truncated: boolean; durationSec: number }> { - const file = await loadSourceVideoFile(videoUrl, options?.signal); - - /** When this returns null, use web-demuxer + AudioDecoder (same as export). */ - const tryDecodeAudioDataPath = async (): Promise<{ - samples: Float32Array; - truncated: boolean; - durationSec: number; - } | null> => { - const audioContext = new AudioContext(); - try { - const ab = await file.arrayBuffer(); - if (options?.signal?.aborted) throw new DOMException("Aborted", "AbortError"); - const audioBuffer = await audioContext.decodeAudioData(ab.slice(0)); - if ( - audioBuffer.numberOfChannels === 0 || - audioBuffer.length === 0 || - !Number.isFinite(audioBuffer.duration) || - audioBuffer.duration <= 0 - ) { - return null; - } - const durationSec = audioBuffer.duration; - const mono = mixToMono(audioBuffer); - const fromRate = audioBuffer.sampleRate; - const out = await truncateAndResampleTo16k(mono, fromRate, durationSec, options?.signal); - // decodeAudioData can resolve for some WebM/Matroska inputs yet yield almost no usable - // PCM, and captions only fall back to the demuxer path on throw, so return null to recover. - if (out.samples.length < 800) { - return null; - } - return out; - } catch { - return null; - } finally { - await audioContext.close().catch(() => undefined); - } - }; - - try { - // Large recordings skip the in-memory decodeAudioData path (it would load - // the whole file) and go straight to the streaming web-demuxer path below. - const isLargeFile = file.size > MAX_IN_MEMORY_SOURCE_BYTES; - const primary = isLargeFile ? null : await tryDecodeAudioDataPath(); - if (primary) { - return primary; - } - - // For oversized sources, also cap how much audio the demuxer path decodes - // — its frame/merge buffers are in-memory and scale with duration. - const pcm = await extractMonoPcmViaWebDemuxer( - file, - options?.signal, - isLargeFile ? LARGE_FILE_CAPTION_SEC : undefined, - ); - const out = await truncateAndResampleTo16k( - pcm.mono, - pcm.sampleRate, - pcm.durationSec, - options?.signal, - ); - return { ...out, truncated: out.truncated || pcm.capped }; - } finally { - // Release the OPFS cache reference taken when streaming a large source. - // The File name is the cache-entry key (no-op for small/remote sources). - releaseLocalSourceFile(file.name); - } -} diff --git a/src/lib/captioning/extractMono16kWebDemuxer.ts b/src/lib/captioning/extractMono16kWebDemuxer.ts deleted file mode 100644 index 0d1a2ba784..0000000000 --- a/src/lib/captioning/extractMono16kWebDemuxer.ts +++ /dev/null @@ -1,198 +0,0 @@ -import { WebDemuxer } from "web-demuxer"; - -import { MAX_CAPTION_AUDIO_SEC } from "./captionConstants"; - -const DECODE_QUEUE_BACKPRESSURE = 20; -const SOURCE_LOAD_TIMEOUT_MS = 60_000; -const READ_END_PADDING_SEC = 0.5; - -function webDemuxerWasmUrl(): string { - return new URL("./wasm/web-demuxer.wasm", window.location.href).href; -} - -/** Mixes one WebCodecs AudioData frame down to mono (averaged across channels). */ -export function audioDataFrameToMono(frame: AudioData): Float32Array { - const frames = frame.numberOfFrames; - const ch = frame.numberOfChannels; - const out = new Float32Array(frames); - const fmt = frame.format || ""; - const planar = fmt.includes("planar"); - - if (planar) { - const plane = new Float32Array(frames); - for (let c = 0; c < ch; c++) { - frame.copyTo(plane, { planeIndex: c }); - for (let i = 0; i < frames; i++) { - out[i] += plane[i]; - } - } - for (let i = 0; i < frames; i++) { - out[i] /= ch; - } - } else { - const interleaved = new Float32Array(frames * ch); - frame.copyTo(interleaved, { planeIndex: 0 }); - for (let i = 0; i < frames; i++) { - let sum = 0; - for (let c = 0; c < ch; c++) { - sum += interleaved[i * ch + c]; - } - out[i] = sum / ch; - } - } - return out; -} - -function mergeAndConsumeDecodedAudioToMonoLinear( - frames: AudioData[], - sampleRate: number, - durationSec: number, -): Float32Array { - const sorted = [...frames].sort((a, b) => a.timestamp - b.timestamp); - const totalSamples = Math.max(1, Math.ceil(durationSec * sampleRate)); - const acc = new Float32Array(totalSamples); - const weight = new Float32Array(totalSamples); - - for (const frame of sorted) { - const startSample = Math.round((frame.timestamp / 1e6) * sampleRate); - const slice = audioDataFrameToMono(frame); - for (let i = 0; i < slice.length; i++) { - const pos = startSample + i; - if (pos >= 0 && pos < totalSamples) { - acc[pos] += slice[i]; - weight[pos] += 1; - } - } - frame.close(); - } - - for (let i = 0; i < totalSamples; i++) { - if (weight[i] > 0) { - acc[i] /= weight[i]; - } - } - return acc; -} - -function withTimeout(promise: Promise, ms: number, message: string): Promise { - return new Promise((resolve, reject) => { - const id = window.setTimeout(() => reject(new Error(message)), ms); - promise - .then((v) => { - window.clearTimeout(id); - resolve(v); - }) - .catch((e) => { - window.clearTimeout(id); - reject(e instanceof Error ? e : new Error(String(e))); - }); - }); -} - -/** - * Demux + WebCodecs audio decode (same stack as export). Use when `decodeAudioData` - * can't handle the container (e.g. WebM with video). - * - * @param maxReadSec Optional cap on how much audio to demux/decode. The decoded - * frames and merge buffers are held in memory, so very long recordings must be - * capped below MAX_CAPTION_AUDIO_SEC to avoid exhausting the renderer heap. - * `capped` reports whether the cap actually cut the track short. - */ -export async function extractMonoPcmViaWebDemuxer( - file: File, - signal?: AbortSignal, - maxReadSec?: number, -): Promise<{ mono: Float32Array; sampleRate: number; durationSec: number; capped: boolean }> { - const demuxer = new WebDemuxer({ wasmFilePath: webDemuxerWasmUrl() }); - await withTimeout( - demuxer.load(file), - SOURCE_LOAD_TIMEOUT_MS, - "Timed out while parsing the source video for captions.", - ); - - if (signal?.aborted) throw new DOMException("Aborted", "AbortError"); - - const mediaInfo = await withTimeout( - demuxer.getMediaInfo(), - SOURCE_LOAD_TIMEOUT_MS, - "Timed out while reading media info for captions.", - ); - - const reportedDurationSec = - Number.isFinite(mediaInfo.duration) && mediaInfo.duration > 0 ? mediaInfo.duration : 0; - - let audioConfig: AudioDecoderConfig; - try { - audioConfig = await demuxer.getDecoderConfig("audio"); - } catch { - throw new Error("No audio track found in this video."); - } - - const codecCheck = await AudioDecoder.isConfigSupported(audioConfig); - if (!codecCheck.supported) { - throw new Error(`Audio codec not supported for captions: ${audioConfig.codec}`); - } - - const sampleRate = audioConfig.sampleRate || 48_000; - - // Many WebM/Matroska files report a too-short duration, so capping read at reported time stops - // demux early and clips everything past that. Read to the caption-decode ceiling instead; the - // demuxer stops when the track ends. - const readCapSec = Math.min(maxReadSec ?? MAX_CAPTION_AUDIO_SEC, MAX_CAPTION_AUDIO_SEC); - const readEndSec = readCapSec + READ_END_PADDING_SEC; - const decodedFrames: AudioData[] = []; - - const decoder = new AudioDecoder({ - output: (data: AudioData) => decodedFrames.push(data), - error: (e: DOMException) => console.error("[captioning] AudioDecoder error:", e), - }); - decoder.configure(audioConfig); - - const reader = demuxer.read("audio", 0, readEndSec).getReader(); - try { - while (!signal?.aborted) { - const { done, value: chunk } = await reader.read(); - if (done || !chunk) break; - decoder.decode(chunk); - while (decoder.decodeQueueSize > DECODE_QUEUE_BACKPRESSURE && !signal?.aborted) { - await new Promise((r) => setTimeout(r, 1)); - } - } - } finally { - try { - await reader.cancel(); - } catch { - /* already closed */ - } - } - - if (decoder.state === "configured") { - await decoder.flush(); - decoder.close(); - } - - if (signal?.aborted) { - for (const f of decodedFrames) f.close(); - throw new DOMException("Aborted", "AbortError"); - } - - if (decodedFrames.length === 0) { - throw new Error("Decoded zero audio frames from this video."); - } - - let maxEndUs = 0; - for (const f of decodedFrames) { - const end = f.timestamp + (f.duration ?? 0); - if (end > maxEndUs) maxEndUs = end; - } - const inferredDurationSec = maxEndUs / 1e6; - // Prefer extent implied by decoded frames (fixes bad container duration); fall back to reported - // metadata when frames lack duration. - const durationSec = inferredDurationSec > 0.02 ? inferredDurationSec : reportedDurationSec; - - // The cap cut the track short when the reported extent exceeds what we read. - const capped = reportedDurationSec > readCapSec + READ_END_PADDING_SEC; - - const mono = mergeAndConsumeDecodedAudioToMonoLinear(decodedFrames, sampleRate, durationSec); - return { mono, sampleRate, durationSec, capped }; -} diff --git a/src/lib/captioning/index.ts b/src/lib/captioning/index.ts index a5ad5dabd6..339782578d 100644 --- a/src/lib/captioning/index.ts +++ b/src/lib/captioning/index.ts @@ -6,10 +6,9 @@ export { mergeAdjacentCaptionSegments, splitMergedCaptionsByWordBounds, } from "./annotationsFromCaptions"; -export { extractMono16kFromVideoUrl, MAX_CAPTION_AUDIO_SEC } from "./extractMono16k"; export type { CaptionSegment, CaptionTimestampGranularity, TranscribeMono16kResult, } from "./transcribe"; -export { transcribeMono16kToSegments, transcribeSourceFileToSegments } from "./transcribe"; +export { transcribeSourceFileToSegments } from "./transcribe"; diff --git a/src/lib/captioning/transcribe.test.ts b/src/lib/captioning/transcribe.test.ts index 505fc59b08..9ef1d94b36 100644 --- a/src/lib/captioning/transcribe.test.ts +++ b/src/lib/captioning/transcribe.test.ts @@ -1,10 +1,10 @@ // @vitest-environment jsdom import { afterEach, beforeEach, describe, expect, it, type Mock, vi } from "vitest"; -import { transcribeMono16kToSegments } from "./transcribe"; +import { transcribeSourceFileToSegments } from "./transcribe"; /** - * The renderer-side `transcribeMono16kToSegments` is a thin adapter on top of + * The renderer-side `transcribeSourceFileToSegments` is a thin adapter on top of * `electronAPI.stt.transcribe`. It must: * - emit each word from `wordSegments` as a single-token `CaptionSegment`, * - tag the run `granularity: "word"`, @@ -75,7 +75,7 @@ const removeMockApi = () => { lastStatusCb = null; }; -describe("transcribeMono16kToSegments", () => { +describe("transcribeSourceFileToSegments", () => { beforeEach(() => { installMockApi(); }); @@ -95,8 +95,7 @@ describe("transcribeMono16kToSegments", () => { backend: "whisper-cpu", }); - const samples = new Float32Array(1600); - const result = await transcribeMono16kToSegments(samples); + const result = await transcribeSourceFileToSegments("/tmp/a.mp4"); expect(result.granularity).toBe("word"); expect(result.segments).toEqual([ @@ -104,7 +103,7 @@ describe("transcribeMono16kToSegments", () => { { text: "world", startSec: 0.31, endSec: 0.65 }, ]); expect(mockApi.transcribe).toHaveBeenCalledWith({ - samples, + sourcePath: "/tmp/a.mp4", language: undefined, }); }); @@ -117,7 +116,7 @@ describe("transcribeMono16kToSegments", () => { backend: "whisper-cpu", }); - const result = await transcribeMono16kToSegments(new Float32Array(1600)); + const result = await transcribeSourceFileToSegments("/tmp/a.mp4"); expect(result.granularity).toBe("phrase"); expect(result.segments).toEqual([{ text: "hello world", startSec: 0, endSec: 0.65 }]); }); @@ -136,7 +135,7 @@ describe("transcribeMono16kToSegments", () => { }; }); - await transcribeMono16kToSegments(new Float32Array(1600), { onStatus }); + await transcribeSourceFileToSegments("/tmp/a.mp4", { onStatus }); expect(onStatus).toHaveBeenCalledWith({ phase: "model" }); // The chunk progress must survive the hop, not just the phase — it is what // drives the progress bar. @@ -151,7 +150,7 @@ describe("transcribeMono16kToSegments", () => { it("returns empty results when running outside Electron (browser tests)", async () => { removeMockApi(); - const result = await transcribeMono16kToSegments(new Float32Array(1600)); + const result = await transcribeSourceFileToSegments("/tmp/a.mp4"); expect(result.granularity).toBe("word"); expect(result.segments).toEqual([]); }); @@ -160,7 +159,7 @@ describe("transcribeMono16kToSegments", () => { const controller = new AbortController(); controller.abort(); await expect( - transcribeMono16kToSegments(new Float32Array(1600), { signal: controller.signal }), + transcribeSourceFileToSegments("/tmp/a.mp4", { signal: controller.signal }), ).rejects.toMatchObject({ name: "AbortError" }); expect(mockApi.transcribe).not.toHaveBeenCalled(); }); diff --git a/src/lib/captioning/transcribe.ts b/src/lib/captioning/transcribe.ts index 806b669141..0d4cf997e9 100644 --- a/src/lib/captioning/transcribe.ts +++ b/src/lib/captioning/transcribe.ts @@ -10,7 +10,7 @@ export interface CaptionSegment { /** * How caption layout should interpret `CaptionSegment` times from - * `transcribeMono16kToSegments`. The native pipeline gets per-word timestamps + * `transcribeSourceFileToSegments`. The native pipeline gets per-word timestamps * from whisper.cpp's own output; `"phrase"` is a fallback for the rare case * where whisper reports zero words for a segment. */ @@ -53,7 +53,7 @@ export interface SttRendererStatus { } interface RendererSttApi { - transcribe: (request: { samples: Float32Array; language?: string; prompt?: string }) => Promise<{ + transcribe: (request: { sourcePath: string; language?: string; prompt?: string }) => Promise<{ segments: CaptionSegment[]; wordSegments: SttWordSegment[]; detectedLanguage: string; @@ -63,42 +63,6 @@ interface RendererSttApi { onStatus?: (callback: (event: SttRendererStatus) => void) => () => void; } -/** - * Transcribes mono 16 kHz audio into per-word timed caption segments. The - * renderer is a thin IPC adapter: it forwards the audio to the Electron main - * process where `whisper-server` runs recognition and emits its own - * per-word timestamps in the same pass. - * - * The previous in-Web-Worker implementation (Transformers.js + ORT-WASM) was - * 0.5× realtime on tiny models and had no word-level accuracy under 50 ms; - * see `technical-documentation/architecture/transcription-and-captions.md` § History for context. - */ -export function transcribeMono16kToSegments( - samples: Float32Array, - options?: TranscribeOptions, -): Promise { - return runTranscription({ samples }, options); -} - -/** - * Same recognition, from a FILE the main process decodes itself. - * - * Preferred over `transcribeMono16kToSegments` wherever there is a path to point at. - * The samples entry point decodes in the renderer — whole file into memory, an - * `arrayBuffer()` copy, a `slice(0)` copy, then a resample loop on the UI thread — - * which is what froze the editor at open on a long import. Here the renderer sends a - * string and gets segments back. - * - * Rejects with a message carrying `STT_NATIVE_EXTRACTION_UNAVAILABLE` when the - * install has no ffmpeg, so the caller can fall back rather than lose the transcript. - */ -export function transcribeSourceFileToSegments( - sourcePath: string, - options?: TranscribeOptions, -): Promise { - return runTranscription({ sourcePath }, options); -} - export interface TranscribeOptions { trimRegions?: TrimRegion[]; onStatus?: (status: SttRendererStatus) => void; @@ -108,8 +72,22 @@ export interface TranscribeOptions { prompt?: string; } -function runTranscription( - payload: { samples: Float32Array } | { sourcePath: string }, +/** + * Transcribes a media FILE into per-word timed caption segments. The renderer is a + * thin IPC adapter: it sends the path, the main process decodes it with ffmpeg and + * `whisper-server` runs recognition, emitting per-word timestamps in the same pass. + * The renderer never holds the audio: decoding it here read the whole file, copied it + * twice and resampled on the UI thread, which is what froze the editor at open. + * + * Rejects with `FfmpegUnavailableError`'s message when the install has no ffmpeg. + * There is no renderer decode to fall back on any more. + * + * The previous in-Web-Worker implementation (Transformers.js + ORT-WASM) was + * 0.5× realtime on tiny models and had no word-level accuracy under 50 ms; + * see `technical-documentation/architecture/transcription-and-captions.md` § History for context. + */ +export function transcribeSourceFileToSegments( + sourcePath: string, options?: TranscribeOptions, ): Promise { if (options?.signal?.aborted) { @@ -138,7 +116,7 @@ function runTranscription( // trimmed leading silence with a peak detector and got false positives on // quiet music intros / room tone. VAD or nothing. return api - .transcribe({ ...payload, language: forcedLanguage, prompt: options?.prompt }) + .transcribe({ sourcePath, language: forcedLanguage, prompt: options?.prompt }) .then((result) => ({ ...segmentsFromStt(result), detectedLanguage: result.detectedLanguage })) .catch((error: unknown) => { // A run the caller cancelled surfaces as an abort, not as an engine diff --git a/technical-documentation/architecture/transcription-and-captions.md b/technical-documentation/architecture/transcription-and-captions.md index a39171c6d1..00c425d65d 100644 --- a/technical-documentation/architecture/transcription-and-captions.md +++ b/technical-documentation/architecture/transcription-and-captions.md @@ -15,8 +15,8 @@ the transcript). flowchart LR Z["Asset added
(import or recording)"] -- "sync()" --> Y["transcriptionStore queue
(src/lib/ai-edition/store/transcriptionStore.ts)"] Y -- "one job at a time" --> A["Recorded audio"] - A -- "extract mono 16 kHz" --> B["transcribeAsset
(src/lib/ai-edition/document/transcribe.ts)"] - B -- "IPC: Float32Array + language" --> C["SttManager
(electron/stt/index.ts)"] + A -- "file path" --> B["transcribeAsset
(src/lib/ai-edition/document/transcribe.ts)"] + B -- "IPC: source path + language" --> C["SttManager
(electron/stt/index.ts)
ffmpeg: mono 16 kHz"] C -- "POST /inference (WAV)" --> D["whisper-stt-server
(electron/native/whisper-stt/)"] D -- "VAD cut + whisper_full() + DTW" --> E["SttTranscribeResponse
(segments + wordSegments)"] E --> C @@ -27,10 +27,11 @@ flowchart LR H -- "annotation path
(scene description)" --> I["Native compositor
(preview AND export)"] ``` -The renderer pieces — `transcribeMono16kToSegments` +The renderer pieces — `transcribeSourceFileToSegments` (`src/lib/captioning/transcribe.ts`) called from `transcribeAsset` (`src/lib/ai-edition/document/transcribe.ts`) — are a thin IPC adapter: the -audio crosses to the main process, the helper does the recognition, and the +file path crosses to the main process, which decodes it with ffmpeg +(`electron/stt/extractAudio.ts`), the helper does the recognition, and the result is mapped back onto the `AxcutTranscript` shape the rest of the editor reads. From the moment that transcript is persisted, captions and the words they show cannot drift — captions are a derived view, not a parallel store. @@ -71,8 +72,7 @@ What the store owns and what it does not: the panes' one button: every timeline asset that still lacks a transcript, skipping the ones already known to be silent and waiting on (rather than duplicating) a run the background pass already has in flight. -- **One run at a time.** whisper-server is a single process and the audio - extraction path holds decoded frames in renderer memory, so the pump is a +- **One run at a time.** whisper-server is a single process, so the pump is a sequential loop, not a fan-out. - **The auto pass never loops.** `sync` enqueues an asset only when it has no transcript, no job entry (queued / running / failed alike) and no persisted @@ -132,12 +132,12 @@ Three edges make that promise hold, and each is load-bearing: clip's word stream swallow Backspace and hover-bin clicks in silence for the whole background pass. -Still synchronous, and still on the main thread: `mixToMono` -([`src/lib/captioning/extractMono16k.ts`](../../src/lib/captioning/extractMono16k.ts)) -now hoists its channel arrays out of the sample loop (it was making one WebIDL -call per sample per channel), but a long recording will still block the renderer -for a moment during "extracting-audio". Moving the mixdown to a worker is the -next step if it shows up in practice. +Audio extraction never runs in the renderer. There is no renderer decode to fall +back on: it read the whole file on the UI thread, froze the editor at open, and +the CLI's hidden runner was not even allowed to read the file. An install with +no ffmpeg fails with "ffmpeg is missing", in the editor, the CLI and MCP alike; +every package ships one, and `before-pack` refuses a macOS or Linux payload +without it. ## The STT engine @@ -606,8 +606,10 @@ For anything user-facing, though, read them off the STATUS events rather than this response: the response lands only once the whole recording is done, which is far too late to explain a wait that is already happening. -The request takes a raw `Float32Array` of mono 16 kHz PCM and an optional -ISO 639-1 `language` code; `"auto"` or absent leaves detection to Whisper. +The renderer sends a `sourcePath`, which the main process decodes with ffmpeg +into mono 16 kHz PCM; `samples`, a raw `Float32Array`, stays as `SttManager`'s +own seam. The request takes an optional ISO 639-1 `language` code; `"auto"` or +absent leaves detection to Whisper. An optional `prompt` carries the project's names and terms (`project.vocabulary`, the media stage's "Names and terms" field, issue #1023). `whisperServer.ts` sends it framed as `Glossary: .`, and the helper sets