fix(stt): drop the renderer audio-decode fallback and say ffmpeg is missing

electron/stt/extractAudio.test.ts

@@ -14,9 +14,7 @@ vi.mock("../media/audioPeaks", () => ({
 
 const { extractMono16kPcm, FfmpegUnavailableError, MediaUnreadableError, NoAudioTrackError } =
 	await import("./extractAudio");
-const { STT_MEDIA_UNREADABLE, STT_NATIVE_EXTRACTION_UNAVAILABLE } = await import(
-	"./transcriptionContract"
-);
+const { STT_MEDIA_UNREADABLE } = await import("./transcriptionContract");
 
 /** A stand-in for the ffmpeg child: two pipes and a close event, nothing more. */
 function fakeChild() {
@@ -179,13 +177,12 @@ describe("extractMono16kPcm", () => {
 		expect(Array.from(await promise)).toEqual([expect.closeTo(0.1, 6), expect.closeTo(0.2, 6)]);
 	});
 
-	it("refuses with the marker the renderer falls back on when ffmpeg is missing", async () => {
-		// The string is the contract across the IPC boundary, which drops the class.
+	it("says ffmpeg is missing when none resolves", async () => {
+		// The message is all that crosses the IPC boundary, and nothing falls back on it:
+		// it is what the editor, the CLI and MCP show.
 		resolveFfmpegMock.mockReturnValue(null);
 		await expect(extractMono16kPcm("/tmp/a.mp3")).rejects.toBeInstanceOf(FfmpegUnavailableError);
-		await expect(extractMono16kPcm("/tmp/a.mp3")).rejects.toThrow(
-			STT_NATIVE_EXTRACTION_UNAVAILABLE,
-		);
+		await expect(extractMono16kPcm("/tmp/a.mp3")).rejects.toThrow(/^ffmpeg is missing/);
 		expect(spawnMock).not.toHaveBeenCalled();
 	});
 

electron/stt/extractAudio.ts

@@ -21,7 +21,7 @@
 
 import { spawn } from "node:child_process";
 import { mediaClockAudioArgs, resolveFfmpeg } from "../media/audioPeaks";
-import { STT_MEDIA_UNREADABLE, STT_NATIVE_EXTRACTION_UNAVAILABLE } from "./transcriptionContract";
+import { STT_MEDIA_UNREADABLE } from "./transcriptionContract";
 
 /** What whisper.cpp wants, and what `decodePeaks` already asks ffmpeg for. */
 const SAMPLE_RATE = 16_000;
@@ -30,14 +30,17 @@ const SAMPLE_RATE = 16_000;
 const EXTRACT_TIMEOUT_MS = 60_000;
 
 /**
- * Thrown when no ffmpeg can be resolved, so the caller can fall back to the
- * renderer pipeline rather than failing the transcription outright. A distinct type
- * because "there is no ffmpeg here" and "this file has no audio" want opposite
- * responses: fall back, versus report a permanent failure for this asset.
+ * Thrown when no ffmpeg can be resolved. There is no fallback behind it: the renderer
+ * decode that used to catch this reads the whole file on the UI thread, and the hidden
+ * CLI runner could not read the file at all. Every package ships ffmpeg, so the message
+ * names what is missing and how to put it back, for the editor, the CLI and MCP alike.
  */
 export class FfmpegUnavailableError extends Error {
 	constructor() {
-		super(`${STT_NATIVE_EXTRACTION_UNAVAILABLE}: no ffmpeg binary for native audio extraction`);
+		super(
+			"ffmpeg is missing: transcription needs it to read the audio, and this install has none. " +
+				"Reinstall OpenScreen, or set OPENSCREEN_FFMPEG_PATH to an ffmpeg executable.",
+		);
 		this.name = "FfmpegUnavailableError";
 	}
 }

electron/stt/index.ts

@@ -53,7 +53,7 @@ import { WhisperServerManager } from "./whisperServer";
  * worth revisiting only if a much smaller model ever becomes the default.
  */
 
-/** The renderer always sends mono 16 kHz (see `extractMono16kFromVideoUrl`). */
+/** What `extractMono16kPcm` decodes to (see `extractAudio.ts`). */
 const SAMPLE_RATE = 16_000;
 
 /** Attempts per chunk before the whole transcription fails. */

electron/stt/transcriptionContract.ts

@@ -107,10 +107,9 @@ export interface SttStatusEvent {
 /** IPC request: renderer → main. */
 export interface SttTranscribeRequest {
 	/**
-	 * Mono-16k samples the CALLER decoded. Optional since native extraction landed:
-	 * pass `sourcePath` instead and the main process decodes with ffmpeg, off the UI
-	 * thread and without the renderer ever holding the audio. Kept for the caption
-	 * path, which already has samples in hand and has no file to point at.
+	 * Mono-16k samples the CALLER decoded. No renderer sends these any more: it passes
+	 * `sourcePath` and the main process decodes with ffmpeg, off the UI thread and
+	 * without the renderer ever holding the audio. Kept as the manager's own seam.
 	 *
 	 * Exactly one of `samples` / `sourcePath` is required.
 	 */
@@ -120,8 +119,7 @@ export interface SttTranscribeRequest {
 	 * Preferred: the renderer's own pipeline read the whole file, copied it twice and
 	 * resampled it on the UI thread, which is what froze the editor at open.
 	 *
-	 * The caller falls back to its own decode when this cannot be honoured — see
-	 * `FfmpegUnavailableError`.
+	 * Fails with `FfmpegUnavailableError` when no ffmpeg can be resolved.
 	 */
 	sourcePath?: string;
 	/**
@@ -136,25 +134,15 @@ export interface SttTranscribeRequest {
 	prompt?: string;
 }
 
-/**
- * Marker carried in the error message when the main process cannot decode a
- * `sourcePath` because no ffmpeg is resolvable on this install.
- *
- * A string rather than an error class because this crosses `ipcRenderer.invoke`,
- * which reconstructs a plain `Error` from the message and drops the prototype and
- * the `name`. Exported so neither side spells it out by hand — a fallback keyed on
- * a literal typed twice is a fallback that silently stops working.
- */
-export const STT_NATIVE_EXTRACTION_UNAVAILABLE = "stt:native-extraction-unavailable";
-
 /**
  * Marker carried in the error message when audio extraction could not read the
  * `sourcePath` itself: the OS refused the read, or ffmpeg hung opening the file
  * (macOS holds the read behind a pending folder-access prompt, issue #968).
  *
  * Only `extractAudio.ts` sets it, so the renderer can tell "this file's folder is
  * not readable" from an engine failure that happens to mention a permission (model
- * loading, the whisper helper). Same IPC reason as above for being a string.
+ * loading, the whisper helper). A string rather than an error class because this
+ * crosses `ipcRenderer.invoke`, which rebuilds a plain `Error` from the message.
  */
 export const STT_MEDIA_UNREADABLE = "stt:media-unreadable";
 

scripts/before-pack.cjs

@@ -134,8 +134,7 @@ const MAC_REQUIRED = [
 		match: (name) => name === "ffmpeg",
 		what: "the LGPL ffmpeg CLI (spawned for waveform peaks and STT audio extraction)",
 		breaks:
-			"transcription falls back to the renderer decode or fails outright on machines with no\n" +
-			'system ffmpeg, shown to the user only as "Failed to fetch" (#616)',
+			'transcription fails in the editor, over MCP and in the CLI with "ffmpeg is missing" (#616)',
 		fix: "Build it with:\n\n    npm run build:native:compositor:mac\n\nwhich stages the SDK's ffmpeg beside the vendored dylibs.",
 	},
 ];
@@ -194,9 +193,7 @@ const LINUX_REQUIRED = [
 	{
 		match: (name) => name === "ffmpeg",
 		what: "the LGPL ffmpeg CLI (spawned for waveform peaks and STT audio extraction)",
-		breaks:
-			"transcription fails over MCP and the CLI with stt:native-extraction-unavailable,\n" +
-			"and the editor falls back to a slow renderer decode",
+		breaks: 'transcription fails in the editor, over MCP and in the CLI with "ffmpeg is missing"',
 		fix: `${FIX_LINUX_HELPER}\n\nwhich stages the SDK's shared ffmpeg beside helper-ffmpeg/.`,
 	},
 	{

scripts/before-pack.test.mjs

@@ -359,7 +359,7 @@ describe("MAC_REQUIRED", () => {
 });
 
 // The Linux twin of #616: packages shipped the libav shared objects but no ffmpeg CLI, so
-// transcription over MCP and the CLI failed with stt:native-extraction-unavailable.
+// transcription over MCP and the CLI failed for want of ffmpeg.
 describe("LINUX_REQUIRED", () => {
 	it("refuses a payload that has everything except the ffmpeg binary", () => {
 		const { LINUX_REQUIRED, checkNativePayload } = testing();

src/hooks/streamingAudioPeaks.ts

@@ -1,5 +1,4 @@
 import { WebDemuxer } from "web-demuxer";
-import { audioDataFrameToMono } from "@/lib/captioning/extractMono16kWebDemuxer";
 
 /**
  * Streaming trim-waveform peaks for recordings too large to load into memory.
@@ -17,6 +16,39 @@ import { audioDataFrameToMono } from "@/lib/captioning/extractMono16kWebDemuxer"
  * min/max starting from 0 (silence baseline) and channels averaged per sample.
  */
 
+/** Mixes one WebCodecs AudioData frame down to mono (averaged across channels). */
+function audioDataFrameToMono(frame: AudioData): Float32Array {
+	const frames = frame.numberOfFrames;
+	const ch = frame.numberOfChannels;
+	const out = new Float32Array(frames);
+	const fmt = frame.format || "";
+	const planar = fmt.includes("planar");
+
+	if (planar) {
+		const plane = new Float32Array(frames);
+		for (let c = 0; c < ch; c++) {
+			frame.copyTo(plane, { planeIndex: c });
+			for (let i = 0; i < frames; i++) {
+				out[i] += plane[i];
+			}
+		}
+		for (let i = 0; i < frames; i++) {
+			out[i] /= ch;
+		}
+	} else {
+		const interleaved = new Float32Array(frames * ch);
+		frame.copyTo(interleaved, { planeIndex: 0 });
+		for (let i = 0; i < frames; i++) {
+			let sum = 0;
+			for (let c = 0; c < ch; c++) {
+				sum += interleaved[i * ch + c];
+			}
+			out[i] = sum / ch;
+		}
+	}
+	return out;
+}
+
 const DECODE_QUEUE_BACKPRESSURE = 20;
 const LOAD_TIMEOUT_MS = 60_000;
 const READ_END_PADDING_SEC = 0.5;

src/lib/ai-edition/document/transcribe.test.ts

@@ -1,29 +1,15 @@
 import { beforeEach, describe, expect, it, vi } from "vitest";
-import { STT_NATIVE_EXTRACTION_UNAVAILABLE } from "../../../../electron/stt/transcriptionContract";
 import { type AxcutDocument, axcutSchemaVersion, transcriptSchema } from "../schema";
+import { classifyTranscriptionError } from "../transcription/status";
 import { transcribeAsset } from "./transcribe";
 
-vi.mock("@/components/video-editor/projectPersistence", () => ({
-	toFileUrl: (path: string) => `file://${path}`,
-}));
-
 vi.mock("@/lib/captioning", () => ({
-	extractMono16kFromVideoUrl: vi.fn(async () => ({
-		samples: new Float32Array([0, 0, 0]),
-		sampleRate: 16_000,
-	})),
-	transcribeMono16kToSegments: vi.fn(),
 	transcribeSourceFileToSegments: vi.fn(),
 }));
 
-const { extractMono16kFromVideoUrl, transcribeMono16kToSegments, transcribeSourceFileToSegments } =
-	await import("@/lib/captioning");
-// `transcribeAsset` sends the PATH now and lets the main process decode; the samples
-// entry point is only reached when no ffmpeg can be resolved. The assertions below
-// therefore target the native call, and the fallback has tests of its own at the end.
+const { transcribeSourceFileToSegments } = await import("@/lib/captioning");
+// `transcribeAsset` sends the PATH and lets the main process decode.
 const transcribeMock = vi.mocked(transcribeSourceFileToSegments);
-const rendererMock = vi.mocked(transcribeMono16kToSegments);
-const extractMock = vi.mocked(extractMono16kFromVideoUrl);
 
 function makeDoc(): AxcutDocument {
 	return {
@@ -183,62 +169,46 @@ describe("transcribeAsset native extraction", () => {
 		vi.clearAllMocks();
 	});
 
-	it("hands the main process a path instead of decoding in the renderer", async () => {
-		// The point of the change: the renderer must not touch the audio at all on the
-		// happy path. `extractMono16kFromVideoUrl` reads the whole file, copies it twice
-		// and resamples on the UI thread — that is the freeze this avoids.
+	it("hands the main process a path", async () => {
 		transcribeMock.mockResolvedValueOnce({
 			segments: [{ startSec: 0, endSec: 1, text: "hi" }],
 			granularity: "word",
 			detectedLanguage: "en",
 		});
 		await transcribeAsset(makeDoc(), "asset_1");
 		expect(transcribeMock).toHaveBeenCalledWith("/tmp/demo.mp4", expect.anything());
-		expect(extractMock).not.toHaveBeenCalled();
-		expect(rendererMock).not.toHaveBeenCalled();
 	});
 
-	it("falls back to the renderer decode when the install has no ffmpeg", async () => {
-		// A dev checkout that never fetched ffmpeg, or a platform build missing it, must
-		// still transcribe rather than lose the feature.
+	it("says ffmpeg is missing instead of falling back to a renderer decode", async () => {
+		// The fallback used to catch this and decode in the renderer, which the CLI's
+		// hidden runner was not allowed to read: "File path is not approved or is not a
+		// supported video file". Now the reason reaches the caller as the IPC delivers it.
+		const missing =
+			"ffmpeg is missing: transcription needs it to read the audio, and this install has none.";
 		transcribeMock.mockRejectedValueOnce(
-			new Error(`${STT_NATIVE_EXTRACTION_UNAVAILABLE}: no ffmpeg binary`),
+			new Error(
+				`Error invoking remote method 'stt:transcribe': FfmpegUnavailableError: ${missing}`,
+			),
 		);
-		rendererMock.mockResolvedValueOnce({
-			segments: [{ startSec: 0, endSec: 1, text: "hi" }],
-			granularity: "word",
-			detectedLanguage: "en",
-		});
-		const transcript = await transcribeAsset(makeDoc(), "asset_1");
-		expect(extractMock).toHaveBeenCalled();
-		expect(rendererMock).toHaveBeenCalled();
-		expect(transcript.segments.length).toBeGreaterThan(0);
+		const error = await transcribeAsset(makeDoc(), "asset_1").catch((e: unknown) => e);
+		expect(transcribeMock).toHaveBeenCalledTimes(1);
+		// What the CLI prints after "Transcription failed: ", and the editor shows.
+		expect(classifyTranscriptionError(error)).toEqual({ kind: "error", message: missing });
 	});
 
-	it("passes the project's names and terms on both paths", async () => {
+	it("passes the project's names and terms", async () => {
 		const doc = makeDoc();
 		doc.project.vocabulary = "OpenScreen, Kubernetes";
-		transcribeMock.mockRejectedValueOnce(
-			new Error(`${STT_NATIVE_EXTRACTION_UNAVAILABLE}: no ffmpeg binary`),
-		);
-		rendererMock.mockResolvedValueOnce({
+		transcribeMock.mockResolvedValueOnce({
 			segments: [],
 			granularity: "word",
 			detectedLanguage: "en",
 		});
 		await transcribeAsset(doc, "asset_1");
-		const withPrompt = expect.objectContaining({ prompt: "OpenScreen, Kubernetes" });
-		expect(transcribeMock).toHaveBeenCalledWith("/tmp/demo.mp4", withPrompt);
-		expect(rendererMock).toHaveBeenCalledWith(expect.anything(), withPrompt);
-	});
-
-	it("does NOT fall back on any other failure", async () => {
-		// "This file has no audio" is a verdict. Re-deriving it in the renderer would buy
-		// the same answer for the price of the decode this change exists to avoid.
-		transcribeMock.mockRejectedValueOnce(new Error("No decodable audio in /tmp/demo.mp4"));
-		await expect(transcribeAsset(makeDoc(), "asset_1")).rejects.toThrow("No decodable audio");
-		expect(extractMock).not.toHaveBeenCalled();
-		expect(rendererMock).not.toHaveBeenCalled();
+		expect(transcribeMock).toHaveBeenCalledWith(
+			"/tmp/demo.mp4",
+			expect.objectContaining({ prompt: "OpenScreen, Kubernetes" }),
+		);
 	});
 });
 

src/lib/ai-edition/document/transcribe.ts

@@ -2,17 +2,11 @@
 // src/lib/captioning/) as a transcribeAsset function that returns an
 // AxcutTranscript and persists it into the document.
 //
-// ponytail: reuses extractMono16kFromVideoUrl + transcribeMono16kToSegments
-// verbatim. No Python, no faster-whisper, no network calls. Privacy-safe.
+// ponytail: reuses transcribeSourceFileToSegments verbatim. No Python, no
+// faster-whisper, no network calls. Privacy-safe.
 
-import { toFileUrl } from "@/components/video-editor/projectPersistence";
-import {
-	extractMono16kFromVideoUrl,
-	transcribeMono16kToSegments,
-	transcribeSourceFileToSegments,
-} from "@/lib/captioning";
+import { transcribeSourceFileToSegments } from "@/lib/captioning";
 import type { SttRendererStatus } from "@/lib/captioning/transcribe";
-import { STT_NATIVE_EXTRACTION_UNAVAILABLE } from "../../../../electron/stt/transcriptionContract";
 import type { AxcutDocument, AxcutTranscript } from "../schema";
 import { transcriptFromSegments } from "../transcription/fromStt";
 
@@ -51,8 +45,6 @@ export async function transcribeAsset(
 		throw new Error(`Asset ${assetId} not found in document.`);
 	}
 
-	const videoUrl = toFileUrl(asset.originalPath);
-
 	options.onStatus?.({ phase: "extracting-audio" });
 
 	// Stay on loading-model until the main process reports inference chunks.
@@ -86,38 +78,17 @@ export async function transcribeAsset(
 			...(status.totalBytes !== undefined ? { totalBytes: status.totalBytes } : {}),
 		});
 
-	// Native first. `extractMono16kFromVideoUrl` runs in the RENDERER: it reads the
-	// whole media into memory, copies it twice and resamples on the UI thread, which
-	// is what froze the editor at project open on a long import (measured on a
-	// four-minute bed: ~86 MB of decoded float32 there against 15.7 MB in the main
-	// process). Handing the path over keeps every byte on the other side of the IPC,
-	// and the whisper helper it feeds was already a separate process.
-	//
-	// The fallback is not decoration: an install with no resolvable ffmpeg — a dev
-	// checkout that never fetched it, a platform build missing the binary — must still
-	// transcribe rather than lose the feature. Only THAT case falls back. "This file
-	// has no audio" is a verdict, and re-deriving it in the renderer would buy the same
-	// answer for the price of the decode this exists to avoid.
+	// The main process decodes the file with ffmpeg; the renderer never holds the audio.
+	// No renderer decode behind it: that path read the whole file on the UI thread,
+	// and the CLI's hidden runner was not even allowed to read it. Every package ships
+	// ffmpeg, so a missing one surfaces as `FfmpegUnavailableError`'s message, which
+	// says so, in the editor, the CLI and MCP alike.
 	const result = await transcribeSourceFileToSegments(asset.originalPath, {
 		trimRegions: [],
 		signal: options.signal,
 		language: forcedLanguage,
 		prompt,
 		onStatus: forwardStatus,
-	}).catch(async (error: unknown) => {
-		const message = error instanceof Error ? error.message : String(error);
-		if (!message.includes(STT_NATIVE_EXTRACTION_UNAVAILABLE)) throw error;
-		const audioResult = await extractMono16kFromVideoUrl(videoUrl, {
-			signal: options.signal,
-		});
-		options.onStatus?.({ phase: "transcribing" });
-		return transcribeMono16kToSegments(audioResult.samples, {
-			trimRegions: [],
-			signal: options.signal,
-			language: forcedLanguage,
-			prompt,
-			onStatus: forwardStatus,
-		});
 	});
 
 	// Prefer the model-reported language (covers both forced picks and auto-detect),