feat: put transcription on one contract and route every Whisper call through it

packages/ai-providers/src/index.ts

@@ -76,6 +76,7 @@ export {
 export * from "./video/index.js";
 export * from "./image/index.js";
 export * from "./speech/index.js";
+export * from "./transcription/index.js";
 export { VIDEO_GENERATOR_PROVIDERS, createVideoGenerator, type VideoGeneratorProvider } from "./video/registry.js";
 export { OpenAIProvider, openaiProvider } from "./openai/index.js";
 export { ClaudeProvider, claudeProvider } from "./claude/index.js";

packages/ai-providers/src/transcription/contract.ts

@@ -0,0 +1,50 @@
+/**
+ * The transcription (speech-to-text) contract.
+ *
+ * Audio bytes go in and a timed transcript comes back, or a `ProviderError`
+ * is thrown. Providers that detect the format from the upload's filename
+ * get one derived from the bytes, so callers never have to name it.
+ */
+
+import type { TranscriptGranularity, TranscriptSegment, TranscriptWord } from "../interface/types.js";
+
+export interface TranscriptionRequest {
+  audio: Uint8Array;
+  /** Upload filename; inferred from the bytes when omitted. */
+  filename?: string;
+  /** BCP-47 language hint ("en", "ko"). */
+  language?: string;
+  /** Timestamp granularity. Default "segment". */
+  granularity?: TranscriptGranularity;
+  /** Catalog model ID or alias; the provider's default when omitted. */
+  model?: string;
+}
+
+export interface Transcript {
+  text: string;
+  /** The language the provider detected. */
+  language?: string;
+  segments?: TranscriptSegment[];
+  words?: TranscriptWord[];
+  /** The catalog model that transcribed it. */
+  model: string;
+}
+
+export interface Transcriber {
+  /** Catalog provider id ("openai"). */
+  readonly transcriptionProvider: string;
+  /** Throws `ProviderError`. */
+  transcribeAudio(request: TranscriptionRequest): Promise<Transcript>;
+}
+
+/** A filename whose extension matches the audio container, from its magic bytes. */
+export function audioFilename(bytes: Uint8Array): string {
+  const ascii = (start: number, length: number) => String.fromCharCode(...bytes.subarray(start, start + length));
+  if (ascii(0, 4) === "RIFF" && ascii(8, 4) === "WAVE") return "audio.wav";
+  if (ascii(0, 3) === "ID3" || (bytes[0] === 0xff && (bytes[1] & 0xe0) === 0xe0)) return "audio.mp3";
+  if (ascii(4, 4) === "ftyp") return "audio.m4a";
+  if (ascii(0, 4) === "OggS") return "audio.ogg";
+  if (ascii(0, 4) === "fLaC") return "audio.flac";
+  if (bytes[0] === 0x1a && bytes[1] === 0x45 && bytes[2] === 0xdf && bytes[3] === 0xa3) return "audio.webm";
+  return "audio.mp3";
+}

packages/ai-providers/src/transcription/index.ts

@@ -0,0 +1,2 @@
+export * from "./contract.js";
+export * from "./registry.js";

packages/ai-providers/src/transcription/registry.ts

@@ -0,0 +1,21 @@
+/** One way to get a transcription provider by its catalog id. */
+
+import { ProviderError } from "../shared/errors.js";
+import { WhisperProvider } from "../whisper/WhisperProvider.js";
+import type { Transcriber } from "./contract.js";
+
+export const TRANSCRIBER_PROVIDERS = ["openai"] as const;
+
+/** A ready transcriber for `provider` (today: OpenAI Whisper). */
+export async function createTranscriber(provider: string, apiKey: string): Promise<Transcriber> {
+  if (provider !== "openai") {
+    throw new ProviderError({
+      kind: "invalid-request",
+      provider,
+      message: `Unknown transcription provider "${provider}". Available: ${TRANSCRIBER_PROVIDERS.join(", ")}.`,
+    });
+  }
+  const whisper = new WhisperProvider();
+  await whisper.initialize({ apiKey });
+  return whisper;
+}

packages/ai-providers/src/transcription/transcription-contract.test.ts

@@ -0,0 +1,65 @@
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+
+import { FakeProviderNetwork, jsonResponse } from "../testing/fake-provider-network.js";
+import { audioFilename } from "./contract.js";
+import { createTranscriber } from "./registry.js";
+
+const bytes = (...values: number[]) => new Uint8Array(values);
+const text = (s: string) => new Uint8Array(Buffer.from(s, "latin1"));
+
+describe("audioFilename", () => {
+  it("names the upload by the container in the bytes", () => {
+    expect(audioFilename(text("RIFF\0\0\0\0WAVEfmt "))).toBe("audio.wav");
+    expect(audioFilename(text("ID3\x04\0\0"))).toBe("audio.mp3");
+    expect(audioFilename(bytes(0xff, 0xfb, 0x90, 0x00))).toBe("audio.mp3");
+    expect(audioFilename(text("\0\0\0\x20ftypM4A "))).toBe("audio.m4a");
+    expect(audioFilename(text("OggS\0\x02"))).toBe("audio.ogg");
+    expect(audioFilename(text("fLaC\0\0"))).toBe("audio.flac");
+    expect(audioFilename(bytes(0x1a, 0x45, 0xdf, 0xa3))).toBe("audio.webm");
+  });
+});
+
+describe("Whisper transcription contract", () => {
+  let net: FakeProviderNetwork;
+  beforeEach(() => {
+    net = new FakeProviderNetwork();
+    net.install();
+  });
+  afterEach(() => net.uninstall());
+
+  const reply = {
+    text: "A paper boat drifts.",
+    language: "english",
+    segments: [{ id: 0, start: 0, end: 1.8, text: " A paper boat drifts." }],
+    words: [{ word: "A", start: 0, end: 0.2 }, { word: "paper", start: 0.2, end: 0.6 }],
+  };
+
+  it("transcribes with word timings on the catalog's default model", async () => {
+    net.on("POST", "api.openai.com", /^\/v1\/audio\/transcriptions$/, () => jsonResponse(reply));
+    const whisper = await createTranscriber("openai", "k");
+
+    const result = await whisper.transcribeAudio({ audio: text("RIFF\0\0\0\0WAVE"), granularity: "both", language: "en" });
+
+    expect(result).toMatchObject({ text: "A paper boat drifts.", language: "english", model: "whisper-1" });
+    expect(result.segments?.[0]).toMatchObject({ startTime: 0, endTime: 1.8, text: "A paper boat drifts." });
+    expect(result.words).toEqual([{ text: "A", start: 0, end: 0.2 }, { text: "paper", start: 0.2, end: 0.6 }]);
+  });
+
+  it("refuses audio over 25 MB before sending", async () => {
+    const whisper = await createTranscriber("openai", "k");
+    await expect(whisper.transcribeAudio({ audio: new Uint8Array(26 * 1024 * 1024) })).rejects.toMatchObject({
+      kind: "invalid-request",
+    });
+    expect(net.requests).toHaveLength(0);
+  });
+
+  it("classifies a bad key", async () => {
+    net.on("POST", "api.openai.com", /transcriptions$/, () => jsonResponse({ error: { message: "Incorrect API key provided" } }, 401));
+    const whisper = await createTranscriber("openai", "k");
+    await expect(whisper.transcribeAudio({ audio: text("ID3") })).rejects.toMatchObject({ kind: "auth" });
+  });
+
+  it("refuses an unknown provider", async () => {
+    await expect(createTranscriber("deepgram", "k")).rejects.toMatchObject({ kind: "invalid-request" });
+  });
+});

packages/ai-providers/src/whisper/WhisperProvider.test.ts

@@ -160,12 +160,18 @@ describe("WhisperProvider", () => {
       expect(result.error).toContain("rate limit exceeded");
     });
 
-    it("returns failed result on network error", async () => {
-      mockFetch.mockRejectedValueOnce(new Error("network down"));
+    it("returns failed result on network error, after the shared layer's retries", async () => {
+      vi.useFakeTimers();
+      mockFetch.mockRejectedValue(new Error("network down"));
+
+      const pending = provider.transcribe(audio);
+      await vi.advanceTimersByTimeAsync(5_000);
+      const result = await pending;
+      vi.useRealTimers();
 
-      const result = await provider.transcribe(audio);
       expect(result.status).toBe("failed");
-      expect(result.error).toBe("network down");
+      expect(result.error).toContain("network down");
+      expect(mockFetch).toHaveBeenCalledTimes(3);
     });
   });
 });

packages/ai-providers/src/whisper/WhisperProvider.ts

@@ -6,6 +6,10 @@ import type {
   TranscriptResult,
   TranscriptWord,
 } from "../interface/types.js";
+import { audioFilename, type Transcriber, type Transcript, type TranscriptionRequest } from "../transcription/contract.js";
+import { ProviderError } from "../shared/errors.js";
+import { providerRequest } from "../shared/http.js";
+import { resolveCatalogModel } from "../video/models.js";
 
 /**
  * OpenAI Whisper provider for speech-to-text.
@@ -15,8 +19,9 @@ import type {
  * endpoint. Word-level output mirrors the Hyperframes `transcript.json`
  * shape (`{text, start, end}`) so it can drive scene HTML GSAP timelines.
  */
-export class WhisperProvider implements AIProvider {
+export class WhisperProvider implements AIProvider, Transcriber {
   id = "whisper";
+  readonly transcriptionProvider = "openai";
   name = "OpenAI Whisper";
   description = "Speech-to-text transcription using OpenAI Whisper API";
   capabilities: AICapability[] = ["speech-to-text"];
@@ -37,96 +42,83 @@ export class WhisperProvider implements AIProvider {
     return !!this.apiKey;
   }
 
-  async transcribe(
-    audio: Blob,
-    language?: string,
-    options?: TranscribeOptions,
-  ): Promise<TranscriptResult> {
+  /** Whisper takes uploads up to 25 MB (docs). */
+  private static readonly MAX_BYTES = 25 * 1024 * 1024;
+
+  // ── Transcriber ───────────────────────────────────────────────────────
+
+  async transcribeAudio(request: TranscriptionRequest): Promise<Transcript> {
     if (!this.apiKey) {
-      return {
-        id: "",
-        status: "failed",
-        error: "Whisper API key not configured",
-      };
+      throw new ProviderError({ kind: "auth", provider: this.transcriptionProvider, message: "OpenAI API key not configured for Whisper" });
     }
+    if (request.audio.byteLength > WhisperProvider.MAX_BYTES) {
+      throw new ProviderError({
+        kind: "invalid-request",
+        provider: this.transcriptionProvider,
+        message: `Whisper takes audio up to 25 MB, not ${(request.audio.byteLength / 1024 / 1024).toFixed(1)} MB. Extract a lower-bitrate audio track first.`,
+      });
+    }
+    const model = resolveCatalogModel(this.transcriptionProvider, "transcription", request.model).id;
+    const granularity = request.granularity ?? "segment";
+    const form = new FormData();
+    // Whisper reads the format from the filename's extension.
+    form.append("file", new Blob([new Uint8Array(request.audio)]), request.filename ?? audioFilename(request.audio));
+    form.append("model", model);
+    form.append("response_format", "verbose_json");
+    if (granularity === "segment" || granularity === "both") form.append("timestamp_granularities[]", "segment");
+    if (granularity === "word" || granularity === "both") form.append("timestamp_granularities[]", "word");
+    if (request.language) form.append("language", request.language);
+
+    const response = await providerRequest(this.transcriptionProvider, `${this.baseUrl}/audio/transcriptions`, {
+      method: "POST",
+      headers: { Authorization: `Bearer ${this.apiKey}` },
+      body: form,
+    }, { timeoutMs: 10 * 60_000 });
+    const data = (await response.json()) as {
+      text: string;
+      language?: string;
+      segments?: Array<{ id: number; start: number; end: number; text: string }>;
+      words?: Array<{ word: string; start: number; end: number }>;
+    };
+    return {
+      text: data.text,
+      language: data.language,
+      model,
+      segments:
+        granularity === "segment" || granularity === "both"
+          ? data.segments?.map((seg, index) => ({
+              id: `segment-${index}`,
+              startTime: seg.start,
+              endTime: seg.end,
+              text: seg.text.trim(),
+              confidence: 1, // Whisper doesn't provide per-segment confidence
+            }))
+          : undefined,
+      words:
+        granularity === "word" || granularity === "both"
+          ? data.words?.map((w): TranscriptWord => ({ text: w.word, start: w.start, end: w.end }))
+          : undefined,
+    };
+  }
 
-    const granularity = options?.granularity ?? "segment";
-
+  /** Older interface; `transcribeAudio` is the contract. */
+  async transcribe(audio: Blob, language?: string, options?: TranscribeOptions): Promise<TranscriptResult> {
     try {
-      const formData = new FormData();
-      formData.append("file", audio, "audio.webm");
-      formData.append("model", "whisper-1");
-      formData.append("response_format", "verbose_json");
-
-      // Whisper API accepts multiple `timestamp_granularities[]` values.
-      if (granularity === "segment" || granularity === "both") {
-        formData.append("timestamp_granularities[]", "segment");
-      }
-      if (granularity === "word" || granularity === "both") {
-        formData.append("timestamp_granularities[]", "word");
-      }
-
-      const lang = language ?? options?.language;
-      if (lang) {
-        formData.append("language", lang);
-      }
-
-      const response = await fetch(`${this.baseUrl}/audio/transcriptions`, {
-        method: "POST",
-        headers: {
-          Authorization: `Bearer ${this.apiKey}`,
-        },
-        body: formData,
+      const transcript = await this.transcribeAudio({
+        audio: new Uint8Array(await audio.arrayBuffer()),
+        language: language ?? options?.language,
+        granularity: options?.granularity,
       });
-
-      if (!response.ok) {
-        const error = await response.text();
-        return {
-          id: "",
-          status: "failed",
-          error: `Transcription failed: ${error}`,
-        };
-      }
-
-      const data = await response.json() as {
-        text: string;
-        language?: string;
-        segments?: Array<{ id: number; start: number; end: number; text: string }>;
-        words?: Array<{ word: string; start: number; end: number }>;
-      };
-
-      const result: TranscriptResult = {
+      return {
         id: crypto.randomUUID(),
         status: "completed",
-        fullText: data.text,
-        detectedLanguage: data.language,
+        fullText: transcript.text,
+        detectedLanguage: transcript.language,
+        segments: transcript.segments,
+        words: transcript.words,
       };
-
-      if (granularity === "segment" || granularity === "both") {
-        result.segments = data.segments?.map((seg, index) => ({
-          id: `segment-${index}`,
-          startTime: seg.start,
-          endTime: seg.end,
-          text: seg.text.trim(),
-          confidence: 1, // Whisper doesn't provide per-segment confidence
-        }));
-      }
-
-      if (granularity === "word" || granularity === "both") {
-        result.words = data.words?.map((w): TranscriptWord => ({
-          text: w.word,
-          start: w.start,
-          end: w.end,
-        }));
-      }
-
-      return result;
     } catch (error) {
-      return {
-        id: "",
-        status: "failed",
-        error: error instanceof Error ? error.message : "Unknown error",
-      };
+      return { id: "", status: "failed", error: error instanceof Error ? error.message : "Unknown error" };
     }
   }
 }

packages/cli/src/commands/_shared/edit/caption.ts

@@ -6,13 +6,13 @@
  */
 
 import { existsSync } from "node:fs";
-import { readFile, writeFile, mkdir } from "node:fs/promises";
+import { writeFile, mkdir } from "node:fs/promises";
 import { dirname, basename, extname, join } from "node:path";
-import { WhisperProvider } from "@vibeframe/ai-providers";
 import { getVideoDuration } from "../../../utils/audio.js";
 import { formatSRT } from "../../../utils/subtitle.js";
 import { execSafe, commandExists } from "../../../utils/exec-safe.js";
 import { getVideoResolution } from "./_helpers.js";
+import { transcribeAudioFile } from "../transcription.js";
 
 /** Visual style preset for burned-in captions. */
 export type CaptionStyle = "minimal" | "bold" | "outline" | "karaoke";
@@ -133,12 +133,8 @@ export async function executeCaption(options: CaptionOptions): Promise<CaptionRe
       );
 
       // Step 2: Transcribe with Whisper
-      const whisper = new WhisperProvider();
-      await whisper.initialize({ apiKey: openaiKey });
 
-      const audioBuffer = await readFile(audioPath);
-      const audioBlob = new Blob([audioBuffer]);
-      const transcriptResult = await whisper.transcribe(audioBlob, language);
+      const transcriptResult = await transcribeAudioFile(audioPath, { apiKey: openaiKey, language: language });
 
       if (
         transcriptResult.status === "failed" ||