From 30dfd94273fe55be44f6417978c57fb5273a2f3e Mon Sep 17 00:00:00 2001 From: Kiyeon Jeon Date: Mon, 5 Oct 2026 16:42:25 +0900 Subject: [PATCH] feat: put transcription on one contract and route every Whisper call through it Twelve places in the CLI built their own Whisper client (create, initialize, read the file, wrap a Blob, transcribe), and jump-cut posted to the OpenAI API itself with a hardcoded "whisper-1". - `Transcriber` (ai-providers/src/transcription): `transcribeAudio(request)` returns a timed transcript or throws `ProviderError`; `createTranscriber(provider, key)` opens one. WhisperProvider implements it over `providerRequest`, takes its model from the catalog, refuses audio over Whisper's 25 MB limit before sending, and names the upload from the audio's magic bytes (it sent every file as "audio.webm"). - `transcribeAudioFile(path, ...)` in the CLI replaces all twelve call sites (audio, captions, highlights, edit, scene narration) and jump-cut's raw request. - `vibe audio transcribe` reports a missing file as NOT_FOUND instead of an API error. Claude-Session: https://claude.ai/code/session_011S1BLTaFctZNQzGERpkCuA --- packages/ai-providers/src/index.ts | 1 + .../src/transcription/contract.ts | 50 ++++++ .../ai-providers/src/transcription/index.ts | 2 + .../src/transcription/registry.ts | 21 +++ .../transcription-contract.test.ts | 65 ++++++++ .../src/whisper/WhisperProvider.test.ts | 14 +- .../src/whisper/WhisperProvider.ts | 156 +++++++++--------- .../cli/src/commands/_shared/edit/caption.ts | 10 +- .../cli/src/commands/_shared/edit/jump-cut.ts | 44 +---- .../commands/_shared/transcribe-narration.ts | 11 +- .../cli/src/commands/_shared/transcription.ts | 56 +++++++ packages/cli/src/commands/ai-audio.ts | 14 +- packages/cli/src/commands/ai-highlights.ts | 26 +-- packages/cli/src/commands/audio.ts | 18 +- packages/cli/src/commands/edit-cmd.ts | 14 +- 15 files changed, 312 insertions(+), 190 deletions(-) create mode 100644 packages/ai-providers/src/transcription/contract.ts create mode 100644 packages/ai-providers/src/transcription/index.ts create mode 100644 packages/ai-providers/src/transcription/registry.ts create mode 100644 packages/ai-providers/src/transcription/transcription-contract.test.ts create mode 100644 packages/cli/src/commands/_shared/transcription.ts diff --git a/packages/ai-providers/src/index.ts b/packages/ai-providers/src/index.ts index 04a0b608..310e1d68 100644 --- a/packages/ai-providers/src/index.ts +++ b/packages/ai-providers/src/index.ts @@ -76,6 +76,7 @@ export { export * from "./video/index.js"; export * from "./image/index.js"; export * from "./speech/index.js"; +export * from "./transcription/index.js"; export { VIDEO_GENERATOR_PROVIDERS, createVideoGenerator, type VideoGeneratorProvider } from "./video/registry.js"; export { OpenAIProvider, openaiProvider } from "./openai/index.js"; export { ClaudeProvider, claudeProvider } from "./claude/index.js"; diff --git a/packages/ai-providers/src/transcription/contract.ts b/packages/ai-providers/src/transcription/contract.ts new file mode 100644 index 00000000..01960c95 --- /dev/null +++ b/packages/ai-providers/src/transcription/contract.ts @@ -0,0 +1,50 @@ +/** + * The transcription (speech-to-text) contract. + * + * Audio bytes go in and a timed transcript comes back, or a `ProviderError` + * is thrown. Providers that detect the format from the upload's filename + * get one derived from the bytes, so callers never have to name it. + */ + +import type { TranscriptGranularity, TranscriptSegment, TranscriptWord } from "../interface/types.js"; + +export interface TranscriptionRequest { + audio: Uint8Array; + /** Upload filename; inferred from the bytes when omitted. */ + filename?: string; + /** BCP-47 language hint ("en", "ko"). */ + language?: string; + /** Timestamp granularity. Default "segment". */ + granularity?: TranscriptGranularity; + /** Catalog model ID or alias; the provider's default when omitted. */ + model?: string; +} + +export interface Transcript { + text: string; + /** The language the provider detected. */ + language?: string; + segments?: TranscriptSegment[]; + words?: TranscriptWord[]; + /** The catalog model that transcribed it. */ + model: string; +} + +export interface Transcriber { + /** Catalog provider id ("openai"). */ + readonly transcriptionProvider: string; + /** Throws `ProviderError`. */ + transcribeAudio(request: TranscriptionRequest): Promise; +} + +/** A filename whose extension matches the audio container, from its magic bytes. */ +export function audioFilename(bytes: Uint8Array): string { + const ascii = (start: number, length: number) => String.fromCharCode(...bytes.subarray(start, start + length)); + if (ascii(0, 4) === "RIFF" && ascii(8, 4) === "WAVE") return "audio.wav"; + if (ascii(0, 3) === "ID3" || (bytes[0] === 0xff && (bytes[1] & 0xe0) === 0xe0)) return "audio.mp3"; + if (ascii(4, 4) === "ftyp") return "audio.m4a"; + if (ascii(0, 4) === "OggS") return "audio.ogg"; + if (ascii(0, 4) === "fLaC") return "audio.flac"; + if (bytes[0] === 0x1a && bytes[1] === 0x45 && bytes[2] === 0xdf && bytes[3] === 0xa3) return "audio.webm"; + return "audio.mp3"; +} diff --git a/packages/ai-providers/src/transcription/index.ts b/packages/ai-providers/src/transcription/index.ts new file mode 100644 index 00000000..fa4577bd --- /dev/null +++ b/packages/ai-providers/src/transcription/index.ts @@ -0,0 +1,2 @@ +export * from "./contract.js"; +export * from "./registry.js"; diff --git a/packages/ai-providers/src/transcription/registry.ts b/packages/ai-providers/src/transcription/registry.ts new file mode 100644 index 00000000..50fc4adb --- /dev/null +++ b/packages/ai-providers/src/transcription/registry.ts @@ -0,0 +1,21 @@ +/** One way to get a transcription provider by its catalog id. */ + +import { ProviderError } from "../shared/errors.js"; +import { WhisperProvider } from "../whisper/WhisperProvider.js"; +import type { Transcriber } from "./contract.js"; + +export const TRANSCRIBER_PROVIDERS = ["openai"] as const; + +/** A ready transcriber for `provider` (today: OpenAI Whisper). */ +export async function createTranscriber(provider: string, apiKey: string): Promise { + if (provider !== "openai") { + throw new ProviderError({ + kind: "invalid-request", + provider, + message: `Unknown transcription provider "${provider}". Available: ${TRANSCRIBER_PROVIDERS.join(", ")}.`, + }); + } + const whisper = new WhisperProvider(); + await whisper.initialize({ apiKey }); + return whisper; +} diff --git a/packages/ai-providers/src/transcription/transcription-contract.test.ts b/packages/ai-providers/src/transcription/transcription-contract.test.ts new file mode 100644 index 00000000..65afe0ab --- /dev/null +++ b/packages/ai-providers/src/transcription/transcription-contract.test.ts @@ -0,0 +1,65 @@ +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import { FakeProviderNetwork, jsonResponse } from "../testing/fake-provider-network.js"; +import { audioFilename } from "./contract.js"; +import { createTranscriber } from "./registry.js"; + +const bytes = (...values: number[]) => new Uint8Array(values); +const text = (s: string) => new Uint8Array(Buffer.from(s, "latin1")); + +describe("audioFilename", () => { + it("names the upload by the container in the bytes", () => { + expect(audioFilename(text("RIFF\0\0\0\0WAVEfmt "))).toBe("audio.wav"); + expect(audioFilename(text("ID3\x04\0\0"))).toBe("audio.mp3"); + expect(audioFilename(bytes(0xff, 0xfb, 0x90, 0x00))).toBe("audio.mp3"); + expect(audioFilename(text("\0\0\0\x20ftypM4A "))).toBe("audio.m4a"); + expect(audioFilename(text("OggS\0\x02"))).toBe("audio.ogg"); + expect(audioFilename(text("fLaC\0\0"))).toBe("audio.flac"); + expect(audioFilename(bytes(0x1a, 0x45, 0xdf, 0xa3))).toBe("audio.webm"); + }); +}); + +describe("Whisper transcription contract", () => { + let net: FakeProviderNetwork; + beforeEach(() => { + net = new FakeProviderNetwork(); + net.install(); + }); + afterEach(() => net.uninstall()); + + const reply = { + text: "A paper boat drifts.", + language: "english", + segments: [{ id: 0, start: 0, end: 1.8, text: " A paper boat drifts." }], + words: [{ word: "A", start: 0, end: 0.2 }, { word: "paper", start: 0.2, end: 0.6 }], + }; + + it("transcribes with word timings on the catalog's default model", async () => { + net.on("POST", "api.openai.com", /^\/v1\/audio\/transcriptions$/, () => jsonResponse(reply)); + const whisper = await createTranscriber("openai", "k"); + + const result = await whisper.transcribeAudio({ audio: text("RIFF\0\0\0\0WAVE"), granularity: "both", language: "en" }); + + expect(result).toMatchObject({ text: "A paper boat drifts.", language: "english", model: "whisper-1" }); + expect(result.segments?.[0]).toMatchObject({ startTime: 0, endTime: 1.8, text: "A paper boat drifts." }); + expect(result.words).toEqual([{ text: "A", start: 0, end: 0.2 }, { text: "paper", start: 0.2, end: 0.6 }]); + }); + + it("refuses audio over 25 MB before sending", async () => { + const whisper = await createTranscriber("openai", "k"); + await expect(whisper.transcribeAudio({ audio: new Uint8Array(26 * 1024 * 1024) })).rejects.toMatchObject({ + kind: "invalid-request", + }); + expect(net.requests).toHaveLength(0); + }); + + it("classifies a bad key", async () => { + net.on("POST", "api.openai.com", /transcriptions$/, () => jsonResponse({ error: { message: "Incorrect API key provided" } }, 401)); + const whisper = await createTranscriber("openai", "k"); + await expect(whisper.transcribeAudio({ audio: text("ID3") })).rejects.toMatchObject({ kind: "auth" }); + }); + + it("refuses an unknown provider", async () => { + await expect(createTranscriber("deepgram", "k")).rejects.toMatchObject({ kind: "invalid-request" }); + }); +}); diff --git a/packages/ai-providers/src/whisper/WhisperProvider.test.ts b/packages/ai-providers/src/whisper/WhisperProvider.test.ts index 845dd811..be8963fe 100644 --- a/packages/ai-providers/src/whisper/WhisperProvider.test.ts +++ b/packages/ai-providers/src/whisper/WhisperProvider.test.ts @@ -160,12 +160,18 @@ describe("WhisperProvider", () => { expect(result.error).toContain("rate limit exceeded"); }); - it("returns failed result on network error", async () => { - mockFetch.mockRejectedValueOnce(new Error("network down")); + it("returns failed result on network error, after the shared layer's retries", async () => { + vi.useFakeTimers(); + mockFetch.mockRejectedValue(new Error("network down")); + + const pending = provider.transcribe(audio); + await vi.advanceTimersByTimeAsync(5_000); + const result = await pending; + vi.useRealTimers(); - const result = await provider.transcribe(audio); expect(result.status).toBe("failed"); - expect(result.error).toBe("network down"); + expect(result.error).toContain("network down"); + expect(mockFetch).toHaveBeenCalledTimes(3); }); }); }); diff --git a/packages/ai-providers/src/whisper/WhisperProvider.ts b/packages/ai-providers/src/whisper/WhisperProvider.ts index 270acb8b..9d0bd252 100644 --- a/packages/ai-providers/src/whisper/WhisperProvider.ts +++ b/packages/ai-providers/src/whisper/WhisperProvider.ts @@ -6,6 +6,10 @@ import type { TranscriptResult, TranscriptWord, } from "../interface/types.js"; +import { audioFilename, type Transcriber, type Transcript, type TranscriptionRequest } from "../transcription/contract.js"; +import { ProviderError } from "../shared/errors.js"; +import { providerRequest } from "../shared/http.js"; +import { resolveCatalogModel } from "../video/models.js"; /** * OpenAI Whisper provider for speech-to-text. @@ -15,8 +19,9 @@ import type { * endpoint. Word-level output mirrors the Hyperframes `transcript.json` * shape (`{text, start, end}`) so it can drive scene HTML GSAP timelines. */ -export class WhisperProvider implements AIProvider { +export class WhisperProvider implements AIProvider, Transcriber { id = "whisper"; + readonly transcriptionProvider = "openai"; name = "OpenAI Whisper"; description = "Speech-to-text transcription using OpenAI Whisper API"; capabilities: AICapability[] = ["speech-to-text"]; @@ -37,96 +42,83 @@ export class WhisperProvider implements AIProvider { return !!this.apiKey; } - async transcribe( - audio: Blob, - language?: string, - options?: TranscribeOptions, - ): Promise { + /** Whisper takes uploads up to 25 MB (docs). */ + private static readonly MAX_BYTES = 25 * 1024 * 1024; + + // ── Transcriber ─────────────────────────────────────────────────────── + + async transcribeAudio(request: TranscriptionRequest): Promise { if (!this.apiKey) { - return { - id: "", - status: "failed", - error: "Whisper API key not configured", - }; + throw new ProviderError({ kind: "auth", provider: this.transcriptionProvider, message: "OpenAI API key not configured for Whisper" }); } + if (request.audio.byteLength > WhisperProvider.MAX_BYTES) { + throw new ProviderError({ + kind: "invalid-request", + provider: this.transcriptionProvider, + message: `Whisper takes audio up to 25 MB, not ${(request.audio.byteLength / 1024 / 1024).toFixed(1)} MB. Extract a lower-bitrate audio track first.`, + }); + } + const model = resolveCatalogModel(this.transcriptionProvider, "transcription", request.model).id; + const granularity = request.granularity ?? "segment"; + const form = new FormData(); + // Whisper reads the format from the filename's extension. + form.append("file", new Blob([new Uint8Array(request.audio)]), request.filename ?? audioFilename(request.audio)); + form.append("model", model); + form.append("response_format", "verbose_json"); + if (granularity === "segment" || granularity === "both") form.append("timestamp_granularities[]", "segment"); + if (granularity === "word" || granularity === "both") form.append("timestamp_granularities[]", "word"); + if (request.language) form.append("language", request.language); + + const response = await providerRequest(this.transcriptionProvider, `${this.baseUrl}/audio/transcriptions`, { + method: "POST", + headers: { Authorization: `Bearer ${this.apiKey}` }, + body: form, + }, { timeoutMs: 10 * 60_000 }); + const data = (await response.json()) as { + text: string; + language?: string; + segments?: Array<{ id: number; start: number; end: number; text: string }>; + words?: Array<{ word: string; start: number; end: number }>; + }; + return { + text: data.text, + language: data.language, + model, + segments: + granularity === "segment" || granularity === "both" + ? data.segments?.map((seg, index) => ({ + id: `segment-${index}`, + startTime: seg.start, + endTime: seg.end, + text: seg.text.trim(), + confidence: 1, // Whisper doesn't provide per-segment confidence + })) + : undefined, + words: + granularity === "word" || granularity === "both" + ? data.words?.map((w): TranscriptWord => ({ text: w.word, start: w.start, end: w.end })) + : undefined, + }; + } - const granularity = options?.granularity ?? "segment"; - + /** Older interface; `transcribeAudio` is the contract. */ + async transcribe(audio: Blob, language?: string, options?: TranscribeOptions): Promise { try { - const formData = new FormData(); - formData.append("file", audio, "audio.webm"); - formData.append("model", "whisper-1"); - formData.append("response_format", "verbose_json"); - - // Whisper API accepts multiple `timestamp_granularities[]` values. - if (granularity === "segment" || granularity === "both") { - formData.append("timestamp_granularities[]", "segment"); - } - if (granularity === "word" || granularity === "both") { - formData.append("timestamp_granularities[]", "word"); - } - - const lang = language ?? options?.language; - if (lang) { - formData.append("language", lang); - } - - const response = await fetch(`${this.baseUrl}/audio/transcriptions`, { - method: "POST", - headers: { - Authorization: `Bearer ${this.apiKey}`, - }, - body: formData, + const transcript = await this.transcribeAudio({ + audio: new Uint8Array(await audio.arrayBuffer()), + language: language ?? options?.language, + granularity: options?.granularity, }); - - if (!response.ok) { - const error = await response.text(); - return { - id: "", - status: "failed", - error: `Transcription failed: ${error}`, - }; - } - - const data = await response.json() as { - text: string; - language?: string; - segments?: Array<{ id: number; start: number; end: number; text: string }>; - words?: Array<{ word: string; start: number; end: number }>; - }; - - const result: TranscriptResult = { + return { id: crypto.randomUUID(), status: "completed", - fullText: data.text, - detectedLanguage: data.language, + fullText: transcript.text, + detectedLanguage: transcript.language, + segments: transcript.segments, + words: transcript.words, }; - - if (granularity === "segment" || granularity === "both") { - result.segments = data.segments?.map((seg, index) => ({ - id: `segment-${index}`, - startTime: seg.start, - endTime: seg.end, - text: seg.text.trim(), - confidence: 1, // Whisper doesn't provide per-segment confidence - })); - } - - if (granularity === "word" || granularity === "both") { - result.words = data.words?.map((w): TranscriptWord => ({ - text: w.word, - start: w.start, - end: w.end, - })); - } - - return result; } catch (error) { - return { - id: "", - status: "failed", - error: error instanceof Error ? error.message : "Unknown error", - }; + return { id: "", status: "failed", error: error instanceof Error ? error.message : "Unknown error" }; } } } diff --git a/packages/cli/src/commands/_shared/edit/caption.ts b/packages/cli/src/commands/_shared/edit/caption.ts index 4e7ff411..bd3c6a68 100644 --- a/packages/cli/src/commands/_shared/edit/caption.ts +++ b/packages/cli/src/commands/_shared/edit/caption.ts @@ -6,13 +6,13 @@ */ import { existsSync } from "node:fs"; -import { readFile, writeFile, mkdir } from "node:fs/promises"; +import { writeFile, mkdir } from "node:fs/promises"; import { dirname, basename, extname, join } from "node:path"; -import { WhisperProvider } from "@vibeframe/ai-providers"; import { getVideoDuration } from "../../../utils/audio.js"; import { formatSRT } from "../../../utils/subtitle.js"; import { execSafe, commandExists } from "../../../utils/exec-safe.js"; import { getVideoResolution } from "./_helpers.js"; +import { transcribeAudioFile } from "../transcription.js"; /** Visual style preset for burned-in captions. */ export type CaptionStyle = "minimal" | "bold" | "outline" | "karaoke"; @@ -133,12 +133,8 @@ export async function executeCaption(options: CaptionOptions): Promise { - const audioBuffer = await readFile(audioPath); - const audioBlob = new Blob([audioBuffer]); - - const formData = new FormData(); - formData.append("file", audioBlob, "audio.wav"); - formData.append("model", "whisper-1"); - formData.append("response_format", "verbose_json"); - formData.append("timestamp_granularities[]", "word"); - - if (language) { - formData.append("language", language); - } - - const response = await fetch("https://api.openai.com/v1/audio/transcriptions", { - method: "POST", - headers: { - Authorization: `Bearer ${apiKey}`, - }, - body: formData, - }); - - if (!response.ok) { - const error = await response.text(); - throw new Error(`Whisper transcription failed: ${error}`); + const result = await transcribeAudioFile(audioPath, { apiKey, language, granularity: "word" }); + if (result.status !== "completed") { + throw new Error(`Whisper transcription failed: ${result.error ?? "unknown error"}`); } - - const data = (await response.json()) as { - text: string; - words?: Array<{ word: string; start: number; end: number }>; - }; - return { - words: data.words || [], - text: data.text, + words: (result.words ?? []).map((w) => ({ word: w.text, start: w.start, end: w.end })), + text: result.fullText ?? "", }; } diff --git a/packages/cli/src/commands/_shared/transcribe-narration.ts b/packages/cli/src/commands/_shared/transcribe-narration.ts index 58c80a9d..84bbaca0 100644 --- a/packages/cli/src/commands/_shared/transcribe-narration.ts +++ b/packages/cli/src/commands/_shared/transcribe-narration.ts @@ -15,9 +15,9 @@ import { existsSync } from "node:fs"; import { readFile } from "node:fs/promises"; import { join } from "node:path"; -import { WhisperProvider } from "@vibeframe/ai-providers"; import type { SceneTranscriptWord } from "./scene-html-emit.js"; +import { transcribeAudioFile } from "./transcription.js"; export interface TranscribeNarrationOptions { /** OpenAI API key for Whisper. */ @@ -36,14 +36,7 @@ export async function transcribeNarrationWords( opts: TranscribeNarrationOptions ): Promise { try { - const whisper = new WhisperProvider(); - await whisper.initialize({ apiKey: opts.apiKey }); - const audioBytes = await readFile(audioAbsPath); - const audioBlob = new Blob([new Uint8Array(audioBytes)]); - const transcript = await whisper.transcribe(audioBlob, undefined, { - granularity: "word", - language: opts.language, - }); + const transcript = await transcribeAudioFile(audioAbsPath, { apiKey: opts.apiKey, granularity: "word", language: opts.language }); if (transcript.status === "completed" && transcript.words?.length) { return transcript.words.map((w) => ({ text: w.text, start: w.start, end: w.end })); } diff --git a/packages/cli/src/commands/_shared/transcription.ts b/packages/cli/src/commands/_shared/transcription.ts new file mode 100644 index 00000000..d4effb63 --- /dev/null +++ b/packages/cli/src/commands/_shared/transcription.ts @@ -0,0 +1,56 @@ +/** + * The CLI side of the transcription contract: transcribe an audio file on + * disk and get the result shape the commands already use. Every command + * that needs a transcript (audio, captions, highlights, jump-cut, scene + * narration) goes through here. + */ + +import { readFile } from "node:fs/promises"; +import { basename } from "node:path"; +import { + createTranscriber, + isProviderError, + type ProviderErrorKind, + type TranscriptGranularity, + type TranscriptResult, +} from "@vibeframe/ai-providers"; + +export interface TranscribeFileOptions { + apiKey: string; + language?: string; + granularity?: TranscriptGranularity; +} + +/** Transcribe `path` with Whisper. Failures come back as `status: "failed"` with an error kind. */ +export async function transcribeAudioFile( + path: string, + options: TranscribeFileOptions +): Promise { + try { + const transcriber = await createTranscriber("openai", options.apiKey); + const audio = new Uint8Array(await readFile(path)); + const name = basename(path); + const transcript = await transcriber.transcribeAudio({ + audio, + // A real extension helps Whisper; temp files without one are sniffed from the bytes. + filename: /\.(mp3|wav|m4a|mp4|ogg|flac|webm|mpeg|mpga)$/i.test(name) ? name : undefined, + language: options.language, + granularity: options.granularity, + }); + return { + id: `transcript-${Date.now()}`, + status: "completed", + fullText: transcript.text, + detectedLanguage: transcript.language, + segments: transcript.segments, + words: transcript.words, + }; + } catch (error) { + return { + id: "", + status: "failed", + error: error instanceof Error ? error.message : String(error), + errorKind: isProviderError(error) ? error.kind : undefined, + }; + } +} diff --git a/packages/cli/src/commands/ai-audio.ts b/packages/cli/src/commands/ai-audio.ts index 7159f487..c75b9dd0 100644 --- a/packages/cli/src/commands/ai-audio.ts +++ b/packages/cli/src/commands/ai-audio.ts @@ -17,12 +17,12 @@ import { resolve, dirname, basename, extname } from "node:path"; import { readFile, writeFile } from "node:fs/promises"; import { existsSync } from "node:fs"; import { - WhisperProvider, ElevenLabsProvider, ClaudeProvider, } from "@vibeframe/ai-providers"; import { execSafe, execSafeSync, commandExists } from "../utils/exec-safe.js"; import { detectFormat, formatTranscript } from "../utils/subtitle.js"; +import { transcribeAudioFile } from "./_shared/transcription.js"; // ============================================================================ // Transcribe @@ -55,13 +55,9 @@ export async function executeTranscribe(options: TranscribeOptions): Promise { audioPath = tempAudioPath; } - const whisper = new WhisperProvider(); - await whisper.initialize({ apiKey: openaiKey }); - const audioBuffer = await readFile(audioPath); - const audioBlob = new Blob([audioBuffer]); - const transcriptResult = await whisper.transcribe(audioBlob, source); + const transcriptResult = await transcribeAudioFile(audioPath, { apiKey: openaiKey, language: source }); if (transcriptResult.status === "failed" || !transcriptResult.segments) { return { success: false, error: `Transcription failed: ${transcriptResult.error}` }; diff --git a/packages/cli/src/commands/ai-highlights.ts b/packages/cli/src/commands/ai-highlights.ts index 6c44320f..c64b5785 100644 --- a/packages/cli/src/commands/ai-highlights.ts +++ b/packages/cli/src/commands/ai-highlights.ts @@ -20,7 +20,6 @@ import chalk from "chalk"; import ora from "ora"; import { GeminiProvider, - WhisperProvider, ClaudeProvider, type Highlight, type HighlightCriteria, @@ -32,6 +31,7 @@ import { formatTime } from "./ai-helpers.js"; import { execSafe, commandExists, ffprobeDuration } from "../utils/exec-safe.js"; import { exitWithError, outputSuccess, authError, notFoundError, apiError, generalError } from "./output.js"; import { validateOutputPath } from "./validate.js"; +import { transcribeAudioFile } from "./_shared/transcription.js"; // ============================================================================ // Shared helpers @@ -274,12 +274,8 @@ Analyze both what is SHOWN (visual cues, actions, expressions) and what is SAID sourceDuration = await ffprobeDuration(absPath); } - const whisper = new WhisperProvider(); - await whisper.initialize({ apiKey: openaiApiKey }); - const audioBuffer = await readFile(audioPath); - const audioBlob = new Blob([audioBuffer]); - const transcriptResult = await whisper.transcribe(audioBlob, options.language); + const transcriptResult = await transcribeAudioFile(audioPath, { apiKey: openaiApiKey, language: options.language }); if (tempAudioPath && existsSync(tempAudioPath)) { const { unlink: unlinkFile } = await import("node:fs/promises"); @@ -551,12 +547,8 @@ Analyze both VISUALS (expressions, actions, scene changes) and AUDIO (speech, re const tempAudio = absPath.replace(/(\.[^.]+)$/, "-temp-audio.mp3"); await execSafe("ffmpeg", ["-i", absPath, "-vn", "-acodec", "libmp3lame", "-q:a", "2", tempAudio, "-y"]); - const whisper = new WhisperProvider(); - await whisper.initialize({ apiKey: openaiApiKey }); - const audioBuffer = await readFile(tempAudio); - const audioBlob = new Blob([audioBuffer]); - const transcript = await whisper.transcribe(audioBlob, options.language); + const transcript = await transcribeAudioFile(tempAudio, { apiKey: openaiApiKey, language: options.language }); try { const { unlink: unlinkFile } = await import("node:fs/promises"); @@ -890,12 +882,8 @@ Analyze both what is SHOWN (visual cues, actions, expressions) and what is SAID const transcribeSpinner = ora("📝 Transcribing with Whisper...").start(); - const whisper = new WhisperProvider(); - await whisper.initialize({ apiKey: openaiApiKey }); - const audioBuffer = await readFile(audioPath); - const audioBlob = new Blob([audioBuffer]); - const transcriptResult = await whisper.transcribe(audioBlob, options.language); + const transcriptResult = await transcribeAudioFile(audioPath, { apiKey: openaiApiKey, language: options.language }); if (transcriptResult.status === "failed" || !transcriptResult.segments) { transcribeSpinner.fail("Transcription failed"); @@ -1215,12 +1203,8 @@ Analyze both VISUALS (expressions, actions, scene changes) and AUDIO (speech, re spinner.text = "Transcribing audio..."; - const whisper = new WhisperProvider(); - await whisper.initialize({ apiKey: openaiApiKey }); - const audioBuffer = await readFile(tempAudio); - const audioBlob = new Blob([audioBuffer]); - const transcript = await whisper.transcribe(audioBlob, options.language); + const transcript = await transcribeAudioFile(tempAudio, { apiKey: openaiApiKey, language: options.language }); try { const { unlink: unlinkFile } = await import("node:fs/promises"); diff --git a/packages/cli/src/commands/audio.ts b/packages/cli/src/commands/audio.ts index 14111df4..926623b7 100644 --- a/packages/cli/src/commands/audio.ts +++ b/packages/cli/src/commands/audio.ts @@ -20,7 +20,7 @@ import { readFile, writeFile } from "node:fs/promises"; import { existsSync } from "node:fs"; import chalk from "chalk"; import ora from "ora"; -import { WhisperProvider, ElevenLabsProvider, ClaudeProvider } from "@vibeframe/ai-providers"; +import { ElevenLabsProvider, ClaudeProvider } from "@vibeframe/ai-providers"; import { getApiKey, requireApiKey } from "../utils/api-key.js"; import { execSafe, commandExists, execSafeSync } from "../utils/exec-safe.js"; import { detectFormat, formatTranscript } from "../utils/subtitle.js"; @@ -36,6 +36,7 @@ import { generalError, } from "./output.js"; import { rejectControlChars, validateOutputPath } from "./validate.js"; +import { transcribeAudioFile } from "./_shared/transcription.js"; export const audioCommand = new Command("audio") .alias("au") @@ -81,21 +82,20 @@ audioCommand if (options.output) { validateOutputPath(options.output); } + if (!existsSync(resolve(process.cwd(), audioPath))) { + exitWithError(notFoundError(resolve(process.cwd(), audioPath))); + } const apiKey = await requireApiKey("OPENAI_API_KEY", "OpenAI", options.apiKey); const spinner = ora("Initializing Whisper...").start(); - const whisper = new WhisperProvider(); - await whisper.initialize({ apiKey }); spinner.text = "Reading audio file..."; const absPath = resolve(process.cwd(), audioPath); - const audioBuffer = await readFile(absPath); - const audioBlob = new Blob([audioBuffer]); spinner.text = "Transcribing..."; - const result = await whisper.transcribe(audioBlob, options.language); + const result = await transcribeAudioFile(absPath, { apiKey, language: options.language }); if (result.status === "failed") { spinner.fail("Transcription failed"); @@ -463,13 +463,9 @@ audioCommand // Step 2: Transcribe with Whisper spinner.text = "Transcribing audio..."; - const whisper = new WhisperProvider(); - await whisper.initialize({ apiKey: openaiKey }); - const audioBuffer = await readFile(audioPath); - const audioBlob = new Blob([audioBuffer]); - const transcriptResult = await whisper.transcribe(audioBlob, options.source); + const transcriptResult = await transcribeAudioFile(audioPath, { apiKey: openaiKey, language: options.source }); if (transcriptResult.status === "failed" || !transcriptResult.segments) { spinner.fail("Transcription failed"); diff --git a/packages/cli/src/commands/edit-cmd.ts b/packages/cli/src/commands/edit-cmd.ts index 39384a67..34841a39 100644 --- a/packages/cli/src/commands/edit-cmd.ts +++ b/packages/cli/src/commands/edit-cmd.ts @@ -28,7 +28,7 @@ import { resolve } from "node:path"; import { readFile, writeFile } from "node:fs/promises"; import chalk from "chalk"; import ora from "ora"; -import { WhisperProvider, ClaudeProvider, findModel, modelAliases } from "@vibeframe/ai-providers"; +import { ClaudeProvider, findModel, modelAliases } from "@vibeframe/ai-providers"; import { requireApiKey } from "../utils/api-key.js"; import { execSafe, commandExists } from "../utils/exec-safe.js"; import { formatTime } from "./ai-helpers.js"; @@ -41,6 +41,7 @@ import { rejectControlChars, validateOutputPath } from "./validate.js"; import { applyTiers } from "./_shared/cost-tier.js"; import { executeImageEdit } from "./ai-image.js"; import { IMAGE_PROVIDER_ENV, IMAGE_PROVIDER_LABELS } from "./_shared/image-jobs.js"; +import { transcribeAudioFile } from "./_shared/transcription.js"; export const editCommand = new Command("edit") .alias("ed") @@ -376,12 +377,8 @@ editCommand // Step 2: Transcribe spinner.text = "Transcribing audio..."; - const whisper = new WhisperProvider(); - await whisper.initialize({ apiKey: openaiApiKey }); - const audioBuffer = await readFile(tempAudio); - const audioBlob = new Blob([audioBuffer]); - const transcript = await whisper.transcribe(audioBlob, options.language); + const transcript = await transcribeAudioFile(tempAudio, { apiKey: openaiApiKey, language: options.language }); if (!transcript.segments || transcript.segments.length === 0) { spinner.fail("No transcript segments found"); @@ -1108,10 +1105,7 @@ export async function executeSpeedRamp(options: SpeedRampOptions): Promise