From 8b4195dfc26ddd80192b0daa258ef3586305e042 Mon Sep 17 00:00:00 2001 From: Kiyeon Jeon Date: Mon, 5 Oct 2026 17:12:54 +0900 Subject: [PATCH] fix: dub with a real translation, any TTS provider, and lines placed on time `vibe audio dub` "translated" by calling Claude's storyboard analysis and reading each scene description as a translated line, so output was garbage or silently the source text. It then concatenated the spoken lines, ignoring when each was said, and required ElevenLabs and Anthropic keys. - `translateTexts` (extracted from translate-srt) translates in batches with Claude, or OpenAI without an Anthropic key, matching replies by their [N] index so a skipped or reordered line cannot shift later ones. translate-srt uses it too. - Each line is spoken through `tts-resolve` (new `--tts auto|elevenlabs| openai|kokoro`), placed at its segment's start time with ffmpeg, and sped up (up to 1.5x) when it would overrun its slot. - Only OPENAI_API_KEY is required (Whisper); temp files live in a temp dir that is always removed. Claude-Session: https://claude.ai/code/session_011S1BLTaFctZNQzGERpkCuA --- docs/cli-reference.md | 3 +- .../envelope-snapshots.test.ts.snap | 13 +- .../_shared/edit/translate-srt.test.ts | 2 +- .../commands/_shared/edit/translate-srt.ts | 112 +--------- .../commands/_shared/translate-texts.test.ts | 64 ++++++ .../src/commands/_shared/translate-texts.ts | 92 ++++++++ packages/cli/src/commands/ai-audio.ts | 199 ++++++++---------- packages/cli/src/commands/audio.ts | 8 +- packages/cli/src/tools/manifest/audio.ts | 7 +- 9 files changed, 279 insertions(+), 221 deletions(-) create mode 100644 packages/cli/src/commands/_shared/translate-texts.test.ts create mode 100644 packages/cli/src/commands/_shared/translate-texts.ts diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 4a7a74cd..1b48fb45 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -1098,7 +1098,8 @@ Cost tier: `high` - `media` _(string)_ **required** - Input media file (video or audio) - `language` _(string)_ - Target language code (e.g., es, ko, ja) (required) - `source` _(string)_ - Source language code (default: auto-detect) -- `voice` _(string)_ - ElevenLabs voice ID for output +- `voice` _(string)_ - Voice for the TTS provider (ElevenLabs name/ID, OpenAI voice, Kokoro voice) +- `tts` _(string)_ _(auto \| elevenlabs \| openai \| kokoro)_ _(default: `"auto"`)_ - TTS provider: auto (default), elevenlabs, openai, kokoro - `analyzeOnly` _(boolean)_ - Only analyze and show timing, don't generate audio - `output` _(string)_ - Output file path - `dryRun` _(boolean)_ - Preview parameters without executing diff --git a/packages/cli/src/commands/__snapshots__/envelope-snapshots.test.ts.snap b/packages/cli/src/commands/__snapshots__/envelope-snapshots.test.ts.snap index 35d0b700..a4b5f1ee 100644 --- a/packages/cli/src/commands/__snapshots__/envelope-snapshots.test.ts.snap +++ b/packages/cli/src/commands/__snapshots__/envelope-snapshots.test.ts.snap @@ -149,8 +149,19 @@ exports[`CLI --describe schemas (drift detection) > vibe audio dub --describe 1` "description": "Source language code (default: auto-detect)", "type": "string", }, + "tts": { + "default": "auto", + "description": "TTS provider: auto (default), elevenlabs, openai, kokoro", + "enum": [ + "auto", + "elevenlabs", + "openai", + "kokoro", + ], + "type": "string", + }, "voice": { - "description": "ElevenLabs voice ID for output", + "description": "Voice for the TTS provider (ElevenLabs name/ID, OpenAI voice, Kokoro voice)", "type": "string", }, }, diff --git a/packages/cli/src/commands/_shared/edit/translate-srt.test.ts b/packages/cli/src/commands/_shared/edit/translate-srt.test.ts index d179e9bc..e2baf052 100644 --- a/packages/cli/src/commands/_shared/edit/translate-srt.test.ts +++ b/packages/cli/src/commands/_shared/edit/translate-srt.test.ts @@ -67,7 +67,7 @@ describe("executeTranslateSrt", () => { const result = await run("claude"); expect(result.success).toBe(false); - expect(result.error).toContain("404 Not Found"); + expect(result.error).toContain("404"); expect(result.error).toContain("model: not_found"); }); }); diff --git a/packages/cli/src/commands/_shared/edit/translate-srt.ts b/packages/cli/src/commands/_shared/edit/translate-srt.ts index ce8edb15..b1659822 100644 --- a/packages/cli/src/commands/_shared/edit/translate-srt.ts +++ b/packages/cli/src/commands/_shared/edit/translate-srt.ts @@ -9,7 +9,7 @@ import { existsSync } from "node:fs"; import { readFile, writeFile } from "node:fs/promises"; import { formatSRT, parseSRT } from "../../../utils/subtitle.js"; -import { defaultModel } from "@vibeframe/ai-providers"; +import { translateTexts } from "../translate-texts.js"; export interface TranslateSrtOptions { /** Path to the source SRT subtitle file */ @@ -69,106 +69,16 @@ export async function executeTranslateSrt( return { success: false, error: "No subtitle segments found in SRT file" }; } - // Batch translate segments (~30 at a time) - const batchSize = 30; - const translatedSegments: { startTime: number; endTime: number; text: string }[] = []; - - for (let i = 0; i < segments.length; i += batchSize) { - const batch = segments.slice(i, i + batchSize); - const textsToTranslate = batch.map((s, idx) => `[${idx}] ${s.text}`).join("\n"); - - const translatePrompt = - `Translate the following subtitle texts to ${targetLanguage}.` + - (sourceLanguage ? ` The source language is ${sourceLanguage}.` : "") + - ` Return ONLY the translated texts, one per line, preserving the [N] prefix format exactly. ` + - `Do not add explanations.\n\n${textsToTranslate}`; - - let translatedText: string; - - if (provider === "openai") { - const openaiKey = apiKey || process.env.OPENAI_API_KEY; - if (!openaiKey) { - return { - success: false, - error: - "OpenAI API key required for translation. Run 'vibe setup' or set OPENAI_API_KEY in .env", - }; - } - const response = await fetch("https://api.openai.com/v1/chat/completions", { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${openaiKey}`, - }, - body: JSON.stringify({ - model: defaultModel("openai", "llm").id, - messages: [{ role: "user", content: translatePrompt }], - }), - }); - if (!response.ok) { - return { - success: false, - error: `OpenAI API error: ${response.status} ${response.statusText}: ${await response.text()}`, - }; - } - const data = (await response.json()) as { - choices: Array<{ message: { content: string } }>; - }; - translatedText = data.choices[0]?.message?.content || ""; - } else { - const claudeKey = apiKey || process.env.ANTHROPIC_API_KEY; - if (!claudeKey) { - return { - success: false, - error: - "Anthropic API key required for translation. Run 'vibe setup' or set ANTHROPIC_API_KEY in .env", - }; - } - const response = await fetch("https://api.anthropic.com/v1/messages", { - method: "POST", - headers: { - "Content-Type": "application/json", - "x-api-key": claudeKey, - "anthropic-version": "2023-06-01", - }, - body: JSON.stringify({ - model: defaultModel("claude", "llm").id, - max_tokens: 16000, - messages: [{ role: "user", content: translatePrompt }], - }), - }); - if (!response.ok) { - return { - success: false, - error: `Claude API error: ${response.status} ${response.statusText}: ${await response.text()}`, - }; - } - const data = (await response.json()) as { - content: Array<{ type: string; text: string }>; - }; - translatedText = data.content?.find((c) => c.type === "text")?.text || ""; - } - - // Parse translated lines - const translatedLines = translatedText.trim().split("\n"); - for (let j = 0; j < batch.length; j++) { - const seg = batch[j]; - // Try to match [N] prefix - const line = translatedLines[j]; - let text: string; - if (line) { - text = line.replace(/^\[\d+\]\s*/, "").trim(); - } else { - // Fallback: use original text if translation is missing - text = seg.text; - } - translatedSegments.push({ - startTime: seg.startTime, - endTime: seg.endTime, - text, - }); - } - } + const translated = await translateTexts( + segments.map((s) => s.text), + { targetLanguage, sourceLanguage, provider, apiKey }, + ); + if (!translated.success) return { success: false, error: translated.error }; + const translatedSegments = segments.map((s, i) => ({ + startTime: s.startTime, + endTime: s.endTime, + text: translated.texts[i], + })); // Format as SRT and write const translatedSrt = formatSRT(translatedSegments); diff --git a/packages/cli/src/commands/_shared/translate-texts.test.ts b/packages/cli/src/commands/_shared/translate-texts.test.ts new file mode 100644 index 00000000..73f262bd --- /dev/null +++ b/packages/cli/src/commands/_shared/translate-texts.test.ts @@ -0,0 +1,64 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +// Keys come only from process.env here, never this machine's config or .env. +vi.mock("../../utils/api-key.js", () => ({ + getConfiguredApiKey: async (envVar: string, optionValue?: string) => optionValue ?? process.env[envVar], +})); + +import { FakeProviderNetwork, jsonResponse } from "../../testing/fake-provider-network.js"; +import { languageName, translateTexts } from "./translate-texts.js"; + +let net: FakeProviderNetwork; +const savedEnv = { ...process.env }; + +beforeEach(() => { + net = new FakeProviderNetwork(); + net.install(); + delete process.env.ANTHROPIC_API_KEY; + delete process.env.OPENAI_API_KEY; +}); + +afterEach(() => { + net.uninstall(); + process.env = { ...savedEnv }; +}); + +describe("translateTexts", () => { + it("matches lines by their [N] index, keeping the source text for any line left out", async () => { + process.env.ANTHROPIC_API_KEY = "k"; + // Out of order, line 1 missing, and a chatty preamble. + net.on("POST", "api.anthropic.com", /\/v1\/messages$/, () => + jsonResponse({ content: [{ type: "text", text: "Here you go:\n[2] 셋\n[0] 하나" }] }) + ); + + const result = await translateTexts(["one", "two", "three"], { targetLanguage: "ko" }); + + expect(result).toEqual({ success: true, provider: "claude", texts: ["하나", "two", "셋"] }); + expect(String((net.requests[0].body as { messages: Array<{ content: string }> }).messages[0].content)).toContain("to Korean"); + }); + + it("uses OpenAI when there is no Anthropic key", async () => { + process.env.OPENAI_API_KEY = "k"; + net.on("POST", "api.openai.com", /\/v1\/chat\/completions$/, () => + jsonResponse({ choices: [{ message: { content: "[0] Hola" } }] }) + ); + + await expect(translateTexts(["Hello"], { targetLanguage: "es" })).resolves.toMatchObject({ success: true, provider: "openai", texts: ["Hola"] }); + }); + + it("batches long lists and reports a missing key as auth", async () => { + await expect(translateTexts(["x"], { targetLanguage: "ko" })).resolves.toMatchObject({ success: false, errorKind: "auth" }); + + process.env.ANTHROPIC_API_KEY = "k"; + net.on("POST", "api.anthropic.com", /\/v1\/messages$/, () => jsonResponse({ content: [{ type: "text", text: "[0] a\n[1] b" }] })); + const result = await translateTexts(["1", "2", "3", "4"], { targetLanguage: "ko", batchSize: 2 }); + + expect(net.requests).toHaveLength(2); + expect(result).toMatchObject({ texts: ["a", "b", "a", "b"] }); + }); + + it("names language codes for the prompt", () => { + expect(languageName("ja")).toBe("Japanese"); + expect(languageName("Swahili")).toBe("Swahili"); + }); +}); diff --git a/packages/cli/src/commands/_shared/translate-texts.ts b/packages/cli/src/commands/_shared/translate-texts.ts new file mode 100644 index 00000000..c4319a15 --- /dev/null +++ b/packages/cli/src/commands/_shared/translate-texts.ts @@ -0,0 +1,92 @@ +/** + * Translate a list of short texts (subtitle cues, dub segments) with Claude + * or OpenAI, in batches, keeping each text's position. Lines come back + * tagged `[N]` and are matched by that index, so a skipped or merged line + * cannot shift every later translation; anything missing keeps its source + * text. + */ + +import { defaultModel, isProviderError, providerRequest, type ProviderErrorKind } from "@vibeframe/ai-providers"; +import { getConfiguredApiKey } from "../../utils/api-key.js"; + +export interface TranslateTextsOptions { + /** Target language name or code ("Korean", "ko"). */ + targetLanguage: string; + sourceLanguage?: string; + /** Default: Claude when ANTHROPIC_API_KEY is set, else OpenAI. */ + provider?: "claude" | "openai"; + apiKey?: string; + /** Texts per request. Default 30. */ + batchSize?: number; +} + +export type TranslateTextsResult = + | { success: true; texts: string[]; provider: "claude" | "openai" } + | { success: false; error: string; errorKind?: ProviderErrorKind }; + +const LANGUAGE_NAMES: Readonly> = { + en: "English", es: "Spanish", fr: "French", de: "German", it: "Italian", pt: "Portuguese", + ja: "Japanese", ko: "Korean", zh: "Chinese", ar: "Arabic", ru: "Russian", hi: "Hindi", +}; + +/** "ko" → "Korean"; names pass through. */ +export function languageName(code: string): string { + return LANGUAGE_NAMES[code.toLowerCase()] ?? code; +} + +export async function translateTexts(texts: string[], options: TranslateTextsOptions): Promise { + const claudeKey = options.provider !== "openai" ? await getConfiguredApiKey("ANTHROPIC_API_KEY", options.provider === "claude" ? options.apiKey : undefined) : undefined; + const provider = options.provider ?? (claudeKey ? "claude" : "openai"); + const key = provider === "claude" ? claudeKey : await getConfiguredApiKey("OPENAI_API_KEY", options.apiKey); + if (!key) { + const envVar = provider === "claude" ? "ANTHROPIC_API_KEY" : "OPENAI_API_KEY"; + return { success: false, error: `${envVar} required for translation. Run 'vibe setup' or set it in .env`, errorKind: "auth" }; + } + + const target = languageName(options.targetLanguage); + const batchSize = options.batchSize ?? 30; + const out = [...texts]; + try { + for (let start = 0; start < texts.length; start += batchSize) { + const batch = texts.slice(start, start + batchSize); + const prompt = + `Translate each line below to ${target}.` + + (options.sourceLanguage ? ` The source language is ${languageName(options.sourceLanguage)}.` : "") + + " Return exactly one line per input, keeping its [N] prefix, and nothing else.\n\n" + + batch.map((text, i) => `[${i}] ${text.replace(/\s+/g, " ").trim()}`).join("\n"); + const reply = provider === "claude" ? await askClaude(key, prompt) : await askOpenAI(key, prompt); + for (const line of reply.split("\n")) { + const m = line.match(/^\s*\[(\d+)\]\s*(.*)$/); + const index = m ? Number(m[1]) : -1; + if (m && index >= 0 && index < batch.length && m[2].trim()) out[start + index] = m[2].trim(); + } + } + return { success: true, texts: out, provider }; + } catch (error) { + return { + success: false, + error: `Translation failed: ${error instanceof Error ? error.message : String(error)}`, + errorKind: isProviderError(error) ? error.kind : undefined, + }; + } +} + +async function askClaude(apiKey: string, prompt: string): Promise { + const response = await providerRequest("claude", "https://api.anthropic.com/v1/messages", { + method: "POST", + headers: { "Content-Type": "application/json", "x-api-key": apiKey, "anthropic-version": "2023-06-01" }, + body: JSON.stringify({ model: defaultModel("claude", "llm").id, max_tokens: 16000, messages: [{ role: "user", content: prompt }] }), + }); + const data = (await response.json()) as { content?: Array<{ type: string; text: string }> }; + return data.content?.find((c) => c.type === "text")?.text ?? ""; +} + +async function askOpenAI(apiKey: string, prompt: string): Promise { + const response = await providerRequest("openai", "https://api.openai.com/v1/chat/completions", { + method: "POST", + headers: { "Content-Type": "application/json", Authorization: `Bearer ${apiKey}` }, + body: JSON.stringify({ model: defaultModel("openai", "llm").id, messages: [{ role: "user", content: prompt }] }), + }); + const data = (await response.json()) as { choices?: Array<{ message?: { content?: string } }> }; + return data.choices?.[0]?.message?.content ?? ""; +} diff --git a/packages/cli/src/commands/ai-audio.ts b/packages/cli/src/commands/ai-audio.ts index 171d763e..6fc79747 100644 --- a/packages/cli/src/commands/ai-audio.ts +++ b/packages/cli/src/commands/ai-audio.ts @@ -13,17 +13,18 @@ * @see MODELS.md for AI model configuration */ -import { resolve, dirname, basename, extname } from "node:path"; -import { readFile, writeFile } from "node:fs/promises"; +import { resolve, dirname, basename, extname, join } from "node:path"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; import { existsSync } from "node:fs"; -import { - ElevenLabsProvider, - ClaudeProvider, -} from "@vibeframe/ai-providers"; -import { execSafe, execSafeSync, commandExists } from "../utils/exec-safe.js"; +import { ElevenLabsProvider } from "@vibeframe/ai-providers"; +import { execSafe, commandExists } from "../utils/exec-safe.js"; import { detectFormat, formatTranscript } from "../utils/subtitle.js"; import { transcribeAudioFile } from "./_shared/transcription.js"; import { getConfiguredApiKey } from "../utils/api-key.js"; +import { getAudioDuration } from "../utils/audio.js"; +import { translateTexts } from "./_shared/translate-texts.js"; +import { parseTtsProviderName, resolveTtsProvider } from "./_shared/tts-resolve.js"; import type { ProviderErrorKind } from "@vibeframe/ai-providers"; // ============================================================================ @@ -194,9 +195,13 @@ export async function executeVoiceClone(options: VoiceCloneOptions): Promise { const { mediaPath, language, source, voice, analyzeOnly, output } = options; + const absPath = resolve(process.cwd(), mediaPath); + if (!existsSync(absPath)) return { success: false, error: `File not found: ${absPath}`, errorKind: "not-found" }; + const openaiKey = await getConfiguredApiKey("OPENAI_API_KEY"); + if (!openaiKey) return { success: false, error: "OPENAI_API_KEY required for Whisper transcription", errorKind: "auth" }; + const workDir = await mkdtemp(join(tmpdir(), "vibe-dub-")); try { - const absPath = resolve(process.cwd(), mediaPath); - if (!existsSync(absPath)) return { success: false, error: `File not found: ${absPath}` }; - - const openaiKey = await getConfiguredApiKey("OPENAI_API_KEY"); - const anthropicKey = await getConfiguredApiKey("ANTHROPIC_API_KEY"); - const elevenlabsKey = await getConfiguredApiKey("ELEVENLABS_API_KEY"); - - if (!openaiKey) return { success: false, error: "OPENAI_API_KEY required for Whisper transcription" }; - if (!anthropicKey) return { success: false, error: "ANTHROPIC_API_KEY required for Claude translation" }; - if (!analyzeOnly && !elevenlabsKey) return { success: false, error: "ELEVENLABS_API_KEY required for TTS" }; - const ext = extname(absPath).toLowerCase(); const isVideo = [".mp4", ".mov", ".avi", ".mkv", ".webm"].includes(ext); - let audioPath = absPath; if (isVideo) { - const tempAudioPath = resolve(dirname(absPath), `temp-audio-${Date.now()}.mp3`); - execSafeSync("ffmpeg", ["-i", absPath, "-vn", "-acodec", "mp3", "-y", tempAudioPath]); - audioPath = tempAudioPath; + audioPath = join(workDir, "source.mp3"); + await execSafe("ffmpeg", ["-y", "-loglevel", "error", "-i", absPath, "-vn", "-acodec", "libmp3lame", "-q:a", "4", audioPath]); } - const transcriptResult = await transcribeAudioFile(audioPath, { apiKey: openaiKey, language: source }); - - if (transcriptResult.status === "failed" || !transcriptResult.segments) { - return { success: false, error: `Transcription failed: ${transcriptResult.error}`, errorKind: transcriptResult.errorKind }; + const transcript = await transcribeAudioFile(audioPath, { apiKey: openaiKey, language: source }); + if (transcript.status === "failed" || !transcript.segments?.length) { + return { + success: false, + error: transcript.status === "failed" ? `Transcription failed: ${transcript.error}` : "No speech found to dub", + errorKind: transcript.errorKind, + }; } - - const segments = transcriptResult.segments; - - const claude = new ClaudeProvider(); - await claude.initialize({ apiKey: anthropicKey }); - - const languageNames: Record = { - en: "English", es: "Spanish", fr: "French", de: "German", - it: "Italian", pt: "Portuguese", ja: "Japanese", ko: "Korean", - zh: "Chinese", ar: "Arabic", ru: "Russian", hi: "Hindi", + const segments = transcript.segments; + const translation = await translateTexts(segments.map((s) => s.text), { targetLanguage: language, sourceLanguage: source }); + if (!translation.success) return { success: false, error: translation.error, errorKind: translation.errorKind }; + + const sourceLanguage = transcript.detectedLanguage || source || "auto"; + const report = segments.map((s, i) => ({ + startTime: s.startTime, + endTime: s.endTime, + original: s.text, + translated: translation.texts[i], + })); + const base: DubResult = { + success: true, + sourceLanguage, + targetLanguage: language, + segmentCount: segments.length, + segments: report, }; - const targetLangName = languageNames[language] || language; - const segmentTexts = segments.map((s, i) => `[${i}] ${s.text}`).join("\n"); - - let translatedSegments: Array<{ text: string; startTime: number; endTime: number }> = []; - try { - const storyboard = await claude.analyzeContent( - `TRANSLATE to ${targetLangName}. Return the translated text only, preserving segment numbers:\n\n${segmentTexts}`, - segments[segments.length - 1]?.endTime || 60 - ); - translatedSegments = segments.map((s, i) => ({ - text: storyboard[i]?.description || s.text, - startTime: s.startTime, - endTime: s.endTime, - })); - } catch { - translatedSegments = segments.map(s => ({ text: s.text, startTime: s.startTime, endTime: s.endTime })); - } if (analyzeOnly) { - if (output) { - const timingPath = resolve(process.cwd(), output); - const timingData = { - sourcePath: absPath, - sourceLanguage: transcriptResult.detectedLanguage || source || "auto", - targetLanguage: language, - segments: segments.map((s, i) => ({ - startTime: s.startTime, endTime: s.endTime, - original: s.text, translated: translatedSegments[i]?.text || s.text, - })), - }; - await writeFile(timingPath, JSON.stringify(timingData, null, 2)); - } - return { - success: true, - sourceLanguage: transcriptResult.detectedLanguage || source || "auto", - targetLanguage: language, - segmentCount: segments.length, - segments: segments.map((s, i) => ({ - startTime: s.startTime, - endTime: s.endTime, - original: s.text, - translated: translatedSegments[i]?.text || s.text, - })), - outputPath: output ? resolve(process.cwd(), output) : undefined, - }; + if (!output) return base; + const timingPath = resolve(process.cwd(), output); + await writeFile(timingPath, JSON.stringify({ sourcePath: absPath, sourceLanguage, targetLanguage: language, segments: report }, null, 2)); + return { ...base, outputPath: timingPath }; } - const elevenlabs = new ElevenLabsProvider(); - await elevenlabs.initialize({ apiKey: elevenlabsKey! }); - - const dubbedBuffers: Buffer[] = []; - for (const seg of translatedSegments) { - const ttsResult = await elevenlabs.textToSpeech(seg.text, { voiceId: voice }); - if (ttsResult.success && ttsResult.audioBuffer) { - dubbedBuffers.push(ttsResult.audioBuffer); + const tts = await resolveTtsProvider(parseTtsProviderName(options.tts)); + const clips: Array<{ path: string; startTime: number; tempo: number }> = []; + for (const [i, seg] of report.entries()) { + const spoken = await tts.call(seg.translated, { voice }); + if (!spoken.success || !spoken.audioBuffer) { + return { success: false, error: `Speech failed on segment ${i + 1}: ${spoken.error}`, errorKind: spoken.errorKind }; } + const clipPath = join(workDir, `line-${i}.${tts.audioExtension}`); + await writeFile(clipPath, spoken.audioBuffer); + const slot = Math.max(0.1, (report[i + 1]?.startTime ?? seg.endTime) - seg.startTime); + const length = await getAudioDuration(clipPath); + clips.push({ path: clipPath, startTime: seg.startTime, tempo: Math.min(MAX_DUB_TEMPO, Math.max(1, length / slot)) }); } - const combinedBuffer = Buffer.concat(dubbedBuffers); - const outputExt = isVideo ? ".mp3" : extname(absPath); - const defaultOutputPath = resolve(dirname(absPath), `${basename(absPath, extname(absPath))}-${language}${outputExt}`); - const finalOutputPath = resolve(process.cwd(), output || defaultOutputPath); - await writeFile(finalOutputPath, combinedBuffer); - - if (isVideo && audioPath !== absPath) { - try { const { unlink } = await import("node:fs/promises"); await unlink(audioPath); } catch { /* cleanup best-effort */ } - } - - return { - success: true, - outputPath: finalOutputPath, - sourceLanguage: transcriptResult.detectedLanguage || source || "auto", - targetLanguage: language, - segmentCount: segments.length, - segments: segments.map((s, i) => ({ - startTime: s.startTime, - endTime: s.endTime, - original: s.text, - translated: translatedSegments[i]?.text || s.text, - })), - }; + const outputExt = isVideo ? ".mp3" : ext || ".mp3"; + const finalOutputPath = resolve( + process.cwd(), + output || resolve(dirname(absPath), `${basename(absPath, extname(absPath))}-${language}${outputExt}`) + ); + await mixAtOffsets(clips, finalOutputPath); + return { ...base, outputPath: finalOutputPath }; } catch (error) { return { success: false, error: `Dubbing failed: ${error instanceof Error ? error.message : String(error)}` }; + } finally { + await rm(workDir, { recursive: true, force: true }); } } +/** Mix clips into one track, each starting at its offset (and sped up by its tempo). */ +async function mixAtOffsets(clips: Array<{ path: string; startTime: number; tempo: number }>, outputPath: string): Promise { + const inputs = clips.flatMap((clip) => ["-i", clip.path]); + const chains = clips.map((clip, i) => { + const delay = Math.round(clip.startTime * 1000); + const tempo = clip.tempo > 1.01 ? `atempo=${clip.tempo.toFixed(3)},` : ""; + return `[${i}:a]${tempo}adelay=${delay}|${delay}[a${i}]`; + }); + const mix = `${clips.map((_, i) => `[a${i}]`).join("")}amix=inputs=${clips.length}:normalize=0:dropout_transition=0[out]`; + await mkdir(dirname(outputPath), { recursive: true }); + await execSafe("ffmpeg", ["-y", "-loglevel", "error", ...inputs, "-filter_complex", [...chains, mix].join(";"), "-map", "[out]", outputPath]); +} + // ============================================================================ // Audio Duck (FFmpeg) // ============================================================================ diff --git a/packages/cli/src/commands/audio.ts b/packages/cli/src/commands/audio.ts index f172df86..81fe6052 100644 --- a/packages/cli/src/commands/audio.ts +++ b/packages/cli/src/commands/audio.ts @@ -312,7 +312,8 @@ audioCommand .argument("", "Input media file (video or audio)") .option("-l, --language ", "Target language code (e.g., es, ko, ja) (required)") .option("--source ", "Source language code (default: auto-detect)") - .option("--voice ", "ElevenLabs voice ID for output") + .option("--voice ", "Voice for the TTS provider (ElevenLabs name/ID, OpenAI voice, Kokoro voice)") + .option("--tts ", "TTS provider: auto (default), elevenlabs, openai, kokoro", "auto") .option("--analyze-only", "Only analyze and show timing, don't generate audio") .option("-o, --output ", "Output file path") .option("--dry-run", "Preview parameters without executing") @@ -330,10 +331,8 @@ audioCommand if (options.output) validateOutputPath(options.output); if (!options.language) exitWithError(usageError("Target language is required. Use -l or --language")); if (!existsSync(resolve(process.cwd(), mediaPath))) exitWithError(notFoundError(mediaPath)); - // Ask for every key up front, so a run never fails halfway for a missing one. + // Whisper needs OpenAI; translation uses Claude when its key is set, else OpenAI. await requireApiKey("OPENAI_API_KEY", "OpenAI"); - await requireApiKey("ANTHROPIC_API_KEY", "Anthropic"); - if (!options.analyzeOnly) await requireApiKey("ELEVENLABS_API_KEY", "ElevenLabs"); const spinner = isJsonMode() ? null : ora("Transcribing, translating, and dubbing...").start(); const result = await executeDub({ @@ -341,6 +340,7 @@ audioCommand language: options.language, source: options.source, voice: options.voice, + tts: options.tts, analyzeOnly: options.analyzeOnly, output: options.output, }); diff --git a/packages/cli/src/tools/manifest/audio.ts b/packages/cli/src/tools/manifest/audio.ts index 45ab7079..3dd79c5d 100644 --- a/packages/cli/src/tools/manifest/audio.ts +++ b/packages/cli/src/tools/manifest/audio.ts @@ -2,7 +2,7 @@ * @module manifest/audio * @description Audio post-processing tools. * audio_transcribe (Whisper), audio_isolate (ElevenLabs vocals isolation), - * audio_clone_voice (ElevenLabs IVC), audio_dub (Whisper+Claude+ElevenLabs), + * audio_clone_voice (ElevenLabs IVC), audio_dub (Whisper + translation + any TTS), * audio_duck (FFmpeg sidechain). */ @@ -114,12 +114,13 @@ export const audioDubTool = defineTool({ title: "Dub Audio to Another Language", annotations: { readOnly: false, openWorld: true }, description: - "Dub audio/video to another language (transcribe + translate + TTS). Requires OPENAI_API_KEY, ANTHROPIC_API_KEY, ELEVENLABS_API_KEY.", + "Dub audio/video to another language: transcribe (Whisper), translate (Claude, or OpenAI without an Anthropic key), speak each line with a TTS provider, and place it at its original time. Requires OPENAI_API_KEY.", schema: z.object({ mediaPath: z.string().describe("Input media file (video or audio)"), language: z.string().describe("Target language code (e.g., es, ko, ja)"), source: z.string().optional().describe("Source language code (default: auto-detect)"), - voice: z.string().optional().describe("ElevenLabs voice ID for output"), + voice: z.string().optional().describe("Voice for the TTS provider (ElevenLabs name/ID, OpenAI voice, Kokoro voice)"), + tts: z.enum(["auto", "elevenlabs", "openai", "kokoro"]).optional().describe("TTS provider (default: auto)"), analyzeOnly: z.boolean().optional().describe("Only analyze timing, don't generate audio"), output: z.string().optional().describe("Output file path"), }),