From b2edc45b42ed79225adb752ebc1363068c0b57d4 Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Thu, 8 Oct 2026 11:03:47 +0200 Subject: [PATCH 1/6] feat(stt): pass a prompt field to whisper as its initial prompt The helper reads an optional prompt multipart field and sets it as initial_prompt with carry_initial_prompt, so every 30 s window is conditioned on the project's names and terms. Part of #1023. --- electron/native/whisper-stt/src/main.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/electron/native/whisper-stt/src/main.cpp b/electron/native/whisper-stt/src/main.cpp index 77440e54d..5fc4061b9 100644 --- a/electron/native/whisper-stt/src/main.cpp +++ b/electron/native/whisper-stt/src/main.cpp @@ -12,6 +12,8 @@ // POST /inference multipart/form-data, fields: // - file (WAV: 16 kHz mono PCM16) // - language ("en" | "fr" | ... | "auto") +// - prompt (optional: names and terms to bias the +// decode toward, issue #1023) // - response_format ("verbose_json" — accepted for compat // with the previous CT2 client) // GET / 200 "ok" once the model is loaded — readiness probe. @@ -452,6 +454,8 @@ int main(int argc, char** argv) { else if (auto kv = req.params.find("language"); kv != req.params.end()) language = kv->second; // "auto" → empty string tells whisper.cpp to detect; matches the // Node contract (electron/stt/whisperServer.ts) and OpenAI convention. + // The project's names and terms (issue #1023). Empty = no prompt. + const std::string prompt = req.get_file_value("prompt").content; const std::lock_guard lk(infer_mu); @@ -462,6 +466,13 @@ int main(int argc, char** argv) { wparams.print_realtime = false; wparams.print_timestamps = false; wparams.n_threads = threads; + // Carried into every 30 s window whisper_full decodes, not only the first: + // a name said at minute 3 is as likely to be misheard as one said at 0:05. + // whisper.cpp keeps the last n_text_ctx/2 tokens of an over-long prompt. + if (!prompt.empty()) { + wparams.initial_prompt = prompt.c_str(); + wparams.carry_initial_prompt = true; + } // ---- Speech only (Silero VAD), cut here rather than by whisper_full ---- // whisper_full's own `vad` param maps SEGMENT times back onto the upload From d56610c6e2e3c15b60412bc32171f65b374aa9e2 Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Thu, 8 Oct 2026 11:03:50 +0200 Subject: [PATCH 2/6] feat(transcription): per-project names and terms for Whisper A Names and terms field in the media stage, saved as project.vocabulary (additive, no schema bump) and sent with every chunk of every transcription run. Part of #1023. --- electron/stt/index.test.ts | 9 +++ electron/stt/index.ts | 9 ++- electron/stt/transcriptionContract.ts | 5 ++ electron/stt/whisperServer.test.ts | 19 +++++- electron/stt/whisperServer.ts | 11 +++- .../ai-edition/v4/MediaStage.test.ts | 40 +++++++++++- src/components/ai-edition/v4/MediaStage.tsx | 61 ++++++++++++++++++- src/i18n/locales/ar/editor.json | 1 + src/i18n/locales/cs/editor.json | 1 + src/i18n/locales/de/editor.json | 1 + src/i18n/locales/en/editor.json | 1 + src/i18n/locales/es/editor.json | 1 + src/i18n/locales/fr/editor.json | 1 + src/i18n/locales/it/editor.json | 1 + src/i18n/locales/ja-JP/editor.json | 1 + src/i18n/locales/ko-KR/editor.json | 1 + src/i18n/locales/pt-BR/editor.json | 1 + src/i18n/locales/ru/editor.json | 1 + src/i18n/locales/tr/editor.json | 1 + src/i18n/locales/vi/editor.json | 1 + src/i18n/locales/zh-CN/editor.json | 1 + src/i18n/locales/zh-TW/editor.json | 1 + .../ai-edition/document/transcribe.test.ts | 17 ++++++ src/lib/ai-edition/document/transcribe.ts | 4 ++ src/lib/ai-edition/schema/index.ts | 3 + src/lib/captioning/transcribe.ts | 6 +- .../transcription-and-captions.md | 4 ++ 27 files changed, 194 insertions(+), 9 deletions(-) diff --git a/electron/stt/index.test.ts b/electron/stt/index.test.ts index c5af87fe9..02a071a42 100644 --- a/electron/stt/index.test.ts +++ b/electron/stt/index.test.ts @@ -331,6 +331,15 @@ describe("SttManager", () => { expect(languages.slice(1).every((l) => l === "en")).toBe(true); }); + it("hands the names and terms to every chunk, not only the first", async () => { + const mgr = new SttManager(); + await mgr.init({ modelsBaseDir: "/tmp/fake-stt-models" }); + await mgr.transcribe({ samples: new Float32Array(200 * 16000), prompt: "OpenScreen, Vulkan" }); + const prompts = fakeWhisperServer.transcribe.mock.calls.map(([req]) => req.prompt); + expect(prompts.length).toBeGreaterThan(1); + expect(prompts.every((p) => p === "OpenScreen, Vulkan")).toBe(true); + }); + it("retries a failed chunk instead of losing the whole transcription", async () => { fakeWhisperServer.transcribe.mockRejectedValueOnce(new Error("helper died")).mockResolvedValue({ segments: [{ text: "hello", startSec: 0, endSec: 0.5 }], diff --git a/electron/stt/index.ts b/electron/stt/index.ts index 11c760f0b..7a536409b 100644 --- a/electron/stt/index.ts +++ b/electron/stt/index.ts @@ -291,12 +291,18 @@ export class SttManager { private async transcribeChunk( samples: Float32Array, language: string | undefined, + prompt: string | undefined, ): Promise>> { let lastError: unknown; for (let attempt = 1; attempt <= CHUNK_ATTEMPTS; attempt++) { if (this.shuttingDown) throw cancelledError(); try { - return await this.server.transcribe({ samples, language, alignerFor: this.alignerFor }); + return await this.server.transcribe({ + samples, + language, + prompt, + alignerFor: this.alignerFor, + }); } catch (error) { lastError = error; if (this.shuttingDown) throw cancelledError(); @@ -387,6 +393,7 @@ export class SttManager { const result = await this.transcribeChunk( samples.subarray(chunk.startSample, chunk.endSample), language, + req.prompt, ).catch((error) => { if (error instanceof Error && error.name === "AbortError") throw error; // Say where it died. Without this the user gets "Transcription diff --git a/electron/stt/transcriptionContract.ts b/electron/stt/transcriptionContract.ts index 4a2563a78..d30181487 100644 --- a/electron/stt/transcriptionContract.ts +++ b/electron/stt/transcriptionContract.ts @@ -129,6 +129,11 @@ export interface SttTranscribeRequest { * The spec locks language detection on by default; we only honour an explicit value. */ language?: string; + /** + * The project's names and terms (issue #1023), handed to Whisper as its + * initial prompt on every chunk. Omit or leave blank for none. + */ + prompt?: string; } /** diff --git a/electron/stt/whisperServer.test.ts b/electron/stt/whisperServer.test.ts index 773235577..1d66e0f06 100644 --- a/electron/stt/whisperServer.test.ts +++ b/electron/stt/whisperServer.test.ts @@ -255,7 +255,11 @@ describe("WhisperServerManager", () => { }); describe("WhisperServerManager language normalization", () => { - function captureFormField(language: string | undefined): Promise { + function captureFormField( + language: string | undefined, + prompt?: string, + field = "language", + ): Promise { return new Promise((resolve, reject) => { let resolvedText: string | null = null; const fakeJson = { @@ -268,7 +272,7 @@ describe("WhisperServerManager", () => { vi.fn(async (_url: string, init: RequestInit) => { const body = init?.body as FormData | undefined; if (body && typeof (body as FormData).get === "function") { - resolvedText = (body as FormData).get("language") as string | null; + resolvedText = (body as FormData).get(field) as string | null; } return new Response(JSON.stringify(fakeJson), { status: 200 }); }), @@ -281,6 +285,7 @@ describe("WhisperServerManager", () => { await mgr.transcribe({ samples: new Float32Array(1600), language, + prompt, }); resolve(resolvedText); } catch (e) { @@ -305,6 +310,16 @@ describe("WhisperServerManager", () => { expect(sent).toBe("fr"); }); + it("sends the project's names and terms as the prompt field, trimmed", async () => { + const sent = await captureFormField("auto", " OpenScreen, whisper.cpp \n", "prompt"); + expect(sent).toBe("OpenScreen, whisper.cpp"); + }); + + it("sends no prompt field when the names and terms are blank", async () => { + expect(await captureFormField("auto", undefined, "prompt")).toBeNull(); + expect(await captureFormField("auto", " ", "prompt")).toBeNull(); + }); + afterEach(() => { vi.unstubAllGlobals(); }); diff --git a/electron/stt/whisperServer.ts b/electron/stt/whisperServer.ts index 2543a79bb..a1065fecd 100644 --- a/electron/stt/whisperServer.ts +++ b/electron/stt/whisperServer.ts @@ -85,6 +85,8 @@ export interface WhisperServerStatus { export interface TranscribeOptions { samples: Float32Array; language?: string; + /** Names and terms to bias the decode toward (whisper's initial prompt). */ + prompt?: string; /** The aligner model for a language, or null for none (see `transcribe`). */ alignerFor?: (language: string) => Promise; } @@ -484,6 +486,7 @@ export class WhisperServerManager { private async runMultipartInfer(opts: { wavPath: string; language?: string; + prompt?: string; }): Promise { await this.ensureReady(); const url = `${this.baseUrl()}/inference`; @@ -493,6 +496,8 @@ export class WhisperServerManager { form.set("file", blob, path.basename(opts.wavPath)); form.set("response_format", "verbose_json"); form.set("language", opts.language && opts.language !== "auto" ? opts.language : "auto"); + const prompt = opts.prompt?.trim(); + if (prompt) form.set("prompt", prompt); let res: Response; try { res = await fetch(url, { @@ -612,7 +617,11 @@ export class WhisperServerManager { }> { const wavPath = await writeSamplesAsWav(opts.samples); try { - const json = await this.runMultipartInfer({ wavPath, language: opts.language }); + const json = await this.runMultipartInfer({ + wavPath, + language: opts.language, + prompt: opts.prompt, + }); const raw = json.segments ?? []; const segments: SttPhraseSegment[] = raw .map((seg) => { diff --git a/src/components/ai-edition/v4/MediaStage.test.ts b/src/components/ai-edition/v4/MediaStage.test.ts index 6fa306e83..51d9c789e 100644 --- a/src/components/ai-edition/v4/MediaStage.test.ts +++ b/src/components/ai-edition/v4/MediaStage.test.ts @@ -1,6 +1,15 @@ import { describe, expect, it, vi } from "vitest"; -import type { AxcutTranscript } from "@/lib/ai-edition/schema"; -import { addSelectedAssetToTimeline, detectedLanguageForTranscript } from "./MediaStage"; +import { + type AxcutTranscript, + createEmptyDocument, + parseDocumentFile, +} from "@/lib/ai-edition/schema"; +import { + addSelectedAssetToTimeline, + detectedLanguageForTranscript, + VOCABULARY_MAX_CHARS, + withVocabulary, +} from "./MediaStage"; function transcript(language: string, text = ""): AxcutTranscript { return { @@ -85,3 +94,30 @@ describe("addSelectedAssetToTimeline", () => { expect(onSuccess).not.toHaveBeenCalled(); }); }); + +describe("withVocabulary", () => { + const empty = createEmptyDocument({ projectId: "p1", title: "Demo" }); + + it("stores the trimmed names and terms on the project, and they survive a save", () => { + const next = withVocabulary(empty, " OpenScreen, whisper.cpp\n"); + expect(next?.project.vocabulary).toBe("OpenScreen, whisper.cpp"); + const reloaded = parseDocumentFile(JSON.parse(JSON.stringify(next))); + expect(reloaded.project.vocabulary).toBe("OpenScreen, whisper.cpp"); + }); + + it("returns null when nothing changed, so a blur does not add an undo step", () => { + expect(withVocabulary(empty, " ")).toBeNull(); + const set = withVocabulary(empty, "Vulkan"); + expect(set && withVocabulary(set, "Vulkan ")).toBeNull(); + }); + + it("clears the field back to no prompt", () => { + const set = withVocabulary(empty, "Vulkan"); + expect(set && withVocabulary(set, "")?.project.vocabulary).toBeUndefined(); + }); + + it("caps what reaches Whisper", () => { + const next = withVocabulary(empty, "x".repeat(VOCABULARY_MAX_CHARS + 50)); + expect(next?.project.vocabulary).toHaveLength(VOCABULARY_MAX_CHARS); + }); +}); diff --git a/src/components/ai-edition/v4/MediaStage.tsx b/src/components/ai-edition/v4/MediaStage.tsx index 7b0558084..d370b9ee2 100644 --- a/src/components/ai-edition/v4/MediaStage.tsx +++ b/src/components/ai-edition/v4/MediaStage.tsx @@ -3,7 +3,12 @@ import { useMemo, useState } from "react"; import { toast } from "sonner"; import { Tooltip } from "@/components/ui/tooltip"; import { useI18n, useScopedT } from "@/contexts/I18nContext"; -import type { AxcutAsset, AxcutTranscript, TranscriptLanguageCode } from "@/lib/ai-edition/schema"; +import type { + AxcutAsset, + AxcutDocument, + AxcutTranscript, + TranscriptLanguageCode, +} from "@/lib/ai-edition/schema"; import { useProjectStore } from "@/lib/ai-edition/store/projectStore"; import { useAssetTranscriptions, @@ -48,6 +53,19 @@ export function detectedLanguageForTranscript(transcript: AxcutTranscript | null return transcript.language; } +/** + * Whisper reads at most the last 223 tokens of its prompt; 600 characters of + * names and terms stays under that for Latin scripts (issue #1023). + */ +export const VOCABULARY_MAX_CHARS = 600; + +/** The document with its names and terms set, or null when nothing changed. */ +export function withVocabulary(document: AxcutDocument, text: string): AxcutDocument | null { + const vocabulary = text.trim().slice(0, VOCABULARY_MAX_CHARS) || undefined; + if (vocabulary === document.project.vocabulary) return null; + return { ...document, project: { ...document.project, vocabulary } }; +} + export async function addSelectedAssetToTimeline( selected: Pick | null, onAddToTimeline: (assetId: string) => Promise, @@ -68,6 +86,7 @@ export function MediaStage({ const projectId = useProjectStore((s) => s.projectId); const document = useProjectStore((s) => s.document); const addAsset = useProjectStore((s) => s.addAsset); + const saveDocument = useProjectStore((s) => s.saveDocument); // Transcripts are produced in the background as soon as a media lands here // (see transcriptionStore) — this stage only reports where each one is at, // and lets the user force a re-run in another language. @@ -388,6 +407,46 @@ export function MediaStage({

) : null} + {/* Project-wide: whisper's initial prompt for every run (issue #1023). + Uncontrolled and keyed on the saved value, so an undo that restores + an older list shows it; the save happens on blur, one undo step. */} +