From 114cf84fe32fe2557c03ffb9a2eff0eb94221a6a Mon Sep 17 00:00:00 2001 From: Wassim Gharbi Date: Tue, 4 Aug 2026 23:29:56 -0700 Subject: [PATCH 1/9] new model + transcription improvements --- .gitignore | 5 + components/ExportDialog.tsx | 2 +- components/SpeakerLabel.tsx | 4 +- components/TranscriptPanel.tsx | 2 +- lib/models.ts | 182 +++++++++++++- tests/models-test.ts | 74 +++++- tests/vad-regression-test.ts | 14 +- tools/crisperwhisper-onnx/.gitignore | 12 + tools/crisperwhisper-onnx/MODEL_CARD.md | 97 ++++++++ tools/crisperwhisper-onnx/README.md | 233 ++++++++++++++++++ tools/crisperwhisper-onnx/export.py | 176 ++++++++++++++ tools/crisperwhisper-onnx/install_local.py | 117 +++++++++ tools/crisperwhisper-onnx/onnx_config.py | 73 ++++++ tools/crisperwhisper-onnx/quantize.py | 175 ++++++++++++++ tools/crisperwhisper-onnx/requirements.txt | 28 +++ tools/crisperwhisper-onnx/upload.py | 85 +++++++ tools/crisperwhisper-onnx/verify.py | 219 +++++++++++++++++ workers/transcription.worker.ts | 267 +++++++++++++++++++-- 18 files changed, 1714 insertions(+), 51 deletions(-) create mode 100644 tools/crisperwhisper-onnx/.gitignore create mode 100644 tools/crisperwhisper-onnx/MODEL_CARD.md create mode 100644 tools/crisperwhisper-onnx/README.md create mode 100644 tools/crisperwhisper-onnx/export.py create mode 100644 tools/crisperwhisper-onnx/install_local.py create mode 100644 tools/crisperwhisper-onnx/onnx_config.py create mode 100644 tools/crisperwhisper-onnx/quantize.py create mode 100644 tools/crisperwhisper-onnx/requirements.txt create mode 100644 tools/crisperwhisper-onnx/upload.py create mode 100644 tools/crisperwhisper-onnx/verify.py diff --git a/.gitignore b/.gitignore index 10f0dc8..7f0fda5 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,11 @@ /electron-dist/ /dist/ +# Locally-served speech models (tools/crisperwhisper-onnx/install_local.py). +# Hundreds of MB of ONNX weights, reproducible from the export tooling, and +# non-commercially licensed — they must not be committed. +/public/models/ + # misc .DS_Store *.pem diff --git a/components/ExportDialog.tsx b/components/ExportDialog.tsx index c2c59a0..74c5c4d 100644 --- a/components/ExportDialog.tsx +++ b/components/ExportDialog.tsx @@ -407,7 +407,7 @@ export default function ExportDialog() { )} {error && ( -

+

{error}

)} diff --git a/components/SpeakerLabel.tsx b/components/SpeakerLabel.tsx index 1b0d52e..08a5697 100644 --- a/components/SpeakerLabel.tsx +++ b/components/SpeakerLabel.tsx @@ -275,7 +275,7 @@ export default function SpeakerLabel({ ? "Find a speaker…" : "Search or create…" } - className="w-full rounded-lg border border-zinc-200 bg-zinc-50 px-2 py-1.5 text-[13px] text-zinc-800 outline-none focus:border-zinc-400 focus:bg-white dark:border-zinc-700 dark:bg-zinc-950 dark:text-zinc-100 dark:focus:border-zinc-500" + className="w-full rounded-lg border border-zinc-200 bg-zinc-50 px-2 py-1.5 text-[13px] text-zinc-800 outline-none focus:border-zinc-400 dark:border-zinc-700 dark:bg-zinc-950 dark:text-zinc-100 dark:focus:border-zinc-500" /> @@ -481,7 +481,7 @@ export function SelectionSpeakerPopover({ else if (canCreate) apply("new", trimmedQuery); }} placeholder="Search or create…" - className="w-full rounded-lg border border-zinc-200 bg-zinc-50 px-2 py-1.5 text-[13px] text-zinc-800 outline-none focus:border-zinc-400 focus:bg-white dark:border-zinc-600 dark:bg-zinc-900 dark:text-zinc-100 dark:focus:border-zinc-400" + className="w-full rounded-lg border border-zinc-200 bg-zinc-50 px-2 py-1.5 text-[13px] text-zinc-800 outline-none focus:border-zinc-400 dark:border-zinc-600 dark:bg-zinc-900 dark:text-zinc-100 dark:focus:bg-zinc-900 dark:focus:border-zinc-400" />
diff --git a/components/TranscriptPanel.tsx b/components/TranscriptPanel.tsx index fde58d2..766c704 100644 --- a/components/TranscriptPanel.tsx +++ b/components/TranscriptPanel.tsx @@ -419,7 +419,7 @@ export default function TranscriptPanel() { )} {status === "error" && ( -
+
{error}
)} diff --git a/lib/models.ts b/lib/models.ts index 3756943..6273e32 100644 --- a/lib/models.ts +++ b/lib/models.ts @@ -1,7 +1,14 @@ import type { TranscriptLanguage } from "./languages"; /** Local speech models offered on the upload screen. */ -export type WhisperModel = "base" | "small"; +export type WhisperModel = + | "base" + | "small" + | "medium" + /** CrisperWhisper 2.0 Small, exported by tools/crisperwhisper-onnx. */ + | "crisperSmall" + /** CrisperWhisper 2.0 Turbo, published ONNX export. */ + | "crisperTurbo"; /** NVIDIA Parakeet TDT 0.6B v3 via parakeet.js (ONNX / WebGPU). */ export type ParakeetModel = "parakeet"; export type ModelId = WhisperModel | ParakeetModel; @@ -29,11 +36,64 @@ export type WhisperModelInfo = ModelDisplay & { * When true, condition Whisper on a short filler-rich initial prompt so * "Remove fillers" has tokens to act on. See {@link whisperFillerPrompt}. * - * (A dedicated verbatim model — CrisperWhisper — was evaluated, but its - * only browser-runnable ONNX export lacks the cross-attention outputs - * required for word-level timestamps, which this editor depends on.) + * **Currently unset on every model — the prompt does more harm than good.** + * Measured on an 11.5 s clip, decoding each VAD segment the way the worker + * does, Whisper Small with the prompt attached: + * + * | slice | plain | + prompt | + * |--------------|-----------------------------|-----------------------| + * | full 11.5 s | complete, includes "uh" | "Nice. How does it, uh," | + * | 2.5–5.0 s | "Nice. How does it work?" | "Nice. How does it" | + * | 5.0–11.5 s | complete sentence | **"Um,"** | + * + * The long tail segment collapses into an echo of the prompt. Medium behaves + * the same way, and Base is only marginally more robust. This is the + * truncation {@link MAX_VERBATIM_PROMPT_LENGTH} was meant to bound, but the + * length cap does not prevent it — a 20-character prompt still triggers it. + * + * Little is lost by dropping it: plain decoding already yields "uh" on this + * clip, and anything the model does swallow is still recovered as a timed + * `...` placeholder by {@link insertDisfluencyPlaceholders}, so it stays + * cuttable. Kept as a one-flag switch for re-testing on other material. */ keepFillers?: boolean; + /** + * A CrisperWhisper checkpoint. Its vocabulary extends 31 tokens past + * Whisper's timestamp block (`[UM]`, `[UH]`, vocal events, prompt + * scaffolding), which transformers.js misreads as timestamps unless the + * tokenizer is patched at load — see `markCrisperPromptTokensSpecial` in the + * transcription worker. Independent of {@link WhisperModelInfo.verbatimTags}, + * because the extra tokens can be emitted whether or not a mode prefix is + * sent. + */ + crisper?: boolean; + /** + * CrisperWhisper mode prefix: how many `[verbatim_N]` tags to prime the + * decoder with. Verbatim vs. intended is selected purely by this prefix — the + * encoder output is identical either way — and unlike Whisper's initial + * prompt there is no `<|startofprev|>`; the tags lead and the standard prefix + * follows. Source: `crisperwhisper==2.0.1`, `crisperwhisper/prompt.py`. + * + * Currently unset on both CrisperWhisper entries, deliberately. Measured + * against the PyTorch checkpoint on an 11.5 s two-speaker clip: + * + * - verbatim prefix → dropped a whole clause ("Nice. How does it,") and + * emitted `[breath]` where the speaker hesitated + * - intended prefix → complete, but no disfluencies at all + * - **no prefix → complete _and_ kept the "uh"**, beating every other model + * tested including Whisper Small and Parakeet + * + * Adding `<|notimestamps|>` to match upstream exactly changed nothing. The + * plumbing is verified — the tags do reach the decoder as ids 51880–51884 — + * so this is the mode itself underperforming, not a wiring bug. Kept as a + * one-line switch because the finding rests on a single clip. + */ + verbatimTags?: number; + /** + * Load from `public/models//` instead of the Hub. Used for exports that + * have not been published yet — see tools/crisperwhisper-onnx. + */ + local?: boolean; }; export type ParakeetModelInfo = ModelDisplay & { @@ -47,7 +107,14 @@ export type ParakeetModelInfo = ModelDisplay & { export type ModelInfo = WhisperModelInfo | ParakeetModelInfo; /** Display order for model rows in the source dropdown. */ -export const MODEL_ORDER: ModelId[] = ["base", "small", "parakeet"]; +export const MODEL_ORDER: ModelId[] = [ + "base", + "small", + "medium", + "parakeet", + "crisperSmall", + "crisperTurbo", +]; /** * Hard cap for Whisper filler prompts. Longer `<|startofprev|>` strings @@ -89,6 +156,29 @@ const WHISPER_DTYPE = { wasm: { encoder_model: "fp32", decoder_model_merged: "q4" }, } satisfies WhisperModelInfo["dtype"]; +/** + * Medium cannot share {@link WHISPER_DTYPE}: its fp32 encoder export is 1.2 GB, + * which no browser tab survives instantiating. Splits per device the same way + * {@link MODELS.parakeet} already does — fp16 encoder on WebGPU, int8 on WASM — + * and keeps the q4 merged decoder that Base and Small are proven on. + */ +const WHISPER_MEDIUM_DTYPE = { + webgpu: { encoder_model: "fp16", decoder_model_merged: "q4" }, + wasm: { encoder_model: "int8", decoder_model_merged: "q4" }, +} satisfies WhisperModelInfo["dtype"]; + +/** + * Both CrisperWhisper exports run q4 on either device — it is the only + * quantisation available for the merged decoder in both repos, because + * `quantize_dynamic` cannot reach weights inside the merged decoder's + * control-flow subgraphs (see tools/crisperwhisper-onnx/README.md). The q4 pair + * is also the combination verified end-to-end for the local export. + */ +const CRISPER_DTYPE = { + webgpu: { encoder_model: "q4", decoder_model_merged: "q4" }, + wasm: { encoder_model: "q4", decoder_model_merged: "q4" }, +} satisfies WhisperModelInfo["dtype"]; + /** * Local speech models that can run in the transcription worker. * Shared display fields live on every entry; backend-specific knobs @@ -97,7 +187,10 @@ const WHISPER_DTYPE = { export const MODELS: { base: WhisperModelInfo; small: WhisperModelInfo; + medium: WhisperModelInfo; parakeet: ParakeetModelInfo; + crisperSmall: WhisperModelInfo; + crisperTurbo: WhisperModelInfo; } = { base: { backend: "whisper", @@ -106,8 +199,6 @@ export const MODELS: { description: "Faster download and transcription. Good for most clips.", size: "~200 MB", dtype: WHISPER_DTYPE, - // Short filler prompt only — long prompts truncate long-form ASR. - keepFillers: true, }, small: { backend: "whisper", @@ -116,7 +207,23 @@ export const MODELS: { description: "More accurate on longer or noisier audio. Larger download.", size: "~600 MB", dtype: WHISPER_DTYPE, - keepFillers: true, + }, + medium: { + backend: "whisper", + id: "onnx-community/whisper-medium_timestamped", + label: "Whisper Medium", + description: + "Best accuracy on accents, crosstalk and poor recordings. Slow, and a big download.", + // WASM int8 encoder + q4 decoder ~780 MB; WebGPU fp16 encoder ~1.1 GB. + size: "~1.1 GB", + dtype: WHISPER_MEDIUM_DTYPE, + // No filler prompt, unlike Base and Small. Medium is far more sensitive to + // <|startofprev|> conditioning than they are: measured on an 11.5 s clip, + // both dtype configs transcribe it in full unprompted, and both collapse to + // the fragment "Nice. How does it, uh..." with the prompt attached — the + // truncation failure {@link MAX_VERBATIM_PROMPT_LENGTH} describes, except + // triggered by a prompt already inside that cap. It costs nothing here: + // Medium emits "uh" on its own, which is all the prompt was there to buy. }, parakeet: { backend: "parakeet", @@ -125,13 +232,66 @@ export const MODELS: { label: "Parakeet TDT v3", description: "NVIDIA FastConformer — faster on WebGPU, strong EU-language accuracy. Auto-detects language.", - // WASM int8 ~670 MB; WebGPU fp16 ~1.2 GB. - size: "~700 MB", + // WASM int8 encoder + fp16 decoder ~690 MB; WebGPU fp16 + fp32 ~1.3 GB. + size: "~1.3 GB", + }, + crisperSmall: { + backend: "whisper", + // Local folder under public/models — not published yet. Install with + // `python tools/crisperwhisper-onnx/install_local.py`. + id: "crisperwhisper-2.0-small-onnx", + local: true, + label: "CrisperWhisper Small (local)", + description: + "Keeps fillers other models drop, with clean punctuation. Self-exported, unpublished. Non-commercial licence.", + // q4 encoder 66 MB + q4 merged decoder 258 MB. + size: "~324 MB", + dtype: CRISPER_DTYPE, + crisper: true, + }, + crisperTurbo: { + backend: "whisper", + id: "Masterx/CrisperWhisper2.0-turbo-ONNX", + label: "CrisperWhisper Turbo", + description: + "CrisperWhisper on a large-v3 encoder. Largest download. Non-commercial licence.", + // q4 encoder 425 MB + q4 merged decoder 600 MB. + size: "~1.0 GB", + dtype: CRISPER_DTYPE, + crisper: true, }, }; +/** Whether `model` is steered with CrisperWhisper's `[verbatim_N]` prefix. */ +export function verbatimTagCount(model: ModelId): number | null { + const info = MODELS[model]; + return info.backend === "whisper" ? (info.verbatimTags ?? null) : null; +} + +/** + * Whether `model` is a CrisperWhisper checkpoint, and so needs the tokenizer + * fix-up for its extended vocabulary. True regardless of whether a mode prefix + * is sent: `[UM]`, `[UH]` and the vocal-event tokens can be emitted either way. + */ +export function isCrisperModel(model: ModelId): boolean { + const info = MODELS[model]; + return info.backend === "whisper" && info.crisper === true; +} + +/** Whether `model` loads from public/models rather than the Hub. */ +export function isLocalModel(model: ModelId): boolean { + const info = MODELS[model]; + return info.backend === "whisper" && info.local === true; +} + export function isWhisperModel(value: unknown): value is WhisperModel { - return value === "base" || value === "small"; + return ( + value === "base" || + value === "small" || + value === "medium" || + value === "crisperSmall" || + value === "crisperTurbo" + ); } export function isParakeetModel(value: unknown): value is ParakeetModel { diff --git a/tests/models-test.ts b/tests/models-test.ts index f582587..10a112b 100644 --- a/tests/models-test.ts +++ b/tests/models-test.ts @@ -2,12 +2,14 @@ * Model + transcript-source helpers. */ import { - MAX_VERBATIM_PROMPT_LENGTH, MODEL_ORDER, MODELS, + isCrisperModel, + isLocalModel, isModelId, isParakeetModel, isWhisperModel, + verbatimTagCount, whisperFillerPrompt, } from "../lib/models"; import { isTranscriptSource } from "../lib/source"; @@ -18,6 +20,7 @@ function assert(cond: boolean, msg: string) { assert(isWhisperModel("base"), "base is Whisper"); assert(isWhisperModel("small"), "small is Whisper"); +assert(isWhisperModel("medium"), "medium is Whisper"); assert(!isWhisperModel("parakeet"), "parakeet is not Whisper"); assert(isParakeetModel("parakeet"), "parakeet is Parakeet"); assert(isModelId("parakeet"), "parakeet is a model id"); @@ -37,9 +40,31 @@ assert(typeof MODELS.parakeet.label === "string", "parakeet label"); assert(MODELS.base.backend === "whisper", "base backend"); assert(typeof MODELS.base.id === "string", "whisper base id"); assert(typeof MODELS.small.id === "string", "whisper small id"); +assert(typeof MODELS.medium.id === "string", "whisper medium id"); assert(MODELS.base.dtype.webgpu.encoder_model === "fp32", "whisper dtype"); -assert(MODELS.base.keepFillers === true, "base keeps fillers"); -assert(MODELS.small.keepFillers === true, "small keeps fillers"); +// No model uses the <|startofprev|> filler prompt any more. Measured per VAD +// segment, it collapses Whisper Small's longest slice to a bare "Um," and +// truncates Medium to a fragment, while plain decoding transcribes every +// segment correctly and still yields "uh". See WhisperModelInfo.keepFillers. +for (const id of MODEL_ORDER) { + const info = MODELS[id]; + if (info.backend !== "whisper") continue; + assert(!info.keepFillers, `${id} must not use the filler prompt`); + assert( + whisperFillerPrompt(id, "en") === null, + `${id} must resolve no filler prompt` + ); +} +// Medium deviates from WHISPER_DTYPE on purpose: its fp32 encoder export is +// 1.2 GB. Pin the split so a later dtype tidy-up cannot quietly reinstate it. +assert( + MODELS.medium.dtype.wasm.encoder_model === "int8", + "medium encoder is int8 on wasm" +); +assert( + MODELS.medium.dtype.webgpu.encoder_model === "fp16", + "medium encoder is fp16 on webgpu" +); assert( MODEL_ORDER.includes("parakeet") && MODEL_ORDER.includes("base"), @@ -51,12 +76,45 @@ for (const id of MODEL_ORDER) { assert(typeof MODELS[id].size === "string", `${id} has size`); } -assert(whisperFillerPrompt("base", "en") === "Um, uh, hmm, er, ah.", "en prompt"); -assert(whisperFillerPrompt("small", "de") === "Äh, ähm, öhm, mhh.", "de prompt"); -assert(whisperFillerPrompt("parakeet", "en") === null, "parakeet has no prompt"); +// CrisperWhisper: verbatim comes from the [verbatim_N] mode prefix, so these +// models must NOT also get Whisper's filler-bias prompt — the two are different +// mechanisms and stacking them would corrupt the decoder prefix. +// The mode prefix is off on both: measured, it drops a clause and emits +// [breath] where the speaker hesitated, while no prefix keeps the filler. +assert(verbatimTagCount("crisperSmall") === null, "crisperSmall sends no mode prefix"); +assert(verbatimTagCount("crisperTurbo") === null, "crisperTurbo sends no mode prefix"); +assert(verbatimTagCount("base") === null, "base has no verbatim tags"); +// The tokenizer fix-up must still run without a mode prefix — the extended +// vocabulary tokens that break word collation are emitted either way. +assert(isCrisperModel("crisperSmall"), "crisperSmall needs the tokenizer fix-up"); +assert(isCrisperModel("crisperTurbo"), "crisperTurbo needs the tokenizer fix-up"); +assert(!isCrisperModel("base"), "base needs no tokenizer fix-up"); +assert(!isCrisperModel("parakeet"), "parakeet needs no tokenizer fix-up"); +assert( + whisperFillerPrompt("crisperSmall", "en") === null, + "crisperSmall does not use the filler-bias prompt" +); +assert( + whisperFillerPrompt("crisperTurbo", "en") === null, + "crisperTurbo does not use the filler-bias prompt" +); + +// Only the unpublished export is served from public/models; a stray `local` +// flag on a Hub model would send transformers.js to a path that 404s. +assert(isLocalModel("crisperSmall"), "crisperSmall is served locally"); +assert(!isLocalModel("crisperTurbo"), "crisperTurbo loads from the Hub"); +assert(!isLocalModel("base"), "base loads from the Hub"); assert( - (whisperFillerPrompt("base", "en")?.length ?? 0) <= MAX_VERBATIM_PROMPT_LENGTH, - "en prompt within length cap" + !MODELS.crisperSmall.id.includes("/"), + "local model id is a public/models folder name, not a Hub repo id" ); +assert( + MODELS.crisperTurbo.id === "Masterx/CrisperWhisper2.0-turbo-ONNX", + "crisperTurbo Hub id" +); + +assert(whisperFillerPrompt("parakeet", "en") === null, "parakeet has no prompt"); +// The prompts themselves are still checked for length and content in +// vad-regression-test.ts, which constrains whatever any model opts back into. console.log("models-test: ok"); diff --git a/tests/vad-regression-test.ts b/tests/vad-regression-test.ts index 37c5fdc..b7f64e4 100644 --- a/tests/vad-regression-test.ts +++ b/tests/vad-regression-test.ts @@ -39,10 +39,15 @@ function ffmpegAvailable(): boolean { } // Short filler prompts are OK; long ones truncate multi-speaker / long-form ASR. +// Not every Whisper model opts in: Medium truncates to a fragment even inside +// the length cap, and the CrisperWhisper checkpoints transcribe fillers without +// prompting. Those are checked in models-test.ts; here we only constrain the +// prompts that do get sent. +let promptedModels = 0; for (const id of Object.keys(MODELS) as (keyof typeof MODELS)[]) { const info = MODELS[id]; - if (info.backend !== "whisper") continue; - assert(info.keepFillers === true, `${id} should keep fillers via a short prompt`); + if (info.backend !== "whisper" || !info.keepFillers) continue; + promptedModels++; for (const language of TRANSCRIPT_LANGUAGE_ORDER) { const prompt = whisperFillerPrompt(id, language); assert(!!prompt, `${id}/${language} must resolve a filler prompt`); @@ -56,6 +61,11 @@ for (const id of Object.keys(MODELS) as (keyof typeof MODELS)[]) { ); } } +// Currently zero: the prompt truncates long segments badly enough that every +// model opts out (see WhisperModelInfo.keepFillers). The loop above is a +// standing constraint on anything that opts back in, not a claim that something +// does — so it is deliberately allowed to be empty. +void promptedModels; assert( whisperFillerPrompt("parakeet", "en" as TranscriptLanguage) === null, "parakeet must not use Whisper filler prompts" diff --git a/tools/crisperwhisper-onnx/.gitignore b/tools/crisperwhisper-onnx/.gitignore new file mode 100644 index 0000000..205a035 --- /dev/null +++ b/tools/crisperwhisper-onnx/.gitignore @@ -0,0 +1,12 @@ +# Virtualenv and Python caches +.venv/ +__pycache__/ +*.pyc + +# Export artefacts: multi-GB ONNX graphs, external weight blobs, and the staging +# copy of the upstream checkpoint. All reproducible via export.py, and the +# weights are non-commercially licensed — they must not be committed here. +out/ +*.onnx +*.onnx_data +*.safetensors diff --git a/tools/crisperwhisper-onnx/MODEL_CARD.md b/tools/crisperwhisper-onnx/MODEL_CARD.md new file mode 100644 index 0000000..fdfdefb --- /dev/null +++ b/tools/crisperwhisper-onnx/MODEL_CARD.md @@ -0,0 +1,97 @@ +--- +license: other +license_name: nyra-health-non-commercial-research +license_link: https://huggingface.co/nyralabs/CrisperWhisper2.0_small/blob/main/LICENSE.md +base_model: nyralabs/CrisperWhisper2.0_small +base_model_relation: quantized +library_name: transformers.js +pipeline_tag: automatic-speech-recognition +language: + - en + - de +tags: + - onnx + - transformers.js + - whisper + - verbatim + - disfluency + - word-timestamps +--- + +# CrisperWhisper 2.0 Small — ONNX (word timestamps) + +ONNX export of [`nyralabs/CrisperWhisper2.0_small`](https://huggingface.co/nyralabs/CrisperWhisper2.0_small) +for [transformers.js](https://github.com/huggingface/transformers.js), exported +**with per-layer cross-attention outputs** so word-level timestamps work in the +browser. + +## Why re-export + +Every other CrisperWhisper ONNX export is large-v3-scale. Even the "turbo" +export keeps large-v3's 32-layer, 2.5 GB fp32 encoder and only shrinks the +decoder, so the cheapest published option that supports word timestamps is +around 1 GB in 4-bit. This export is ~324 MB (66 MB encoder + 258 MB merged +decoder in q4), which is browser-viable — `CrisperWhisper2.0_small` had no ONNX +export at all before this. + +It is exported **with** per-layer cross-attention outputs, which +`return_timestamps: "word"` requires: transformers.js computes word timings by +running DTW over `cross_attentions.{i}`. Note that the `alignment_heads` in +`generation_config.json` are not sufficient on their own — they select which +attention tensors to align over, so they are useless if the decoder does not +emit any. Some published CrisperWhisper exports ship the heads without the +tensors. See `CrossAttentionWhisperOnnxConfig` in the export tooling, adapted +from `transformers.js/scripts/extra/whisper.py`. + +If you need a larger, more accurate variant with the same capability, +[`Masterx/CrisperWhisper2.0-turbo-ONNX`](https://huggingface.co/Masterx/CrisperWhisper2.0-turbo-ONNX) +already provides one. + +## Usage + +Verbatim mode is selected by the **decoder prompt prefix**, not a flag — the +encoder output is identical for both modes: + +``` +tokenize("[verbatim_1][verbatim_2][verbatim_3][verbatim_4][verbatim_5]") + + [<|startoftranscript|>, <|LANG|>, <|transcribe|>] +``` + +Substitute `[intended_N]` for the clean, non-verbatim transcript. Each tag is a +single vocabulary token (51880–51884 verbatim, 51885–51889 intended) absent from +`added_tokens.json`, so the stock tokenizer encodes them without special +handling, and there is **no `<|startofprev|>`** — the tags precede the standard +prefix. Verbatim output spells fillers as `[UH]` / `[UM]`. + +The upstream `crisperwhisper` package appends `<|notimestamps|>` to this prefix, +because it computes word timings itself. **Omit it under transformers.js**, +which segments chunked audio on timestamp tokens; suppressing them makes long +transcriptions fail part-way with `token_ids must be a non-empty array of +integers`. + +## Licence — non-commercial + +The upstream weights are dual-licensed: inference code under MIT, but **model +weights, configuration, tokenizers, and any Outputs the model generates** under +the [nyra health Non-Commercial Research License](https://huggingface.co/nyralabs/CrisperWhisper2.0_small/blob/main/LICENSE.md). + +This export is a derivative of those weights and **inherits that licence in +full**. Non-commercial research use only. The restriction extends to transcripts +produced by the model. Commercial use requires a separate licence from nyra +health GmbH. + +## Attribution + +- Model: CrisperWhisper 2.0 by [nyra health GmbH](https://www.nyra-labs.com/) — + [paper](https://arxiv.org/abs/2607.18934) +- Export tooling: [`tools/crisperwhisper-onnx`](https://github.com/wassgha/rescript/tree/main/tools/crisperwhisper-onnx) + in Rescript +- Cross-attention export config adapted from + [transformers.js](https://github.com/huggingface/transformers.js/blob/v3/scripts/extra/whisper.py) (Apache-2.0) + +## Known limitation + +CrisperWhisper's published ~30 ms word-boundary accuracy comes from its own +timing method (Viterbi over the explicit space token's cross-attention). +transformers.js instead runs its standard DTW with `alignment_heads` and a median +filter. Word timestamps work; the upstream benchmark figures do not transfer. diff --git a/tools/crisperwhisper-onnx/README.md b/tools/crisperwhisper-onnx/README.md new file mode 100644 index 0000000..ac48988 --- /dev/null +++ b/tools/crisperwhisper-onnx/README.md @@ -0,0 +1,233 @@ +# CrisperWhisper → ONNX (with cross-attentions) + +Exports [CrisperWhisper 2.0](https://huggingface.co/nyralabs) to ONNX in a +transformers.js-compatible layout, **including the cross-attention outputs that +word-level timestamps need**, so the Rescript editor can use it as a verbatim +speech model. + +## Why this exists + +Rescript needs two things from a speech model that no published CrisperWhisper +ONNX export provides together: + +1. **Verbatim output** — `um` / `uh` / stutters transcribed, not silently + dropped. Whisper omits them (its training data is cleaned subtitles) and + Parakeet TDT omits them too (Granary is pseudo-labelled and filtered). + `lib/models.ts` works around this with a short filler-bias prompt, which is + capped at 32 characters because longer prompts truncate long-form output. +2. **Word-level timestamps** — the editor's entire model is cutting video by + selecting words, so per-word `start`/`end` is non-negotiable. + +Two published exports satisfy (2); none satisfies it at a size this app can +afford. Verified by loading each decoder graph and reading its outputs — +`onnx.load(..., load_external_data=False)`, which is cheap regardless of weight +size, and is what `verify.py` automates: + +| Export | `cross_attentions.*` outputs | smallest q4 pair | +| --- | --- | ---: | +| `Masterx/CrisperWhisper2.0-turbo-ONNX` | **`.0` … `.3`** (4 layers) | ~1025 MB | +| `Masterx/CrisperWhisper2.0-large-ONNX` | **`.0` … `.31`** (32 layers) | ~1487 MB | +| `onnx-community/CrisperWhisper-ONNX` | none (0 of 129 outputs) | — | +| `Prince-1/CrisperWhisper` | none (0 of 65 outputs) | — | +| *this export* (`CrisperWhisper2.0_small`) | **`.0` … `.11`** (12 layers) | **324 MB** | +| `onnx-community/whisper-base_timestamped` *(Rescript today)* | `.0` … `.5` | ~200 MB | + +**The gap this folder fills is size, not capability.** Every published +CrisperWhisper ONNX is large-v3-scale — even "turbo" keeps large-v3's 32-layer, +2.5 GB fp32 encoder and only shrinks the decoder — which puts the cheapest +working option at ~1 GB. Rescript is currently having Safari tabs killed for +memory while loading a 200 MB model, so 1 GB is not viable there. +`CrisperWhisper2.0_small` had no ONNX export at all before this one. + +If you can afford ~1 GB, **`Masterx/CrisperWhisper2.0-turbo-ONNX` works off the +shelf**, is a stronger model than small, and its card documents the same +verbatim prompt prefix. Try it before spending anything on integration — it +answers "is CrisperWhisper's verbatim output good enough to justify the licence +question?" at zero cost. + +One trap worth naming: `alignment_heads` in `generation_config.json` proves +nothing. All five exports above ship it, including the two that cannot use it. +The heads select *which* cross-attention tensors to run DTW over; with no such +tensors emitted there is nothing to select. Check graph outputs, not config. + +## What this exports instead + +`nyralabs/CrisperWhisper2.0_small` — 484 MB of bf16 safetensors, plain +whisper-small geometry (12 encoder / 12 decoder layers, `d_model` 768). Quantised +to q4 it should land in the 150–250 MB range, comparable to the Whisper Base +export Rescript ships today. + +The export path is the same one that produced `onnx-community/*_timestamped`: +a `WhisperOnnxConfig` subclass that appends `cross_attentions.{i}` to the decoder +outputs, combined with `model_kwargs={"output_attentions": True}`. See +`onnx_config.py`, which is adapted from +[`transformers.js/scripts/extra/whisper.py`](https://github.com/huggingface/transformers.js/blob/v3/scripts/extra/whisper.py). + +## ⚠️ Licensing — read before publishing + +The CrisperWhisper 2.0 weights are **not** open source. `LICENSE.md` in the +upstream repos splits into two parts: + +- **Part A** — inference code, pre/post-processing, scripts: MIT. +- **Part B** — *model weights, checkpoints, configuration, tokenizers, **and any + Outputs the model generates***: nyra health Non-Commercial Research License. + Commercial use requires a separate licence from nyra health GmbH. + +CrisperWhisper v1 (`nyralabs/CrisperWhisper`) is CC-BY-NC-4.0. Both are +non-commercial. + +Consequences that matter here: + +- An ONNX export is a **derivative of the weights**, so it inherits Part B. It + cannot be relicensed under Rescript's own licence. +- The "and any Outputs" clause reaches the *transcripts users generate*, not just + the weights we redistribute. +- Anything published from this folder must carry the upstream licence and + attribution. `MODEL_CARD.md` is pre-filled to do that; do not strip it. + +Rescript ships signed desktop builds, so whether this model can be offered in the +product at all is a licensing decision, not a technical one. **This folder +deliberately stops at "produces a working export" and does not wire the model +into the app.** + +## Requirements + +Python **3.11+** — `onnxruntime` requires 3.11 and `onnx` requires 3.10, so the +`python3` that ships with macOS (3.9.6) will not work. Install a newer +interpreter first, e.g. `brew install python@3.11`. + +```sh +cd tools/crisperwhisper-onnx +python3.11 -m venv .venv +source .venv/bin/activate +pip install -r requirements.txt +``` + +`requirements.txt` pins `optimum` to the 1.x line, where +`optimum.exporters.onnx` still lives in the main package and the +`WhisperOnnxConfig` API matches what `onnx_config.py` subclasses. Optimum 2.x +moved the exporters into a separate `optimum-onnx` distribution; both scripts +fall back to that import path, so upgrading is possible, but the 1.x pin is the +combination this was written against. + +## Usage + +```sh +# 1. Export encoder + decoder(+merged) to ONNX with cross-attentions. +# Downloads ~500 MB of weights, writes ~1.5 GB of fp32 ONNX. +python export.py --model nyralabs/CrisperWhisper2.0_small --out out/ + +# 2. Prove the export is actually usable for word timestamps. +# This is the check every published export fails — run it before anything else. +python verify.py out/ + +# 3. Quantise for the browser (q4 / int8 / fp16). +python quantize.py out/ + +# 4. Publish (requires `huggingface-cli login` and a licence decision). +python upload.py out/ --repo /CrisperWhisper2.0_small-ONNX-timestamped +``` + +## Results (verified 2026-08-04) + +Exported from `nyralabs/CrisperWhisper2.0_small` on macOS arm64, Python 3.11.15. + +| Graph | fp32 | fp16 | int8 | q4 | +| --- | ---: | ---: | ---: | ---: | +| `encoder_model` | 352.8 MB | 176.5 MB | 92.7 MB | **66.1 MB** | +| `decoder_model_merged` | 774.6 MB | 388.0 MB | — *(see below)* | **257.9 MB** | + +The q4 pair is **324 MB** — comparable to the ~200 MB Whisper Base export +Rescript ships today, and against ~980 MB for the smallest published +CrisperWhisper ONNX. + +`verify.py` passes, including a live onnxruntime run of the q4 pair: +`cross_attentions.0` comes back `(1, 12, 1, 1500)` — batch × heads × decoder +positions × encoder frames, which is what DTW consumes — and `logits` +`(1, 1, 51896)` confirms the extended CrisperWhisper vocabulary survived. + +### Gotchas hit along the way + +Each of these is pinned or handled in code; they are recorded so a future +version bump does not rediscover them. + +- **`torch<2.6`.** Optimum 1.x's `exporters/onnx/model_patcher.py` imports + `_attention_scale` from `torch.onnx.symbolic_opset14`, which torch removed with + the rest of the legacy TorchScript exporter. Newer torch makes importing the + exporter fail outright. +- **`onnx-ir` is an undeclared dependency.** onnxruntime ≥1.28's + `matmul_nbits_quantizer` imports it without declaring it, so q4 fails with + `ModuleNotFoundError`. Also note the module and class were renamed from + `matmul_4bits_quantizer` / `MatMul4BitsQuantizer` in 1.28. +- **int8 cannot quantise the merged decoder.** Optimum emits + `decoder_model_merged` as one top-level `If` node with all 3889 nodes (314 + MatMuls) inside its branches. `quantize_dynamic` only walks the top level, + finds nothing, and writes a file the same size as its input under an `_int8` + name. `quantize.py` now detects this and skips rather than emitting a + mislabelled file. + + This is not a local quirk — `onnx-community/whisper-medium_timestamped` ships + `decoder_model_merged_int8.onnx` and `_uint8.onnx` at 1828 MB, byte-for-byte + the size of its fp32, while `_q4` is 469 MB. Use q4 or fp16 for merged + decoders. It is also the likely root of the note in `lib/models.ts:86-87` + about q8 decoders failing. +- **`encoder_blank_head.{weight,bias}` are dropped on load.** That is + CrisperWhisper's training-time attention-loss head; it plays no part in + inference, and the warning from `WhisperForConditionalGeneration` is expected. + +## Driving it from transformers.js + +Verbatim mode is **not** a flag — CrisperWhisper 2.0 selects it via the decoder +prompt prefix, and the encoder output is identical either way. From +`crisperwhisper/prompt.py` (PyPI `crisperwhisper==2.0.1`), the sequence is: + +``` +# upstream (crisperwhisper/prompt.py) +tokenize("[verbatim_1][verbatim_2][verbatim_3][verbatim_4][verbatim_5]") + + [<|startoftranscript|>, <|LANG|>, <|transcribe|>, <|notimestamps|>] + +# what to send through transformers.js — same, minus <|notimestamps|> +tokenize("[verbatim_1]…[verbatim_5]") + + [<|startoftranscript|>, <|LANG|>, <|transcribe|>] +``` + +`[intended_N]` in place of `[verbatim_N]` gives the clean transcript instead. +Notes: + +- Each mode tag is a **single token already in the vocabulary** — 51880–51884 + for `[verbatim_1…5]` and 51885–51889 for `[intended_1…5]` in the small + checkpoint. They are absent from `added_tokens.json`, so they need no special + handling; the stock tokenizer encodes the five-tag string to exactly five ids. + The full English verbatim prefix for this checkpoint is + `[51880, 51881, 51882, 51883, 51884, 50258, 50259, 50359, 50363]`. +- Do not hardcode the trailing four. The turbo export uses different ids for + `<|transcribe|>` (50360) and `<|notimestamps|>` (50364) because it carries an + extra language; read them from `generation_config.json`. +- Unlike vanilla Whisper prompting there is **no `<|startofprev|>`**; the tags + come first and the standard prefix follows. +- `<|notimestamps|>` is part of the *upstream* prefix, but **omit it under + transformers.js**. Upstream suppresses timestamp tokens because it derives + word timings itself, via Viterbi over the space token's cross-attention. + transformers.js instead splits chunked audio on timestamp tokens inside + `_decode_asr`; with them suppressed, a long transcript never segments, and the + stride-overlap merge can resolve to an empty token list. That surfaces + part-way through a transcription as `token_ids must be a non-empty array of + integers`. Mode selection and timestamps are orthogonal, so dropping it does + not affect verbatim output. +- The ``/``/`` tokens in `added_tokens.json` are for verbatimize, + hotwords and long-form continuation context respectively — *not* mode control. +- Verbatim output spells fillers as bracketed uppercase (`[UH]`, `[UM]`), so + `lib/hallucinations.ts` and the "Remove fillers" matcher would need to + recognise that form. + +Rescript already builds custom `decoder_input_ids` for its Whisper filler prompt, +so the same mechanism applies — see the `decoder_input_ids` construction in +`workers/transcription.worker.ts`. + +## Caveat on timing quality + +CrisperWhisper's headline ~30 ms boundary accuracy comes from its own timing +method — Viterbi over the explicit space token's cross-attention, per the +upstream docs. transformers.js will instead run its standard DTW with +`alignment_heads` and a median filter. That still yields word timestamps, but do +not expect the upstream benchmark numbers through this path. diff --git a/tools/crisperwhisper-onnx/export.py b/tools/crisperwhisper-onnx/export.py new file mode 100644 index 0000000..0609536 --- /dev/null +++ b/tools/crisperwhisper-onnx/export.py @@ -0,0 +1,176 @@ +"""Export a CrisperWhisper checkpoint to ONNX with cross-attention outputs. + +Writes a transformers.js-compatible layout: + + out/ + onnx/encoder_model.onnx + onnx/decoder_model_merged.onnx + config.json, generation_config.json, tokenizer.json, preprocessor_config.json, ... + +Run `verify.py` afterwards. Do not trust an export that has not been verified — +every CrisperWhisper ONNX repo on the Hub looks correct by file listing and is +unusable for word timestamps. +""" + +from __future__ import annotations + +import argparse +import json +import shutil +from pathlib import Path + +DEFAULT_MODEL = "nyralabs/CrisperWhisper2.0_small" + +# CrisperWhisper checkpoints declare a training-time subclass that carries an +# extra attention-loss head. Nothing in it is needed for inference, and Optimum +# resolves the exportable model from `model_type` ("whisper") anyway — but the +# name in `architectures` trips `AutoModel` resolution, so it is rewritten in the +# staging copy. The upstream repo is never modified. +TRAINING_ARCHITECTURE = "WhisperForConditionalGenerationWithAttentionLoss" +INFERENCE_ARCHITECTURE = "WhisperForConditionalGeneration" + +# Files transformers.js loads from the repo root (everything that is not weights). +SUPPORT_FILES = ( + "config.json", + "generation_config.json", + "preprocessor_config.json", + "tokenizer.json", + "tokenizer_config.json", + "special_tokens_map.json", + "added_tokens.json", + "vocab.json", + "merges.txt", + "normalizer.json", +) + + +def stage_checkpoint(model_id: str, staging: Path) -> Path: + """Download the checkpoint and normalise its config for export.""" + from huggingface_hub import snapshot_download + + print(f"[1/4] Downloading {model_id} …") + local = Path( + snapshot_download( + repo_id=model_id, + allow_patterns=["*.json", "*.txt", "*.safetensors", "*.model"], + ) + ) + + print(f"[2/4] Staging a normalised copy in {staging} …") + if staging.exists(): + shutil.rmtree(staging) + shutil.copytree(local, staging, symlinks=False) + + config_path = staging / "config.json" + config = json.loads(config_path.read_text()) + + architectures = config.get("architectures") or [] + if TRAINING_ARCHITECTURE in architectures: + config["architectures"] = [INFERENCE_ARCHITECTURE] + print(f" architectures: {TRAINING_ARCHITECTURE} → {INFERENCE_ARCHITECTURE}") + + # Weights ship as bf16; ONNX export needs fp32. Clearing the hint stops + # transformers from loading in bf16 and tracing a bf16 graph. + for key in ("dtype", "torch_dtype"): + if config.pop(key, None) is not None: + print(f" cleared config.{key} (exporting fp32)") + + config_path.write_text(json.dumps(config, indent=2)) + return staging + + +def copy_support_files(staging: Path, out: Path) -> None: + """Carry tokenizer / preprocessor / generation config into the output repo. + + `generation_config.json` matters most: CrisperWhisper ships its own + `alignment_heads`, and transformers.js needs them to pick which + cross-attention heads to run DTW over. Optimum writes a generation config of + its own, so this runs last and wins. + """ + for name in SUPPORT_FILES: + src = staging / name + if src.exists(): + shutil.copy2(src, out / name) + + generation_config = out / "generation_config.json" + if generation_config.exists(): + config = json.loads(generation_config.read_text()) + heads = config.get("alignment_heads") + if heads: + print(f" alignment_heads preserved ({len(heads)} pairs)") + else: + print( + " WARNING: no alignment_heads in generation_config.json — " + "word timestamps will not work even with cross-attentions present" + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", default=DEFAULT_MODEL, help="HF repo id or local path") + parser.add_argument("--out", default="out", type=Path, help="output directory") + parser.add_argument("--opset", type=int, default=None, help="ONNX opset override") + parser.add_argument( + "--keep-staging", + action="store_true", + help="keep the normalised checkpoint copy for debugging", + ) + args = parser.parse_args() + + try: + from optimum.exporters.onnx import main_export + except ImportError: # optimum 2.x moved the exporters into `optimum-onnx` + from optimum_onnx.exporters.onnx import main_export # type: ignore[import-not-found] + from transformers import AutoConfig + + from onnx_config import main_export_kwargs + + out: Path = args.out + out.mkdir(parents=True, exist_ok=True) + staging = out / "_staging" + + source = Path(args.model) + staged = source if source.is_dir() else stage_checkpoint(args.model, staging) + + config = AutoConfig.from_pretrained(staged) + print( + f" {config.encoder_layers} encoder / {config.decoder_layers} decoder layers, " + f"d_model={config.d_model} → expecting cross_attentions.0 … .{config.decoder_layers - 1}" + ) + + print("[3/4] Exporting to ONNX (this is the slow part) …") + main_export( + model_name_or_path=str(staged), + output=out, + task="automatic-speech-recognition-with-past", + library_name="transformers", + # Validation compares ONNX outputs against PyTorch, but Optimum does not + # know how to compare the cross-attention outputs we just added, so it + # reports spurious mismatches. verify.py checks the graph instead. + do_validation=False, + **({"opset": args.opset} if args.opset else {}), + **main_export_kwargs(config), + ) + + print("[4/4] Arranging transformers.js layout …") + onnx_dir = out / "onnx" + onnx_dir.mkdir(exist_ok=True) + for model_file in sorted(out.glob("*.onnx")): + shutil.move(str(model_file), onnx_dir / model_file.name) + print(f" onnx/{model_file.name}") + # External weight blobs travel with their graph file. + for data_file in sorted(out.glob("*.onnx_data")): + shutil.move(str(data_file), onnx_dir / data_file.name) + print(f" onnx/{data_file.name}") + + copy_support_files(staged, out) + + if staging.exists() and not args.keep_staging: + shutil.rmtree(staging) + + print(f"\nDone → {out}\nNext: python verify.py {out}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/crisperwhisper-onnx/install_local.py b/tools/crisperwhisper-onnx/install_local.py new file mode 100644 index 0000000..855c7f5 --- /dev/null +++ b/tools/crisperwhisper-onnx/install_local.py @@ -0,0 +1,117 @@ +"""Install an export into the app's public/models/ so the browser can load it. + +Rescript's `crisperSmall` entry is flagged `local`, which makes the worker point +transformers.js at `/models//` for the duration of that model's load. This +copies the files that entry actually needs into place. + +Only the q4 pair is copied by default: the fp32 graphs are 1.1 GB and nothing +references them at runtime, so shipping them into public/ would bloat every +`next build` for no reason. + + python install_local.py # from tools/crisperwhisper-onnx + python install_local.py --dtype fp16 --name my-export +""" + +from __future__ import annotations + +import argparse +import shutil +import sys +from pathlib import Path + +import verify + +# Must match MODELS.crisperSmall.id in lib/models.ts. +DEFAULT_NAME = "crisperwhisper-2.0-small-onnx" + +# Everything transformers.js reads from the repo root. Missing optional entries +# are skipped; a missing required one is a hard error, because the failure would +# otherwise surface as an opaque tokenizer crash in the worker. +REQUIRED_FILES = ("config.json", "generation_config.json", "preprocessor_config.json") +OPTIONAL_FILES = ( + "tokenizer.json", + "tokenizer_config.json", + "special_tokens_map.json", + "added_tokens.json", + "vocab.json", + "merges.txt", + "normalizer.json", +) + + +def repo_root() -> Path: + return Path(__file__).resolve().parents[2] + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--out", type=Path, default=Path("out"), help="export directory") + parser.add_argument("--name", default=DEFAULT_NAME, help="folder name under public/models") + parser.add_argument( + "--dtype", + default="q4", + help="graph suffix to install (q4, fp16, int8, or 'fp32' for unsuffixed)", + ) + parser.add_argument("--skip-verify", action="store_true", help="install without verifying") + args = parser.parse_args() + + out: Path = args.out + if not (out / "config.json").exists(): + print(f"No export in {out} — run export.py first") + return 1 + + # Installing an export whose decoder has no cross-attentions would produce a + # model that loads fine and silently returns no word timings. + if not args.skip_verify: + argv = sys.argv + sys.argv = ["verify.py", str(out), "--no-smoke"] + try: + if verify.main() != 0: + print("\nRefusing to install: verification failed.") + return 1 + finally: + sys.argv = argv + print() + + suffix = "" if args.dtype == "fp32" else f"_{args.dtype}" + graphs = [f"encoder_model{suffix}.onnx", f"decoder_model_merged{suffix}.onnx"] + missing = [g for g in graphs if not (out / "onnx" / g).exists()] + if missing: + print(f"Missing graph(s) for --dtype {args.dtype}: {', '.join(missing)}") + print("Run quantize.py, or pick a dtype that exists.") + return 1 + + dest = repo_root() / "public" / "models" / args.name + if dest.exists(): + shutil.rmtree(dest) + (dest / "onnx").mkdir(parents=True) + + total = 0 + for name in REQUIRED_FILES: + src = out / name + if not src.exists(): + print(f"Missing required file: {name}") + return 1 + shutil.copy2(src, dest / name) + for name in OPTIONAL_FILES: + src = out / name + if src.exists(): + shutil.copy2(src, dest / name) + for name in graphs: + src = out / "onnx" / name + shutil.copy2(src, dest / "onnx" / name) + size = src.stat().st_size + total += size + print(f" onnx/{name} {size / 1e6:.1f} MB") + + rel = dest.relative_to(repo_root()) + print(f"\nInstalled {total / 1e6:.0f} MB → {rel}") + print( + "Served at /models/" + f"{args.name}/ — select \"CrisperWhisper Small (local)\" in the model menu." + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/crisperwhisper-onnx/onnx_config.py b/tools/crisperwhisper-onnx/onnx_config.py new file mode 100644 index 0000000..4fcda4e --- /dev/null +++ b/tools/crisperwhisper-onnx/onnx_config.py @@ -0,0 +1,73 @@ +"""ONNX export config that makes Whisper decoders emit cross-attentions. + +Adapted from transformers.js `scripts/extra/whisper.py` (v3 branch), which is +what produced the `onnx-community/*_timestamped` repos. Kept as a local copy +rather than a dependency because that file lives in a scripts/ directory that is +not published to PyPI, and it moved between branches once already. + +The only thing that matters here: `WhisperOnnxConfig.outputs` does not include +attention tensors, so a stock Optimum export produces a decoder whose outputs +are `logits` plus the KV cache. transformers.js derives word timestamps by +running DTW over `cross_attentions.{i}`, so such an export can never do +word-level timing no matter what `alignment_heads` says. +""" + +from __future__ import annotations + +from typing import Dict + +# Optimum 2.x split the ONNX exporters into the separate `optimum-onnx` +# distribution. It re-exports under the original path in most installs, but not +# all, so try both rather than pinning the ecosystem to 1.x forever. +try: + from optimum.exporters.onnx.base import ConfigBehavior + from optimum.exporters.onnx.model_configs import WhisperOnnxConfig +except ImportError: # pragma: no cover - depends on which distribution is installed + from optimum_onnx.exporters.onnx.base import ConfigBehavior # type: ignore[import-not-found] + from optimum_onnx.exporters.onnx.model_configs import ( # type: ignore[import-not-found] + WhisperOnnxConfig, + ) + + +class CrossAttentionWhisperOnnxConfig(WhisperOnnxConfig): + """Whisper ONNX config that additionally exports per-layer cross-attentions.""" + + @property + def outputs(self) -> Dict[str, Dict[int, str]]: + common_outputs = super().outputs + + # Encoder behaviour has no cross-attention to report; only the decoder + # (both the plain and with-past variants) gains outputs here. + if self._behavior is ConfigBehavior.DECODER: + for i in range(self._config.decoder_layers): + common_outputs[f"cross_attentions.{i}"] = { + 0: "batch_size", + 2: "decoder_sequence_length", + 3: "encoder_sequence_length_out", + } + return common_outputs + + +def main_export_kwargs(config, task: str = "automatic-speech-recognition") -> dict: + """Extra kwargs for `optimum.exporters.onnx.main_export`. + + `output_attentions=True` makes the traced forward actually compute the + tensors; the custom configs declare them as graph outputs. Both are required + — either alone silently produces the same useless export. + """ + custom_config = CrossAttentionWhisperOnnxConfig(config=config, task=task) + + custom_onnx_configs = { + "encoder_model": custom_config.with_behavior("encoder"), + "decoder_model": custom_config.with_behavior( + "decoder", use_past=True, use_past_in_inputs=False + ), + "decoder_with_past_model": custom_config.with_behavior( + "decoder", use_past=True, use_past_in_inputs=True + ), + } + + return { + "model_kwargs": {"output_attentions": True}, + "custom_onnx_configs": custom_onnx_configs, + } diff --git a/tools/crisperwhisper-onnx/quantize.py b/tools/crisperwhisper-onnx/quantize.py new file mode 100644 index 0000000..5f52d36 --- /dev/null +++ b/tools/crisperwhisper-onnx/quantize.py @@ -0,0 +1,175 @@ +"""Quantise an exported CrisperWhisper for the browser. + +Produces the dtype-suffixed variants transformers.js resolves from a `dtype` +config — `_fp16`, `_int8`, `_q4` — alongside the fp32 originals. + +Rescript's `lib/models.ts` picks encoder and decoder dtypes independently, and +that split matters: Whisper's encoder degrades under aggressive quantisation +(which is why Base and Small pin `encoder_model: "fp32"`), while the merged +decoder tolerates q4. Emitting every variant here keeps that choice open at +integration time instead of baking it in. + + python quantize.py out/ --modes fp16 int8 q4 +""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +# transformers.js dtype → filename suffix (mirrors its DEFAULT_DTYPE_SUFFIX_MAP). +SUFFIXES = {"fp16": "_fp16", "int8": "_int8", "q4": "_q4"} + + +class SkipVariant(Exception): + """Raised when a (graph, mode) pair cannot produce a genuinely smaller model.""" + + +def _weights_are_in_subgraphs(src: Path) -> bool: + """True when the graph's MatMuls live inside control-flow subgraphs. + + Optimum emits `decoder_model_merged` as a single top-level `If` node — the + with-past and without-past branches — so every MatMul sits one level down. + `quantize_dynamic` only walks the top-level graph, finds nothing, and writes + a file the same size as its input under an `_int8` name. + + This is not hypothetical: `onnx-community/whisper-medium_timestamped` ships + `decoder_model_merged_int8.onnx` and `_uint8.onnx` at exactly the fp32 size + of 1828 MB, while its `_q4` is 469 MB. Refusing to write the file is better + than shipping a mislabelled one. + """ + import onnx + + model = onnx.load(str(src), load_external_data=False) + nodes = model.graph.node + top_level_matmuls = sum(1 for n in nodes if n.op_type in ("MatMul", "Gemm")) + if top_level_matmuls: + return False + return any(n.op_type in ("If", "Loop", "Scan") for n in nodes) + + +def quantize_int8(src: Path, dst: Path) -> None: + from onnxruntime.quantization import QuantType, quantize_dynamic + + if _weights_are_in_subgraphs(src): + raise SkipVariant( + "weights live inside control-flow subgraphs, which quantize_dynamic " + "cannot reach — use q4 (MatMulNBits descends into subgraphs) or fp16" + ) + + quantize_dynamic( + model_input=str(src), + model_output=str(dst), + weight_type=QuantType.QInt8, + # Conv/attention kernels in Whisper are the accuracy-sensitive part; + # per-channel weights cost nothing at runtime and lose noticeably less. + per_channel=True, + reduce_range=False, + ) + + +def quantize_fp16(src: Path, dst: Path) -> None: + import onnx + from onnxconverter_common import float16 + + model = onnx.load(str(src)) + converted = float16.convert_float_to_float16( + model, + keep_io_types=True, + disable_shape_infer=False, + ) + onnx.save(converted, str(dst), save_as_external_data=False) + + +def quantize_q4(src: Path, dst: Path) -> None: + import onnx + + # Renamed in onnxruntime 1.28 (matmul_4bits_quantizer/MatMul4BitsQuantizer → + # matmul_nbits_quantizer/MatMulNBitsQuantizer). Try the current name first. + try: + from onnxruntime.quantization.matmul_nbits_quantizer import ( + MatMulNBitsQuantizer as Quantizer, + ) + except ImportError: # onnxruntime < 1.28 + from onnxruntime.quantization.matmul_4bits_quantizer import ( # type: ignore[import-not-found] + MatMul4BitsQuantizer as Quantizer, + ) + + model = onnx.load(str(src)) + quantizer = Quantizer(model, block_size=32, is_symmetric=True) + quantizer.process() + onnx.save(quantizer.model.model, str(dst), save_as_external_data=False) + + +MODES = {"int8": quantize_int8, "fp16": quantize_fp16, "q4": quantize_q4} + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("out", type=Path, help="export directory produced by export.py") + parser.add_argument( + "--modes", + nargs="+", + default=["fp16", "int8", "q4"], + choices=sorted(MODES), + ) + parser.add_argument( + "--include", + nargs="+", + # transformers.js loads the merged decoder; the split decoder_model / + # decoder_with_past_model pair is redundant for it and each is ~750 MB, + # so quantising them by default triples the work for nothing. + default=["encoder_model", "decoder_model_merged"], + help="graph stems to quantise ('all' for every graph)", + ) + args = parser.parse_args() + + onnx_dir: Path = args.out / "onnx" + if not onnx_dir.is_dir(): + print(f"No onnx/ directory in {args.out} — run export.py first") + return 1 + + # Only the fp32 originals; never quantise an already-quantised file. + sources = [ + path + for path in sorted(onnx_dir.glob("*.onnx")) + if not any(path.stem.endswith(s) for s in SUFFIXES.values()) + and ("all" in args.include or path.stem in args.include) + ] + if not sources: + print(f"No source graphs in {onnx_dir}") + return 1 + + failures = 0 + for mode in args.modes: + for src in sources: + dst = src.with_name(f"{src.stem}{SUFFIXES[mode]}.onnx") + print(f"{mode:>5} {src.name} → {dst.name}", flush=True) + try: + MODES[mode](src, dst) + except SkipVariant as reason: + print(f" skipped: {reason}") + continue + except Exception as err: # noqa: BLE001 - report and continue + failures += 1 + print(f" FAILED: {type(err).__name__}: {err}") + continue + + before = src.stat().st_size + after = dst.stat().st_size + print(f" {after / 1e6:.1f} MB ({after / before:.0%} of fp32)") + # A "quantised" file the size of its input is the failure mode that + # put un-quantised int8 decoders on the Hub. Do not ship one. + if after > before * 0.95: + failures += 1 + print(" FAILED: no meaningful size reduction — removing") + dst.unlink() + + if failures: + print(f"\n{failures} variant(s) failed — the rest are usable") + print(f"\nRe-run verification: python verify.py {args.out}") + return 1 if failures else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/crisperwhisper-onnx/requirements.txt b/tools/crisperwhisper-onnx/requirements.txt new file mode 100644 index 0000000..a2f350f --- /dev/null +++ b/tools/crisperwhisper-onnx/requirements.txt @@ -0,0 +1,28 @@ +# Python 3.11+ required (onnxruntime floor). See README.md. + +# Optimum 1.x keeps the ONNX exporters in-package and matches the +# WhisperOnnxConfig API that onnx_config.py subclasses. 2.x moved them to the +# separate `optimum-onnx` distribution; the scripts fall back to that import +# path, but this is the combination they were written against. +optimum[exporters]>=1.24,<2 + +# Pinned below 4.50: optimum 1.x tracks the pre-4.50 Whisper modelling code, and +# the export traces that module directly. +transformers>=4.44,<4.50 + +# Upper bound is not cosmetic: optimum 1.x's `exporters/onnx/model_patcher.py` +# imports `_attention_scale` from `torch.onnx.symbolic_opset14`, and torch +# removed it along with the rest of the legacy TorchScript ONNX exporter. With a +# newer torch, importing optimum's exporter fails outright. +torch>=2.2,<2.6 + +onnx>=1.17 +onnxruntime>=1.20 +# fp16 conversion in quantize.py +onnxconverter-common>=1.16 +# onnxruntime >=1.28 imports onnx_ir from matmul_nbits_quantizer but does not +# declare it, so q4 quantisation fails with ModuleNotFoundError without this. +onnx-ir>=0.1 + +huggingface_hub>=0.26 +numpy>=1.26 diff --git a/tools/crisperwhisper-onnx/upload.py b/tools/crisperwhisper-onnx/upload.py new file mode 100644 index 0000000..40cc6b8 --- /dev/null +++ b/tools/crisperwhisper-onnx/upload.py @@ -0,0 +1,85 @@ +"""Publish a verified export to the Hugging Face Hub. + +Refuses to upload unless `verify.py` passes. An export that looks right but has +no cross-attention outputs is exactly what is already on the Hub several times +over; adding another is worse than publishing nothing. + + huggingface-cli login + python upload.py out/ --repo /CrisperWhisper2.0_small-ONNX-timestamped +""" + +from __future__ import annotations + +import argparse +import shutil +from pathlib import Path + +import verify + +CARD_TEMPLATE = Path(__file__).with_name("MODEL_CARD.md") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("out", type=Path, help="export directory produced by export.py") + parser.add_argument("--repo", required=True, help="target repo id, e.g. org/name") + parser.add_argument("--private", action="store_true", help="create as private") + parser.add_argument( + "--message", default="Add ONNX export with cross-attentions", help="commit message" + ) + parser.add_argument( + "--dry-run", action="store_true", help="verify and stage the card, but do not push" + ) + args = parser.parse_args() + + out: Path = args.out + + print("Verifying before upload …") + if _verify(out) != 0: + print("\nRefusing to upload: verification failed.") + return 1 + + # The licence notice is not optional — the upstream terms cover the weights + # and their outputs, and this export is a derivative. + card = out / "README.md" + if not card.exists(): + shutil.copy2(CARD_TEMPLATE, card) + print(f"Staged model card → {card}") + else: + print(f"Keeping existing {card}") + + if args.dry_run: + print("\n--dry-run: nothing pushed.") + return 0 + + from huggingface_hub import HfApi + + api = HfApi() + api.create_repo(args.repo, private=args.private, exist_ok=True, repo_type="model") + print(f"\nUploading {out} → {args.repo} …") + api.upload_folder( + folder_path=str(out), + repo_id=args.repo, + repo_type="model", + commit_message=args.message, + # Staging leftovers and virtualenvs must never reach the Hub. + ignore_patterns=["_staging/*", ".venv/*", "**/__pycache__/*"], + ) + print(f"Done → https://huggingface.co/{args.repo}") + return 0 + + +def _verify(out: Path) -> int: + """Run verify.py's checks in-process; returns 0 on success.""" + import sys + + argv = sys.argv + sys.argv = ["verify.py", str(out)] + try: + return verify.main() + finally: + sys.argv = argv + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/crisperwhisper-onnx/verify.py b/tools/crisperwhisper-onnx/verify.py new file mode 100644 index 0000000..32aa2f7 --- /dev/null +++ b/tools/crisperwhisper-onnx/verify.py @@ -0,0 +1,219 @@ +"""Verify an export can actually produce word-level timestamps. + +This is the check that matters. Every CrisperWhisper ONNX repo on the Hub has a +plausible file listing and `alignment_heads` in its generation config, and none +of them emit cross-attentions — so the only way to tell a usable export from a +useless one is to read the decoder's graph outputs. + +Exits non-zero on failure so it can gate `upload.py`. + + python verify.py out/ +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +CROSS_ATTENTION_PREFIX = "cross_attentions." + + +def graph_output_names(model_path: Path) -> list[str]: + """Output names of an ONNX graph, without touching external weight blobs. + + `load_external_data=False` keeps this cheap and constant-time regardless of + model size — graph metadata lives in the .onnx file even when the weights sit + in a sibling .onnx_data of several GB. + """ + import onnx + + model = onnx.load(str(model_path), load_external_data=False) + return [output.name for output in model.graph.output] + + +def check_decoder(model_path: Path, decoder_layers: int) -> bool: + names = graph_output_names(model_path) + cross = sorted( + int(n[len(CROSS_ATTENTION_PREFIX) :]) + for n in names + if n.startswith(CROSS_ATTENTION_PREFIX) + ) + + print(f"\n {model_path.name}") + print(f" outputs: {len(names)} total") + + if not cross: + print(" FAIL: no cross_attentions.* outputs") + print(f" got: {', '.join(names[:6])}{' …' if len(names) > 6 else ''}") + print( + " → this is the same defect as every published CrisperWhisper export;" + " word timestamps are impossible" + ) + return False + + expected = list(range(decoder_layers)) + if cross != expected: + print( + f" FAIL: expected cross_attentions.0 … .{decoder_layers - 1}, " + f"got indices {cross}" + ) + return False + + print(f" OK: cross_attentions.0 … .{cross[-1]} ({len(cross)} layers)") + return True + + +def check_alignment_heads(out: Path, decoder_layers: int, decoder_heads: int) -> bool: + path = out / "generation_config.json" + if not path.exists(): + print("\n generation_config.json\n FAIL: missing") + return False + + heads = json.loads(path.read_text()).get("alignment_heads") + print("\n generation_config.json") + if not heads: + print(" FAIL: no alignment_heads — transformers.js cannot select DTW heads") + return False + + # A head reference outside the model's real shape silently produces garbage + # timings rather than an error, so bound-check it here. + bad = [ + pair + for pair in heads + if not (0 <= pair[0] < decoder_layers and 0 <= pair[1] < decoder_heads) + ] + if bad: + print( + f" FAIL: {len(bad)} head(s) out of range for " + f"{decoder_layers} layers × {decoder_heads} heads: {bad[:4]}" + ) + return False + + print(f" OK: {len(heads)} alignment heads, all within bounds") + return True + + +def smoke_test(out: Path, config: dict) -> bool: + """Load the graphs in onnxruntime and run one decoder step. + + Graph outputs can be declared correctly and still fail at runtime — a + quantised Whisper decoder previously failed session creation outright with + "Missing required scale … MatMulNBits" (see the note in Rescript's + `lib/models.ts`). Only an actual InferenceSession catches that. + """ + import numpy as np + import onnxruntime as ort + + onnx_dir = out / "onnx" + # Prefer the quantised pair — it is what ships, and the variant most likely + # to fail session creation. + encoder = next( + (onnx_dir / f"encoder_model{s}.onnx" for s in ("_q4", "_fp16", "") + if (onnx_dir / f"encoder_model{s}.onnx").exists()), + None, + ) + decoder = next( + (onnx_dir / f"decoder_model_merged{s}.onnx" for s in ("_q4", "_fp16", "") + if (onnx_dir / f"decoder_model_merged{s}.onnx").exists()), + None, + ) + print("\n smoke test") + if encoder is None or decoder is None: + print(" FAIL: need an encoder_model and a decoder_model_merged") + return False + + options = ort.SessionOptions() + options.log_severity_level = 3 + layers = config["decoder_layers"] + heads = config["decoder_attention_heads"] + head_dim = config["d_model"] // heads + + try: + enc = ort.InferenceSession(str(encoder), options, providers=["CPUExecutionProvider"]) + hidden = enc.run( + None, + {"input_features": np.zeros((1, config["num_mel_bins"], 3000), dtype=np.float32)}, + )[0] + + dec = ort.InferenceSession(str(decoder), options, providers=["CPUExecutionProvider"]) + names = {i.name for i in dec.get_inputs()} + feed = { + "input_ids": np.array([[config["decoder_start_token_id"]]], dtype=np.int64), + "encoder_hidden_states": hidden, + } + if "use_cache_branch" in names: + feed["use_cache_branch"] = np.array([False]) + for i in range(layers): + for kind, length in (("decoder", 0), ("encoder", hidden.shape[1])): + for kv in ("key", "value"): + key = f"past_key_values.{i}.{kind}.{kv}" + if key in names: + feed[key] = np.zeros((1, heads, length, head_dim), dtype=np.float32) + + outputs = dec.run(None, feed) + by_name = dict(zip((o.name for o in dec.get_outputs()), outputs)) + except Exception as err: # noqa: BLE001 - the point is to report, not raise + print(f" FAIL: {type(err).__name__}: {err}") + return False + + attention = by_name.get("cross_attentions.0") + if attention is None: + print(" FAIL: no cross_attentions.0 at runtime") + return False + + # (batch, heads, decoder positions, encoder frames) is what DTW consumes. + if attention.shape[1] != heads or attention.shape[3] != hidden.shape[1]: + print(f" FAIL: cross_attentions.0 has unexpected shape {attention.shape}") + return False + + print(f" OK: {encoder.name} + {decoder.name} run") + print(f" cross_attentions.0 {attention.shape}, logits {by_name['logits'].shape}") + return True + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("out", type=Path, help="export directory produced by export.py") + parser.add_argument( + "--no-smoke", + action="store_true", + help="skip the onnxruntime load/run check (graph inspection only)", + ) + args = parser.parse_args() + + out: Path = args.out + config_path = out / "config.json" + if not config_path.exists(): + print(f"No config.json in {out} — did export.py finish?") + return 1 + + config = json.loads(config_path.read_text()) + decoder_layers = config["decoder_layers"] + decoder_heads = config["decoder_attention_heads"] + print( + f"Verifying {out} — {decoder_layers} decoder layers, " + f"{decoder_heads} attention heads" + ) + + decoders = sorted((out / "onnx").glob("decoder*.onnx")) + if not decoders: + print(f"\nNo decoder graphs in {out / 'onnx'}") + return 1 + + ok = all(check_decoder(path, decoder_layers) for path in decoders) + ok &= check_alignment_heads(out, decoder_layers, decoder_heads) + + encoder = out / "onnx" / "encoder_model.onnx" + print(f"\n encoder_model.onnx\n {'OK: present' if encoder.exists() else 'FAIL: missing'}") + ok &= encoder.exists() + + if not args.no_smoke: + ok &= smoke_test(out, config) + + print("\n" + ("PASS — export supports word timestamps" if ok else "FAIL")) + return 0 if ok else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/workers/transcription.worker.ts b/workers/transcription.worker.ts index 5402d11..ca4497b 100644 --- a/workers/transcription.worker.ts +++ b/workers/transcription.worker.ts @@ -31,7 +31,9 @@ import type { Word, WorkerRequest, WorkerResponse } from "@/lib/types"; import { MODELS, isParakeetModel, + isCrisperModel, isWhisperModel, + verbatimTagCount, whisperFillerPrompt, type ModelId, type WhisperModel, @@ -49,6 +51,14 @@ import { import { isWebGpuDeviceLostError } from "@/lib/webgpu"; env.allowLocalModels = false; +/** + * Where {@link MODELS} entries flagged `local` are served from — an export that + * has not been published to the Hub yet, sitting in public/models//. + * Enabled only for the duration of such a load (see `servedLocally`), because + * `allowLocalModels` is global: left on, every Hub model would probe this path + * and 404 for each of its files before falling back. + */ +const LOCAL_MODEL_PATH = "/models/"; const ORT_WASM_PATHS = "/vendor/ort/"; /** Parakeet.js pins onnxruntime-web@1.24.1 — keep its WASM on a separate path. */ const PARAKEET_ORT_WASM_PATHS = "/vendor/ort-parakeet/"; @@ -57,6 +67,25 @@ if (env.backends?.onnx?.wasm) { env.backends.onnx.wasm.wasmPaths = ORT_WASM_PATHS; } +/** + * WebKit — Safari everywhere, plus every browser on iOS — kills the tab for + * memory far sooner than Chromium ("This webpage was reloaded because it was + * using significant memory"), and onnxruntime's WebGPU path is what pushes it + * over. That path loads the JSEP build (26 MB of wasm against 13 MB for the + * plain threaded one, all compiled up front by JSC) and then uploads every + * weight into Metal buffers during session creation, which is precisely where + * the reload lands. Staying on WASM costs throughput but is the difference + * between finishing a transcript and losing the tab mid-run. + * + * Sniffed rather than feature-detected on purpose: there is nothing to detect. + * WebGPU is present and functional here — it is the memory ceiling around it + * that differs, and no API reports that. `vendor` is frozen to Apple's string + * across WebKit, which is the exact set of engines affected. + */ +if (/apple/i.test(navigator.vendor)) { + fallbackDevicePolicy.preferWasm(); +} + const DIARIZATION_MODEL = "onnx-community/pyannote-segmentation-3.0"; const VAD_MODEL = "onnx-community/silero-vad"; /** Gaps longer than this split speech into separate Whisper jobs. */ @@ -166,21 +195,35 @@ function parakeetModel(): ModelDefinition { }); }; const common = { - preprocessorBackend: "js" as const, + // nemo128.onnx is NeMo's own featurisation graph (0.1 MB). The "js" + // alternative is a hand-written mel — its own FFT and slaney filterbank — + // and any drift from NeMo's exact features degrades every prediction in + // a way that reads as "the model is just worse". + preprocessorBackend: "onnx" as const, progress: onProgress, wasmPaths: PARAKEET_ORT_WASM_PATHS, }; - // WebGPU cannot run the int8 encoder; fp16 (~1.2 GB) is the practical - // WebGPU path. WASM int8 (~670 MB) is the compatibility / size fallback. + /** + * Encoder quantisation is a size decision; decoder quantisation is not. + * + * In a TDT model the decoder/joint network is what emits tokens, so + * quantising it lands directly on word accuracy — and it is tiny next to + * the encoder (fp32 72 MB, fp16 36 MB, int8 18 MB, against 1239 MB for + * the fp16 encoder). parakeet.js defaults both to int8; taking that + * default for the decoder traded measurable accuracy for ~1% of the + * download, so it is set explicitly here instead. + */ const device = await fallbackDevicePolicy.pickDevice(); if (device === "webgpu") { try { + // fp16 encoder + fp32 decoder ≈ 1.31 GB. WebGPU cannot run the int8 + // encoder at all, so fp16 is the only practical encoder here. const model = await fromHub(MODELS.parakeet.id, { ...common, backend: "webgpu", encoderQuant: "fp16", - decoderQuant: "int8", + decoderQuant: "fp32", }); asrDevice = "webgpu"; return model as ParakeetInstance; @@ -193,11 +236,13 @@ function parakeetModel(): ModelDefinition { } } + // int8 encoder + fp16 decoder ≈ 690 MB: the compatibility / size fallback, + // keeping the decoder off int8 for the reason above. const model = await fromHub(MODELS.parakeet.id, { ...common, backend: "wasm", encoderQuant: "int8", - decoderQuant: "int8", + decoderQuant: "fp16", }); asrDevice = "wasm"; return model as ParakeetInstance; @@ -205,6 +250,37 @@ function parakeetModel(): ModelDefinition { }; } +/** + * Flip `env.allowLocalModels` on for one model's load and back afterwards. + * + * transformers.js resolves local-vs-Hub from global state, so a model served + * from public/models can only be reached by enabling it — but leaving it + * enabled makes every Hub model try the local path first and 404 once per + * file. Scoping it to the load keeps both paths clean. + */ +function servedLocally(definition: ModelDefinition): ModelDefinition { + const withLocalPath = async (fn: () => Promise): Promise => { + const previousAllow = env.allowLocalModels; + const previousPath = env.localModelPath; + env.allowLocalModels = true; + env.localModelPath = LOCAL_MODEL_PATH; + try { + return await fn(); + } finally { + env.allowLocalModels = previousAllow; + env.localModelPath = previousPath; + } + }; + + return { + ...definition, + load: (ctx) => withLocalPath(() => definition.load(ctx)), + ...(definition.isCached + ? { isCached: () => withLocalPath(async () => definition.isCached!()) } + : {}), + }; +} + /** * ASR registry keyed by each model's `id` from MODELS. Definitions are * registered up front; loaders only take an id. unloadAll() after a WebGPU @@ -217,19 +293,17 @@ const models = new ModelManager({ if (info.backend === "parakeet") { return [info.id, parakeetModel()]; } - return [ - info.id, - transformersModel({ - pipeline, - task: "automatic-speech-recognition", - modelId: info.id, - dtype: info.dtype, - cacheKey: env.cacheKey ?? "transformers-cache", - onDevice: (device) => { - asrDevice = device; - }, - }), - ]; + const definition = transformersModel({ + pipeline, + task: "automatic-speech-recognition", + modelId: info.id, + dtype: info.dtype, + cacheKey: env.cacheKey ?? "transformers-cache", + onDevice: (device) => { + asrDevice = device; + }, + }); + return [info.id, info.local ? servedLocally(definition) : definition]; }) ), }); @@ -248,8 +322,86 @@ models.subscribe((snap) => { }); }); +/** + * CrisperWhisper's prompt scaffolding: mode tags plus the verbatimize / hotword + * / continuation markers. All sit at the very top of the vocabulary, above the + * timestamp block. + */ +const CRISPER_PROMPT_TOKENS = [ + ...[1, 2, 3, 4, 5].map((i) => `[verbatim_${i}]`), + ...[1, 2, 3, 4, 5].map((i) => `[intended_${i}]`), + "", + "", + "", + "", + "", + "", +]; + +/** + * Register CrisperWhisper's prompt tokens as special, working around a + * transformers.js assumption that otherwise crashes every verbatim transcript. + * + * WhisperTokenizer decides what counts as a timestamp token in two different, + * disagreeing ways. `_decode_asr` reads the real boundary + * (`token_to_id("<|notimestamps|>") + 1`, and an upper bound 1500 tokens + * above), but `decodeWithTimestamps` — reached through `collateWordTimestamps` + * when `return_timestamps: "word"` — infers it as `all_special_ids.at(-1) + 1` + * and applies no upper bound at all. + * + * That is fine for stock Whisper, whose vocabulary ends at the timestamp block. + * CrisperWhisper appends 31 tokens past it — `[UM]`, `[UH]`, vocal events, and + * this scaffolding. Each one is above the inferred threshold, so + * `decodeWithTimestamps` treats it as a timestamp, emits a `<|…|>` marker, and + * starts a fresh empty token bucket; that empty array then reaches `decode([])` + * and throws "token_ids must be a non-empty array of integers". In other words + * it crashed on the first `[UM]` — precisely the token the model is here for. + * + * Marking only the scaffolding special raises `all_special_ids.at(-1)` to the + * top of the vocabulary, so nothing below it is mistaken for a timestamp: + * `[UM]`, `[UH]` and the vocal events decode as ordinary text. `_decode_asr` is + * unaffected because its boundary comes from that independent getter, and it + * additionally skips these ids now — which stops the forced `[verbatim_N]` + * prefix from leaking into the transcript. + * + * Idempotent, so re-running it after a WebGPU-to-WASM reload is harmless. + */ +function markCrisperPromptTokensSpecial( + transcriber: AutomaticSpeechRecognitionPipeline +): void { + try { + // Tokenizer internals are untyped in transformers.js. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const tokenizer = transcriber.tokenizer as any; + const ids = new Set(tokenizer.all_special_ids ?? []); + const before = ids.size; + for (const token of CRISPER_PROMPT_TOKENS) { + const encoded = tokenizer.encode(token, { add_special_tokens: false }); + // Anything that does not map to exactly one id is not the atomic token we + // are looking for — skip rather than guess. + if (encoded?.length === 1) ids.add(encoded[0]); + } + if (ids.size === before) return; + // Sorted because the workaround depends on `.at(-1)` being the maximum. + tokenizer.all_special_ids = [...ids].sort((a, b) => a - b); + } catch { + console.warn( + "Could not mark CrisperWhisper prompt tokens as special; " + + "word timestamps may fail on the first filler token." + ); + } +} + async function getAsr(choice: WhisperModel) { - return models.load(MODELS[choice].id); + const transcriber = await models.load( + MODELS[choice].id + ); + // Not gated on the mode prefix: the extended-vocabulary tokens that break + // word collation can be emitted with or without one. + if (isCrisperModel(choice)) { + markCrisperPromptTokensSpecial(transcriber); + } + return transcriber; } async function getParakeet() { @@ -272,6 +424,57 @@ async function fallbackAsrToWasm() { }); } +/** + * Build decoder input tokens for CrisperWhisper's verbatim mode: + * `[verbatim_1]…[verbatim_N] <|startoftranscript|> <|lang|> <|transcribe|>`. + * + * From `crisperwhisper/prompt.py` (PyPI `crisperwhisper==2.0.1`), with one + * deliberate deviation: + * + * - No `<|startofprev|>`. The tags lead and the standard prefix follows; this + * is a trained-in mode selector, not Whisper's initial-prompt conditioning. + * - Each tag is a single vocabulary token (51880–51884 for verbatim in the + * small checkpoint), absent from `added_tokens.json`, so the stock tokenizer + * encodes the five-tag string to exactly five ids. + * - **`<|notimestamps|>` is omitted, though upstream includes it.** Upstream can + * afford to, because it derives word timings itself via Viterbi over the + * space token's cross-attention. We go through transformers.js, whose + * `_decode_asr` splits chunked audio *on timestamp tokens*: suppress them and + * a long transcript accumulates into one unsegmented run whose + * stride-overlap merge can resolve to nothing, surfacing mid-transcription as + * "token_ids must be a non-empty array of integers". Timestamps are + * orthogonal to mode selection, so leaving them on costs nothing here. + * + * Substituting `[intended_N]` would request the cleaned-up transcript instead; + * this editor always wants verbatim. + */ +function buildVerbatimDecoderIds( + transcriber: AutomaticSpeechRecognitionPipeline, + tagCount: number, + language: string +): number[] | null { + try { + /* eslint-disable @typescript-eslint/no-explicit-any */ + const tokenizer = transcriber.tokenizer as any; + const genCfg = (transcriber.model as any).generation_config; + /* eslint-enable @typescript-eslint/no-explicit-any */ + const tags = Array.from( + { length: tagCount }, + (_, i) => `[verbatim_${i + 1}]` + ).join(""); + const tagIds = tokenizer.encode(tags, { add_special_tokens: false }); + const startId = genCfg?.decoder_start_token_id; + const langId = genCfg?.lang_to_id?.[`<|${language}|>`]; + const taskId = genCfg?.task_to_id?.["transcribe"]; + if (!tagIds?.length || startId == null || langId == null || taskId == null) { + return null; + } + return [...tagIds, startId, langId, taskId]; + } catch { + return null; + } +} + /** * Build decoder input tokens implementing Whisper's "initial prompt" * conditioning: `<|startofprev|> …prompt… <|startoftranscript|> <|lang|> @@ -688,13 +891,25 @@ async function runWhisper( const { segments: speechSegments, frames: speechFrames } = await detectSpeechSegments(audio, vad); - // Short filler-rich <|startofprev|> prompt so um/uh survive for "Remove - // fillers". Long prompts truncate multi-speaker clips — see whisperFillerPrompt. - const fillerPrompt = whisperFillerPrompt(choice, transcriptLanguage); - const promptedIds = fillerPrompt - ? buildPromptedDecoderIds(transcriber, fillerPrompt, transcriptLanguage) - : null; - if (fillerPrompt && !promptedIds) { + // Two different ways of asking for disfluencies, depending on the model. + // CrisperWhisper was trained verbatim and selects it with a mode prefix; + // stock Whisper was not, and only gets a nudge from a filler-rich + // <|startofprev|> prompt (kept short — long prompts truncate multi-speaker + // clips, see whisperFillerPrompt). + const tagCount = verbatimTagCount(choice); + const fillerPrompt = tagCount ? null : whisperFillerPrompt(choice, transcriptLanguage); + const promptedIds = tagCount + ? buildVerbatimDecoderIds(transcriber, tagCount, transcriptLanguage) + : fillerPrompt + ? buildPromptedDecoderIds(transcriber, fillerPrompt, transcriptLanguage) + : null; + if (tagCount && !promptedIds) { + // Falling through to plain decoding would silently produce a non-verbatim + // transcript, which is the entire reason for picking this model. + console.warn( + "Could not build CrisperWhisper verbatim prefix; output will not be verbatim." + ); + } else if (fillerPrompt && !promptedIds) { console.warn("Could not build filler prompt tokens; using default decoding."); } From 21f8271f2344af6dcedfbe2426639ee2e60a150d Mon Sep 17 00:00:00 2001 From: Wassim Gharbi Date: Tue, 4 Aug 2026 23:55:21 -0700 Subject: [PATCH 2/9] bring back verbatim tag count --- lib/models.ts | 59 ++++++--------------------------- workers/transcription.worker.ts | 5 +-- 2 files changed, 11 insertions(+), 53 deletions(-) diff --git a/lib/models.ts b/lib/models.ts index 6273e32..057ef0e 100644 --- a/lib/models.ts +++ b/lib/models.ts @@ -57,36 +57,14 @@ export type WhisperModelInfo = ModelDisplay & { * cuttable. Kept as a one-flag switch for re-testing on other material. */ keepFillers?: boolean; - /** - * A CrisperWhisper checkpoint. Its vocabulary extends 31 tokens past - * Whisper's timestamp block (`[UM]`, `[UH]`, vocal events, prompt - * scaffolding), which transformers.js misreads as timestamps unless the - * tokenizer is patched at load — see `markCrisperPromptTokensSpecial` in the - * transcription worker. Independent of {@link WhisperModelInfo.verbatimTags}, - * because the extra tokens can be emitted whether or not a mode prefix is - * sent. - */ - crisper?: boolean; /** * CrisperWhisper mode prefix: how many `[verbatim_N]` tags to prime the - * decoder with. Verbatim vs. intended is selected purely by this prefix — the - * encoder output is identical either way — and unlike Whisper's initial - * prompt there is no `<|startofprev|>`; the tags lead and the standard prefix - * follows. Source: `crisperwhisper==2.0.1`, `crisperwhisper/prompt.py`. - * - * Currently unset on both CrisperWhisper entries, deliberately. Measured - * against the PyTorch checkpoint on an 11.5 s two-speaker clip: - * - * - verbatim prefix → dropped a whole clause ("Nice. How does it,") and - * emitted `[breath]` where the speaker hesitated - * - intended prefix → complete, but no disfluencies at all - * - **no prefix → complete _and_ kept the "uh"**, beating every other model - * tested including Whisper Small and Parakeet - * - * Adding `<|notimestamps|>` to match upstream exactly changed nothing. The - * plumbing is verified — the tags do reach the decoder as ids 51880–51884 — - * so this is the mode itself underperforming, not a wiring bug. Kept as a - * one-line switch because the finding rests on a single clip. + * decoder with. CrisperWhisper 2.0 picks verbatim vs. intended output purely + * from this prefix — the encoder output is identical either way — and unlike + * Whisper's initial prompt there is no `<|startofprev|>`; the tags come + * first and the standard prefix follows. Verbatim output spells fillers as + * `[UM]` / `[UH]`, which `lib/fillers.ts` already matches once punctuation is + * stripped. Source: `crisperwhisper==2.0.1`, `crisperwhisper/prompt.py`. */ verbatimTags?: number; /** @@ -116,13 +94,6 @@ export const MODEL_ORDER: ModelId[] = [ "crisperTurbo", ]; -/** - * Hard cap for Whisper filler prompts. Longer `<|startofprev|>` strings - * forced via `decoder_input_ids` have truncated multi-speaker / long-form - * transcripts (second speaker dropped). Keep prompts to a short filler list. - */ -export const MAX_VERBATIM_PROMPT_LENGTH = 32; - /** * Short, language-specific filler prompts for Whisper. Deliberately tiny and * free of "I'm …" openers (those seeded "I'm sorry" hallucination loops). @@ -243,22 +214,22 @@ export const MODELS: { local: true, label: "CrisperWhisper Small (local)", description: - "Keeps fillers other models drop, with clean punctuation. Self-exported, unpublished. Non-commercial licence.", + "Verbatim: transcribes fillers as [UM] / [UH] instead of dropping them. Self-exported, unpublished. Non-commercial licence.", // q4 encoder 66 MB + q4 merged decoder 258 MB. size: "~324 MB", dtype: CRISPER_DTYPE, - crisper: true, + verbatimTags: 5, }, crisperTurbo: { backend: "whisper", id: "Masterx/CrisperWhisper2.0-turbo-ONNX", label: "CrisperWhisper Turbo", description: - "CrisperWhisper on a large-v3 encoder. Largest download. Non-commercial licence.", + "Verbatim, on a large-v3 encoder. The most accurate verbatim option, and the largest download. Non-commercial licence.", // q4 encoder 425 MB + q4 merged decoder 600 MB. size: "~1.0 GB", dtype: CRISPER_DTYPE, - crisper: true, + verbatimTags: 5, }, }; @@ -268,16 +239,6 @@ export function verbatimTagCount(model: ModelId): number | null { return info.backend === "whisper" ? (info.verbatimTags ?? null) : null; } -/** - * Whether `model` is a CrisperWhisper checkpoint, and so needs the tokenizer - * fix-up for its extended vocabulary. True regardless of whether a mode prefix - * is sent: `[UM]`, `[UH]` and the vocal-event tokens can be emitted either way. - */ -export function isCrisperModel(model: ModelId): boolean { - const info = MODELS[model]; - return info.backend === "whisper" && info.crisper === true; -} - /** Whether `model` loads from public/models rather than the Hub. */ export function isLocalModel(model: ModelId): boolean { const info = MODELS[model]; diff --git a/workers/transcription.worker.ts b/workers/transcription.worker.ts index ca4497b..ab1f363 100644 --- a/workers/transcription.worker.ts +++ b/workers/transcription.worker.ts @@ -31,7 +31,6 @@ import type { Word, WorkerRequest, WorkerResponse } from "@/lib/types"; import { MODELS, isParakeetModel, - isCrisperModel, isWhisperModel, verbatimTagCount, whisperFillerPrompt, @@ -396,9 +395,7 @@ async function getAsr(choice: WhisperModel) { const transcriber = await models.load( MODELS[choice].id ); - // Not gated on the mode prefix: the extended-vocabulary tokens that break - // word collation can be emitted with or without one. - if (isCrisperModel(choice)) { + if (verbatimTagCount(choice) !== null) { markCrisperPromptTokensSpecial(transcriber); } return transcriber; From b545fc07240db9b89e16dde7f0de3f3313b18190 Mon Sep 17 00:00:00 2001 From: Wassim Gharbi Date: Wed, 5 Aug 2026 00:54:29 -0700 Subject: [PATCH 3/9] fix turbo --- lib/models.ts | 88 ++++++++++++++++++++++++++++----- tests/models-test.ts | 15 ++++++ tests/vad-regression-test.ts | 17 +++++-- workers/transcription.worker.ts | 5 +- 4 files changed, 106 insertions(+), 19 deletions(-) diff --git a/lib/models.ts b/lib/models.ts index 057ef0e..fe4421f 100644 --- a/lib/models.ts +++ b/lib/models.ts @@ -67,6 +67,20 @@ export type WhisperModelInfo = ModelDisplay & { * stripped. Source: `crisperwhisper==2.0.1`, `crisperwhisper/prompt.py`. */ verbatimTags?: number; + /** + * A CrisperWhisper checkpoint, whose vocabulary extends past Whisper's + * timestamp block: `[UM]`, `[UH]`, vocal events and the prompt scaffolding all + * sit above it. transformers.js's `decodeWithTimestamps` infers the timestamp + * boundary as `all_special_ids.at(-1) + 1` with no upper bound, so every one of + * those tokens is mistaken for a timestamp and word-timestamp collation dies on + * the first filler. The worker repairs this per model — see + * `markCrisperPromptTokensSpecial`. + * + * Deliberately separate from {@link WhisperModelInfo.verbatimTags}: the repair + * is required by the vocabulary layout, so it must keep running even for a + * CrisperWhisper model decoded without the mode prefix. + */ + crisper?: boolean; /** * Load from `public/models//` instead of the Hub. Used for exports that * have not been published yet — see tools/crisperwhisper-onnx. @@ -139,17 +153,39 @@ const WHISPER_MEDIUM_DTYPE = { } satisfies WhisperModelInfo["dtype"]; /** - * Both CrisperWhisper exports run q4 on either device — it is the only - * quantisation available for the merged decoder in both repos, because - * `quantize_dynamic` cannot reach weights inside the merged decoder's - * control-flow subgraphs (see tools/crisperwhisper-onnx/README.md). The q4 pair - * is also the combination verified end-to-end for the local export. + * The local Small export ships only q4 for the merged decoder: int8 cannot + * reach weights inside the decoder's control-flow subgraphs, so + * `quantize_dynamic` silently emits an un-quantised file and the export tooling + * refuses it (see tools/crisperwhisper-onnx/README.md). This q4 pair is the + * combination verified end-to-end — encoder and decoder loaded in onnxruntime, + * cross-attentions returned at the right shape. */ -const CRISPER_DTYPE = { +const CRISPER_SMALL_DTYPE = { webgpu: { encoder_model: "q4", decoder_model_merged: "q4" }, wasm: { encoder_model: "q4", decoder_model_merged: "q4" }, } satisfies WhisperModelInfo["dtype"]; +/** + * Turbo takes fp16 for the decoder rather than q4. + * + * Turbo collapsed after the first VAD segment on q4/q4. Bisecting against the + * fp32 checkpoint cleared everything else: fp32 transcribes the clip in full, + * and the q4 encoder is faithful — swapping it in under an fp32 decoder gives a + * byte-identical transcript (cosine 0.93 against fp32 hidden states, but no + * effect on output). That leaves the merged decoder as the only component not + * exonerated, so it gets the precision. + * + * It is also cheaper: fp16 is 477 MB against q4's 600 MB, because q4 leaves the + * embedding and lm_head — most of a 4-layer decoder's weight — unquantised. + * fp32 would be better still but ships as a 953 MB external-data sidecar, which + * transformers.js only fetches when `use_external_data_format` is declared, and + * this repo's config.json does not declare it. + */ +const CRISPER_TURBO_DTYPE = { + webgpu: { encoder_model: "q4", decoder_model_merged: "fp16" }, + wasm: { encoder_model: "q4", decoder_model_merged: "fp16" }, +} satisfies WhisperModelInfo["dtype"]; + /** * Local speech models that can run in the transcription worker. * Shared display fields live on every entry; backend-specific knobs @@ -217,22 +253,48 @@ export const MODELS: { "Verbatim: transcribes fillers as [UM] / [UH] instead of dropping them. Self-exported, unpublished. Non-commercial licence.", // q4 encoder 66 MB + q4 merged decoder 258 MB. size: "~324 MB", - dtype: CRISPER_DTYPE, - verbatimTags: 5, + dtype: CRISPER_SMALL_DTYPE, + crisper: true, + // No `verbatimTags` — see the note on crisperTurbo. Measured against the + // fp32 checkpoint on an 11.5 s clip, the mode prefix drops "Nice. How does + // it," and emits [breath] where the speaker hesitated, while plain decoding + // keeps the hesitation as "uh". }, crisperTurbo: { backend: "whisper", id: "Masterx/CrisperWhisper2.0-turbo-ONNX", label: "CrisperWhisper Turbo", description: - "Verbatim, on a large-v3 encoder. The most accurate verbatim option, and the largest download. Non-commercial licence.", - // q4 encoder 425 MB + q4 merged decoder 600 MB. - size: "~1.0 GB", - dtype: CRISPER_DTYPE, - verbatimTags: 5, + "Keeps fillers, on a large-v3 encoder. The largest download. Non-commercial licence.", + // q4 encoder 425 MB + fp16 merged decoder 477 MB. + size: "~900 MB", + dtype: CRISPER_TURBO_DTYPE, + crisper: true, + // No `verbatimTags`, for the same reason Medium carries no filler prompt: + // prefix conditioning collapses short segments, and the worker decodes VAD + // segments rather than whole files. Measured on an 11.5 s clip against the + // fp32 checkpoint, one segment at a time: + // + // slice | plain | + [verbatim_1..5] + // ------------|-----------------------------|--------------------- + // full 11.5 s | complete, includes "[UH]" | drops "Nice." + // 2.5–5.0 s | "Nice. How does it work?" | "Nice. How does it-" + // + // Nothing is lost by dropping it: this checkpoint emits `[UH]` unprompted, + // so plain decoding is both more complete and still verbatim. }, }; +/** + * Whether `model` is a CrisperWhisper checkpoint, and so needs the tokenizer + * repair in {@link WhisperModelInfo.crisper}. Independent of whether it is + * decoded with a mode prefix. + */ +export function isCrisperModel(model: ModelId): boolean { + const info = MODELS[model]; + return info.backend === "whisper" && info.crisper === true; +} + /** Whether `model` is steered with CrisperWhisper's `[verbatim_N]` prefix. */ export function verbatimTagCount(model: ModelId): number | null { const info = MODELS[model]; diff --git a/tests/models-test.ts b/tests/models-test.ts index 10a112b..4cb1567 100644 --- a/tests/models-test.ts +++ b/tests/models-test.ts @@ -86,6 +86,21 @@ assert(verbatimTagCount("crisperTurbo") === null, "crisperTurbo sends no mode pr assert(verbatimTagCount("base") === null, "base has no verbatim tags"); // The tokenizer fix-up must still run without a mode prefix — the extended // vocabulary tokens that break word collation are emitted either way. +// Turbo runs fp16 on the merged decoder, not q4: on q4 it collapsed after the +// first VAD segment, and bisection cleared the encoder (a q4 encoder under an +// fp32 decoder reproduces the fp32 transcript exactly). fp16 is also 477 MB +// against q4's 600 MB. Pinned so a dtype tidy-up cannot fold it back into the +// Small config. +assert( + MODELS.crisperTurbo.dtype.wasm.decoder_model_merged === "fp16" && + MODELS.crisperTurbo.dtype.webgpu.decoder_model_merged === "fp16", + "crisperTurbo decodes in fp16" +); +assert( + MODELS.crisperSmall.dtype.wasm.decoder_model_merged === "q4", + "crisperSmall keeps the verified q4 decoder" +); + assert(isCrisperModel("crisperSmall"), "crisperSmall needs the tokenizer fix-up"); assert(isCrisperModel("crisperTurbo"), "crisperTurbo needs the tokenizer fix-up"); assert(!isCrisperModel("base"), "base needs no tokenizer fix-up"); diff --git a/tests/vad-regression-test.ts b/tests/vad-regression-test.ts index b7f64e4..e6812b1 100644 --- a/tests/vad-regression-test.ts +++ b/tests/vad-regression-test.ts @@ -14,11 +14,7 @@ import { speechSegmentsFromFrames, VAD_SAMPLE_RATE, } from "../lib/vad"; -import { - MAX_VERBATIM_PROMPT_LENGTH, - MODELS, - whisperFillerPrompt, -} from "../lib/models"; +import { MODELS, whisperFillerPrompt } from "../lib/models"; import { TRANSCRIPT_LANGUAGE_ORDER, type TranscriptLanguage, @@ -38,6 +34,17 @@ function ffmpegAvailable(): boolean { return probe.status === 0; } +/** + * Longest `<|startofprev|>` filler prompt considered safe. Lived in lib/models + * until every model stopped opting in; kept here because the constraint is + * still real for anything that opts back in, and prod no longer reads it. + * + * Note the cap is necessary but not sufficient — Whisper Small collapses a long + * VAD segment to a bare "Um," with a 20-character prompt, well inside it. That + * is why `keepFillers` is unset everywhere rather than merely bounded. + */ +const MAX_VERBATIM_PROMPT_LENGTH = 32; + // Short filler prompts are OK; long ones truncate multi-speaker / long-form ASR. // Not every Whisper model opts in: Medium truncates to a fragment even inside // the length cap, and the CrisperWhisper checkpoints transcribe fillers without diff --git a/workers/transcription.worker.ts b/workers/transcription.worker.ts index ab1f363..b30e011 100644 --- a/workers/transcription.worker.ts +++ b/workers/transcription.worker.ts @@ -32,6 +32,7 @@ import { MODELS, isParakeetModel, isWhisperModel, + isCrisperModel, verbatimTagCount, whisperFillerPrompt, type ModelId, @@ -395,7 +396,9 @@ async function getAsr(choice: WhisperModel) { const transcriber = await models.load( MODELS[choice].id ); - if (verbatimTagCount(choice) !== null) { + // Keyed on the checkpoint, not on the prefix: the repair is required by + // CrisperWhisper's vocabulary layout, so it applies even without one. + if (isCrisperModel(choice)) { markCrisperPromptTokensSpecial(transcriber); } return transcriber; From 52d36dacbe94fe7f6e93bc6bafc45be49aa474a2 Mon Sep 17 00:00:00 2001 From: Wassim Gharbi Date: Wed, 5 Aug 2026 01:37:16 -0700 Subject: [PATCH 4/9] patch transformers.js to handle extra tokens --- lib/models.ts | 20 +- package-lock.json | 176 +++++++++++ package.json | 3 +- patches/@huggingface+transformers+4.2.0.patch | 296 ++++++++++++++++++ patches/README.md | 62 ++++ workers/transcription.worker.ts | 51 +-- 6 files changed, 576 insertions(+), 32 deletions(-) create mode 100644 patches/@huggingface+transformers+4.2.0.patch create mode 100644 patches/README.md diff --git a/lib/models.ts b/lib/models.ts index fe4421f..afb2e3b 100644 --- a/lib/models.ts +++ b/lib/models.ts @@ -70,15 +70,19 @@ export type WhisperModelInfo = ModelDisplay & { /** * A CrisperWhisper checkpoint, whose vocabulary extends past Whisper's * timestamp block: `[UM]`, `[UH]`, vocal events and the prompt scaffolding all - * sit above it. transformers.js's `decodeWithTimestamps` infers the timestamp - * boundary as `all_special_ids.at(-1) + 1` with no upper bound, so every one of - * those tokens is mistaken for a timestamp and word-timestamp collation dies on - * the first filler. The worker repairs this per model — see - * `markCrisperPromptTokensSpecial`. + * sit above it. * - * Deliberately separate from {@link WhisperModelInfo.verbatimTags}: the repair - * is required by the vocabulary layout, so it must keep running even for a - * CrisperWhisper model decoded without the mode prefix. + * These models only work at all because of + * patches/@huggingface+transformers+4.2.0.patch, which bounds the timestamp + * range at both ends. Unpatched, transformers.js reads every token above the + * block as a timestamp: word-timestamp collation crashes on the first `[UM]`, + * and the logits processor suppresses all text after the first `[UH]`, cutting + * the transcript off mid-sentence with no error. See patches/README.md. + * + * The flag itself drives `markCrisperPromptTokensSpecial`, keeping the mode + * tags out of transcript text. Deliberately separate from + * {@link WhisperModelInfo.verbatimTags} so it still applies to a CrisperWhisper + * model decoded without the mode prefix. */ crisper?: boolean; /** diff --git a/package-lock.json b/package-lock.json index 9c08de2..97f1f82 100644 --- a/package-lock.json +++ b/package-lock.json @@ -45,6 +45,7 @@ "esbuild": "^0.28.1", "eslint": "^9", "eslint-config-next": "16.2.12", + "patch-package": "^8.0.1", "tailwindcss": "^4", "tsx": "^4.22.4", "typescript": "^5" @@ -4104,6 +4105,13 @@ "node": ">=10.0.0" } }, + "node_modules/@yarnpkg/lockfile": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@yarnpkg/lockfile/-/lockfile-1.1.0.tgz", + "integrity": "sha512-GpSwvyXOcOOlV70vbnzjj4fW5xW/FdUF6nQEt1ENy7m4ZCczi1+/buVUPAqmGfqznsORNFzUMjctTIp8a9tuCQ==", + "dev": true, + "license": "BSD-2-Clause" + }, "node_modules/abbrev": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-4.0.0.tgz", @@ -6857,6 +6865,16 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/find-yarn-workspace-root": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/find-yarn-workspace-root/-/find-yarn-workspace-root-2.0.0.tgz", + "integrity": "sha512-1IMnbjt4KzsQfnhnzNd8wUEgXZ44IzZaZmnLYx7D5FZlaHt2gW20Cri8Q+E/t5tIj4+epTBub+2Zxu/vNILzqQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "micromatch": "^4.0.2" + } + }, "node_modules/flat-cache": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-4.0.1.tgz", @@ -7699,6 +7717,22 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-docker": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-2.2.1.tgz", + "integrity": "sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ==", + "dev": true, + "license": "MIT", + "bin": { + "is-docker": "cli.js" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/is-document.all": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-document.all/-/is-document.all-1.0.0.tgz", @@ -7982,6 +8016,19 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-wsl": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz", + "integrity": "sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-docker": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/isarray": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.5.tgz", @@ -8119,6 +8166,26 @@ "dev": true, "license": "MIT" }, + "node_modules/json-stable-stringify": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/json-stable-stringify/-/json-stable-stringify-1.3.0.tgz", + "integrity": "sha512-qtYiSSFlwot9XHtF9bD9c7rwKjr+RecWT//ZnPvSmEjpV5mmPOCN4j8UjY5hbjNkOwZ/jQv3J6R1/pL7RwgMsg==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bind": "^1.0.8", + "call-bound": "^1.0.4", + "isarray": "^2.0.5", + "jsonify": "^0.0.1", + "object-keys": "^1.1.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/json-stable-stringify-without-jsonify": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", @@ -8158,6 +8225,16 @@ "graceful-fs": "^4.1.6" } }, + "node_modules/jsonify": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/jsonify/-/jsonify-0.0.1.tgz", + "integrity": "sha512-2/Ki0GcmuqSrgFyelQq9M05y7PS0mEwuIzrf3f1fPqkVDVRvZrPZtVSMHxdgo8Aq0sxAOb/cr2aqqA3LeWHVPg==", + "dev": true, + "license": "Public Domain", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/jsx-ast-utils": { "version": "3.3.5", "resolved": "https://registry.npmjs.org/jsx-ast-utils/-/jsx-ast-utils-3.3.5.tgz", @@ -8184,6 +8261,16 @@ "json-buffer": "3.0.1" } }, + "node_modules/klaw-sync": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/klaw-sync/-/klaw-sync-6.0.0.tgz", + "integrity": "sha512-nIeuVSzdCCs6TDPTqI8w1Yre34sSq7AkZ4B3sfOBbI2CgVSB4Du4aLQijFU2+lhAFCwt9+42Hel6lQNIv6AntQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.1.11" + } + }, "node_modules/language-subtag-registry": { "version": "0.3.23", "resolved": "https://registry.npmjs.org/language-subtag-registry/-/language-subtag-registry-0.3.23.tgz", @@ -9230,6 +9317,23 @@ "integrity": "sha512-BOoomdHYmNRL5r4iQ4bMvsl2t0/hzVQ3OM3PHD0gxeXu1PmggqBv3puZicEUVOA3AtHHYmqZtjMj9FOfGrATTw==", "license": "MIT" }, + "node_modules/open": { + "version": "7.4.2", + "resolved": "https://registry.npmjs.org/open/-/open-7.4.2.tgz", + "integrity": "sha512-MVHddDVweXZF3awtlAS+6pgKLlm/JgxZ90+/NBurBoQctVOOB/zDdVjcyPzQ+0laDGbsWgrRkflI65sQeOgT9Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-docker": "^2.0.0", + "is-wsl": "^2.1.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/optionator": { "version": "0.9.4", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", @@ -9351,6 +9455,52 @@ "node": ">=6" } }, + "node_modules/patch-package": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/patch-package/-/patch-package-8.0.1.tgz", + "integrity": "sha512-VsKRIA8f5uqHQ7NGhwIna6Bx6D9s/1iXlA1hthBVBEbkq+t4kXD0HHt+rJhf/Z+Ci0F/HCB2hvn0qLdLG+Qxlw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@yarnpkg/lockfile": "^1.1.0", + "chalk": "^4.1.2", + "ci-info": "^3.7.0", + "cross-spawn": "^7.0.3", + "find-yarn-workspace-root": "^2.0.0", + "fs-extra": "^10.0.0", + "json-stable-stringify": "^1.0.2", + "klaw-sync": "^6.0.0", + "minimist": "^1.2.6", + "open": "^7.4.2", + "semver": "^7.5.3", + "slash": "^2.0.0", + "tmp": "^0.2.4", + "yaml": "^2.2.2" + }, + "bin": { + "patch-package": "index.js" + }, + "engines": { + "node": ">=14", + "npm": ">5" + } + }, + "node_modules/patch-package/node_modules/ci-info": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.9.0.tgz", + "integrity": "sha512-NIxF55hv4nSqQswkAeiOi1r83xy8JldOFDTWiug55KBu9Jnblncd2U6ViHmYgHf01TPZS77NJBhBMKdWj9HQMQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/sibiraj-s" + } + ], + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", @@ -10388,6 +10538,16 @@ "node": ">=10" } }, + "node_modules/slash": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-2.0.0.tgz", + "integrity": "sha512-ZYKh3Wh2z1PpEXWr0MpSBZ0V6mZHAQfYevttO11c51CaWjGTaadiKZ+wVt1PbMlDV5qhMFslpZCemhwOK7C89A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", @@ -11488,6 +11648,22 @@ "dev": true, "license": "ISC" }, + "node_modules/yaml": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.0.tgz", + "integrity": "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==", + "dev": true, + "license": "ISC", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14.6" + }, + "funding": { + "url": "https://github.com/sponsors/eemeli" + } + }, "node_modules/yargs": { "version": "17.7.2", "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", diff --git a/package.json b/package.json index 63aba16..0bca322 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,7 @@ "build": "next build", "start": "next start", "lint": "eslint", - "postinstall": "node scripts/copy-assets.mjs", + "postinstall": "patch-package && node scripts/copy-assets.mjs", "build:electron": "node scripts/build-electron.mjs", "typecheck:electron": "tsc -p electron/tsconfig.json --noEmit", "electron:dev": "npm run build:electron && concurrently -k -n next,electron -c cyan,magenta \"next dev\" \"node scripts/electron-dev.mjs\"", @@ -68,6 +68,7 @@ "esbuild": "^0.28.1", "eslint": "^9", "eslint-config-next": "16.2.12", + "patch-package": "^8.0.1", "tailwindcss": "^4", "tsx": "^4.22.4", "typescript": "^5" diff --git a/patches/@huggingface+transformers+4.2.0.patch b/patches/@huggingface+transformers+4.2.0.patch new file mode 100644 index 0000000..78b25e9 --- /dev/null +++ b/patches/@huggingface+transformers+4.2.0.patch @@ -0,0 +1,296 @@ +diff --git a/node_modules/@huggingface/transformers/dist/transformers.js b/node_modules/@huggingface/transformers/dist/transformers.js +index 3274d9c..0baa642 100644 +--- a/node_modules/@huggingface/transformers/dist/transformers.js ++++ b/node_modules/@huggingface/transformers/dist/transformers.js +@@ -16280,11 +16280,12 @@ var WhisperTokenizer = class extends PreTrainedTokenizer { + */ + decodeWithTimestamps(token_ids, decode_args) { + const time_precision = decode_args?.time_precision ?? 0.02; +- const timestamp_begin = this.all_special_ids.at(-1) + 1; ++ const timestamp_begin = this.timestamp_begin; ++ const timestamp_end = timestamp_begin + 1500; + let outputs = [[]]; + for (let token of token_ids) { + token = Number(token); +- if (token >= timestamp_begin) { ++ if (token >= timestamp_begin && token <= timestamp_end) { + const timestamp = ((token - timestamp_begin) * time_precision).toFixed(2); + outputs.push(`<|${timestamp}|>`); + outputs.push([]); +@@ -16292,7 +16293,7 @@ var WhisperTokenizer = class extends PreTrainedTokenizer { + outputs[outputs.length - 1].push(token); + } + } +- outputs = outputs.map((s) => typeof s === "string" ? s : super.decode(s, decode_args)); ++ outputs = outputs.map((s) => typeof s === "string" ? s : s.length === 0 ? "" : super.decode(s, decode_args)); + return outputs.join(""); + } + /** +@@ -23586,6 +23587,7 @@ var WhisperTimeStampLogitsProcessor = class extends LogitsProcessor { + this.eos_token_id = Array.isArray(generate_config.eos_token_id) ? generate_config.eos_token_id[0] : generate_config.eos_token_id; + this.no_timestamps_token_id = generate_config.no_timestamps_token_id; + this.timestamp_begin = this.no_timestamps_token_id + 1; ++ this.timestamp_end = this.timestamp_begin + 1500; + this.begin_index = init_tokens.length; + if (init_tokens.at(-1) === this.no_timestamps_token_id) { + this.begin_index -= 1; +@@ -23606,17 +23608,17 @@ var WhisperTimeStampLogitsProcessor = class extends LogitsProcessor { + ); + batch_logits_data[this.no_timestamps_token_id] = -Infinity; + if (input_ids[i].length === this.begin_index) { +- batch_logits_data.subarray(0, this.timestamp_begin).fill(-Infinity); ++ batch_logits_data.subarray(0, this.timestamp_begin).fill(-Infinity), batch_logits_data.subarray(this.timestamp_end + 1).fill(-Infinity); + continue; + } + const seq = input_ids[i].slice(this.begin_index); +- const last_was_timestamp = seq.length >= 1 && seq[seq.length - 1] >= this.timestamp_begin; +- const penultimate_was_timestamp = seq.length < 2 || seq[seq.length - 2] >= this.timestamp_begin; ++ const last_was_timestamp = seq.length >= 1 && seq[seq.length - 1] >= this.timestamp_begin && seq[seq.length - 1] <= this.timestamp_end; ++ const penultimate_was_timestamp = seq.length < 2 || (seq[seq.length - 2] >= this.timestamp_begin && seq[seq.length - 2] <= this.timestamp_end); + if (last_was_timestamp) { + if (penultimate_was_timestamp) { +- batch_logits_data.subarray(this.timestamp_begin).fill(-Infinity); ++ batch_logits_data.subarray(this.timestamp_begin, this.timestamp_end + 1).fill(-Infinity); + } else { +- batch_logits_data.subarray(0, this.eos_token_id).fill(-Infinity); ++ batch_logits_data.subarray(0, this.eos_token_id).fill(-Infinity), batch_logits_data.subarray(this.timestamp_end + 1).fill(-Infinity); + } + } + if (input_ids[i].length === this.begin_index && this.max_initial_timestamp_index !== null) { +@@ -23625,11 +23627,11 @@ var WhisperTimeStampLogitsProcessor = class extends LogitsProcessor { + } + const logprobs = log_softmax(batch_logits_data); + const timestamp_logprob = Math.log( +- logprobs.subarray(this.timestamp_begin).map(Math.exp).reduce((a, b) => a + b) ++ logprobs.subarray(this.timestamp_begin, this.timestamp_end + 1).map(Math.exp).reduce((a, b) => a + b) + ); +- const max_text_token_logprob = max(logprobs.subarray(0, this.timestamp_begin))[0]; ++ const max_text_token_logprob = Math.max(max(logprobs.subarray(0, this.timestamp_begin))[0], logprobs.length > this.timestamp_end + 1 ? max(logprobs.subarray(this.timestamp_end + 1))[0] : -Infinity); + if (timestamp_logprob > max_text_token_logprob) { +- batch_logits_data.subarray(0, this.timestamp_begin).fill(-Infinity); ++ batch_logits_data.subarray(0, this.timestamp_begin).fill(-Infinity), batch_logits_data.subarray(this.timestamp_end + 1).fill(-Infinity); + } + } + return logits; +diff --git a/node_modules/@huggingface/transformers/dist/transformers.node.cjs b/node_modules/@huggingface/transformers/dist/transformers.node.cjs +index 6cb09ae..fcf2117 100644 +--- a/node_modules/@huggingface/transformers/dist/transformers.node.cjs ++++ b/node_modules/@huggingface/transformers/dist/transformers.node.cjs +@@ -16472,11 +16472,12 @@ var WhisperTokenizer = class extends PreTrainedTokenizer { + */ + decodeWithTimestamps(token_ids, decode_args) { + const time_precision = decode_args?.time_precision ?? 0.02; +- const timestamp_begin = this.all_special_ids.at(-1) + 1; ++ const timestamp_begin = this.timestamp_begin; ++ const timestamp_end = timestamp_begin + 1500; + let outputs = [[]]; + for (let token of token_ids) { + token = Number(token); +- if (token >= timestamp_begin) { ++ if (token >= timestamp_begin && token <= timestamp_end) { + const timestamp = ((token - timestamp_begin) * time_precision).toFixed(2); + outputs.push(`<|${timestamp}|>`); + outputs.push([]); +@@ -16484,7 +16485,7 @@ var WhisperTokenizer = class extends PreTrainedTokenizer { + outputs[outputs.length - 1].push(token); + } + } +- outputs = outputs.map((s) => typeof s === "string" ? s : super.decode(s, decode_args)); ++ outputs = outputs.map((s) => typeof s === "string" ? s : s.length === 0 ? "" : super.decode(s, decode_args)); + return outputs.join(""); + } + /** +@@ -23769,6 +23770,7 @@ var WhisperTimeStampLogitsProcessor = class extends LogitsProcessor { + this.eos_token_id = Array.isArray(generate_config.eos_token_id) ? generate_config.eos_token_id[0] : generate_config.eos_token_id; + this.no_timestamps_token_id = generate_config.no_timestamps_token_id; + this.timestamp_begin = this.no_timestamps_token_id + 1; ++ this.timestamp_end = this.timestamp_begin + 1500; + this.begin_index = init_tokens.length; + if (init_tokens.at(-1) === this.no_timestamps_token_id) { + this.begin_index -= 1; +@@ -23789,17 +23791,17 @@ var WhisperTimeStampLogitsProcessor = class extends LogitsProcessor { + ); + batch_logits_data[this.no_timestamps_token_id] = -Infinity; + if (input_ids[i].length === this.begin_index) { +- batch_logits_data.subarray(0, this.timestamp_begin).fill(-Infinity); ++ batch_logits_data.subarray(0, this.timestamp_begin).fill(-Infinity), batch_logits_data.subarray(this.timestamp_end + 1).fill(-Infinity); + continue; + } + const seq = input_ids[i].slice(this.begin_index); +- const last_was_timestamp = seq.length >= 1 && seq[seq.length - 1] >= this.timestamp_begin; +- const penultimate_was_timestamp = seq.length < 2 || seq[seq.length - 2] >= this.timestamp_begin; ++ const last_was_timestamp = seq.length >= 1 && seq[seq.length - 1] >= this.timestamp_begin && seq[seq.length - 1] <= this.timestamp_end; ++ const penultimate_was_timestamp = seq.length < 2 || (seq[seq.length - 2] >= this.timestamp_begin && seq[seq.length - 2] <= this.timestamp_end); + if (last_was_timestamp) { + if (penultimate_was_timestamp) { +- batch_logits_data.subarray(this.timestamp_begin).fill(-Infinity); ++ batch_logits_data.subarray(this.timestamp_begin, this.timestamp_end + 1).fill(-Infinity); + } else { +- batch_logits_data.subarray(0, this.eos_token_id).fill(-Infinity); ++ batch_logits_data.subarray(0, this.eos_token_id).fill(-Infinity), batch_logits_data.subarray(this.timestamp_end + 1).fill(-Infinity); + } + } + if (input_ids[i].length === this.begin_index && this.max_initial_timestamp_index !== null) { +@@ -23808,11 +23810,11 @@ var WhisperTimeStampLogitsProcessor = class extends LogitsProcessor { + } + const logprobs = log_softmax(batch_logits_data); + const timestamp_logprob = Math.log( +- logprobs.subarray(this.timestamp_begin).map(Math.exp).reduce((a, b) => a + b) ++ logprobs.subarray(this.timestamp_begin, this.timestamp_end + 1).map(Math.exp).reduce((a, b) => a + b) + ); +- const max_text_token_logprob = max(logprobs.subarray(0, this.timestamp_begin))[0]; ++ const max_text_token_logprob = Math.max(max(logprobs.subarray(0, this.timestamp_begin))[0], logprobs.length > this.timestamp_end + 1 ? max(logprobs.subarray(this.timestamp_end + 1))[0] : -Infinity); + if (timestamp_logprob > max_text_token_logprob) { +- batch_logits_data.subarray(0, this.timestamp_begin).fill(-Infinity); ++ batch_logits_data.subarray(0, this.timestamp_begin).fill(-Infinity), batch_logits_data.subarray(this.timestamp_end + 1).fill(-Infinity); + } + } + return logits; +diff --git a/node_modules/@huggingface/transformers/dist/transformers.node.mjs b/node_modules/@huggingface/transformers/dist/transformers.node.mjs +index bacb354..e367085 100644 +--- a/node_modules/@huggingface/transformers/dist/transformers.node.mjs ++++ b/node_modules/@huggingface/transformers/dist/transformers.node.mjs +@@ -15512,11 +15512,12 @@ var WhisperTokenizer = class extends PreTrainedTokenizer { + */ + decodeWithTimestamps(token_ids, decode_args) { + const time_precision = decode_args?.time_precision ?? 0.02; +- const timestamp_begin = this.all_special_ids.at(-1) + 1; ++ const timestamp_begin = this.timestamp_begin; ++ const timestamp_end = timestamp_begin + 1500; + let outputs = [[]]; + for (let token of token_ids) { + token = Number(token); +- if (token >= timestamp_begin) { ++ if (token >= timestamp_begin && token <= timestamp_end) { + const timestamp = ((token - timestamp_begin) * time_precision).toFixed(2); + outputs.push(`<|${timestamp}|>`); + outputs.push([]); +@@ -15524,7 +15525,7 @@ var WhisperTokenizer = class extends PreTrainedTokenizer { + outputs[outputs.length - 1].push(token); + } + } +- outputs = outputs.map((s) => typeof s === "string" ? s : super.decode(s, decode_args)); ++ outputs = outputs.map((s) => typeof s === "string" ? s : s.length === 0 ? "" : super.decode(s, decode_args)); + return outputs.join(""); + } + /** +@@ -22809,6 +22810,7 @@ var WhisperTimeStampLogitsProcessor = class extends LogitsProcessor { + this.eos_token_id = Array.isArray(generate_config.eos_token_id) ? generate_config.eos_token_id[0] : generate_config.eos_token_id; + this.no_timestamps_token_id = generate_config.no_timestamps_token_id; + this.timestamp_begin = this.no_timestamps_token_id + 1; ++ this.timestamp_end = this.timestamp_begin + 1500; + this.begin_index = init_tokens.length; + if (init_tokens.at(-1) === this.no_timestamps_token_id) { + this.begin_index -= 1; +@@ -22829,17 +22831,17 @@ var WhisperTimeStampLogitsProcessor = class extends LogitsProcessor { + ); + batch_logits_data[this.no_timestamps_token_id] = -Infinity; + if (input_ids[i].length === this.begin_index) { +- batch_logits_data.subarray(0, this.timestamp_begin).fill(-Infinity); ++ batch_logits_data.subarray(0, this.timestamp_begin).fill(-Infinity), batch_logits_data.subarray(this.timestamp_end + 1).fill(-Infinity); + continue; + } + const seq = input_ids[i].slice(this.begin_index); +- const last_was_timestamp = seq.length >= 1 && seq[seq.length - 1] >= this.timestamp_begin; +- const penultimate_was_timestamp = seq.length < 2 || seq[seq.length - 2] >= this.timestamp_begin; ++ const last_was_timestamp = seq.length >= 1 && seq[seq.length - 1] >= this.timestamp_begin && seq[seq.length - 1] <= this.timestamp_end; ++ const penultimate_was_timestamp = seq.length < 2 || (seq[seq.length - 2] >= this.timestamp_begin && seq[seq.length - 2] <= this.timestamp_end); + if (last_was_timestamp) { + if (penultimate_was_timestamp) { +- batch_logits_data.subarray(this.timestamp_begin).fill(-Infinity); ++ batch_logits_data.subarray(this.timestamp_begin, this.timestamp_end + 1).fill(-Infinity); + } else { +- batch_logits_data.subarray(0, this.eos_token_id).fill(-Infinity); ++ batch_logits_data.subarray(0, this.eos_token_id).fill(-Infinity), batch_logits_data.subarray(this.timestamp_end + 1).fill(-Infinity); + } + } + if (input_ids[i].length === this.begin_index && this.max_initial_timestamp_index !== null) { +@@ -22848,11 +22850,11 @@ var WhisperTimeStampLogitsProcessor = class extends LogitsProcessor { + } + const logprobs = log_softmax(batch_logits_data); + const timestamp_logprob = Math.log( +- logprobs.subarray(this.timestamp_begin).map(Math.exp).reduce((a, b) => a + b) ++ logprobs.subarray(this.timestamp_begin, this.timestamp_end + 1).map(Math.exp).reduce((a, b) => a + b) + ); +- const max_text_token_logprob = max(logprobs.subarray(0, this.timestamp_begin))[0]; ++ const max_text_token_logprob = Math.max(max(logprobs.subarray(0, this.timestamp_begin))[0], logprobs.length > this.timestamp_end + 1 ? max(logprobs.subarray(this.timestamp_end + 1))[0] : -Infinity); + if (timestamp_logprob > max_text_token_logprob) { +- batch_logits_data.subarray(0, this.timestamp_begin).fill(-Infinity); ++ batch_logits_data.subarray(0, this.timestamp_begin).fill(-Infinity), batch_logits_data.subarray(this.timestamp_end + 1).fill(-Infinity); + } + } + return logits; +diff --git a/node_modules/@huggingface/transformers/dist/transformers.web.js b/node_modules/@huggingface/transformers/dist/transformers.web.js +index bb7b327..37a5541 100644 +--- a/node_modules/@huggingface/transformers/dist/transformers.web.js ++++ b/node_modules/@huggingface/transformers/dist/transformers.web.js +@@ -11617,11 +11617,12 @@ var WhisperTokenizer = class extends PreTrainedTokenizer { + */ + decodeWithTimestamps(token_ids, decode_args) { + const time_precision = decode_args?.time_precision ?? 0.02; +- const timestamp_begin = this.all_special_ids.at(-1) + 1; ++ const timestamp_begin = this.timestamp_begin; ++ const timestamp_end = timestamp_begin + 1500; + let outputs = [[]]; + for (let token of token_ids) { + token = Number(token); +- if (token >= timestamp_begin) { ++ if (token >= timestamp_begin && token <= timestamp_end) { + const timestamp = ((token - timestamp_begin) * time_precision).toFixed(2); + outputs.push(`<|${timestamp}|>`); + outputs.push([]); +@@ -11629,7 +11630,7 @@ var WhisperTokenizer = class extends PreTrainedTokenizer { + outputs[outputs.length - 1].push(token); + } + } +- outputs = outputs.map((s) => typeof s === "string" ? s : super.decode(s, decode_args)); ++ outputs = outputs.map((s) => typeof s === "string" ? s : s.length === 0 ? "" : super.decode(s, decode_args)); + return outputs.join(""); + } + /** +@@ -18923,6 +18924,7 @@ var WhisperTimeStampLogitsProcessor = class extends LogitsProcessor { + this.eos_token_id = Array.isArray(generate_config.eos_token_id) ? generate_config.eos_token_id[0] : generate_config.eos_token_id; + this.no_timestamps_token_id = generate_config.no_timestamps_token_id; + this.timestamp_begin = this.no_timestamps_token_id + 1; ++ this.timestamp_end = this.timestamp_begin + 1500; + this.begin_index = init_tokens.length; + if (init_tokens.at(-1) === this.no_timestamps_token_id) { + this.begin_index -= 1; +@@ -18943,17 +18945,17 @@ var WhisperTimeStampLogitsProcessor = class extends LogitsProcessor { + ); + batch_logits_data[this.no_timestamps_token_id] = -Infinity; + if (input_ids[i].length === this.begin_index) { +- batch_logits_data.subarray(0, this.timestamp_begin).fill(-Infinity); ++ batch_logits_data.subarray(0, this.timestamp_begin).fill(-Infinity), batch_logits_data.subarray(this.timestamp_end + 1).fill(-Infinity); + continue; + } + const seq = input_ids[i].slice(this.begin_index); +- const last_was_timestamp = seq.length >= 1 && seq[seq.length - 1] >= this.timestamp_begin; +- const penultimate_was_timestamp = seq.length < 2 || seq[seq.length - 2] >= this.timestamp_begin; ++ const last_was_timestamp = seq.length >= 1 && seq[seq.length - 1] >= this.timestamp_begin && seq[seq.length - 1] <= this.timestamp_end; ++ const penultimate_was_timestamp = seq.length < 2 || (seq[seq.length - 2] >= this.timestamp_begin && seq[seq.length - 2] <= this.timestamp_end); + if (last_was_timestamp) { + if (penultimate_was_timestamp) { +- batch_logits_data.subarray(this.timestamp_begin).fill(-Infinity); ++ batch_logits_data.subarray(this.timestamp_begin, this.timestamp_end + 1).fill(-Infinity); + } else { +- batch_logits_data.subarray(0, this.eos_token_id).fill(-Infinity); ++ batch_logits_data.subarray(0, this.eos_token_id).fill(-Infinity), batch_logits_data.subarray(this.timestamp_end + 1).fill(-Infinity); + } + } + if (input_ids[i].length === this.begin_index && this.max_initial_timestamp_index !== null) { +@@ -18962,11 +18964,11 @@ var WhisperTimeStampLogitsProcessor = class extends LogitsProcessor { + } + const logprobs = log_softmax(batch_logits_data); + const timestamp_logprob = Math.log( +- logprobs.subarray(this.timestamp_begin).map(Math.exp).reduce((a, b) => a + b) ++ logprobs.subarray(this.timestamp_begin, this.timestamp_end + 1).map(Math.exp).reduce((a, b) => a + b) + ); +- const max_text_token_logprob = max(logprobs.subarray(0, this.timestamp_begin))[0]; ++ const max_text_token_logprob = Math.max(max(logprobs.subarray(0, this.timestamp_begin))[0], logprobs.length > this.timestamp_end + 1 ? max(logprobs.subarray(this.timestamp_end + 1))[0] : -Infinity); + if (timestamp_logprob > max_text_token_logprob) { +- batch_logits_data.subarray(0, this.timestamp_begin).fill(-Infinity); ++ batch_logits_data.subarray(0, this.timestamp_begin).fill(-Infinity), batch_logits_data.subarray(this.timestamp_end + 1).fill(-Infinity); + } + } + return logits; diff --git a/patches/README.md b/patches/README.md new file mode 100644 index 0000000..b52a661 --- /dev/null +++ b/patches/README.md @@ -0,0 +1,62 @@ +# Patches + +Applied by `patch-package` from `postinstall`. If a patch fails to apply after a +dependency bump, install fails loudly rather than silently reverting to the +buggy behaviour. + +## `@huggingface/transformers` — bound the Whisper timestamp-token range + +**Upstream bug.** `WhisperTokenizer` defines the start of the timestamp block as +`token_to_id("<|notimestamps|>") + 1`, and Whisper has exactly 1500 timestamp +tokens above it. `_decode_asr` uses both bounds: + +```js +const timestamp_begin = this.timestamp_begin; +const total_timestamp_tokens = 1500; +const timestamp_end = timestamp_begin + total_timestamp_tokens; +``` + +Two other places test only the lower bound, so **every token above the timestamp +block is mistaken for a timestamp**: + +1. `WhisperTokenizer.decodeWithTimestamps` — additionally re-derives the start as + `all_special_ids.at(-1) + 1` rather than using the getter. +2. `WhisperTimeStampLogitsProcessor._call` — during generation. + +For stock Whisper this is harmless: the vocabulary ends at the timestamp block, +so there is nothing above it to misclassify. It breaks any derivative that +extends the vocabulary. + +**How it broke Rescript.** CrisperWhisper appends 31 tokens past the block — +`[UM]`, `[UH]`, 13 vocal events (`[laughter]`, `[breath]`, …) and its prompt +scaffolding. Two distinct failures, both triggered by the model transcribing a +filler: + +- *Decoding.* `decodeWithTimestamps` split on `[UM]`, leaving an empty token + bucket that reached `decode([])` — `token_ids must be a non-empty array of + integers`, thrown from `combineTokensIntoWords` part-way through a transcript. +- *Generation.* The logits processor saw `[UH]` as a timestamp and ran + `subarray(0, eos_token_id).fill(-Infinity)`, suppressing every text token and + leaving only EOS. Transcription stopped at the first hesitation; the rest of + the audio came back as `...` VAD placeholders. + +The second one is the quieter of the two — no error, just a silently truncated +transcript. It is why CrisperWhisper appeared to *never* emit fillers through +transformers.js while emitting them readily in PyTorch: asking for word +timestamps suppressed the very tokens the model was chosen for. + +**The patch.** Adds `timestamp_end = timestamp_begin + 1500` alongside the +existing lower bound and range-checks against both, matching what `_decode_asr` +already does. Also: + +- makes `decodeWithTimestamps` use the `timestamp_begin` getter instead of + re-deriving it from `all_special_ids`; +- skips empty buckets in its output map (a leading or doubled timestamp produces + one even on stock Whisper — a latent crash); +- when forcing a timestamp, suppresses text tokens *above* the block too, not + just below it; +- compares against the max text-token logprob on both sides of the block. + +**Upstreaming.** Worth a PR — the fix is small and the tokenizer already computes +the correct bound. Until then this patch is required for the CrisperWhisper +entries in `lib/models.ts` to work at all. diff --git a/workers/transcription.worker.ts b/workers/transcription.worker.ts index b30e011..3d63ca3 100644 --- a/workers/transcription.worker.ts +++ b/workers/transcription.worker.ts @@ -339,30 +339,19 @@ const CRISPER_PROMPT_TOKENS = [ ]; /** - * Register CrisperWhisper's prompt tokens as special, working around a - * transformers.js assumption that otherwise crashes every verbatim transcript. + * Register CrisperWhisper's prompt scaffolding as special so it can never + * surface as literal text in a transcript. * - * WhisperTokenizer decides what counts as a timestamp token in two different, - * disagreeing ways. `_decode_asr` reads the real boundary - * (`token_to_id("<|notimestamps|>") + 1`, and an upper bound 1500 tokens - * above), but `decodeWithTimestamps` — reached through `collateWordTimestamps` - * when `return_timestamps: "word"` — infers it as `all_special_ids.at(-1) + 1` - * and applies no upper bound at all. + * These are the `[verbatim_N]` / `[intended_N]` mode tags and the + * verbatimize / hotword / context markers — decoder-prompt machinery, not + * speech. `_decode_asr` skips anything in `all_special_ids`, which is the only + * hook for keeping them out of the output. * - * That is fine for stock Whisper, whose vocabulary ends at the timestamp block. - * CrisperWhisper appends 31 tokens past it — `[UM]`, `[UH]`, vocal events, and - * this scaffolding. Each one is above the inferred threshold, so - * `decodeWithTimestamps` treats it as a timestamp, emits a `<|…|>` marker, and - * starts a fresh empty token bucket; that empty array then reaches `decode([])` - * and throws "token_ids must be a non-empty array of integers". In other words - * it crashed on the first `[UM]` — precisely the token the model is here for. - * - * Marking only the scaffolding special raises `all_special_ids.at(-1)` to the - * top of the vocabulary, so nothing below it is mistaken for a timestamp: - * `[UM]`, `[UH]` and the vocal events decode as ordinary text. `_decode_asr` is - * unaffected because its boundary comes from that independent getter, and it - * additionally skips these ids now — which stops the forced `[verbatim_N]` - * prefix from leaking into the transcript. + * This used to carry a second job: raising `all_special_ids.at(-1)` above the + * whole vocabulary so `decodeWithTimestamps` would stop mistaking `[UM]` and + * `[UH]` for timestamps. That was a workaround for an upstream bug, now fixed + * properly in patches/@huggingface+transformers+4.2.0.patch — see + * patches/README.md. Only the narrow purpose above remains. * * Idempotent, so re-running it after a WebGPU-to-WASM reload is harmless. */ @@ -1056,7 +1045,23 @@ async function runWhisper( chunks = await runSlice(); } - rawWords.push(...wordsFromChunks(chunks, offsetS, sliceDuration, duration)); + const words = wordsFromChunks(chunks, offsetS, sliceDuration, duration); + // A segment that decodes to nothing is the signature of a model or prompt + // that has collapsed on this slice — the timeline fills with "..." VAD + // placeholders and the transcript silently loses a stretch of speech. It is + // indistinguishable from genuine silence downstream, so say so here, and + // report enough to tell "ASR returned nothing" apart from "words were + // produced and then dropped in post-processing". + if (chunks.length === 0 || words.length === 0) { + console.warn( + `[asr] ${choice}: segment ${offsetS.toFixed(2)}s +${sliceDuration.toFixed(2)}s ` + + `produced ${chunks.length} chunk(s) → ${words.length} word(s).`, + chunks.length > 0 + ? { text: chunks.map((c) => c.text).join(""), chunks } + : "(model returned no chunks)" + ); + } + rawWords.push(...words); speechDone += segmentSamples; reportProgress(0, 0); } From be27e36db227e160ebe46e17d03dd7fe67e9f6b7 Mon Sep 17 00:00:00 2001 From: Wassim Gharbi Date: Wed, 5 Aug 2026 02:25:38 -0700 Subject: [PATCH 5/9] move crisper to its own repo --- lib/models.ts | 147 +++++-------- package-lock.json | 2 +- package.json | 2 +- patches/README.md | 14 +- tests/models-test.ts | 40 +--- tests/vad-regression-test.ts | 61 +----- tools/crisperwhisper-onnx/.gitignore | 12 -- tools/crisperwhisper-onnx/MODEL_CARD.md | 97 --------- tools/crisperwhisper-onnx/README.md | 233 --------------------- tools/crisperwhisper-onnx/export.py | 176 ---------------- tools/crisperwhisper-onnx/install_local.py | 117 ----------- tools/crisperwhisper-onnx/onnx_config.py | 73 ------- tools/crisperwhisper-onnx/quantize.py | 175 ---------------- tools/crisperwhisper-onnx/requirements.txt | 28 --- tools/crisperwhisper-onnx/upload.py | 85 -------- tools/crisperwhisper-onnx/verify.py | 219 ------------------- workers/transcription.worker.ts | 122 +---------- 17 files changed, 81 insertions(+), 1522 deletions(-) delete mode 100644 tools/crisperwhisper-onnx/.gitignore delete mode 100644 tools/crisperwhisper-onnx/MODEL_CARD.md delete mode 100644 tools/crisperwhisper-onnx/README.md delete mode 100644 tools/crisperwhisper-onnx/export.py delete mode 100644 tools/crisperwhisper-onnx/install_local.py delete mode 100644 tools/crisperwhisper-onnx/onnx_config.py delete mode 100644 tools/crisperwhisper-onnx/quantize.py delete mode 100644 tools/crisperwhisper-onnx/requirements.txt delete mode 100644 tools/crisperwhisper-onnx/upload.py delete mode 100644 tools/crisperwhisper-onnx/verify.py diff --git a/lib/models.ts b/lib/models.ts index afb2e3b..bda1071 100644 --- a/lib/models.ts +++ b/lib/models.ts @@ -1,5 +1,3 @@ -import type { TranscriptLanguage } from "./languages"; - /** Local speech models offered on the upload screen. */ export type WhisperModel = | "base" @@ -32,41 +30,6 @@ export type WhisperModelInfo = ModelDisplay & { webgpu: Record; wasm: Record; }; - /** - * When true, condition Whisper on a short filler-rich initial prompt so - * "Remove fillers" has tokens to act on. See {@link whisperFillerPrompt}. - * - * **Currently unset on every model — the prompt does more harm than good.** - * Measured on an 11.5 s clip, decoding each VAD segment the way the worker - * does, Whisper Small with the prompt attached: - * - * | slice | plain | + prompt | - * |--------------|-----------------------------|-----------------------| - * | full 11.5 s | complete, includes "uh" | "Nice. How does it, uh," | - * | 2.5–5.0 s | "Nice. How does it work?" | "Nice. How does it" | - * | 5.0–11.5 s | complete sentence | **"Um,"** | - * - * The long tail segment collapses into an echo of the prompt. Medium behaves - * the same way, and Base is only marginally more robust. This is the - * truncation {@link MAX_VERBATIM_PROMPT_LENGTH} was meant to bound, but the - * length cap does not prevent it — a 20-character prompt still triggers it. - * - * Little is lost by dropping it: plain decoding already yields "uh" on this - * clip, and anything the model does swallow is still recovered as a timed - * `...` placeholder by {@link insertDisfluencyPlaceholders}, so it stays - * cuttable. Kept as a one-flag switch for re-testing on other material. - */ - keepFillers?: boolean; - /** - * CrisperWhisper mode prefix: how many `[verbatim_N]` tags to prime the - * decoder with. CrisperWhisper 2.0 picks verbatim vs. intended output purely - * from this prefix — the encoder output is identical either way — and unlike - * Whisper's initial prompt there is no `<|startofprev|>`; the tags come - * first and the standard prefix follows. Verbatim output spells fillers as - * `[UM]` / `[UH]`, which `lib/fillers.ts` already matches once punctuation is - * stripped. Source: `crisperwhisper==2.0.1`, `crisperwhisper/prompt.py`. - */ - verbatimTags?: number; /** * A CrisperWhisper checkpoint, whose vocabulary extends past Whisper's * timestamp block: `[UM]`, `[UH]`, vocal events and the prompt scaffolding all @@ -80,9 +43,8 @@ export type WhisperModelInfo = ModelDisplay & { * the transcript off mid-sentence with no error. See patches/README.md. * * The flag itself drives `markCrisperPromptTokensSpecial`, keeping the mode - * tags out of transcript text. Deliberately separate from - * {@link WhisperModelInfo.verbatimTags} so it still applies to a CrisperWhisper - * model decoded without the mode prefix. + * tags out of transcript text even though we never send them — see the note on + * decoder-prefix conditioning above {@link MODELS}. */ crisper?: boolean; /** @@ -112,32 +74,6 @@ export const MODEL_ORDER: ModelId[] = [ "crisperTurbo", ]; -/** - * Short, language-specific filler prompts for Whisper. Deliberately tiny and - * free of "I'm …" openers (those seeded "I'm sorry" hallucination loops). - * The classic OpenAI example prompt is too long for our chunked decoder path. - */ -const WHISPER_FILLER_PROMPTS: Record = { - en: "Um, uh, hmm, er, ah.", - es: "Em, emm, eee.", - fr: "Euh, heu, euhm.", - de: "Äh, ähm, öhm, mhh.", - zh: "嗯, 呃, 额, 唔.", -}; - -/** - * Filler-bias prompt for Whisper when the selected model opts into - * {@link WhisperModelInfo.keepFillers}. Returns null when prompting is off. - */ -export function whisperFillerPrompt( - model: ModelId, - language: TranscriptLanguage -): string | null { - const info = MODELS[model]; - if (info.backend !== "whisper" || !info.keepFillers) return null; - return WHISPER_FILLER_PROMPTS[language]; -} - const WHISPER_DTYPE = { // q4 decoder: q8 fails session creation on onnxruntime-web 1.26 // (Missing required scale … MatMulNBits). @@ -191,9 +127,54 @@ const CRISPER_TURBO_DTYPE = { } satisfies WhisperModelInfo["dtype"]; /** + * No model conditions the decoder on a prefix, and that is a measured decision + * rather than an omission. Two mechanisms were tried and both were removed: + * + * **Whisper's `<|startofprev|>` filler prompt** (a short filler list — "Um, uh, + * hmm, er, ah." — forced via `decoder_input_ids` to give "Remove fillers" + * something to act on). Measured on an 11.5 s clip, decoding each VAD segment + * the way the worker does, Whisper Small: + * + * | slice | plain | + prompt | + * |-------------|---------------------------|--------------------------| + * | full 11.5 s | complete, includes "uh" | "Nice. How does it, uh," | + * | 2.5–5.0 s | "Nice. How does it work?" | "Nice. How does it" | + * | 5.0–11.5 s | complete sentence | **"Um,"** | + * + * The long tail segment collapses into an echo of the prompt. Medium is worse + * (the whole clip truncates to "Nice. How does it, uh…"), Base only marginally + * more robust. A length cap was tried first and does not help — a 20-character + * prompt still triggers it. + * + * **CrisperWhisper's `[verbatim_N]` mode prefix** (its trained-in verbatim + * selector, from `crisperwhisper==2.0.1`, `crisperwhisper/prompt.py`). Same + * failure, same cause — the worker decodes short VAD segments, and prefix + * conditioning collapses them. Measured against the fp32 checkpoints: + * + * | slice | plain | + [verbatim_1..5] | + * |-------------|-----------------------------|------------------------| + * | full 11.5 s | complete, includes "[UH]" | Turbo drops "Nice." | + * | 2.5–5.0 s | "Nice. How does it work?" | "Nice. How does it-" | + * + * On Small the prefix also emitted `[breath]` where the speaker hesitated while + * plain decoding kept it as "uh". + * + * If either is ever re-attempted: do **not** copy upstream's `<|notimestamps|>` + * along with the mode tags. Upstream can suppress timestamps because it derives + * word timings itself via Viterbi over the space token's cross-attention; + * transformers.js instead splits chunked audio *on* timestamp tokens, so + * suppressing them accumulates one unsegmented run whose stride-overlap merge + * can resolve to nothing — surfacing mid-transcription as "token_ids must be a + * non-empty array of integers". + * + * Nothing is lost either way: every checkpoint emits its fillers unprompted + * ("uh" on stock Whisper, `[UH]` on CrisperWhisper), and whatever a model does + * swallow is still recovered as a timed `...` placeholder by + * `insertDisfluencyPlaceholders`, so it stays cuttable. + * * Local speech models that can run in the transcription worker. * Shared display fields live on every entry; backend-specific knobs - * (`dtype` / `keepFillers` vs `repoId`) are gated by `backend`. + * (`dtype` / `crisper` vs `repoId`) are gated by `backend`. */ export const MODELS: { base: WhisperModelInfo; @@ -228,13 +209,6 @@ export const MODELS: { // WASM int8 encoder + q4 decoder ~780 MB; WebGPU fp16 encoder ~1.1 GB. size: "~1.1 GB", dtype: WHISPER_MEDIUM_DTYPE, - // No filler prompt, unlike Base and Small. Medium is far more sensitive to - // <|startofprev|> conditioning than they are: measured on an 11.5 s clip, - // both dtype configs transcribe it in full unprompted, and both collapse to - // the fragment "Nice. How does it, uh..." with the prompt attached — the - // truncation failure {@link MAX_VERBATIM_PROMPT_LENGTH} describes, except - // triggered by a prompt already inside that cap. It costs nothing here: - // Medium emits "uh" on its own, which is all the prompt was there to buy. }, parakeet: { backend: "parakeet", @@ -259,10 +233,6 @@ export const MODELS: { size: "~324 MB", dtype: CRISPER_SMALL_DTYPE, crisper: true, - // No `verbatimTags` — see the note on crisperTurbo. Measured against the - // fp32 checkpoint on an 11.5 s clip, the mode prefix drops "Nice. How does - // it," and emits [breath] where the speaker hesitated, while plain decoding - // keeps the hesitation as "uh". }, crisperTurbo: { backend: "whisper", @@ -274,37 +244,18 @@ export const MODELS: { size: "~900 MB", dtype: CRISPER_TURBO_DTYPE, crisper: true, - // No `verbatimTags`, for the same reason Medium carries no filler prompt: - // prefix conditioning collapses short segments, and the worker decodes VAD - // segments rather than whole files. Measured on an 11.5 s clip against the - // fp32 checkpoint, one segment at a time: - // - // slice | plain | + [verbatim_1..5] - // ------------|-----------------------------|--------------------- - // full 11.5 s | complete, includes "[UH]" | drops "Nice." - // 2.5–5.0 s | "Nice. How does it work?" | "Nice. How does it-" - // - // Nothing is lost by dropping it: this checkpoint emits `[UH]` unprompted, - // so plain decoding is both more complete and still verbatim. }, }; /** * Whether `model` is a CrisperWhisper checkpoint, and so needs the tokenizer - * repair in {@link WhisperModelInfo.crisper}. Independent of whether it is - * decoded with a mode prefix. + * repair in {@link WhisperModelInfo.crisper}. */ export function isCrisperModel(model: ModelId): boolean { const info = MODELS[model]; return info.backend === "whisper" && info.crisper === true; } -/** Whether `model` is steered with CrisperWhisper's `[verbatim_N]` prefix. */ -export function verbatimTagCount(model: ModelId): number | null { - const info = MODELS[model]; - return info.backend === "whisper" ? (info.verbatimTags ?? null) : null; -} - /** Whether `model` loads from public/models rather than the Hub. */ export function isLocalModel(model: ModelId): boolean { const info = MODELS[model]; diff --git a/package-lock.json b/package-lock.json index 97f1f82..1dc62f4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,7 +14,7 @@ "@ffmpeg/ffmpeg": "^0.12.15", "@ffmpeg/util": "^0.12.2", "@floating-ui/react": "^0.27.20", - "@huggingface/transformers": "^4.2.0", + "@huggingface/transformers": "4.2.0", "@next/third-parties": "^16.2.12", "@sentry/electron": "^7.16.0", "@sentry/react": "^10.69.0", diff --git a/package.json b/package.json index 0bca322..7d3b3fe 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,7 @@ "@ffmpeg/ffmpeg": "^0.12.15", "@ffmpeg/util": "^0.12.2", "@floating-ui/react": "^0.27.20", - "@huggingface/transformers": "^4.2.0", + "@huggingface/transformers": "4.2.0", "@next/third-parties": "^16.2.12", "@sentry/electron": "^7.16.0", "@sentry/react": "^10.69.0", diff --git a/patches/README.md b/patches/README.md index b52a661..743f75f 100644 --- a/patches/README.md +++ b/patches/README.md @@ -1,8 +1,16 @@ # Patches -Applied by `patch-package` from `postinstall`. If a patch fails to apply after a -dependency bump, install fails loudly rather than silently reverting to the -buggy behaviour. +Applied by `patch-package` from `postinstall`. If a patch fails to apply, install +fails loudly rather than silently reverting to the buggy behaviour. + +**Patched dependencies are pinned to an exact version in `package.json`, not a +caret range.** A patch filename carries the version it was cut against, and on a +mismatch `patch-package` only *warns* — so a range would let a minor bump quietly +drop the fix. That matters more than usual here: the unpatched failure mode below +is a silently truncated transcript, not an error. When bumping +`@huggingface/transformers`, re-cut the patch (`npx patch-package +@huggingface/transformers`) and re-run the CrisperWhisper models against a clip +containing a filler. ## `@huggingface/transformers` — bound the Whisper timestamp-token range diff --git a/tests/models-test.ts b/tests/models-test.ts index 4cb1567..54a9c33 100644 --- a/tests/models-test.ts +++ b/tests/models-test.ts @@ -9,8 +9,6 @@ import { isModelId, isParakeetModel, isWhisperModel, - verbatimTagCount, - whisperFillerPrompt, } from "../lib/models"; import { isTranscriptSource } from "../lib/source"; @@ -42,19 +40,6 @@ assert(typeof MODELS.base.id === "string", "whisper base id"); assert(typeof MODELS.small.id === "string", "whisper small id"); assert(typeof MODELS.medium.id === "string", "whisper medium id"); assert(MODELS.base.dtype.webgpu.encoder_model === "fp32", "whisper dtype"); -// No model uses the <|startofprev|> filler prompt any more. Measured per VAD -// segment, it collapses Whisper Small's longest slice to a bare "Um," and -// truncates Medium to a fragment, while plain decoding transcribes every -// segment correctly and still yields "uh". See WhisperModelInfo.keepFillers. -for (const id of MODEL_ORDER) { - const info = MODELS[id]; - if (info.backend !== "whisper") continue; - assert(!info.keepFillers, `${id} must not use the filler prompt`); - assert( - whisperFillerPrompt(id, "en") === null, - `${id} must resolve no filler prompt` - ); -} // Medium deviates from WHISPER_DTYPE on purpose: its fp32 encoder export is // 1.2 GB. Pin the split so a later dtype tidy-up cannot quietly reinstate it. assert( @@ -76,16 +61,6 @@ for (const id of MODEL_ORDER) { assert(typeof MODELS[id].size === "string", `${id} has size`); } -// CrisperWhisper: verbatim comes from the [verbatim_N] mode prefix, so these -// models must NOT also get Whisper's filler-bias prompt — the two are different -// mechanisms and stacking them would corrupt the decoder prefix. -// The mode prefix is off on both: measured, it drops a clause and emits -// [breath] where the speaker hesitated, while no prefix keeps the filler. -assert(verbatimTagCount("crisperSmall") === null, "crisperSmall sends no mode prefix"); -assert(verbatimTagCount("crisperTurbo") === null, "crisperTurbo sends no mode prefix"); -assert(verbatimTagCount("base") === null, "base has no verbatim tags"); -// The tokenizer fix-up must still run without a mode prefix — the extended -// vocabulary tokens that break word collation are emitted either way. // Turbo runs fp16 on the merged decoder, not q4: on q4 it collapsed after the // first VAD segment, and bisection cleared the encoder (a q4 encoder under an // fp32 decoder reproduces the fp32 transcript exactly). fp16 is also 477 MB @@ -101,18 +76,13 @@ assert( "crisperSmall keeps the verified q4 decoder" ); +// The tokenizer fix-up is required by CrisperWhisper's vocabulary layout: the +// extended tokens that break word collation are emitted whether or not the +// decoder is primed with anything. assert(isCrisperModel("crisperSmall"), "crisperSmall needs the tokenizer fix-up"); assert(isCrisperModel("crisperTurbo"), "crisperTurbo needs the tokenizer fix-up"); assert(!isCrisperModel("base"), "base needs no tokenizer fix-up"); assert(!isCrisperModel("parakeet"), "parakeet needs no tokenizer fix-up"); -assert( - whisperFillerPrompt("crisperSmall", "en") === null, - "crisperSmall does not use the filler-bias prompt" -); -assert( - whisperFillerPrompt("crisperTurbo", "en") === null, - "crisperTurbo does not use the filler-bias prompt" -); // Only the unpublished export is served from public/models; a stray `local` // flag on a Hub model would send transformers.js to a path that 404s. @@ -128,8 +98,4 @@ assert( "crisperTurbo Hub id" ); -assert(whisperFillerPrompt("parakeet", "en") === null, "parakeet has no prompt"); -// The prompts themselves are still checked for length and content in -// vad-regression-test.ts, which constrains whatever any model opts back into. - console.log("models-test: ok"); diff --git a/tests/vad-regression-test.ts b/tests/vad-regression-test.ts index e6812b1..0341ec9 100644 --- a/tests/vad-regression-test.ts +++ b/tests/vad-regression-test.ts @@ -14,11 +14,6 @@ import { speechSegmentsFromFrames, VAD_SAMPLE_RATE, } from "../lib/vad"; -import { MODELS, whisperFillerPrompt } from "../lib/models"; -import { - TRANSCRIPT_LANGUAGE_ORDER, - type TranscriptLanguage, -} from "../lib/languages"; const wav = path.join( process.cwd(), @@ -34,56 +29,22 @@ function ffmpegAvailable(): boolean { return probe.status === 0; } -/** - * Longest `<|startofprev|>` filler prompt considered safe. Lived in lib/models - * until every model stopped opting in; kept here because the constraint is - * still real for anything that opts back in, and prod no longer reads it. - * - * Note the cap is necessary but not sufficient — Whisper Small collapses a long - * VAD segment to a bare "Um," with a 20-character prompt, well inside it. That - * is why `keepFillers` is unset everywhere rather than merely bounded. - */ -const MAX_VERBATIM_PROMPT_LENGTH = 32; - -// Short filler prompts are OK; long ones truncate multi-speaker / long-form ASR. -// Not every Whisper model opts in: Medium truncates to a fragment even inside -// the length cap, and the CrisperWhisper checkpoints transcribe fillers without -// prompting. Those are checked in models-test.ts; here we only constrain the -// prompts that do get sent. -let promptedModels = 0; -for (const id of Object.keys(MODELS) as (keyof typeof MODELS)[]) { - const info = MODELS[id]; - if (info.backend !== "whisper" || !info.keepFillers) continue; - promptedModels++; - for (const language of TRANSCRIPT_LANGUAGE_ORDER) { - const prompt = whisperFillerPrompt(id, language); - assert(!!prompt, `${id}/${language} must resolve a filler prompt`); - assert( - prompt!.length <= MAX_VERBATIM_PROMPT_LENGTH, - `${id}/${language} filler prompt too long (${prompt!.length} > ${MAX_VERBATIM_PROMPT_LENGTH})` - ); - assert( - !/\bI'm\b/i.test(prompt!), - `${id}/${language} filler prompt must not contain "I'm" (seeds hallucination loops)` - ); - } -} -// Currently zero: the prompt truncates long segments badly enough that every -// model opts out (see WhisperModelInfo.keepFillers). The loop above is a -// standing constraint on anything that opts back in, not a claim that something -// does — so it is deliberately allowed to be empty. -void promptedModels; -assert( - whisperFillerPrompt("parakeet", "en" as TranscriptLanguage) === null, - "parakeet must not use Whisper filler prompts" -); -console.log("whisper filler prompts stay short: ok"); - const workerSrc = fs.readFileSync( path.join(process.cwd(), "workers/transcription.worker.ts"), "utf8" ); +// No decoder-prefix conditioning. Both mechanisms tried — Whisper's +// <|startofprev|> filler prompt and CrisperWhisper's [verbatim_N] mode tags — +// collapse short VAD segments: Small's longest slice decoded to a bare "Um,", +// Medium truncated to a fragment, Turbo dropped a leading clause. See the note +// above MODELS in lib/models.ts for the measurements. +assert( + !/decoder_input_ids/.test(workerSrc), + "worker must not force decoder_input_ids (prefix conditioning truncates VAD segments)" +); +console.log("no decoder-prefix conditioning: ok"); + // High repetition_penalty truncates this clip mid-utterance even on full audio. const penaltyMatch = workerSrc.match(/repetition_penalty:\s*([0-9.]+)/); if (!penaltyMatch) throw new Error("worker must set repetition_penalty"); diff --git a/tools/crisperwhisper-onnx/.gitignore b/tools/crisperwhisper-onnx/.gitignore deleted file mode 100644 index 205a035..0000000 --- a/tools/crisperwhisper-onnx/.gitignore +++ /dev/null @@ -1,12 +0,0 @@ -# Virtualenv and Python caches -.venv/ -__pycache__/ -*.pyc - -# Export artefacts: multi-GB ONNX graphs, external weight blobs, and the staging -# copy of the upstream checkpoint. All reproducible via export.py, and the -# weights are non-commercially licensed — they must not be committed here. -out/ -*.onnx -*.onnx_data -*.safetensors diff --git a/tools/crisperwhisper-onnx/MODEL_CARD.md b/tools/crisperwhisper-onnx/MODEL_CARD.md deleted file mode 100644 index fdfdefb..0000000 --- a/tools/crisperwhisper-onnx/MODEL_CARD.md +++ /dev/null @@ -1,97 +0,0 @@ ---- -license: other -license_name: nyra-health-non-commercial-research -license_link: https://huggingface.co/nyralabs/CrisperWhisper2.0_small/blob/main/LICENSE.md -base_model: nyralabs/CrisperWhisper2.0_small -base_model_relation: quantized -library_name: transformers.js -pipeline_tag: automatic-speech-recognition -language: - - en - - de -tags: - - onnx - - transformers.js - - whisper - - verbatim - - disfluency - - word-timestamps ---- - -# CrisperWhisper 2.0 Small — ONNX (word timestamps) - -ONNX export of [`nyralabs/CrisperWhisper2.0_small`](https://huggingface.co/nyralabs/CrisperWhisper2.0_small) -for [transformers.js](https://github.com/huggingface/transformers.js), exported -**with per-layer cross-attention outputs** so word-level timestamps work in the -browser. - -## Why re-export - -Every other CrisperWhisper ONNX export is large-v3-scale. Even the "turbo" -export keeps large-v3's 32-layer, 2.5 GB fp32 encoder and only shrinks the -decoder, so the cheapest published option that supports word timestamps is -around 1 GB in 4-bit. This export is ~324 MB (66 MB encoder + 258 MB merged -decoder in q4), which is browser-viable — `CrisperWhisper2.0_small` had no ONNX -export at all before this. - -It is exported **with** per-layer cross-attention outputs, which -`return_timestamps: "word"` requires: transformers.js computes word timings by -running DTW over `cross_attentions.{i}`. Note that the `alignment_heads` in -`generation_config.json` are not sufficient on their own — they select which -attention tensors to align over, so they are useless if the decoder does not -emit any. Some published CrisperWhisper exports ship the heads without the -tensors. See `CrossAttentionWhisperOnnxConfig` in the export tooling, adapted -from `transformers.js/scripts/extra/whisper.py`. - -If you need a larger, more accurate variant with the same capability, -[`Masterx/CrisperWhisper2.0-turbo-ONNX`](https://huggingface.co/Masterx/CrisperWhisper2.0-turbo-ONNX) -already provides one. - -## Usage - -Verbatim mode is selected by the **decoder prompt prefix**, not a flag — the -encoder output is identical for both modes: - -``` -tokenize("[verbatim_1][verbatim_2][verbatim_3][verbatim_4][verbatim_5]") - + [<|startoftranscript|>, <|LANG|>, <|transcribe|>] -``` - -Substitute `[intended_N]` for the clean, non-verbatim transcript. Each tag is a -single vocabulary token (51880–51884 verbatim, 51885–51889 intended) absent from -`added_tokens.json`, so the stock tokenizer encodes them without special -handling, and there is **no `<|startofprev|>`** — the tags precede the standard -prefix. Verbatim output spells fillers as `[UH]` / `[UM]`. - -The upstream `crisperwhisper` package appends `<|notimestamps|>` to this prefix, -because it computes word timings itself. **Omit it under transformers.js**, -which segments chunked audio on timestamp tokens; suppressing them makes long -transcriptions fail part-way with `token_ids must be a non-empty array of -integers`. - -## Licence — non-commercial - -The upstream weights are dual-licensed: inference code under MIT, but **model -weights, configuration, tokenizers, and any Outputs the model generates** under -the [nyra health Non-Commercial Research License](https://huggingface.co/nyralabs/CrisperWhisper2.0_small/blob/main/LICENSE.md). - -This export is a derivative of those weights and **inherits that licence in -full**. Non-commercial research use only. The restriction extends to transcripts -produced by the model. Commercial use requires a separate licence from nyra -health GmbH. - -## Attribution - -- Model: CrisperWhisper 2.0 by [nyra health GmbH](https://www.nyra-labs.com/) — - [paper](https://arxiv.org/abs/2607.18934) -- Export tooling: [`tools/crisperwhisper-onnx`](https://github.com/wassgha/rescript/tree/main/tools/crisperwhisper-onnx) - in Rescript -- Cross-attention export config adapted from - [transformers.js](https://github.com/huggingface/transformers.js/blob/v3/scripts/extra/whisper.py) (Apache-2.0) - -## Known limitation - -CrisperWhisper's published ~30 ms word-boundary accuracy comes from its own -timing method (Viterbi over the explicit space token's cross-attention). -transformers.js instead runs its standard DTW with `alignment_heads` and a median -filter. Word timestamps work; the upstream benchmark figures do not transfer. diff --git a/tools/crisperwhisper-onnx/README.md b/tools/crisperwhisper-onnx/README.md deleted file mode 100644 index ac48988..0000000 --- a/tools/crisperwhisper-onnx/README.md +++ /dev/null @@ -1,233 +0,0 @@ -# CrisperWhisper → ONNX (with cross-attentions) - -Exports [CrisperWhisper 2.0](https://huggingface.co/nyralabs) to ONNX in a -transformers.js-compatible layout, **including the cross-attention outputs that -word-level timestamps need**, so the Rescript editor can use it as a verbatim -speech model. - -## Why this exists - -Rescript needs two things from a speech model that no published CrisperWhisper -ONNX export provides together: - -1. **Verbatim output** — `um` / `uh` / stutters transcribed, not silently - dropped. Whisper omits them (its training data is cleaned subtitles) and - Parakeet TDT omits them too (Granary is pseudo-labelled and filtered). - `lib/models.ts` works around this with a short filler-bias prompt, which is - capped at 32 characters because longer prompts truncate long-form output. -2. **Word-level timestamps** — the editor's entire model is cutting video by - selecting words, so per-word `start`/`end` is non-negotiable. - -Two published exports satisfy (2); none satisfies it at a size this app can -afford. Verified by loading each decoder graph and reading its outputs — -`onnx.load(..., load_external_data=False)`, which is cheap regardless of weight -size, and is what `verify.py` automates: - -| Export | `cross_attentions.*` outputs | smallest q4 pair | -| --- | --- | ---: | -| `Masterx/CrisperWhisper2.0-turbo-ONNX` | **`.0` … `.3`** (4 layers) | ~1025 MB | -| `Masterx/CrisperWhisper2.0-large-ONNX` | **`.0` … `.31`** (32 layers) | ~1487 MB | -| `onnx-community/CrisperWhisper-ONNX` | none (0 of 129 outputs) | — | -| `Prince-1/CrisperWhisper` | none (0 of 65 outputs) | — | -| *this export* (`CrisperWhisper2.0_small`) | **`.0` … `.11`** (12 layers) | **324 MB** | -| `onnx-community/whisper-base_timestamped` *(Rescript today)* | `.0` … `.5` | ~200 MB | - -**The gap this folder fills is size, not capability.** Every published -CrisperWhisper ONNX is large-v3-scale — even "turbo" keeps large-v3's 32-layer, -2.5 GB fp32 encoder and only shrinks the decoder — which puts the cheapest -working option at ~1 GB. Rescript is currently having Safari tabs killed for -memory while loading a 200 MB model, so 1 GB is not viable there. -`CrisperWhisper2.0_small` had no ONNX export at all before this one. - -If you can afford ~1 GB, **`Masterx/CrisperWhisper2.0-turbo-ONNX` works off the -shelf**, is a stronger model than small, and its card documents the same -verbatim prompt prefix. Try it before spending anything on integration — it -answers "is CrisperWhisper's verbatim output good enough to justify the licence -question?" at zero cost. - -One trap worth naming: `alignment_heads` in `generation_config.json` proves -nothing. All five exports above ship it, including the two that cannot use it. -The heads select *which* cross-attention tensors to run DTW over; with no such -tensors emitted there is nothing to select. Check graph outputs, not config. - -## What this exports instead - -`nyralabs/CrisperWhisper2.0_small` — 484 MB of bf16 safetensors, plain -whisper-small geometry (12 encoder / 12 decoder layers, `d_model` 768). Quantised -to q4 it should land in the 150–250 MB range, comparable to the Whisper Base -export Rescript ships today. - -The export path is the same one that produced `onnx-community/*_timestamped`: -a `WhisperOnnxConfig` subclass that appends `cross_attentions.{i}` to the decoder -outputs, combined with `model_kwargs={"output_attentions": True}`. See -`onnx_config.py`, which is adapted from -[`transformers.js/scripts/extra/whisper.py`](https://github.com/huggingface/transformers.js/blob/v3/scripts/extra/whisper.py). - -## ⚠️ Licensing — read before publishing - -The CrisperWhisper 2.0 weights are **not** open source. `LICENSE.md` in the -upstream repos splits into two parts: - -- **Part A** — inference code, pre/post-processing, scripts: MIT. -- **Part B** — *model weights, checkpoints, configuration, tokenizers, **and any - Outputs the model generates***: nyra health Non-Commercial Research License. - Commercial use requires a separate licence from nyra health GmbH. - -CrisperWhisper v1 (`nyralabs/CrisperWhisper`) is CC-BY-NC-4.0. Both are -non-commercial. - -Consequences that matter here: - -- An ONNX export is a **derivative of the weights**, so it inherits Part B. It - cannot be relicensed under Rescript's own licence. -- The "and any Outputs" clause reaches the *transcripts users generate*, not just - the weights we redistribute. -- Anything published from this folder must carry the upstream licence and - attribution. `MODEL_CARD.md` is pre-filled to do that; do not strip it. - -Rescript ships signed desktop builds, so whether this model can be offered in the -product at all is a licensing decision, not a technical one. **This folder -deliberately stops at "produces a working export" and does not wire the model -into the app.** - -## Requirements - -Python **3.11+** — `onnxruntime` requires 3.11 and `onnx` requires 3.10, so the -`python3` that ships with macOS (3.9.6) will not work. Install a newer -interpreter first, e.g. `brew install python@3.11`. - -```sh -cd tools/crisperwhisper-onnx -python3.11 -m venv .venv -source .venv/bin/activate -pip install -r requirements.txt -``` - -`requirements.txt` pins `optimum` to the 1.x line, where -`optimum.exporters.onnx` still lives in the main package and the -`WhisperOnnxConfig` API matches what `onnx_config.py` subclasses. Optimum 2.x -moved the exporters into a separate `optimum-onnx` distribution; both scripts -fall back to that import path, so upgrading is possible, but the 1.x pin is the -combination this was written against. - -## Usage - -```sh -# 1. Export encoder + decoder(+merged) to ONNX with cross-attentions. -# Downloads ~500 MB of weights, writes ~1.5 GB of fp32 ONNX. -python export.py --model nyralabs/CrisperWhisper2.0_small --out out/ - -# 2. Prove the export is actually usable for word timestamps. -# This is the check every published export fails — run it before anything else. -python verify.py out/ - -# 3. Quantise for the browser (q4 / int8 / fp16). -python quantize.py out/ - -# 4. Publish (requires `huggingface-cli login` and a licence decision). -python upload.py out/ --repo /CrisperWhisper2.0_small-ONNX-timestamped -``` - -## Results (verified 2026-08-04) - -Exported from `nyralabs/CrisperWhisper2.0_small` on macOS arm64, Python 3.11.15. - -| Graph | fp32 | fp16 | int8 | q4 | -| --- | ---: | ---: | ---: | ---: | -| `encoder_model` | 352.8 MB | 176.5 MB | 92.7 MB | **66.1 MB** | -| `decoder_model_merged` | 774.6 MB | 388.0 MB | — *(see below)* | **257.9 MB** | - -The q4 pair is **324 MB** — comparable to the ~200 MB Whisper Base export -Rescript ships today, and against ~980 MB for the smallest published -CrisperWhisper ONNX. - -`verify.py` passes, including a live onnxruntime run of the q4 pair: -`cross_attentions.0` comes back `(1, 12, 1, 1500)` — batch × heads × decoder -positions × encoder frames, which is what DTW consumes — and `logits` -`(1, 1, 51896)` confirms the extended CrisperWhisper vocabulary survived. - -### Gotchas hit along the way - -Each of these is pinned or handled in code; they are recorded so a future -version bump does not rediscover them. - -- **`torch<2.6`.** Optimum 1.x's `exporters/onnx/model_patcher.py` imports - `_attention_scale` from `torch.onnx.symbolic_opset14`, which torch removed with - the rest of the legacy TorchScript exporter. Newer torch makes importing the - exporter fail outright. -- **`onnx-ir` is an undeclared dependency.** onnxruntime ≥1.28's - `matmul_nbits_quantizer` imports it without declaring it, so q4 fails with - `ModuleNotFoundError`. Also note the module and class were renamed from - `matmul_4bits_quantizer` / `MatMul4BitsQuantizer` in 1.28. -- **int8 cannot quantise the merged decoder.** Optimum emits - `decoder_model_merged` as one top-level `If` node with all 3889 nodes (314 - MatMuls) inside its branches. `quantize_dynamic` only walks the top level, - finds nothing, and writes a file the same size as its input under an `_int8` - name. `quantize.py` now detects this and skips rather than emitting a - mislabelled file. - - This is not a local quirk — `onnx-community/whisper-medium_timestamped` ships - `decoder_model_merged_int8.onnx` and `_uint8.onnx` at 1828 MB, byte-for-byte - the size of its fp32, while `_q4` is 469 MB. Use q4 or fp16 for merged - decoders. It is also the likely root of the note in `lib/models.ts:86-87` - about q8 decoders failing. -- **`encoder_blank_head.{weight,bias}` are dropped on load.** That is - CrisperWhisper's training-time attention-loss head; it plays no part in - inference, and the warning from `WhisperForConditionalGeneration` is expected. - -## Driving it from transformers.js - -Verbatim mode is **not** a flag — CrisperWhisper 2.0 selects it via the decoder -prompt prefix, and the encoder output is identical either way. From -`crisperwhisper/prompt.py` (PyPI `crisperwhisper==2.0.1`), the sequence is: - -``` -# upstream (crisperwhisper/prompt.py) -tokenize("[verbatim_1][verbatim_2][verbatim_3][verbatim_4][verbatim_5]") - + [<|startoftranscript|>, <|LANG|>, <|transcribe|>, <|notimestamps|>] - -# what to send through transformers.js — same, minus <|notimestamps|> -tokenize("[verbatim_1]…[verbatim_5]") - + [<|startoftranscript|>, <|LANG|>, <|transcribe|>] -``` - -`[intended_N]` in place of `[verbatim_N]` gives the clean transcript instead. -Notes: - -- Each mode tag is a **single token already in the vocabulary** — 51880–51884 - for `[verbatim_1…5]` and 51885–51889 for `[intended_1…5]` in the small - checkpoint. They are absent from `added_tokens.json`, so they need no special - handling; the stock tokenizer encodes the five-tag string to exactly five ids. - The full English verbatim prefix for this checkpoint is - `[51880, 51881, 51882, 51883, 51884, 50258, 50259, 50359, 50363]`. -- Do not hardcode the trailing four. The turbo export uses different ids for - `<|transcribe|>` (50360) and `<|notimestamps|>` (50364) because it carries an - extra language; read them from `generation_config.json`. -- Unlike vanilla Whisper prompting there is **no `<|startofprev|>`**; the tags - come first and the standard prefix follows. -- `<|notimestamps|>` is part of the *upstream* prefix, but **omit it under - transformers.js**. Upstream suppresses timestamp tokens because it derives - word timings itself, via Viterbi over the space token's cross-attention. - transformers.js instead splits chunked audio on timestamp tokens inside - `_decode_asr`; with them suppressed, a long transcript never segments, and the - stride-overlap merge can resolve to an empty token list. That surfaces - part-way through a transcription as `token_ids must be a non-empty array of - integers`. Mode selection and timestamps are orthogonal, so dropping it does - not affect verbatim output. -- The ``/``/`` tokens in `added_tokens.json` are for verbatimize, - hotwords and long-form continuation context respectively — *not* mode control. -- Verbatim output spells fillers as bracketed uppercase (`[UH]`, `[UM]`), so - `lib/hallucinations.ts` and the "Remove fillers" matcher would need to - recognise that form. - -Rescript already builds custom `decoder_input_ids` for its Whisper filler prompt, -so the same mechanism applies — see the `decoder_input_ids` construction in -`workers/transcription.worker.ts`. - -## Caveat on timing quality - -CrisperWhisper's headline ~30 ms boundary accuracy comes from its own timing -method — Viterbi over the explicit space token's cross-attention, per the -upstream docs. transformers.js will instead run its standard DTW with -`alignment_heads` and a median filter. That still yields word timestamps, but do -not expect the upstream benchmark numbers through this path. diff --git a/tools/crisperwhisper-onnx/export.py b/tools/crisperwhisper-onnx/export.py deleted file mode 100644 index 0609536..0000000 --- a/tools/crisperwhisper-onnx/export.py +++ /dev/null @@ -1,176 +0,0 @@ -"""Export a CrisperWhisper checkpoint to ONNX with cross-attention outputs. - -Writes a transformers.js-compatible layout: - - out/ - onnx/encoder_model.onnx - onnx/decoder_model_merged.onnx - config.json, generation_config.json, tokenizer.json, preprocessor_config.json, ... - -Run `verify.py` afterwards. Do not trust an export that has not been verified — -every CrisperWhisper ONNX repo on the Hub looks correct by file listing and is -unusable for word timestamps. -""" - -from __future__ import annotations - -import argparse -import json -import shutil -from pathlib import Path - -DEFAULT_MODEL = "nyralabs/CrisperWhisper2.0_small" - -# CrisperWhisper checkpoints declare a training-time subclass that carries an -# extra attention-loss head. Nothing in it is needed for inference, and Optimum -# resolves the exportable model from `model_type` ("whisper") anyway — but the -# name in `architectures` trips `AutoModel` resolution, so it is rewritten in the -# staging copy. The upstream repo is never modified. -TRAINING_ARCHITECTURE = "WhisperForConditionalGenerationWithAttentionLoss" -INFERENCE_ARCHITECTURE = "WhisperForConditionalGeneration" - -# Files transformers.js loads from the repo root (everything that is not weights). -SUPPORT_FILES = ( - "config.json", - "generation_config.json", - "preprocessor_config.json", - "tokenizer.json", - "tokenizer_config.json", - "special_tokens_map.json", - "added_tokens.json", - "vocab.json", - "merges.txt", - "normalizer.json", -) - - -def stage_checkpoint(model_id: str, staging: Path) -> Path: - """Download the checkpoint and normalise its config for export.""" - from huggingface_hub import snapshot_download - - print(f"[1/4] Downloading {model_id} …") - local = Path( - snapshot_download( - repo_id=model_id, - allow_patterns=["*.json", "*.txt", "*.safetensors", "*.model"], - ) - ) - - print(f"[2/4] Staging a normalised copy in {staging} …") - if staging.exists(): - shutil.rmtree(staging) - shutil.copytree(local, staging, symlinks=False) - - config_path = staging / "config.json" - config = json.loads(config_path.read_text()) - - architectures = config.get("architectures") or [] - if TRAINING_ARCHITECTURE in architectures: - config["architectures"] = [INFERENCE_ARCHITECTURE] - print(f" architectures: {TRAINING_ARCHITECTURE} → {INFERENCE_ARCHITECTURE}") - - # Weights ship as bf16; ONNX export needs fp32. Clearing the hint stops - # transformers from loading in bf16 and tracing a bf16 graph. - for key in ("dtype", "torch_dtype"): - if config.pop(key, None) is not None: - print(f" cleared config.{key} (exporting fp32)") - - config_path.write_text(json.dumps(config, indent=2)) - return staging - - -def copy_support_files(staging: Path, out: Path) -> None: - """Carry tokenizer / preprocessor / generation config into the output repo. - - `generation_config.json` matters most: CrisperWhisper ships its own - `alignment_heads`, and transformers.js needs them to pick which - cross-attention heads to run DTW over. Optimum writes a generation config of - its own, so this runs last and wins. - """ - for name in SUPPORT_FILES: - src = staging / name - if src.exists(): - shutil.copy2(src, out / name) - - generation_config = out / "generation_config.json" - if generation_config.exists(): - config = json.loads(generation_config.read_text()) - heads = config.get("alignment_heads") - if heads: - print(f" alignment_heads preserved ({len(heads)} pairs)") - else: - print( - " WARNING: no alignment_heads in generation_config.json — " - "word timestamps will not work even with cross-attentions present" - ) - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--model", default=DEFAULT_MODEL, help="HF repo id or local path") - parser.add_argument("--out", default="out", type=Path, help="output directory") - parser.add_argument("--opset", type=int, default=None, help="ONNX opset override") - parser.add_argument( - "--keep-staging", - action="store_true", - help="keep the normalised checkpoint copy for debugging", - ) - args = parser.parse_args() - - try: - from optimum.exporters.onnx import main_export - except ImportError: # optimum 2.x moved the exporters into `optimum-onnx` - from optimum_onnx.exporters.onnx import main_export # type: ignore[import-not-found] - from transformers import AutoConfig - - from onnx_config import main_export_kwargs - - out: Path = args.out - out.mkdir(parents=True, exist_ok=True) - staging = out / "_staging" - - source = Path(args.model) - staged = source if source.is_dir() else stage_checkpoint(args.model, staging) - - config = AutoConfig.from_pretrained(staged) - print( - f" {config.encoder_layers} encoder / {config.decoder_layers} decoder layers, " - f"d_model={config.d_model} → expecting cross_attentions.0 … .{config.decoder_layers - 1}" - ) - - print("[3/4] Exporting to ONNX (this is the slow part) …") - main_export( - model_name_or_path=str(staged), - output=out, - task="automatic-speech-recognition-with-past", - library_name="transformers", - # Validation compares ONNX outputs against PyTorch, but Optimum does not - # know how to compare the cross-attention outputs we just added, so it - # reports spurious mismatches. verify.py checks the graph instead. - do_validation=False, - **({"opset": args.opset} if args.opset else {}), - **main_export_kwargs(config), - ) - - print("[4/4] Arranging transformers.js layout …") - onnx_dir = out / "onnx" - onnx_dir.mkdir(exist_ok=True) - for model_file in sorted(out.glob("*.onnx")): - shutil.move(str(model_file), onnx_dir / model_file.name) - print(f" onnx/{model_file.name}") - # External weight blobs travel with their graph file. - for data_file in sorted(out.glob("*.onnx_data")): - shutil.move(str(data_file), onnx_dir / data_file.name) - print(f" onnx/{data_file.name}") - - copy_support_files(staged, out) - - if staging.exists() and not args.keep_staging: - shutil.rmtree(staging) - - print(f"\nDone → {out}\nNext: python verify.py {out}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tools/crisperwhisper-onnx/install_local.py b/tools/crisperwhisper-onnx/install_local.py deleted file mode 100644 index 855c7f5..0000000 --- a/tools/crisperwhisper-onnx/install_local.py +++ /dev/null @@ -1,117 +0,0 @@ -"""Install an export into the app's public/models/ so the browser can load it. - -Rescript's `crisperSmall` entry is flagged `local`, which makes the worker point -transformers.js at `/models//` for the duration of that model's load. This -copies the files that entry actually needs into place. - -Only the q4 pair is copied by default: the fp32 graphs are 1.1 GB and nothing -references them at runtime, so shipping them into public/ would bloat every -`next build` for no reason. - - python install_local.py # from tools/crisperwhisper-onnx - python install_local.py --dtype fp16 --name my-export -""" - -from __future__ import annotations - -import argparse -import shutil -import sys -from pathlib import Path - -import verify - -# Must match MODELS.crisperSmall.id in lib/models.ts. -DEFAULT_NAME = "crisperwhisper-2.0-small-onnx" - -# Everything transformers.js reads from the repo root. Missing optional entries -# are skipped; a missing required one is a hard error, because the failure would -# otherwise surface as an opaque tokenizer crash in the worker. -REQUIRED_FILES = ("config.json", "generation_config.json", "preprocessor_config.json") -OPTIONAL_FILES = ( - "tokenizer.json", - "tokenizer_config.json", - "special_tokens_map.json", - "added_tokens.json", - "vocab.json", - "merges.txt", - "normalizer.json", -) - - -def repo_root() -> Path: - return Path(__file__).resolve().parents[2] - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--out", type=Path, default=Path("out"), help="export directory") - parser.add_argument("--name", default=DEFAULT_NAME, help="folder name under public/models") - parser.add_argument( - "--dtype", - default="q4", - help="graph suffix to install (q4, fp16, int8, or 'fp32' for unsuffixed)", - ) - parser.add_argument("--skip-verify", action="store_true", help="install without verifying") - args = parser.parse_args() - - out: Path = args.out - if not (out / "config.json").exists(): - print(f"No export in {out} — run export.py first") - return 1 - - # Installing an export whose decoder has no cross-attentions would produce a - # model that loads fine and silently returns no word timings. - if not args.skip_verify: - argv = sys.argv - sys.argv = ["verify.py", str(out), "--no-smoke"] - try: - if verify.main() != 0: - print("\nRefusing to install: verification failed.") - return 1 - finally: - sys.argv = argv - print() - - suffix = "" if args.dtype == "fp32" else f"_{args.dtype}" - graphs = [f"encoder_model{suffix}.onnx", f"decoder_model_merged{suffix}.onnx"] - missing = [g for g in graphs if not (out / "onnx" / g).exists()] - if missing: - print(f"Missing graph(s) for --dtype {args.dtype}: {', '.join(missing)}") - print("Run quantize.py, or pick a dtype that exists.") - return 1 - - dest = repo_root() / "public" / "models" / args.name - if dest.exists(): - shutil.rmtree(dest) - (dest / "onnx").mkdir(parents=True) - - total = 0 - for name in REQUIRED_FILES: - src = out / name - if not src.exists(): - print(f"Missing required file: {name}") - return 1 - shutil.copy2(src, dest / name) - for name in OPTIONAL_FILES: - src = out / name - if src.exists(): - shutil.copy2(src, dest / name) - for name in graphs: - src = out / "onnx" / name - shutil.copy2(src, dest / "onnx" / name) - size = src.stat().st_size - total += size - print(f" onnx/{name} {size / 1e6:.1f} MB") - - rel = dest.relative_to(repo_root()) - print(f"\nInstalled {total / 1e6:.0f} MB → {rel}") - print( - "Served at /models/" - f"{args.name}/ — select \"CrisperWhisper Small (local)\" in the model menu." - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tools/crisperwhisper-onnx/onnx_config.py b/tools/crisperwhisper-onnx/onnx_config.py deleted file mode 100644 index 4fcda4e..0000000 --- a/tools/crisperwhisper-onnx/onnx_config.py +++ /dev/null @@ -1,73 +0,0 @@ -"""ONNX export config that makes Whisper decoders emit cross-attentions. - -Adapted from transformers.js `scripts/extra/whisper.py` (v3 branch), which is -what produced the `onnx-community/*_timestamped` repos. Kept as a local copy -rather than a dependency because that file lives in a scripts/ directory that is -not published to PyPI, and it moved between branches once already. - -The only thing that matters here: `WhisperOnnxConfig.outputs` does not include -attention tensors, so a stock Optimum export produces a decoder whose outputs -are `logits` plus the KV cache. transformers.js derives word timestamps by -running DTW over `cross_attentions.{i}`, so such an export can never do -word-level timing no matter what `alignment_heads` says. -""" - -from __future__ import annotations - -from typing import Dict - -# Optimum 2.x split the ONNX exporters into the separate `optimum-onnx` -# distribution. It re-exports under the original path in most installs, but not -# all, so try both rather than pinning the ecosystem to 1.x forever. -try: - from optimum.exporters.onnx.base import ConfigBehavior - from optimum.exporters.onnx.model_configs import WhisperOnnxConfig -except ImportError: # pragma: no cover - depends on which distribution is installed - from optimum_onnx.exporters.onnx.base import ConfigBehavior # type: ignore[import-not-found] - from optimum_onnx.exporters.onnx.model_configs import ( # type: ignore[import-not-found] - WhisperOnnxConfig, - ) - - -class CrossAttentionWhisperOnnxConfig(WhisperOnnxConfig): - """Whisper ONNX config that additionally exports per-layer cross-attentions.""" - - @property - def outputs(self) -> Dict[str, Dict[int, str]]: - common_outputs = super().outputs - - # Encoder behaviour has no cross-attention to report; only the decoder - # (both the plain and with-past variants) gains outputs here. - if self._behavior is ConfigBehavior.DECODER: - for i in range(self._config.decoder_layers): - common_outputs[f"cross_attentions.{i}"] = { - 0: "batch_size", - 2: "decoder_sequence_length", - 3: "encoder_sequence_length_out", - } - return common_outputs - - -def main_export_kwargs(config, task: str = "automatic-speech-recognition") -> dict: - """Extra kwargs for `optimum.exporters.onnx.main_export`. - - `output_attentions=True` makes the traced forward actually compute the - tensors; the custom configs declare them as graph outputs. Both are required - — either alone silently produces the same useless export. - """ - custom_config = CrossAttentionWhisperOnnxConfig(config=config, task=task) - - custom_onnx_configs = { - "encoder_model": custom_config.with_behavior("encoder"), - "decoder_model": custom_config.with_behavior( - "decoder", use_past=True, use_past_in_inputs=False - ), - "decoder_with_past_model": custom_config.with_behavior( - "decoder", use_past=True, use_past_in_inputs=True - ), - } - - return { - "model_kwargs": {"output_attentions": True}, - "custom_onnx_configs": custom_onnx_configs, - } diff --git a/tools/crisperwhisper-onnx/quantize.py b/tools/crisperwhisper-onnx/quantize.py deleted file mode 100644 index 5f52d36..0000000 --- a/tools/crisperwhisper-onnx/quantize.py +++ /dev/null @@ -1,175 +0,0 @@ -"""Quantise an exported CrisperWhisper for the browser. - -Produces the dtype-suffixed variants transformers.js resolves from a `dtype` -config — `_fp16`, `_int8`, `_q4` — alongside the fp32 originals. - -Rescript's `lib/models.ts` picks encoder and decoder dtypes independently, and -that split matters: Whisper's encoder degrades under aggressive quantisation -(which is why Base and Small pin `encoder_model: "fp32"`), while the merged -decoder tolerates q4. Emitting every variant here keeps that choice open at -integration time instead of baking it in. - - python quantize.py out/ --modes fp16 int8 q4 -""" - -from __future__ import annotations - -import argparse -from pathlib import Path - -# transformers.js dtype → filename suffix (mirrors its DEFAULT_DTYPE_SUFFIX_MAP). -SUFFIXES = {"fp16": "_fp16", "int8": "_int8", "q4": "_q4"} - - -class SkipVariant(Exception): - """Raised when a (graph, mode) pair cannot produce a genuinely smaller model.""" - - -def _weights_are_in_subgraphs(src: Path) -> bool: - """True when the graph's MatMuls live inside control-flow subgraphs. - - Optimum emits `decoder_model_merged` as a single top-level `If` node — the - with-past and without-past branches — so every MatMul sits one level down. - `quantize_dynamic` only walks the top-level graph, finds nothing, and writes - a file the same size as its input under an `_int8` name. - - This is not hypothetical: `onnx-community/whisper-medium_timestamped` ships - `decoder_model_merged_int8.onnx` and `_uint8.onnx` at exactly the fp32 size - of 1828 MB, while its `_q4` is 469 MB. Refusing to write the file is better - than shipping a mislabelled one. - """ - import onnx - - model = onnx.load(str(src), load_external_data=False) - nodes = model.graph.node - top_level_matmuls = sum(1 for n in nodes if n.op_type in ("MatMul", "Gemm")) - if top_level_matmuls: - return False - return any(n.op_type in ("If", "Loop", "Scan") for n in nodes) - - -def quantize_int8(src: Path, dst: Path) -> None: - from onnxruntime.quantization import QuantType, quantize_dynamic - - if _weights_are_in_subgraphs(src): - raise SkipVariant( - "weights live inside control-flow subgraphs, which quantize_dynamic " - "cannot reach — use q4 (MatMulNBits descends into subgraphs) or fp16" - ) - - quantize_dynamic( - model_input=str(src), - model_output=str(dst), - weight_type=QuantType.QInt8, - # Conv/attention kernels in Whisper are the accuracy-sensitive part; - # per-channel weights cost nothing at runtime and lose noticeably less. - per_channel=True, - reduce_range=False, - ) - - -def quantize_fp16(src: Path, dst: Path) -> None: - import onnx - from onnxconverter_common import float16 - - model = onnx.load(str(src)) - converted = float16.convert_float_to_float16( - model, - keep_io_types=True, - disable_shape_infer=False, - ) - onnx.save(converted, str(dst), save_as_external_data=False) - - -def quantize_q4(src: Path, dst: Path) -> None: - import onnx - - # Renamed in onnxruntime 1.28 (matmul_4bits_quantizer/MatMul4BitsQuantizer → - # matmul_nbits_quantizer/MatMulNBitsQuantizer). Try the current name first. - try: - from onnxruntime.quantization.matmul_nbits_quantizer import ( - MatMulNBitsQuantizer as Quantizer, - ) - except ImportError: # onnxruntime < 1.28 - from onnxruntime.quantization.matmul_4bits_quantizer import ( # type: ignore[import-not-found] - MatMul4BitsQuantizer as Quantizer, - ) - - model = onnx.load(str(src)) - quantizer = Quantizer(model, block_size=32, is_symmetric=True) - quantizer.process() - onnx.save(quantizer.model.model, str(dst), save_as_external_data=False) - - -MODES = {"int8": quantize_int8, "fp16": quantize_fp16, "q4": quantize_q4} - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("out", type=Path, help="export directory produced by export.py") - parser.add_argument( - "--modes", - nargs="+", - default=["fp16", "int8", "q4"], - choices=sorted(MODES), - ) - parser.add_argument( - "--include", - nargs="+", - # transformers.js loads the merged decoder; the split decoder_model / - # decoder_with_past_model pair is redundant for it and each is ~750 MB, - # so quantising them by default triples the work for nothing. - default=["encoder_model", "decoder_model_merged"], - help="graph stems to quantise ('all' for every graph)", - ) - args = parser.parse_args() - - onnx_dir: Path = args.out / "onnx" - if not onnx_dir.is_dir(): - print(f"No onnx/ directory in {args.out} — run export.py first") - return 1 - - # Only the fp32 originals; never quantise an already-quantised file. - sources = [ - path - for path in sorted(onnx_dir.glob("*.onnx")) - if not any(path.stem.endswith(s) for s in SUFFIXES.values()) - and ("all" in args.include or path.stem in args.include) - ] - if not sources: - print(f"No source graphs in {onnx_dir}") - return 1 - - failures = 0 - for mode in args.modes: - for src in sources: - dst = src.with_name(f"{src.stem}{SUFFIXES[mode]}.onnx") - print(f"{mode:>5} {src.name} → {dst.name}", flush=True) - try: - MODES[mode](src, dst) - except SkipVariant as reason: - print(f" skipped: {reason}") - continue - except Exception as err: # noqa: BLE001 - report and continue - failures += 1 - print(f" FAILED: {type(err).__name__}: {err}") - continue - - before = src.stat().st_size - after = dst.stat().st_size - print(f" {after / 1e6:.1f} MB ({after / before:.0%} of fp32)") - # A "quantised" file the size of its input is the failure mode that - # put un-quantised int8 decoders on the Hub. Do not ship one. - if after > before * 0.95: - failures += 1 - print(" FAILED: no meaningful size reduction — removing") - dst.unlink() - - if failures: - print(f"\n{failures} variant(s) failed — the rest are usable") - print(f"\nRe-run verification: python verify.py {args.out}") - return 1 if failures else 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tools/crisperwhisper-onnx/requirements.txt b/tools/crisperwhisper-onnx/requirements.txt deleted file mode 100644 index a2f350f..0000000 --- a/tools/crisperwhisper-onnx/requirements.txt +++ /dev/null @@ -1,28 +0,0 @@ -# Python 3.11+ required (onnxruntime floor). See README.md. - -# Optimum 1.x keeps the ONNX exporters in-package and matches the -# WhisperOnnxConfig API that onnx_config.py subclasses. 2.x moved them to the -# separate `optimum-onnx` distribution; the scripts fall back to that import -# path, but this is the combination they were written against. -optimum[exporters]>=1.24,<2 - -# Pinned below 4.50: optimum 1.x tracks the pre-4.50 Whisper modelling code, and -# the export traces that module directly. -transformers>=4.44,<4.50 - -# Upper bound is not cosmetic: optimum 1.x's `exporters/onnx/model_patcher.py` -# imports `_attention_scale` from `torch.onnx.symbolic_opset14`, and torch -# removed it along with the rest of the legacy TorchScript ONNX exporter. With a -# newer torch, importing optimum's exporter fails outright. -torch>=2.2,<2.6 - -onnx>=1.17 -onnxruntime>=1.20 -# fp16 conversion in quantize.py -onnxconverter-common>=1.16 -# onnxruntime >=1.28 imports onnx_ir from matmul_nbits_quantizer but does not -# declare it, so q4 quantisation fails with ModuleNotFoundError without this. -onnx-ir>=0.1 - -huggingface_hub>=0.26 -numpy>=1.26 diff --git a/tools/crisperwhisper-onnx/upload.py b/tools/crisperwhisper-onnx/upload.py deleted file mode 100644 index 40cc6b8..0000000 --- a/tools/crisperwhisper-onnx/upload.py +++ /dev/null @@ -1,85 +0,0 @@ -"""Publish a verified export to the Hugging Face Hub. - -Refuses to upload unless `verify.py` passes. An export that looks right but has -no cross-attention outputs is exactly what is already on the Hub several times -over; adding another is worse than publishing nothing. - - huggingface-cli login - python upload.py out/ --repo /CrisperWhisper2.0_small-ONNX-timestamped -""" - -from __future__ import annotations - -import argparse -import shutil -from pathlib import Path - -import verify - -CARD_TEMPLATE = Path(__file__).with_name("MODEL_CARD.md") - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("out", type=Path, help="export directory produced by export.py") - parser.add_argument("--repo", required=True, help="target repo id, e.g. org/name") - parser.add_argument("--private", action="store_true", help="create as private") - parser.add_argument( - "--message", default="Add ONNX export with cross-attentions", help="commit message" - ) - parser.add_argument( - "--dry-run", action="store_true", help="verify and stage the card, but do not push" - ) - args = parser.parse_args() - - out: Path = args.out - - print("Verifying before upload …") - if _verify(out) != 0: - print("\nRefusing to upload: verification failed.") - return 1 - - # The licence notice is not optional — the upstream terms cover the weights - # and their outputs, and this export is a derivative. - card = out / "README.md" - if not card.exists(): - shutil.copy2(CARD_TEMPLATE, card) - print(f"Staged model card → {card}") - else: - print(f"Keeping existing {card}") - - if args.dry_run: - print("\n--dry-run: nothing pushed.") - return 0 - - from huggingface_hub import HfApi - - api = HfApi() - api.create_repo(args.repo, private=args.private, exist_ok=True, repo_type="model") - print(f"\nUploading {out} → {args.repo} …") - api.upload_folder( - folder_path=str(out), - repo_id=args.repo, - repo_type="model", - commit_message=args.message, - # Staging leftovers and virtualenvs must never reach the Hub. - ignore_patterns=["_staging/*", ".venv/*", "**/__pycache__/*"], - ) - print(f"Done → https://huggingface.co/{args.repo}") - return 0 - - -def _verify(out: Path) -> int: - """Run verify.py's checks in-process; returns 0 on success.""" - import sys - - argv = sys.argv - sys.argv = ["verify.py", str(out)] - try: - return verify.main() - finally: - sys.argv = argv - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tools/crisperwhisper-onnx/verify.py b/tools/crisperwhisper-onnx/verify.py deleted file mode 100644 index 32aa2f7..0000000 --- a/tools/crisperwhisper-onnx/verify.py +++ /dev/null @@ -1,219 +0,0 @@ -"""Verify an export can actually produce word-level timestamps. - -This is the check that matters. Every CrisperWhisper ONNX repo on the Hub has a -plausible file listing and `alignment_heads` in its generation config, and none -of them emit cross-attentions — so the only way to tell a usable export from a -useless one is to read the decoder's graph outputs. - -Exits non-zero on failure so it can gate `upload.py`. - - python verify.py out/ -""" - -from __future__ import annotations - -import argparse -import json -from pathlib import Path - -CROSS_ATTENTION_PREFIX = "cross_attentions." - - -def graph_output_names(model_path: Path) -> list[str]: - """Output names of an ONNX graph, without touching external weight blobs. - - `load_external_data=False` keeps this cheap and constant-time regardless of - model size — graph metadata lives in the .onnx file even when the weights sit - in a sibling .onnx_data of several GB. - """ - import onnx - - model = onnx.load(str(model_path), load_external_data=False) - return [output.name for output in model.graph.output] - - -def check_decoder(model_path: Path, decoder_layers: int) -> bool: - names = graph_output_names(model_path) - cross = sorted( - int(n[len(CROSS_ATTENTION_PREFIX) :]) - for n in names - if n.startswith(CROSS_ATTENTION_PREFIX) - ) - - print(f"\n {model_path.name}") - print(f" outputs: {len(names)} total") - - if not cross: - print(" FAIL: no cross_attentions.* outputs") - print(f" got: {', '.join(names[:6])}{' …' if len(names) > 6 else ''}") - print( - " → this is the same defect as every published CrisperWhisper export;" - " word timestamps are impossible" - ) - return False - - expected = list(range(decoder_layers)) - if cross != expected: - print( - f" FAIL: expected cross_attentions.0 … .{decoder_layers - 1}, " - f"got indices {cross}" - ) - return False - - print(f" OK: cross_attentions.0 … .{cross[-1]} ({len(cross)} layers)") - return True - - -def check_alignment_heads(out: Path, decoder_layers: int, decoder_heads: int) -> bool: - path = out / "generation_config.json" - if not path.exists(): - print("\n generation_config.json\n FAIL: missing") - return False - - heads = json.loads(path.read_text()).get("alignment_heads") - print("\n generation_config.json") - if not heads: - print(" FAIL: no alignment_heads — transformers.js cannot select DTW heads") - return False - - # A head reference outside the model's real shape silently produces garbage - # timings rather than an error, so bound-check it here. - bad = [ - pair - for pair in heads - if not (0 <= pair[0] < decoder_layers and 0 <= pair[1] < decoder_heads) - ] - if bad: - print( - f" FAIL: {len(bad)} head(s) out of range for " - f"{decoder_layers} layers × {decoder_heads} heads: {bad[:4]}" - ) - return False - - print(f" OK: {len(heads)} alignment heads, all within bounds") - return True - - -def smoke_test(out: Path, config: dict) -> bool: - """Load the graphs in onnxruntime and run one decoder step. - - Graph outputs can be declared correctly and still fail at runtime — a - quantised Whisper decoder previously failed session creation outright with - "Missing required scale … MatMulNBits" (see the note in Rescript's - `lib/models.ts`). Only an actual InferenceSession catches that. - """ - import numpy as np - import onnxruntime as ort - - onnx_dir = out / "onnx" - # Prefer the quantised pair — it is what ships, and the variant most likely - # to fail session creation. - encoder = next( - (onnx_dir / f"encoder_model{s}.onnx" for s in ("_q4", "_fp16", "") - if (onnx_dir / f"encoder_model{s}.onnx").exists()), - None, - ) - decoder = next( - (onnx_dir / f"decoder_model_merged{s}.onnx" for s in ("_q4", "_fp16", "") - if (onnx_dir / f"decoder_model_merged{s}.onnx").exists()), - None, - ) - print("\n smoke test") - if encoder is None or decoder is None: - print(" FAIL: need an encoder_model and a decoder_model_merged") - return False - - options = ort.SessionOptions() - options.log_severity_level = 3 - layers = config["decoder_layers"] - heads = config["decoder_attention_heads"] - head_dim = config["d_model"] // heads - - try: - enc = ort.InferenceSession(str(encoder), options, providers=["CPUExecutionProvider"]) - hidden = enc.run( - None, - {"input_features": np.zeros((1, config["num_mel_bins"], 3000), dtype=np.float32)}, - )[0] - - dec = ort.InferenceSession(str(decoder), options, providers=["CPUExecutionProvider"]) - names = {i.name for i in dec.get_inputs()} - feed = { - "input_ids": np.array([[config["decoder_start_token_id"]]], dtype=np.int64), - "encoder_hidden_states": hidden, - } - if "use_cache_branch" in names: - feed["use_cache_branch"] = np.array([False]) - for i in range(layers): - for kind, length in (("decoder", 0), ("encoder", hidden.shape[1])): - for kv in ("key", "value"): - key = f"past_key_values.{i}.{kind}.{kv}" - if key in names: - feed[key] = np.zeros((1, heads, length, head_dim), dtype=np.float32) - - outputs = dec.run(None, feed) - by_name = dict(zip((o.name for o in dec.get_outputs()), outputs)) - except Exception as err: # noqa: BLE001 - the point is to report, not raise - print(f" FAIL: {type(err).__name__}: {err}") - return False - - attention = by_name.get("cross_attentions.0") - if attention is None: - print(" FAIL: no cross_attentions.0 at runtime") - return False - - # (batch, heads, decoder positions, encoder frames) is what DTW consumes. - if attention.shape[1] != heads or attention.shape[3] != hidden.shape[1]: - print(f" FAIL: cross_attentions.0 has unexpected shape {attention.shape}") - return False - - print(f" OK: {encoder.name} + {decoder.name} run") - print(f" cross_attentions.0 {attention.shape}, logits {by_name['logits'].shape}") - return True - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("out", type=Path, help="export directory produced by export.py") - parser.add_argument( - "--no-smoke", - action="store_true", - help="skip the onnxruntime load/run check (graph inspection only)", - ) - args = parser.parse_args() - - out: Path = args.out - config_path = out / "config.json" - if not config_path.exists(): - print(f"No config.json in {out} — did export.py finish?") - return 1 - - config = json.loads(config_path.read_text()) - decoder_layers = config["decoder_layers"] - decoder_heads = config["decoder_attention_heads"] - print( - f"Verifying {out} — {decoder_layers} decoder layers, " - f"{decoder_heads} attention heads" - ) - - decoders = sorted((out / "onnx").glob("decoder*.onnx")) - if not decoders: - print(f"\nNo decoder graphs in {out / 'onnx'}") - return 1 - - ok = all(check_decoder(path, decoder_layers) for path in decoders) - ok &= check_alignment_heads(out, decoder_layers, decoder_heads) - - encoder = out / "onnx" / "encoder_model.onnx" - print(f"\n encoder_model.onnx\n {'OK: present' if encoder.exists() else 'FAIL: missing'}") - ok &= encoder.exists() - - if not args.no_smoke: - ok &= smoke_test(out, config) - - print("\n" + ("PASS — export supports word timestamps" if ok else "FAIL")) - return 0 if ok else 1 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/workers/transcription.worker.ts b/workers/transcription.worker.ts index 3d63ca3..6303851 100644 --- a/workers/transcription.worker.ts +++ b/workers/transcription.worker.ts @@ -33,8 +33,6 @@ import { isParakeetModel, isWhisperModel, isCrisperModel, - verbatimTagCount, - whisperFillerPrompt, type ModelId, type WhisperModel, } from "@/lib/models"; @@ -413,96 +411,6 @@ async function fallbackAsrToWasm() { }); } -/** - * Build decoder input tokens for CrisperWhisper's verbatim mode: - * `[verbatim_1]…[verbatim_N] <|startoftranscript|> <|lang|> <|transcribe|>`. - * - * From `crisperwhisper/prompt.py` (PyPI `crisperwhisper==2.0.1`), with one - * deliberate deviation: - * - * - No `<|startofprev|>`. The tags lead and the standard prefix follows; this - * is a trained-in mode selector, not Whisper's initial-prompt conditioning. - * - Each tag is a single vocabulary token (51880–51884 for verbatim in the - * small checkpoint), absent from `added_tokens.json`, so the stock tokenizer - * encodes the five-tag string to exactly five ids. - * - **`<|notimestamps|>` is omitted, though upstream includes it.** Upstream can - * afford to, because it derives word timings itself via Viterbi over the - * space token's cross-attention. We go through transformers.js, whose - * `_decode_asr` splits chunked audio *on timestamp tokens*: suppress them and - * a long transcript accumulates into one unsegmented run whose - * stride-overlap merge can resolve to nothing, surfacing mid-transcription as - * "token_ids must be a non-empty array of integers". Timestamps are - * orthogonal to mode selection, so leaving them on costs nothing here. - * - * Substituting `[intended_N]` would request the cleaned-up transcript instead; - * this editor always wants verbatim. - */ -function buildVerbatimDecoderIds( - transcriber: AutomaticSpeechRecognitionPipeline, - tagCount: number, - language: string -): number[] | null { - try { - /* eslint-disable @typescript-eslint/no-explicit-any */ - const tokenizer = transcriber.tokenizer as any; - const genCfg = (transcriber.model as any).generation_config; - /* eslint-enable @typescript-eslint/no-explicit-any */ - const tags = Array.from( - { length: tagCount }, - (_, i) => `[verbatim_${i + 1}]` - ).join(""); - const tagIds = tokenizer.encode(tags, { add_special_tokens: false }); - const startId = genCfg?.decoder_start_token_id; - const langId = genCfg?.lang_to_id?.[`<|${language}|>`]; - const taskId = genCfg?.task_to_id?.["transcribe"]; - if (!tagIds?.length || startId == null || langId == null || taskId == null) { - return null; - } - return [...tagIds, startId, langId, taskId]; - } catch { - return null; - } -} - -/** - * Build decoder input tokens implementing Whisper's "initial prompt" - * conditioning: `<|startofprev|> …prompt… <|startoftranscript|> <|lang|> - * <|transcribe|>`. transformers.js documents `prompt_ids` but does not - * implement it, so the tokens are constructed manually and passed as - * `decoder_input_ids` (which `generate` honors for every chunk). Returns - * null if any required token cannot be resolved. - */ -function buildPromptedDecoderIds( - transcriber: AutomaticSpeechRecognitionPipeline, - prompt: string, - language: string -): number[] | null { - try { - // Tokenizer/config internals are untyped in transformers.js. - /* eslint-disable @typescript-eslint/no-explicit-any */ - const tokenizer = transcriber.tokenizer as any; - const genCfg = (transcriber.model as any).generation_config; - /* eslint-enable @typescript-eslint/no-explicit-any */ - const startOfPrev = tokenizer.encode("<|startofprev|>", { add_special_tokens: false }); - const promptIds = tokenizer.encode(" " + prompt.trim(), { add_special_tokens: false }); - const startId = genCfg?.decoder_start_token_id; - const langId = genCfg?.lang_to_id?.[`<|${language}|>`]; - const taskId = genCfg?.task_to_id?.["transcribe"]; - if ( - startOfPrev?.length !== 1 || - !promptIds?.length || - startId == null || - langId == null || - taskId == null - ) { - return null; - } - return [startOfPrev[0], ...promptIds, startId, langId, taskId]; - } catch { - return null; - } -} - interface DiarizationSegment { id: number; start: number; @@ -880,28 +788,6 @@ async function runWhisper( const { segments: speechSegments, frames: speechFrames } = await detectSpeechSegments(audio, vad); - // Two different ways of asking for disfluencies, depending on the model. - // CrisperWhisper was trained verbatim and selects it with a mode prefix; - // stock Whisper was not, and only gets a nudge from a filler-rich - // <|startofprev|> prompt (kept short — long prompts truncate multi-speaker - // clips, see whisperFillerPrompt). - const tagCount = verbatimTagCount(choice); - const fillerPrompt = tagCount ? null : whisperFillerPrompt(choice, transcriptLanguage); - const promptedIds = tagCount - ? buildVerbatimDecoderIds(transcriber, tagCount, transcriptLanguage) - : fillerPrompt - ? buildPromptedDecoderIds(transcriber, fillerPrompt, transcriptLanguage) - : null; - if (tagCount && !promptedIds) { - // Falling through to plain decoding would silently produce a non-verbatim - // transcript, which is the entire reason for picking this model. - console.warn( - "Could not build CrisperWhisper verbatim prefix; output will not be verbatim." - ); - } else if (fillerPrompt && !promptedIds) { - console.warn("Could not build filler prompt tokens; using default decoding."); - } - const speechSamples = speechSegments.reduce( (n, s) => n + (s.endSample - s.startSample), 0 @@ -969,9 +855,11 @@ async function runWhisper( // mid-utterance (second speaker dropped on continuous speech). no_repeat_ngram_size: 4, repetition_penalty: 1.05, - ...(promptedIds - ? { decoder_input_ids: promptedIds } - : { language: transcriptLanguage }), + // Plain decoding — no forced decoder prefix. Priming the decoder collapses + // short VAD segments on every model here, whether with Whisper's + // <|startofprev|> filler prompt or CrisperWhisper's mode tags. See the note + // above MODELS in lib/models.ts; vad-regression-test.ts guards it. + language: transcriptLanguage, }; const rawWords: Word[] = []; From 61c22bdf8c9b2b7396cf999245ff0e5c7776574b Mon Sep 17 00:00:00 2001 From: Wassim Gharbi Date: Wed, 5 Aug 2026 02:31:41 -0700 Subject: [PATCH 6/9] hide crisper + medium whisper --- lib/models.ts | 90 +++++++++++++++++++++++++-------------------------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/lib/models.ts b/lib/models.ts index bda1071..84c9c0d 100644 --- a/lib/models.ts +++ b/lib/models.ts @@ -2,11 +2,11 @@ export type WhisperModel = | "base" | "small" - | "medium" + // | "medium" /** CrisperWhisper 2.0 Small, exported by tools/crisperwhisper-onnx. */ - | "crisperSmall" + // | "crisperSmall" /** CrisperWhisper 2.0 Turbo, published ONNX export. */ - | "crisperTurbo"; + // | "crisperTurbo"; /** NVIDIA Parakeet TDT 0.6B v3 via parakeet.js (ONNX / WebGPU). */ export type ParakeetModel = "parakeet"; export type ModelId = WhisperModel | ParakeetModel; @@ -68,10 +68,10 @@ export type ModelInfo = WhisperModelInfo | ParakeetModelInfo; export const MODEL_ORDER: ModelId[] = [ "base", "small", - "medium", + // "medium", "parakeet", - "crisperSmall", - "crisperTurbo", + // "crisperSmall", + // "crisperTurbo", ]; const WHISPER_DTYPE = { @@ -179,10 +179,10 @@ const CRISPER_TURBO_DTYPE = { export const MODELS: { base: WhisperModelInfo; small: WhisperModelInfo; - medium: WhisperModelInfo; + // medium: WhisperModelInfo; parakeet: ParakeetModelInfo; - crisperSmall: WhisperModelInfo; - crisperTurbo: WhisperModelInfo; + // crisperSmall: WhisperModelInfo; + // crisperTurbo: WhisperModelInfo; } = { base: { backend: "whisper", @@ -200,51 +200,51 @@ export const MODELS: { size: "~600 MB", dtype: WHISPER_DTYPE, }, - medium: { - backend: "whisper", - id: "onnx-community/whisper-medium_timestamped", - label: "Whisper Medium", - description: - "Best accuracy on accents, crosstalk and poor recordings. Slow, and a big download.", - // WASM int8 encoder + q4 decoder ~780 MB; WebGPU fp16 encoder ~1.1 GB. - size: "~1.1 GB", - dtype: WHISPER_MEDIUM_DTYPE, - }, + // medium: { + // backend: "whisper", + // id: "onnx-community/whisper-medium_timestamped", + // label: "Whisper Medium", + // description: + // "Best accuracy on accents, crosstalk and poor recordings. Slow, and a big download.", + // // WASM int8 encoder + q4 decoder ~780 MB; WebGPU fp16 encoder ~1.1 GB. + // size: "~1.1 GB", + // dtype: WHISPER_MEDIUM_DTYPE, + // }, parakeet: { backend: "parakeet", id: "parakeet-tdt-0.6b-v3", repoId: "ysdede/parakeet-tdt-0.6b-v3-onnx", - label: "Parakeet TDT v3", + label: "Parakeet v3", description: "NVIDIA FastConformer — faster on WebGPU, strong EU-language accuracy. Auto-detects language.", // WASM int8 encoder + fp16 decoder ~690 MB; WebGPU fp16 + fp32 ~1.3 GB. size: "~1.3 GB", }, - crisperSmall: { - backend: "whisper", - // Local folder under public/models — not published yet. Install with - // `python tools/crisperwhisper-onnx/install_local.py`. - id: "crisperwhisper-2.0-small-onnx", - local: true, - label: "CrisperWhisper Small (local)", - description: - "Verbatim: transcribes fillers as [UM] / [UH] instead of dropping them. Self-exported, unpublished. Non-commercial licence.", - // q4 encoder 66 MB + q4 merged decoder 258 MB. - size: "~324 MB", - dtype: CRISPER_SMALL_DTYPE, - crisper: true, - }, - crisperTurbo: { - backend: "whisper", - id: "Masterx/CrisperWhisper2.0-turbo-ONNX", - label: "CrisperWhisper Turbo", - description: - "Keeps fillers, on a large-v3 encoder. The largest download. Non-commercial licence.", - // q4 encoder 425 MB + fp16 merged decoder 477 MB. - size: "~900 MB", - dtype: CRISPER_TURBO_DTYPE, - crisper: true, - }, + // crisperSmall: { + // backend: "whisper", + // // Local folder under public/models — not published yet. Install with + // // `python tools/crisperwhisper-onnx/install_local.py`. + // id: "crisperwhisper-2.0-small-onnx", + // local: true, + // label: "CrisperWhisper Small (local)", + // description: + // "Verbatim: transcribes fillers as [UM] / [UH] instead of dropping them. Self-exported, unpublished. Non-commercial licence.", + // // q4 encoder 66 MB + q4 merged decoder 258 MB. + // size: "~324 MB", + // dtype: CRISPER_SMALL_DTYPE, + // crisper: true, + // }, + // crisperTurbo: { + // backend: "whisper", + // id: "Masterx/CrisperWhisper2.0-turbo-ONNX", + // label: "CrisperWhisper Turbo", + // description: + // "Keeps fillers, on a large-v3 encoder. The largest download. Non-commercial licence.", + // // q4 encoder 425 MB + fp16 merged decoder 477 MB. + // size: "~900 MB", + // dtype: CRISPER_TURBO_DTYPE, + // crisper: true, + // }, }; /** From cebd43182f415810851a4a8d8efe7c6819f2f273 Mon Sep 17 00:00:00 2001 From: Wassim Gharbi Date: Wed, 5 Aug 2026 02:43:33 -0700 Subject: [PATCH 7/9] better model icons --- components/ModelSelector.tsx | 53 +++++++++++++++++++--------- components/SignalBars.tsx | 67 ++++++++++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+), 16 deletions(-) create mode 100644 components/SignalBars.tsx diff --git a/components/ModelSelector.tsx b/components/ModelSelector.tsx index 11c6409..639fe0b 100644 --- a/components/ModelSelector.tsx +++ b/components/ModelSelector.tsx @@ -8,6 +8,7 @@ import { useId, useMemo, useState, + type ComponentType, type ReactNode, } from "react"; import { @@ -18,8 +19,12 @@ import { FileText, Languages, Loader2, - type LucideIcon, } from "lucide-react"; +import { + SignalBarsHigh, + SignalBarsLow, + SignalBarsMedium, +} from "./SignalBars"; import { TRANSCRIPT_LANGUAGE_ORDER, TRANSCRIPT_LANGUAGES, @@ -30,6 +35,7 @@ import { MODELS, isModelId, isWhisperModel, + type ModelId, } from "@/lib/models"; import type { TranscriptSource } from "@/lib/source"; import { @@ -50,9 +56,15 @@ export type ModelOptionContextValue = { keepMenuOpen: () => void; }; +/** Any `size` + `className` icon: lucide's, or a local one like {@link SignalBarsLow}. */ +export type IconComponent = ComponentType<{ + size?: number | string; + className?: string; +}>; + export type OptionTrigger = { label: ReactNode; - icon?: LucideIcon; + icon?: IconComponent; iconClassName?: string; /** Show a spinner instead of the icon in the closed trigger. */ busy?: boolean; @@ -301,12 +313,24 @@ export default function ModelSelector({ ); } +/** Signal bars stand in for relative model strength on the default rows. */ +const MODEL_ICONS: Record = { + base: SignalBarsLow, + small: SignalBarsMedium, + parakeet: SignalBarsHigh, +}; + +function iconForSource(id: TranscriptSource): IconComponent { + if (isModelId(id)) return MODEL_ICONS[id]; + return id === "import" ? FileText : AudioLines; +} + /** Default option row: icon + label + optional meta. ASR ids fill in from MODELS. */ export function ModelOption({ id, label, meta, - icon: Icon = AudioLines, + icon, children, onSelect, /** When false, a child owns the closed trigger via `useOptionTrigger`. */ @@ -315,7 +339,7 @@ export function ModelOption({ id: TranscriptSource; label?: string; meta?: string; - icon?: LucideIcon; + icon?: IconComponent; children?: ReactNode; onSelect?: (ctx: ModelOptionContextValue) => void; autoTrigger?: boolean; @@ -323,6 +347,7 @@ export function ModelOption({ const selector = useSelectorCtx(); const selected = selector.value === id; + const Icon = icon ?? iconForSource(id); const resolvedLabel = label ?? (isModelId(id) ? MODELS[id].label : id); const resolvedMeta = meta ?? (isModelId(id) ? MODELS[id].size : undefined); @@ -355,11 +380,10 @@ export function ModelOption({ role="option" aria-selected={selected} onClick={handleClick} - className={`flex w-full flex-col gap-0.5 rounded-lg px-2.5 py-2 text-left transition cursor-pointer ${ - selected + className={`flex w-full flex-col gap-0.5 rounded-lg px-2.5 py-2 text-left transition cursor-pointer ${selected ? "bg-zinc-100 text-zinc-900 dark:bg-zinc-800 dark:text-zinc-50" : "text-zinc-700 hover:bg-zinc-50 dark:text-zinc-300 dark:hover:bg-zinc-800/60" - }`} + }`} > !v); selector.keepMenuOpen(); }} - className={`flex w-full cursor-pointer items-center gap-2.5 rounded-lg px-2.5 py-2 text-left transition ${ - submenuOpen + className={`flex w-full cursor-pointer items-center gap-2.5 rounded-lg px-2.5 py-2 text-left transition ${submenuOpen ? "bg-zinc-100 text-zinc-900 dark:bg-zinc-800 dark:text-zinc-50" : "text-zinc-700 hover:bg-zinc-50 dark:text-zinc-300 dark:hover:bg-zinc-800/60" - }`} + }`} > {active.flag} @@ -439,9 +462,8 @@ export function LanguageSection() { @@ -462,11 +484,10 @@ export function LanguageSection() { role="menuitemradio" aria-checked={selected} onClick={() => select(id)} - className={`flex w-full cursor-pointer items-center gap-2.5 rounded-lg px-2.5 py-2 text-left transition ${ - selected + className={`flex w-full cursor-pointer items-center gap-2.5 rounded-lg px-2.5 py-2 text-left transition ${selected ? "bg-zinc-100 text-zinc-900 dark:bg-zinc-800 dark:text-zinc-50" : "text-zinc-700 hover:bg-zinc-50 dark:text-zinc-300 dark:hover:bg-zinc-800/60" - }`} + }`} > {option.flag} diff --git a/components/SignalBars.tsx b/components/SignalBars.tsx new file mode 100644 index 0000000..5c24dba --- /dev/null +++ b/components/SignalBars.tsx @@ -0,0 +1,67 @@ +/** + * Signal-strength bars for the model rows, drawn here rather than taken from + * lucide: `SignalLow` and friends omit the bars above the level, so the three + * model rows would each get a differently-shaped, differently-sized glyph. + * These always draw all three bars and fade the inactive ones, so the rows share + * one silhouette and read as "level N of 3". + */ + +export type SignalBarsProps = { + size?: number | string; + className?: string; +}; + +/** x position and top y of each bar, shortest first. Bars sit on y = 19. */ +const BARS: ReadonlyArray<{ x: number; top: number }> = [ + { x: 5.75, top: 14.5 }, + { x: 12, top: 10.25 }, + { x: 18.25, top: 6 }, +]; + +const BASELINE = 19; +/** Matches the 2px stroke of the lucide icons beside it, plus round caps. */ +const STROKE = 2.5; +const INACTIVE_OPACITY = 0.25; + +function SignalBars({ + level, + size = 24, + className, +}: SignalBarsProps & { level: 1 | 2 | 3 }) { + return ( + + {BARS.map((bar, i) => ( + + ))} + + ); +} + +export function SignalBarsLow(props: SignalBarsProps) { + return ; +} + +export function SignalBarsMedium(props: SignalBarsProps) { + return ; +} + +export function SignalBarsHigh(props: SignalBarsProps) { + return ; +} From 7bfb165cfc1e2cda7b2fbb0e331ab302a5237c5a Mon Sep 17 00:00:00 2001 From: Wassim Gharbi Date: Wed, 5 Aug 2026 02:47:24 -0700 Subject: [PATCH 8/9] comment out tests --- tests/models-test.ts | 72 ++++++++++++++++++++++---------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/tests/models-test.ts b/tests/models-test.ts index 54a9c33..302e746 100644 --- a/tests/models-test.ts +++ b/tests/models-test.ts @@ -4,8 +4,8 @@ import { MODEL_ORDER, MODELS, - isCrisperModel, - isLocalModel, + // isCrisperModel, + // isLocalModel, isModelId, isParakeetModel, isWhisperModel, @@ -18,7 +18,7 @@ function assert(cond: boolean, msg: string) { assert(isWhisperModel("base"), "base is Whisper"); assert(isWhisperModel("small"), "small is Whisper"); -assert(isWhisperModel("medium"), "medium is Whisper"); +// assert(isWhisperModel("medium"), "medium is Whisper"); assert(!isWhisperModel("parakeet"), "parakeet is not Whisper"); assert(isParakeetModel("parakeet"), "parakeet is Parakeet"); assert(isModelId("parakeet"), "parakeet is a model id"); @@ -38,18 +38,18 @@ assert(typeof MODELS.parakeet.label === "string", "parakeet label"); assert(MODELS.base.backend === "whisper", "base backend"); assert(typeof MODELS.base.id === "string", "whisper base id"); assert(typeof MODELS.small.id === "string", "whisper small id"); -assert(typeof MODELS.medium.id === "string", "whisper medium id"); +// assert(typeof MODELS.medium.id === "string", "whisper medium id"); assert(MODELS.base.dtype.webgpu.encoder_model === "fp32", "whisper dtype"); // Medium deviates from WHISPER_DTYPE on purpose: its fp32 encoder export is // 1.2 GB. Pin the split so a later dtype tidy-up cannot quietly reinstate it. -assert( - MODELS.medium.dtype.wasm.encoder_model === "int8", - "medium encoder is int8 on wasm" -); -assert( - MODELS.medium.dtype.webgpu.encoder_model === "fp16", - "medium encoder is fp16 on webgpu" -); +// assert( +// MODELS.medium.dtype.wasm.encoder_model === "int8", +// "medium encoder is int8 on wasm" +// ); +// assert( +// MODELS.medium.dtype.webgpu.encoder_model === "fp16", +// "medium encoder is fp16 on webgpu" +// ); assert( MODEL_ORDER.includes("parakeet") && MODEL_ORDER.includes("base"), @@ -66,36 +66,36 @@ for (const id of MODEL_ORDER) { // fp32 decoder reproduces the fp32 transcript exactly). fp16 is also 477 MB // against q4's 600 MB. Pinned so a dtype tidy-up cannot fold it back into the // Small config. -assert( - MODELS.crisperTurbo.dtype.wasm.decoder_model_merged === "fp16" && - MODELS.crisperTurbo.dtype.webgpu.decoder_model_merged === "fp16", - "crisperTurbo decodes in fp16" -); -assert( - MODELS.crisperSmall.dtype.wasm.decoder_model_merged === "q4", - "crisperSmall keeps the verified q4 decoder" -); +// assert( +// MODELS.crisperTurbo.dtype.wasm.decoder_model_merged === "fp16" && +// MODELS.crisperTurbo.dtype.webgpu.decoder_model_merged === "fp16", +// "crisperTurbo decodes in fp16" +// ); +// assert( +// MODELS.crisperSmall.dtype.wasm.decoder_model_merged === "q4", +// "crisperSmall keeps the verified q4 decoder" +// ); // The tokenizer fix-up is required by CrisperWhisper's vocabulary layout: the // extended tokens that break word collation are emitted whether or not the // decoder is primed with anything. -assert(isCrisperModel("crisperSmall"), "crisperSmall needs the tokenizer fix-up"); -assert(isCrisperModel("crisperTurbo"), "crisperTurbo needs the tokenizer fix-up"); -assert(!isCrisperModel("base"), "base needs no tokenizer fix-up"); -assert(!isCrisperModel("parakeet"), "parakeet needs no tokenizer fix-up"); +// assert(isCrisperModel("crisperSmall"), "crisperSmall needs the tokenizer fix-up"); +// assert(isCrisperModel("crisperTurbo"), "crisperTurbo needs the tokenizer fix-up"); +// assert(!isCrisperModel("base"), "base needs no tokenizer fix-up"); +// assert(!isCrisperModel("parakeet"), "parakeet needs no tokenizer fix-up"); // Only the unpublished export is served from public/models; a stray `local` // flag on a Hub model would send transformers.js to a path that 404s. -assert(isLocalModel("crisperSmall"), "crisperSmall is served locally"); -assert(!isLocalModel("crisperTurbo"), "crisperTurbo loads from the Hub"); -assert(!isLocalModel("base"), "base loads from the Hub"); -assert( - !MODELS.crisperSmall.id.includes("/"), - "local model id is a public/models folder name, not a Hub repo id" -); -assert( - MODELS.crisperTurbo.id === "Masterx/CrisperWhisper2.0-turbo-ONNX", - "crisperTurbo Hub id" -); +// assert(isLocalModel("crisperSmall"), "crisperSmall is served locally"); +// assert(!isLocalModel("crisperTurbo"), "crisperTurbo loads from the Hub"); +// assert(!isLocalModel("base"), "base loads from the Hub"); +// assert( +// !MODELS.crisperSmall.id.includes("/"), +// "local model id is a public/models folder name, not a Hub repo id" +// ); +// assert( +// MODELS.crisperTurbo.id === "Masterx/CrisperWhisper2.0-turbo-ONNX", +// "crisperTurbo Hub id" +// ); console.log("models-test: ok"); From daac3888b2192b8edde9a8aeaa12a17027c942dc Mon Sep 17 00:00:00 2001 From: Wassim Gharbi Date: Wed, 5 Aug 2026 02:54:07 -0700 Subject: [PATCH 9/9] lint --- components/ModelSelector.tsx | 19 ++++++++++--------- lib/models.ts | 24 ++++++++++++------------ 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/components/ModelSelector.tsx b/components/ModelSelector.tsx index 639fe0b..6b56c91 100644 --- a/components/ModelSelector.tsx +++ b/components/ModelSelector.tsx @@ -35,7 +35,6 @@ import { MODELS, isModelId, isWhisperModel, - type ModelId, } from "@/lib/models"; import type { TranscriptSource } from "@/lib/source"; import { @@ -313,18 +312,20 @@ export default function ModelSelector({ ); } -/** Signal bars stand in for relative model strength on the default rows. */ -const MODEL_ICONS: Record = { +/** + * Signal bars stand in for relative model strength on the default rows. + * + * A plain lookup rather than a `iconForSource(id)` helper: the row renders this + * value as JSX, and react-hooks/static-components reads any call result used as + * a component type as a component created during render. + */ +const SOURCE_ICONS: Record = { base: SignalBarsLow, small: SignalBarsMedium, parakeet: SignalBarsHigh, + import: FileText, }; -function iconForSource(id: TranscriptSource): IconComponent { - if (isModelId(id)) return MODEL_ICONS[id]; - return id === "import" ? FileText : AudioLines; -} - /** Default option row: icon + label + optional meta. ASR ids fill in from MODELS. */ export function ModelOption({ id, @@ -347,7 +348,7 @@ export function ModelOption({ const selector = useSelectorCtx(); const selected = selector.value === id; - const Icon = icon ?? iconForSource(id); + const Icon = icon ?? SOURCE_ICONS[id]; const resolvedLabel = label ?? (isModelId(id) ? MODELS[id].label : id); const resolvedMeta = meta ?? (isModelId(id) ? MODELS[id].size : undefined); diff --git a/lib/models.ts b/lib/models.ts index 84c9c0d..48f28d7 100644 --- a/lib/models.ts +++ b/lib/models.ts @@ -87,10 +87,10 @@ const WHISPER_DTYPE = { * {@link MODELS.parakeet} already does — fp16 encoder on WebGPU, int8 on WASM — * and keeps the q4 merged decoder that Base and Small are proven on. */ -const WHISPER_MEDIUM_DTYPE = { - webgpu: { encoder_model: "fp16", decoder_model_merged: "q4" }, - wasm: { encoder_model: "int8", decoder_model_merged: "q4" }, -} satisfies WhisperModelInfo["dtype"]; +// const WHISPER_MEDIUM_DTYPE = { +// webgpu: { encoder_model: "fp16", decoder_model_merged: "q4" }, +// wasm: { encoder_model: "int8", decoder_model_merged: "q4" }, +// } satisfies WhisperModelInfo["dtype"]; /** * The local Small export ships only q4 for the merged decoder: int8 cannot @@ -100,10 +100,10 @@ const WHISPER_MEDIUM_DTYPE = { * combination verified end-to-end — encoder and decoder loaded in onnxruntime, * cross-attentions returned at the right shape. */ -const CRISPER_SMALL_DTYPE = { - webgpu: { encoder_model: "q4", decoder_model_merged: "q4" }, - wasm: { encoder_model: "q4", decoder_model_merged: "q4" }, -} satisfies WhisperModelInfo["dtype"]; +// const CRISPER_SMALL_DTYPE = { +// webgpu: { encoder_model: "q4", decoder_model_merged: "q4" }, +// wasm: { encoder_model: "q4", decoder_model_merged: "q4" }, +// } satisfies WhisperModelInfo["dtype"]; /** * Turbo takes fp16 for the decoder rather than q4. @@ -121,10 +121,10 @@ const CRISPER_SMALL_DTYPE = { * transformers.js only fetches when `use_external_data_format` is declared, and * this repo's config.json does not declare it. */ -const CRISPER_TURBO_DTYPE = { - webgpu: { encoder_model: "q4", decoder_model_merged: "fp16" }, - wasm: { encoder_model: "q4", decoder_model_merged: "fp16" }, -} satisfies WhisperModelInfo["dtype"]; +// const CRISPER_TURBO_DTYPE = { +// webgpu: { encoder_model: "q4", decoder_model_merged: "fp16" }, +// wasm: { encoder_model: "q4", decoder_model_merged: "fp16" }, +// } satisfies WhisperModelInfo["dtype"]; /** * No model conditions the decoder on a prefix, and that is a measured decision