From 3e75999ded4c7aa95374349230feada8865452b8 Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Fri, 21 Aug 2026 13:42:03 +0200 Subject: [PATCH 01/45] [codex/vc-workflow] fix(stt): refuse quantized Whisper runtime models --- .env.debug.example | 2 +- .env.example | 2 +- README.md | 13 +- core/build.rs | 3 +- core/config/models.rs | 305 ++++++++++++------ core/stt/onnx_adapter.rs | 2 +- core/stt/whisper/engine.rs | 47 ++- ...05-26-LAYERED_INCREMENTAL_TRANSCRIPTION.md | 4 + docs/ARCHITECTURE.md | 10 +- docs/OVERLAY_STREAMING.md | 2 +- docs/STT_CONTRACT.md | 25 +- docs/TEAM_SETUP.md | 2 +- docs/TRANSCRIPT_LANES.md | 2 +- docs/WHISPER_LIVE.md | 2 +- examples/README.md | 2 +- examples/demo_full_pipeline.rs | 7 +- examples/e2e_stt.rs | 7 +- examples/test_audio.rs | 7 +- examples/test_audio_long.rs | 7 +- examples/transcribe_file.rs | 1 - scripts/bench-stt.sh | 39 +-- scripts/download-model.sh | 43 ++- scripts/ensure-models.sh | 21 +- tests/e2e_full_pipeline.rs | 36 +-- tests/e2e_stt_transcription.rs | 34 +- tests/support/e2e_stt_matrix.rs | 96 +----- 26 files changed, 361 insertions(+), 360 deletions(-) diff --git a/.env.debug.example b/.env.debug.example index b40adf87..228a7d03 100644 --- a/.env.debug.example +++ b/.env.debug.example @@ -38,7 +38,7 @@ # CODESCRIBE_TOGGLE_FINAL_PASS=1 # Default: 1 — Use saved-WAV final-pass adjudication when stopping toggle dictation (0 restores preview-only stop path) # CODESCRIBE_WHISPER_IDLE_UNLOAD_SECS=300 # Default: 300 — Unload idle Whisper engine after this many seconds (0 = explicit keep-warm) # CODESCRIBE_WHISPER_INITIAL_PROMPT= # Default: unset — Initial prompt hint for Whisper decoding (ignored by ONNX adapter) -# LOCAL_MODEL=whisper-large-v3-turbo-mlx-q8 # Default: whisper-large-v3-turbo-mlx-q8 — Local Whisper model id (HF cache / embedded lookup) +# LOCAL_MODEL=whisper-large-v3-turbo # Default fp16 local Whisper model id # STT_API_KEY= # Default: unset — Cloud STT API key; prefer Settings / macOS Keychain # STT_ENDPOINT= # Default: unset — Cloud STT API endpoint (when USE_LOCAL_STT=0) # USE_LOCAL_STT=1 # Default: 1 — Use local Whisper model (vs cloud) diff --git a/.env.example b/.env.example index d166d052..c819f8d7 100644 --- a/.env.example +++ b/.env.example @@ -40,7 +40,7 @@ # CODESCRIBE_TOGGLE_FINAL_PASS=1 # Default: 1 — Use saved-WAV final-pass adjudication when stopping toggle dictation (0 restores preview-only stop path) # CODESCRIBE_WHISPER_IDLE_UNLOAD_SECS=300 # Default: 300 — Unload idle Whisper engine after this many seconds (0 = explicit keep-warm) # CODESCRIBE_WHISPER_INITIAL_PROMPT= # Default: unset — Initial prompt hint for Whisper decoding (ignored by ONNX adapter) -# LOCAL_MODEL=whisper-large-v3-turbo-mlx-q8 # Default: whisper-large-v3-turbo-mlx-q8 — Local Whisper model id (HF cache / embedded lookup) +# LOCAL_MODEL=whisper-large-v3-turbo # Default fp16 local Whisper model id # STT_API_KEY= # Default: unset — Cloud STT API key; prefer Settings / macOS Keychain # STT_ENDPOINT= # Default: unset — Cloud STT API endpoint (when USE_LOCAL_STT=0) # USE_LOCAL_STT=1 # Default: 1 — Use local Whisper model (vs cloud) diff --git a/README.md b/README.md index b02c90ee..23ceac54 100644 --- a/README.md +++ b/README.md @@ -327,8 +327,8 @@ qube-daemon --help Codescribe uses **whisper-large-v3-turbo** (mlx-community, fp16): - 4-layer turbo architecture (vs 32 layers in full model) -- fp16 weights (~1.6 GB): loads without q8→F32 dequantization, roughly - halving cold start; the legacy q8 model stays supported as a fallback +- fp16 weights (~1.6 GB): load without q8→F32 dequantization; quantized Whisper + payloads are rejected before engine load - ~10x faster than whisper-large-v3 - Metal GPU acceleration @@ -342,13 +342,12 @@ Runtime resolution when Whisper is not embedded: 1. `CODESCRIBE_MODEL_PATH` environment variable 2. `~/.codescribe/models/whisper-large-v3-turbo/` (fp16 default) -3. Hugging Face cache snapshots for `mlx-community/whisper-large-v3-turbo` -4. Legacy fallback: `~/.codescribe/models/whisper-large-v3-turbo-mlx-q8/` or - `LibraxisAI/whisper-large-v3-turbo-mlx-q8` snapshots +3. A complete Hugging Face snapshot explicitly configured by repo id The mlx-community repo ships only `config.json` + `weights.safetensors`; -the download paths compose `tokenizer.json` + `mel_filters.npz` from the -legacy repo (both files are quantization-independent). +the download paths compose `tokenizer.json` from the matching official OpenAI +Transformers repo and `mel_filters.npz` from a checksum-pinned OpenAI Whisper +asset. The resulting directory is validated as unquantized before resolution. `CODESCRIBE_EMBED_EMBEDDER=1` is an explicit fat/debug path that compiles MiniLM into Rust artifacts. Normal builds resolve MiniLM from the signed app resource or HF cache. `CODESCRIBE_NO_EMBED=1` disables every optional binary embed; Silero remains embedded. diff --git a/core/build.rs b/core/build.rs index 21d1c0da..a9210c3b 100644 --- a/core/build.rs +++ b/core/build.rs @@ -37,8 +37,7 @@ use license_key_contract::{ const DEFAULT_MODEL_NAME: &str = "whisper-large-v3-turbo"; /// Hugging Face repo id for the default Whisper snapshot (HF cache + download hints). /// The repo ships only config + fp16 weights; `make download-model` composes -/// tokenizer.json + mel_filters.npz from the legacy q8 repo, and runtime keeps -/// a legacy fallback (see core/config/models.rs). +/// the official OpenAI tokenizer and pinned mel filters into the runtime dir. const DEFAULT_WHISPER_REPO: &str = "mlx-community/whisper-large-v3-turbo"; /// Default TTS model to embed diff --git a/core/config/models.rs b/core/config/models.rs index 68d56fb6..74354fae 100644 --- a/core/config/models.rs +++ b/core/config/models.rs @@ -5,7 +5,9 @@ //! model from here instead of re-implementing its own precedence rules. use anyhow::{Context, Result, anyhow}; +use sha2::{Digest, Sha256}; use std::fs; +use std::io::{Read, Seek, SeekFrom}; use std::path::{Path, PathBuf}; use crate::hf_cache; @@ -16,17 +18,13 @@ pub const DEFAULT_MODEL: &str = "whisper-large-v3-turbo"; /// the Settings → Dictation download. fp16 weights: no q8→F32 dequantization /// on load, at the cost of a larger download than the q8 repo. pub const DEFAULT_WHISPER_REPO: &str = "mlx-community/whisper-large-v3-turbo"; -/// Previous default (q8). Kept as a resolution fallback so installs that -/// already carry it keep transcribing without a re-download. -pub const LEGACY_MODEL: &str = "whisper-large-v3-turbo-mlx-q8"; -/// Repo behind [`LEGACY_MODEL`]. Also the companion source for -/// [`COMPANION_MODEL_FILES`], which [`DEFAULT_WHISPER_REPO`] does not ship. -pub const LEGACY_WHISPER_REPO: &str = "LibraxisAI/whisper-large-v3-turbo-mlx-q8"; -/// Files the mlx-community repo does not ship. Both are quantization-independent -/// (same tokenizer and mel filterbank across q8/fp16), so composing them from -/// the legacy repo yields a correct model directory. -const COMPANION_MODEL_FILES: [&str; 2] = ["tokenizer.json", "mel_filters.npz"]; - +/// Official Transformers tokenizer paired with Whisper large-v3-turbo. +pub const TOKENIZER_WHISPER_REPO: &str = "openai/whisper-large-v3-turbo"; +/// Pinned OpenAI Whisper asset. The checksum is asserted by the installer. +pub const MEL_FILTERS_URL: &str = "https://raw.githubusercontent.com/openai/whisper/5f86d1d86363843179951550570367b37c5d6f78/whisper/assets/mel_filters.npz"; +/// SHA-256 of [`MEL_FILTERS_URL`]. +pub const MEL_FILTERS_SHA256: &str = + "7450ae70723a5ef9d341e3cee628c7cb0177f36ce42c44b7ed2bf3325f0f6d4c"; /// Files that must all be present for a directory to count as a usable model. const REQUIRED_MODEL_FILES: [&str; 3] = ["config.json", "tokenizer.json", "mel_filters.npz"]; /// Weight file names, of which **any one** satisfies the completeness check — @@ -57,12 +55,74 @@ fn canonicalize_or_self(path: PathBuf) -> PathBuf { /// [`REQUIRED_MODEL_WEIGHTS`]. This is the gate that keeps half-downloaded /// directories from being advertised or resolved as loadable models. fn is_complete_whisper_model_dir(path: &Path) -> bool { - REQUIRED_MODEL_FILES + let files_present = REQUIRED_MODEL_FILES .iter() .all(|name| path.join(name).exists()) && REQUIRED_MODEL_WEIGHTS .iter() - .any(|name| path.join(name).exists()) + .any(|name| path.join(name).exists()); + files_present && is_unquantized_whisper_model_dir(path) +} + +/// Reject quantized or malformed weights before they can reach the expensive +/// engine loader. The config check catches normal MLX q8 exports; the +/// safetensors header check also catches a q8 payload hidden behind a renamed +/// directory or a config with its `quantization` field removed. +pub(crate) fn is_unquantized_whisper_model_dir(path: &Path) -> bool { + let config = match fs::read_to_string(path.join("config.json")) + .ok() + .and_then(|raw| serde_json::from_str::(&raw).ok()) + { + Some(config) => config, + None => return false, + }; + if config + .get("quantization") + .is_some_and(|value| !value.is_null()) + || config + .get("quantization_config") + .is_some_and(|value| !value.is_null()) + { + return false; + } + + let Some(weights_path) = REQUIRED_MODEL_WEIGHTS + .iter() + .map(|name| path.join(name)) + .find(|candidate| candidate.exists()) + else { + return false; + }; + safetensors_header_is_unquantized(&weights_path).unwrap_or(false) +} + +/// Inspect only the bounded JSON header; model tensor data is never read. +fn safetensors_header_is_unquantized(path: &Path) -> Result { + const MAX_HEADER_BYTES: u64 = 16 * 1024 * 1024; + // nosemgrep: rust.actix.path-traversal.tainted-path.tainted-path -- Read-only model inspection. `path` is an operator-selected local model file or an internally resolved bundle/cache child; no network/request path component reaches it. + let mut file = fs::File::open(path).with_context(|| format!("open {}", path.display()))?; + let mut len_bytes = [0_u8; 8]; + file.read_exact(&mut len_bytes) + .with_context(|| format!("read safetensors header length from {}", path.display()))?; + let header_len = u64::from_le_bytes(len_bytes); + if header_len == 0 || header_len > MAX_HEADER_BYTES { + return Ok(false); + } + let mut header = vec![0_u8; header_len as usize]; + file.seek(SeekFrom::Start(8))?; + file.read_exact(&mut header) + .with_context(|| format!("read safetensors header from {}", path.display()))?; + let metadata: serde_json::Value = serde_json::from_slice(&header) + .with_context(|| format!("parse safetensors header from {}", path.display()))?; + let Some(tensors) = metadata.as_object() else { + return Ok(false); + }; + Ok(tensors.iter().all(|(name, tensor)| { + name == "__metadata__" + || (!name.ends_with(".scales") + && !name.ends_with(".biases") + && tensor.get("dtype").and_then(|value| value.as_str()) != Some("U32")) + })) } /// Whether a candidate models root owns at least one complete Whisper model. @@ -70,7 +130,7 @@ fn is_complete_whisper_model_dir(path: &Path) -> bool { /// A bundled `Resources/models` directory may contain only another model /// family (for example the semantic embedder). Treating mere directory /// existence as Whisper ownership shadows the user-installed fp16 model and -/// incorrectly drops runtime resolution to the legacy q8 cache. +/// could incorrectly select a quantized cache instead. fn models_root_contains_complete_whisper_model(path: &Path) -> bool { fs::read_dir(path).is_ok_and(|entries| { entries @@ -90,31 +150,16 @@ fn hf_snapshot_for_model(model_ref: &str) -> Option { return None; } - if trimmed.contains('/') { - return hf_cache::find_snapshot_with_any( - trimmed, - &REQUIRED_MODEL_FILES, - &REQUIRED_MODEL_WEIGHTS, - ); - } - - if trimmed == DEFAULT_MODEL { - return hf_cache::find_snapshot_with_any( - DEFAULT_WHISPER_REPO, - &REQUIRED_MODEL_FILES, - &REQUIRED_MODEL_WEIGHTS, - ); - } - - if trimmed == LEGACY_MODEL { - return hf_cache::find_snapshot_with_any( - LEGACY_WHISPER_REPO, - &REQUIRED_MODEL_FILES, - &REQUIRED_MODEL_WEIGHTS, - ); - } - - None + let repo = if trimmed.contains('/') { + trimmed + } else if trimmed == DEFAULT_MODEL { + DEFAULT_WHISPER_REPO + } else { + return None; + }; + let snapshot = + hf_cache::find_snapshot_with_any(repo, &REQUIRED_MODEL_FILES, &REQUIRED_MODEL_WEIGHTS)?; + is_complete_whisper_model_dir(&snapshot).then_some(snapshot) } /// Owner of the resolved runtime models directory. @@ -270,8 +315,6 @@ impl ModelManager { /// 3. Configured Hugging Face repo snapshot /// 4. Default models-dir alias (`whisper-large-v3-turbo`) /// 5. Default Hugging Face snapshot (`mlx-community/whisper-large-v3-turbo`) -/// 6. Legacy models-dir alias (`whisper-large-v3-turbo-mlx-q8`) -/// 7. Legacy Hugging Face snapshot (`LibraxisAI/whisper-large-v3-turbo-mlx-q8`) pub fn resolve_runtime_whisper_model_path(configured_model: Option<&str>) -> Result { if let Ok(path) = std::env::var("CODESCRIBE_MODEL_PATH") { let candidate = PathBuf::from(path.trim()); @@ -305,22 +348,13 @@ pub fn resolve_runtime_whisper_model_path(configured_model: Option<&str>) -> Res return Ok(snapshot); } - let legacy_local = manager.get_model_path(LEGACY_MODEL); - if is_complete_whisper_model_dir(&legacy_local) { - return Ok(canonicalize_or_self(legacy_local)); - } - - if let Some(snapshot) = hf_snapshot_for_model(LEGACY_MODEL) { - return Ok(snapshot); - } - Err(anyhow!( - "Whisper runtime fallback model not available.\n\ + "Unquantized Whisper runtime model not available.\n\ Public builds do not embed Whisper; install it from Settings → Dictation,\n\ set CODESCRIBE_MODEL_PATH, configure LOCAL_MODEL, or warm the Hugging Face cache.\n\n\ - Download with: hf download {}\n\ + Quantized q8 models are intentionally refused.\n\n\ + Download with: make download-model\n\ Or: Settings → Dictation → Download Whisper", - DEFAULT_WHISPER_REPO )) } @@ -372,22 +406,18 @@ where return Ok(canonicalize_or_self(dest)); } - // Compose from warm local sources first, so Settings "Download" is a no-op - // when the pieces are already on disk. The primary repo ships only - // config.json + weights; tokenizer.json and mel_filters.npz come from the - // legacy q8 sources. config.json and weights must stay paired to the - // primary repo — legacy weights under the new alias would mislabel q8 as - // fp16, so legacy sources only ever contribute the companion files. + // Compose from warm official sources first, so Settings "Download" is a + // no-op when the pieces are already on disk. Config and weights come from + // mlx-community's fp16 conversion; tokenizer comes from OpenAI's matching + // Transformers repository. The pinned mel filterbank is fetched below. if let Some(snapshot) = hf_cache::find_snapshot(DEFAULT_WHISPER_REPO, &["config.json"]) && snapshot != dest { copy_model_files(&snapshot, &dest, &["config.json"])?; copy_model_files(&snapshot, &dest, &REQUIRED_MODEL_WEIGHTS)?; } - let legacy_local = manager.get_model_path(LEGACY_MODEL); - copy_model_files(&legacy_local, &dest, &COMPANION_MODEL_FILES)?; - if let Some(snapshot) = hf_snapshot_for_model(LEGACY_MODEL) { - copy_model_files(&snapshot, &dest, &COMPANION_MODEL_FILES)?; + if let Some(snapshot) = hf_cache::find_snapshot(TOKENIZER_WHISPER_REPO, &["tokenizer.json"]) { + copy_model_files(&snapshot, &dest, &["tokenizer.json"])?; } if is_complete_whisper_model_dir(&dest) { return Ok(canonicalize_or_self(dest)); @@ -402,8 +432,6 @@ where .context("build HTTP client for Whisper download")?; // Small files first so a failed auth fails fast before multi-GB weights. - // config.json describes the primary weights, so it has no fallback source; - // the companion files fall back to the legacy repo when the primary 404s. download_hf_file( &client, DEFAULT_WHISPER_REPO, @@ -411,29 +439,21 @@ where &dest.join("config.json"), &mut on_progress, )?; - for name in COMPANION_MODEL_FILES { - let target = dest.join(name); - if let Err(err) = download_hf_file( - &client, - DEFAULT_WHISPER_REPO, - name, - &target, - &mut on_progress, - ) { - tracing::warn!( - error = %err, - file = name, - "primary repo does not ship this file; falling back to legacy repo" - ); - download_hf_file( - &client, - LEGACY_WHISPER_REPO, - name, - &target, - &mut on_progress, - )?; - } - } + download_hf_file( + &client, + TOKENIZER_WHISPER_REPO, + "tokenizer.json", + &dest.join("tokenizer.json"), + &mut on_progress, + )?; + download_url_file( + &client, + MEL_FILTERS_URL, + "mel_filters.npz", + &dest.join("mel_filters.npz"), + &mut on_progress, + )?; + verify_sha256(&dest.join("mel_filters.npz"), MEL_FILTERS_SHA256)?; let weights_dest = dest.join("weights.safetensors"); let weights_alt = dest.join("model.safetensors"); @@ -476,7 +496,7 @@ where /// Copy selected model files from a local source into the user models directory. /// /// Lets Settings → Download complete without network traffic when the pieces are -/// already on disk (warm HF cache, legacy q8 install). A missing source is a +/// already on disk (warm official caches). A missing source is a /// clean no-op and existing destination files are left alone, so an interrupted /// composition resumes rather than restarting. fn copy_model_files(src: &Path, dest: &Path, names: &[&str]) -> Result<()> { @@ -488,7 +508,7 @@ fn copy_model_files(src: &Path, dest: &Path, names: &[&str]) -> Result<()> { let from = src.join(name); let to = dest.join(name); if from.exists() && !to.exists() { - // nosemgrep: rust.actix.path-traversal.tainted-path.tainted-path -- Both ends are internal: `name` comes from the REQUIRED_MODEL_* / COMPANION_MODEL_FILES compile-time constants, and callers pass HF cache snapshot dirs or ModelManager::get_model_path outputs. No caller-supplied path component reaches here. + // nosemgrep: rust.actix.path-traversal.tainted-path.tainted-path -- Both ends are internal: `name` comes from compile-time model file constants, and callers pass HF cache snapshot dirs or ModelManager::get_model_path outputs. No caller-supplied path component reaches here. fs::copy(&from, &to) .with_context(|| format!("copy {} → {}", from.display(), to.display()))?; } @@ -522,8 +542,35 @@ where } let url = format!("https://huggingface.co/{repo}/resolve/main/{filename}"); - let mut request = client.get(&url); - if let Ok(token) = std::env::var("HF_TOKEN") { + download_url_file_authenticated(client, &url, filename, dest, on_progress, true) +} + +fn download_url_file( + client: &reqwest::blocking::Client, + url: &str, + filename: &str, + dest: &Path, + on_progress: &mut F, +) -> Result<()> +where + F: FnMut(&str, u64, Option), +{ + download_url_file_authenticated(client, url, filename, dest, on_progress, false) +} + +fn download_url_file_authenticated( + client: &reqwest::blocking::Client, + url: &str, + filename: &str, + dest: &Path, + on_progress: &mut F, + use_hf_token: bool, +) -> Result<()> +where + F: FnMut(&str, u64, Option), +{ + let mut request = client.get(url); + if use_hf_token && let Ok(token) = std::env::var("HF_TOKEN") { let token = token.trim(); if !token.is_empty() { request = request.bearer_auth(token); @@ -575,6 +622,21 @@ where Ok(()) } +fn verify_sha256(path: &Path, expected: &str) -> Result<()> { + // nosemgrep: rust.actix.path-traversal.tainted-path.tainted-path -- Read-only checksum of the fixed mel_filters.npz destination assembled under the internally resolved model directory. + let bytes = fs::read(path).with_context(|| format!("read {} for checksum", path.display()))?; + let actual = format!("{:x}", Sha256::digest(bytes)); + if actual != expected { + return Err(anyhow!( + "SHA-256 mismatch for {}: expected {}, got {}", + path.display(), + expected, + actual + )); + } + Ok(()) +} + /// ModelManager resolution, completeness gates, and env-override isolation tests. #[cfg(test)] mod tests { @@ -626,7 +688,20 @@ mod tests { fs::write(path.join("config.json"), "{}").unwrap(); fs::write(path.join("tokenizer.json"), "{}").unwrap(); fs::write(path.join("mel_filters.npz"), "npz").unwrap(); - fs::write(path.join("model.safetensors"), "weights").unwrap(); + let header = br#"{"model.weight":{"dtype":"F16","shape":[1],"data_offsets":[0,2]}}"#; + let mut safetensors = (header.len() as u64).to_le_bytes().to_vec(); + safetensors.extend_from_slice(header); + safetensors.extend_from_slice(&[0, 0]); + fs::write(path.join("model.safetensors"), safetensors).unwrap(); + } + + fn create_q8_whisper_model(path: &Path) { + create_complete_whisper_model(path); + fs::write( + path.join("config.json"), + r#"{"quantization":{"group_size":32,"bits":8}}"#, + ) + .unwrap(); } /// A bundle containing only the semantic embedder must not claim ownership @@ -639,7 +714,7 @@ mod tests { fs::create_dir_all(&embedder).unwrap(); fs::write(embedder.join("config.json"), "{}").unwrap(); fs::write(embedder.join("tokenizer.json"), "{}").unwrap(); - fs::write(embedder.join("model.safetensors"), "weights").unwrap(); + fs::write(embedder.join("model.safetensors"), "not-a-safetensors-file").unwrap(); assert!(!models_root_contains_complete_whisper_model(&models_dir)); @@ -675,11 +750,7 @@ mod tests { let models_dir = temp_dir.path().join("../../models"); fs::create_dir_all(&models_dir).unwrap(); - let model_names = [ - "whisper-base-mlx-q8", - "whisper-medium-mlx-q8", - "whisper-large-v3-turbo-mlx-q8", - ]; + let model_names = ["whisper-base-fp16", "whisper-medium-fp16", DEFAULT_MODEL]; for name in &model_names { let model_path = models_dir.join(name); @@ -718,6 +789,36 @@ mod tests { assert_eq!(manager.list_models().unwrap(), vec!["complete-whisper"]); } + /// Q8 is refused even when every expected file exists. + #[test] + #[serial] + fn model_manager_rejects_complete_q8_model() { + let temp_dir = TempDir::new().unwrap(); + let models_dir = temp_dir.path().join("models"); + let q8 = models_dir.join("renamed-as-fp16"); + create_q8_whisper_model(&q8); + + let _models_dir = EnvGuard::set("CODESCRIBE_MODELS_DIR", &models_dir); + let manager = ModelManager::new().unwrap(); + assert!(!manager.check_model_exists("renamed-as-fp16")); + assert!(manager.list_models().unwrap().is_empty()); + } + + /// Header-level detection catches packed q8 even if config metadata lies. + #[test] + fn model_manager_rejects_q8_tensor_header_without_quantization_config() { + let temp_dir = TempDir::new().unwrap(); + let model = temp_dir.path().join("model"); + create_complete_whisper_model(&model); + let header = br#"{"encoder.weight":{"dtype":"U32","shape":[1],"data_offsets":[0,4]},"encoder.scales":{"dtype":"F16","shape":[1],"data_offsets":[4,6]}}"#; + let mut safetensors = (header.len() as u64).to_le_bytes().to_vec(); + safetensors.extend_from_slice(header); + safetensors.extend_from_slice(&[0; 6]); + fs::write(model.join("model.safetensors"), safetensors).unwrap(); + + assert!(!is_complete_whisper_model_dir(&model)); + } + /// Complete `CODESCRIBE_MODEL_PATH` wins over the bundled default tier. #[test] #[serial] @@ -810,7 +911,7 @@ mod tests { ); } - /// All-empty fallback chain returns guidance mentioning env and `hf download`. + /// All-empty fallback chain returns guidance mentioning env and the composer. #[test] #[serial] fn resolve_runtime_whisper_model_path_errors_with_guidance_when_all_tiers_empty() { @@ -830,8 +931,8 @@ mod tests { "error must mention the env override knob, got: {message}" ); assert!( - message.contains("hf download"), - "error must hint at HF cache warm-up, got: {message}" + message.contains("make download-model"), + "error must point to the complete-model composer, got: {message}" ); } diff --git a/core/stt/onnx_adapter.rs b/core/stt/onnx_adapter.rs index 6b556cd9..a9ab1a7d 100644 --- a/core/stt/onnx_adapter.rs +++ b/core/stt/onnx_adapter.rs @@ -1,6 +1,6 @@ //! ONNX Whisper adapter — speech-to-text via ort (ONNX Runtime). //! -//! **STATUS: EXPERIMENTAL** — Candle q8 (MLX) remains the production default. +//! **STATUS: EXPERIMENTAL** — Candle fp16 (MLX) is the production local path. //! Benchmark (2026-02-11, 10 Polish files) showed ONNX +3.6–3.9pp WER vs Candle. //! Enable via `CODESCRIBE_STT_ENGINE=onnx` for testing only. //! diff --git a/core/stt/whisper/engine.rs b/core/stt/whisper/engine.rs index f2d8dc52..4a287c6a 100644 --- a/core/stt/whisper/engine.rs +++ b/core/stt/whisper/engine.rs @@ -429,13 +429,18 @@ impl LocalWhisperEngine { /// Load a model from a directory (development / external models). /// /// Expects `config.json` plus `weights.safetensors` or `model.safetensors`. - /// MLX-style quantized weights are dequantized during load — see - /// [`dequantize_q8`] for the cost this implies. + /// Quantized MLX weights are refused. Runtime Whisper is fp16/fp32 only; + /// this keeps q8 dequantization off every product path. /// /// # Errors - /// Missing config or weights, an unreadable tokenizer, or tensor shapes the - /// dequantizer cannot reconcile. + /// Missing config or weights, an unreadable tokenizer, a quantized payload, + /// or tensor shapes the fp16 loader cannot reconcile. pub fn new(model_path: &Path) -> Result { + if !crate::config::models::is_unquantized_whisper_model_dir(model_path) { + anyhow::bail!( + "Quantized or malformed Whisper payload refused before tensor load; install the complete fp16 model" + ); + } let device = process_device(); tracing::debug!("LocalWhisperEngine using device: {:?}", device); @@ -459,6 +464,17 @@ impl LocalWhisperEngine { // Parse MLX config and map to Candle Config let mlx_config: serde_json::Value = serde_json::from_str(&config_str).context("Failed to parse MLX config json")?; + if mlx_config + .get("quantization") + .is_some_and(|value| !value.is_null()) + || mlx_config + .get("quantization_config") + .is_some_and(|value| !value.is_null()) + { + anyhow::bail!( + "Quantized Whisper weights are not supported; install the complete fp16 model" + ); + } let n_mels = mlx_config["n_mels"].as_u64().unwrap_or(80); let new_config_json = serde_json::json!({ @@ -508,6 +524,15 @@ impl LocalWhisperEngine { let loaded = view.load(&Device::Cpu)?; raw_tensors.insert(name.to_string(), loaded); } + if raw_tensors.iter().any(|(name, tensor)| { + tensor.dtype() == DType::U32 + || name.ends_with(".scales") + || name.ends_with(".biases") + }) { + anyhow::bail!( + "Quantized Whisper tensor payload refused; install the complete fp16 model" + ); + } read_secs = read_started.elapsed().as_secs_f64(); let plain_started = std::time::Instant::now(); @@ -642,6 +667,15 @@ impl LocalWhisperEngine { .context("Invalid UTF-8 in embedded config.json")?; let mlx_config: serde_json::Value = serde_json::from_str(config_str).context("Failed to parse embedded config json")?; + if mlx_config + .get("quantization") + .is_some_and(|value| !value.is_null()) + || mlx_config + .get("quantization_config") + .is_some_and(|value| !value.is_null()) + { + anyhow::bail!("Embedded quantized Whisper payload refused; build with fp16 weights"); + } let n_mels = mlx_config["n_mels"].as_u64().unwrap_or(80); let new_config_json = serde_json::json!({ @@ -2014,6 +2048,11 @@ fn build_varbuilder_from_tensors( raw_tensors: HashMap, device: &Device, ) -> Result> { + if raw_tensors.iter().any(|(name, tensor)| { + tensor.dtype() == DType::U32 || name.ends_with(".scales") || name.ends_with(".biases") + }) { + anyhow::bail!("Quantized Whisper tensor payload refused; fp16 weights are required"); + } let mut tensor_map = HashMap::new(); let mut quantized_weights: Vec = Vec::new(); diff --git a/docs/ADR/2026-05-26-LAYERED_INCREMENTAL_TRANSCRIPTION.md b/docs/ADR/2026-05-26-LAYERED_INCREMENTAL_TRANSCRIPTION.md index 0e887eb7..52289aad 100644 --- a/docs/ADR/2026-05-26-LAYERED_INCREMENTAL_TRANSCRIPTION.md +++ b/docs/ADR/2026-05-26-LAYERED_INCREMENTAL_TRANSCRIPTION.md @@ -1,5 +1,9 @@ # ADR 2026-05-26 — Layered Incremental Transcription Pipeline +> **Historical model reference:** the original ADR named a Q8 Whisper artifact. +> Current runtime policy is FP16-only; resolver, filesystem, direct-engine and +> embedded paths reject quantized Whisper payloads before tensor load. + > **Status:** PROPOSED → ACCEPTED (operator-authored vision, 2026-05-26) > **Replaces:** Whisper-as-primary live STT model (see `WHISPER_LIVE.md`, `OVERLAY_STREAMING.md`) > **Owns invariant:** **NEVER REWRITE FROM ZERO.** All layers act incrementally on what was already shown to the user. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 73024f6d..65308670 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -107,7 +107,7 @@ flowchart TB BRIDGE --> APP APP --> CORE - WH --> MODEL[Whisper Model\nlarge-v3-turbo\nmlx-q8\nembedded or runtime-loaded] + WH --> MODEL[Whisper Model\nlarge-v3-turbo\nfp16 only\nembedded or runtime-loaded] subgraph TOOLS[Quality & CLI Tools] TEACH[bin/codescribe-teacher] @@ -313,9 +313,11 @@ may still opt into binary embedding: 1. `CODESCRIBE_MODEL_PATH` environment variable 2. `~/.codescribe/models/whisper-large-v3-turbo/` (fp16 default) -3. Hugging Face cache snapshots for `mlx-community/whisper-large-v3-turbo` -4. Legacy fallback: `whisper-large-v3-turbo-mlx-q8` dir or - `LibraxisAI/whisper-large-v3-turbo-mlx-q8` snapshots +3. A complete explicitly configured Hugging Face snapshot + +Every candidate must contain config, tokenizer, mel filters and safetensors +weights, and must pass config plus safetensors-header checks proving that it is +not quantized. Q8 has no runtime fallback path. MiniLM resolution: `CODESCRIBE_EMBEDDER_PATH`, then `Codescribe.app/Contents/Resources/models/embedder`, then the configured/default diff --git a/docs/OVERLAY_STREAMING.md b/docs/OVERLAY_STREAMING.md index a91646ca..bb50d32c 100644 --- a/docs/OVERLAY_STREAMING.md +++ b/docs/OVERLAY_STREAMING.md @@ -68,7 +68,7 @@ flowchart TD SPEECH[SpeechSession\ncore/audio/chunker.rs\nSupervisor mode] CHUNK[SpeechEvent::Utterance / UtteranceFinal\nclean speech audio segments] WORKER[transcription_session\ncore/pipeline/streaming/session.rs\nunified pipeline] - WHISPER[Whisper Engine\ncore/stt/whisper/engine.rs\nMetal GPU · large-v3-turbo-q8] + WHISPER[Whisper Engine\ncore/stt/whisper/engine.rs\nMetal GPU · large-v3-turbo fp16] POSTPROC[StreamPostProcessor\ncore/pipeline/stream_postprocess.rs\nlexicon + semantic gate] EVENT[EngineEvent\ncore/pipeline/contracts.rs] SINK{EventSink / DeltaSink} diff --git a/docs/STT_CONTRACT.md b/docs/STT_CONTRACT.md index 270925f7..5e989500 100644 --- a/docs/STT_CONTRACT.md +++ b/docs/STT_CONTRACT.md @@ -35,12 +35,12 @@ Recording stopped before a transcript was available. ### Fixed product contract (2026-07-24 ship cut) -| Layer | Rule | -| --------------------- | ------------------------------------------------------------------------------------------------------ | -| Empty `speech.engine` | Load defaults to **`stt_engine=apple`**, **`final_pass_mode=smart`** | -| Settings UI write | **Promoted** to `settings.json` + reconciles process env **and** `.env` (single brain) | -| Record start | **`preflight_apple_live_ready()`** when engine is Apple — refuse before REC if Speech/bridge not ready | -| Live vs final | Live = Apple only; file final = Whisper; recovery path separate from mid-stream swap | +| Layer | Rule | +| --------------------- | -------------------------------------------------------------------------------------------------------- | +| Empty `speech.engine` | Load pins **`stt_engine=apple`**; persist **`final_pass_mode=off`** explicitly on current installs | +| Settings UI write | **Promoted** to `settings.json` + reconciles process env **and** `.env` (single brain) | +| Record start | **`preflight_apple_live_ready()`** when engine is Apple — refuse before REC if Speech/bridge not ready | +| Live vs final | Cloud/Apple-only live fails closed without local weights; explicit HQ/local Retranscribe may use Whisper | --- @@ -130,7 +130,10 @@ button. "language": "pl", "engine": { "stt_engine": "apple", - "final_pass_mode": "smart" + "whisper_model": "whisper-large-v3-turbo", + "final_pass_mode": "off", + "layered_transcription": "off", + "asr_mode": "cloud" }, "formatting": { "enabled": true, "level": "smart" } } @@ -267,7 +270,13 @@ Valid engine labels on verdict: `local_apple`, `local_whisper`, `streaming_whisp CODESCRIBE_STT_ENGINE=apple ``` ```json - "engine": { "stt_engine": "apple", "final_pass_mode": "smart" } + "engine": { + "stt_engine": "apple", + "whisper_model": "whisper-large-v3-turbo", + "final_pass_mode": "off", + "layered_transcription": "off", + "asr_mode": "cloud" + } ``` 2. Full quit + relaunch. 3. Footer / Active STT after a take: **`local_apple`** on happy path. diff --git a/docs/TEAM_SETUP.md b/docs/TEAM_SETUP.md index d624958a..fbc43807 100644 --- a/docs/TEAM_SETUP.md +++ b/docs/TEAM_SETUP.md @@ -59,7 +59,7 @@ Grant in: System Settings > Privacy & Security ## Model -**Runtime Whisper policy**: `whisper-large-v3-turbo` (mlx-community fp16; legacy q8 fallback) +**Runtime Whisper policy**: `whisper-large-v3-turbo` (mlx-community fp16 only; q8 refused) **Runtime Embedder**: `paraphrase-multilingual-MiniLM-L12-v2` (signed app resource or HF cache, for semantic gating) - `core/build.rs` keeps Whisper and MiniLM out of normal Cargo artifacts; explicit diff --git a/docs/TRANSCRIPT_LANES.md b/docs/TRANSCRIPT_LANES.md index 4c23b3bf..aed0704d 100644 --- a/docs/TRANSCRIPT_LANES.md +++ b/docs/TRANSCRIPT_LANES.md @@ -84,7 +84,7 @@ mic ▶ recorder ▶ [J1] ▶ Silero VAD chunker ▶ utterance boundaries | --- | ------------- | ---------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------- | | B1 | VAD filter | `core/audio/chunker.rs` (Silero, embedded, zero-I/O) | detects WORDS, not noise; silence edges close utterances — “fundament stabilności” | doctrine §3.5 | | B2 | scheduling | `core/stt/scheduler.rs :: SttScheduler` | Fast lane = utterance decode; Refine lane = correction re-decodes; per-lane `initial_prompt_for_lane` (⚑ OFF, W13-6A) | — | -| B3 | decode | `core/stt/whisper/singleton.rs` | in-process Whisper (turbo fp16 native; q8 composition for tokenizer+mel); TTL reaper unloads 30 min after last finished decode (`whisper_residency_reclaim`) | fp16 default | +| B3 | decode | `core/stt/whisper/singleton.rs` | in-process Whisper (turbo fp16 only; official OpenAI tokenizer + pinned mel asset); TTL reaper unloads 30 min after last finished decode (`whisper_residency_reclaim`) | fp16 only | | B4 | corrections | `streaming/correction.rs` | Phase-2 Refine: partial passes triggered by finals/speech-ms, **VAD-aligned windows** (`plan_vad_aligned_windows_with_config`) so windows never begin mid-phrase | W1-A | | B5 | postprocess | `core/pipeline/stream_postprocess.rs` | lexicon rewrite table (compiled-in seed/programming/operator/protected), hallucination + SemanticGate + empty-drop gates | — | | B6 | canvas + user | **[J2]** → overlay | same reducer/emitter contract as LINE A | — | diff --git a/docs/WHISPER_LIVE.md b/docs/WHISPER_LIVE.md index d22de86f..cd2d9910 100644 --- a/docs/WHISPER_LIVE.md +++ b/docs/WHISPER_LIVE.md @@ -34,7 +34,7 @@ See the ADR for the full contract. Codescribe’s Whisper layer power-ups: -1. **Embedded-first Whisper model** (`whisper-large-v3-turbo`, mlx-community fp16, by default; legacy q8 as fallback) +1. **FP16-only Whisper model** (`whisper-large-v3-turbo`, mlx-community weights; q8 is rejected before load) - build policy embeds Whisper whenever the model is available at build time - runtime lookup from `CODESCRIBE_MODEL_PATH`, configured model dirs, bundled app resources, or the Hugging Face cache is a fallback path for `CODESCRIBE_NO_EMBED=1` builds or recovery 2. **Live (streaming) transcription** while the user is recording diff --git a/examples/README.md b/examples/README.md index 5b7539c0..b094cad4 100644 --- a/examples/README.md +++ b/examples/README.md @@ -21,7 +21,7 @@ cargo run --release --example demo_full_pipeline -- cargo run --release --example demo_full_pipeline -- --assistive ``` -Requires a local Whisper model (default `~/.codescribe/models/whisper-large-v3-turbo`, legacy q8 dir as fallback, override with `--model`) and `LLM_ENDPOINT`/`LLM_MODEL` (or `LLM_FORMATTING_*` overrides) for the formatting step. +Requires the complete fp16 Whisper model at `~/.codescribe/models/whisper-large-v3-turbo` (override with `--model`) and `LLM_ENDPOINT`/`LLM_MODEL` (or `LLM_FORMATTING_*` overrides) for the formatting step. Quantized Whisper payloads are refused. ### `e2e_stt.rs` diff --git a/examples/demo_full_pipeline.rs b/examples/demo_full_pipeline.rs index ec302153..a0b04790 100644 --- a/examples/demo_full_pipeline.rs +++ b/examples/demo_full_pipeline.rs @@ -50,12 +50,7 @@ async fn main() -> Result<()> { // Parse args // Model path: ~/.codescribe/models/ (unified standard) let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string()); - // fp16 default first, legacy q8 as fallback — mirrors runtime precedence. - let mut model = ["whisper-large-v3-turbo", "whisper-large-v3-turbo-mlx-q8"] - .iter() - .map(|name| PathBuf::from(&home).join(".codescribe/models").join(name)) - .find(|p| p.join("config.json").exists()) - .unwrap_or_else(|| PathBuf::from(&home).join(".codescribe/models/whisper-large-v3-turbo")); + let mut model = PathBuf::from(&home).join(".codescribe/models/whisper-large-v3-turbo"); let mut assistive = false; let mut raw_only = false; let mut audio_file: Option = None; diff --git a/examples/e2e_stt.rs b/examples/e2e_stt.rs index 0379bc31..b4f09973 100644 --- a/examples/e2e_stt.rs +++ b/examples/e2e_stt.rs @@ -22,12 +22,7 @@ async fn main() -> Result<()> { // Model path: ~/.codescribe/models/ (unified standard) let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string()); - // fp16 default first, legacy q8 as fallback — mirrors runtime precedence. - let model_path = ["whisper-large-v3-turbo", "whisper-large-v3-turbo-mlx-q8"] - .iter() - .map(|name| PathBuf::from(&home).join(".codescribe/models").join(name)) - .find(|p| p.join("config.json").exists()) - .unwrap_or_else(|| PathBuf::from(&home).join(".codescribe/models/whisper-large-v3-turbo")); + let model_path = PathBuf::from(&home).join(".codescribe/models/whisper-large-v3-turbo"); // Supply your own sample files via env vars (or edit the placeholder paths below). let audio_medium = PathBuf::from( std::env::var("CODESCRIBE_E2E_AUDIO_MEDIUM") diff --git a/examples/test_audio.rs b/examples/test_audio.rs index a9145737..2fbb8302 100644 --- a/examples/test_audio.rs +++ b/examples/test_audio.rs @@ -6,12 +6,7 @@ use std::path::PathBuf; fn main() -> Result<()> { // Model path: ~/.codescribe/models/ (unified standard) let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string()); - // fp16 default first, legacy q8 as fallback — mirrors runtime precedence. - let model = ["whisper-large-v3-turbo", "whisper-large-v3-turbo-mlx-q8"] - .iter() - .map(|name| PathBuf::from(&home).join(".codescribe/models").join(name)) - .find(|p| p.join("config.json").exists()) - .unwrap_or_else(|| PathBuf::from(&home).join(".codescribe/models/whisper-large-v3-turbo")); + let model = PathBuf::from(&home).join(".codescribe/models/whisper-large-v3-turbo"); println!("Loading model..."); let mut engine = LocalWhisperEngine::new(&model)?; println!("Model loaded.\n"); diff --git a/examples/test_audio_long.rs b/examples/test_audio_long.rs index c06e238b..e54d73fc 100644 --- a/examples/test_audio_long.rs +++ b/examples/test_audio_long.rs @@ -14,12 +14,7 @@ fn main() -> Result<()> { // Model path: ~/.codescribe/models/ (unified standard) let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string()); - // fp16 default first, legacy q8 as fallback — mirrors runtime precedence. - let default_model = ["whisper-large-v3-turbo", "whisper-large-v3-turbo-mlx-q8"] - .iter() - .map(|name| PathBuf::from(&home).join(".codescribe/models").join(name)) - .find(|p| p.join("config.json").exists()) - .unwrap_or_else(|| PathBuf::from(&home).join(".codescribe/models/whisper-large-v3-turbo")); + let default_model = PathBuf::from(&home).join(".codescribe/models/whisper-large-v3-turbo"); let (model, files): (PathBuf, Vec) = if args[0] == "--model" { ( diff --git a/examples/transcribe_file.rs b/examples/transcribe_file.rs index d3bff224..bc5f8f06 100644 --- a/examples/transcribe_file.rs +++ b/examples/transcribe_file.rs @@ -37,7 +37,6 @@ fn main() -> anyhow::Result<()> { let model_candidates = [ env::var("CODESCRIBE_MODEL_PATH").ok().map(PathBuf::from), Some(PathBuf::from(&home).join(".codescribe/models/whisper-large-v3-turbo")), - Some(PathBuf::from(&home).join(".codescribe/models/whisper-large-v3-turbo-mlx-q8")), ]; let model_path = model_candidates diff --git a/scripts/bench-stt.sh b/scripts/bench-stt.sh index 14fba211..32a4df50 100755 --- a/scripts/bench-stt.sh +++ b/scripts/bench-stt.sh @@ -150,38 +150,11 @@ discover_model() { return 0 fi - for candidate in \ - "$home_dir/.codescribe/models/whisper-large-v3-turbo" \ - "$home_dir/.codescribe/models/whisper-large-v3-turbo-mlx-q8" \ - "$home_dir/.codescribe/models/whisper-large-v3-mlx-q8"; do - if model_is_complete "$candidate"; then - printf '%s\n' "$candidate" - return 0 - fi - done - - local hf_base repo snapshot - for hf_base in \ - "${CODESCRIBE_HF_CACHE:-}" \ - "${HUGGINGFACE_HUB_CACHE:-}" \ - "${HF_HUB_CACHE:-}" \ - "${HF_HOME:+$HF_HOME/hub}" \ - "$home_dir/.cache/huggingface/hub"; do - [[ -n "$hf_base" ]] || continue - for repo in \ - models--LibraxisAI--whisper-large-v3-turbo-mlx-q8 \ - models--libraxisai--whisper-large-v3-turbo-mlx-q8 \ - models--LibraxisAI--whisper-large-v3-mlx-q8 \ - models--libraxisai--whisper-large-v3-mlx-q8; do - for snapshot in "$hf_base/$repo/snapshots"/*; do - [[ -d "$snapshot" ]] || continue - if model_is_complete "$snapshot"; then - printf '%s\n' "$snapshot" - return 0 - fi - done - done - done + candidate="$home_dir/.codescribe/models/whisper-large-v3-turbo" + if model_is_complete "$candidate"; then + printf '%s\n' "$candidate" + return 0 + fi return 1 } @@ -1245,7 +1218,7 @@ if [[ "$(($(count_lines "$manifest_tsv") - 1))" -le 0 ]]; then fi if ! model_path="$(discover_model)"; then - write_honest_report "No complete Whisper model found. Checked CODESCRIBE_MODEL_PATH, ~/.codescribe/models/{whisper-large-v3-turbo,whisper-large-v3-turbo-mlx-q8,whisper-large-v3-mlx-q8}, and Hugging Face cache snapshots." + write_honest_report "No complete fp16 Whisper model found. Checked CODESCRIBE_MODEL_PATH and ~/.codescribe/models/whisper-large-v3-turbo." fi export CODESCRIBE_MODEL_PATH="$model_path" diff --git a/scripts/download-model.sh b/scripts/download-model.sh index 18f72cfa..bcc3ef50 100755 --- a/scripts/download-model.sh +++ b/scripts/download-model.sh @@ -1,6 +1,6 @@ #!/bin/bash # Codescribe Model Download Script -# Downloads whisper-large-v3-turbo-mlx-q8 from HuggingFace +# Composes a complete, runtime-verified Whisper large-v3-turbo fp16 directory. # # Prerequisites: # - HF_TOKEN environment variable (for gated models) @@ -18,9 +18,9 @@ ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)" # Configuration DEFAULT_REPO="mlx-community/whisper-large-v3-turbo" -# Companion source for tokenizer.json + mel_filters.npz, which the default -# repo does not ship (both files are quantization-independent). -COMPANION_REPO="LibraxisAI/whisper-large-v3-turbo-mlx-q8" +TOKENIZER_REPO="openai/whisper-large-v3-turbo" +MEL_FILTERS_URL="https://raw.githubusercontent.com/openai/whisper/5f86d1d86363843179951550570367b37c5d6f78/whisper/assets/mel_filters.npz" +MEL_FILTERS_SHA256="7450ae70723a5ef9d341e3cee628c7cb0177f36ce42c44b7ed2bf3325f0f6d4c" MODEL_REPO="${CODESCRIBE_EMBED_MODEL:-$DEFAULT_REPO}" # If CODESCRIBE_EMBED_MODEL points to a local path, skip download. @@ -83,22 +83,43 @@ echo "▶ Downloading model (HF cache)..." echo " This may take a few minutes..." echo "" -"$HF_BIN" download "$MODEL_REPO" +MODEL_SNAPSHOT=$("$HF_BIN" download "$MODEL_REPO" --quiet) -# The default repo ships only config.json + weights; pull the two -# quantization-independent companion files from the companion repo so the -# cache holds everything a complete model dir needs. +# The default conversion ships only config + fp16 weights. Compose one +# self-contained product directory using the matching official OpenAI +# tokenizer and the pinned OpenAI mel filterbank. Q8 is not a fallback or an +# asset source anywhere in this path. if [[ "$MODEL_REPO" == "$DEFAULT_REPO" ]]; then echo "" - echo "▶ Fetching companion files (tokenizer.json, mel_filters.npz)..." - "$HF_BIN" download "$COMPANION_REPO" tokenizer.json mel_filters.npz + echo "▶ Composing verified fp16 runtime directory..." + TOKENIZER_PATH=$("$HF_BIN" download "$TOKENIZER_REPO" tokenizer.json --quiet) + MODEL_DEST="${CODESCRIBE_MODELS_DIR:-$HOME/.codescribe/models}/whisper-large-v3-turbo" + mkdir -p "$MODEL_DEST" + cp -fL "$MODEL_SNAPSHOT/config.json" "$MODEL_DEST/config.json" + if [[ -f "$MODEL_SNAPSHOT/weights.safetensors" ]]; then + cp -fL "$MODEL_SNAPSHOT/weights.safetensors" "$MODEL_DEST/weights.safetensors" + elif [[ -f "$MODEL_SNAPSHOT/model.safetensors" ]]; then + cp -fL "$MODEL_SNAPSHOT/model.safetensors" "$MODEL_DEST/model.safetensors" + else + echo "ERROR: fp16 snapshot has no safetensors weights: $MODEL_SNAPSHOT" >&2 + exit 1 + fi + cp -fL "$TOKENIZER_PATH" "$MODEL_DEST/tokenizer.json" + curl -fsSL "$MEL_FILTERS_URL" -o "$MODEL_DEST/mel_filters.npz.partial" + ACTUAL_MEL_SHA=$(shasum -a 256 "$MODEL_DEST/mel_filters.npz.partial" | awk '{print $1}') + if [[ "$ACTUAL_MEL_SHA" != "$MEL_FILTERS_SHA256" ]]; then + echo "ERROR: mel_filters.npz checksum mismatch" >&2 + exit 1 + fi + mv "$MODEL_DEST/mel_filters.npz.partial" "$MODEL_DEST/mel_filters.npz" + echo " Runtime directory: $MODEL_DEST" fi echo "" echo "═══════════════════════════════════════════════════════════" echo " Download Complete!" echo "═══════════════════════════════════════════════════════════" -echo " Location: HF cache (use: hf cache ls)" +echo " Source cache: $MODEL_SNAPSHOT" echo "" echo " Model ready for use with Codescribe." echo "───────────────────────────────────────────────────────────" diff --git a/scripts/ensure-models.sh b/scripts/ensure-models.sh index 1abe0202..8232ad16 100755 --- a/scripts/ensure-models.sh +++ b/scripts/ensure-models.sh @@ -1,7 +1,6 @@ #!/bin/bash # Ensure required models are present in HF cache for embedding. -# - Whisper (default: mlx-community/whisper-large-v3-turbo, fp16; tokenizer.json -# + mel_filters.npz come from the LibraxisAI q8 companion repo) +# - Whisper (mlx-community fp16 weights + official OpenAI tokenizer and mel filters) # - Embedder (default: sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2) set -euo pipefail @@ -42,7 +41,8 @@ has_snapshot_with_files() { local repo="$1"; shift local required=("$@") for base in "${CACHE_DIRS[@]}"; do - local dir="$base/$(repo_dir "$repo")/snapshots" + local dir + dir="$base/$(repo_dir "$repo")/snapshots" [[ -d "$dir" ]] || continue for snap in "$dir"/*; do [[ -d "$snap" ]] || continue @@ -81,7 +81,6 @@ ensure_repo() { } WHISPER_REPO="mlx-community/whisper-large-v3-turbo" -WHISPER_COMPANION_REPO="LibraxisAI/whisper-large-v3-turbo-mlx-q8" if [[ -n "${CODESCRIBE_EMBED_MODEL:-}" && "${CODESCRIBE_EMBED_MODEL}" == */* ]]; then WHISPER_REPO="$CODESCRIBE_EMBED_MODEL" fi @@ -90,14 +89,14 @@ EMBEDDER_REPO="${CODESCRIBE_EMBEDDER_REPO:-sentence-transformers/paraphrase-mult # If CODESCRIBE_MODEL_PATH already satisfied, skip Whisper cache check if [[ "${WHISPER_OK:-0}" -ne 1 ]]; then if [[ "$WHISPER_REPO" == "mlx-community/whisper-large-v3-turbo" ]]; then - # The default repo ships only config + weights; tokenizer.json and - # mel_filters.npz live in the companion repo (quantization-independent). - # download-model.sh fetches both, so one ensure call covers the pair. - if has_snapshot_with_files "$WHISPER_REPO" config.json __ANY_SAFETENSORS__ \ - && has_snapshot_with_files "$WHISPER_COMPANION_REPO" tokenizer.json mel_filters.npz; then - echo "✓ Whisper cached (${WHISPER_REPO} + companion files)" + COMPOSED_MODEL="${CODESCRIBE_MODELS_DIR:-$HOME/.codescribe/models}/whisper-large-v3-turbo" + if [[ -f "$COMPOSED_MODEL/config.json" \ + && -f "$COMPOSED_MODEL/tokenizer.json" \ + && -f "$COMPOSED_MODEL/mel_filters.npz" \ + && ( -f "$COMPOSED_MODEL/weights.safetensors" || -f "$COMPOSED_MODEL/model.safetensors" ) ]]; then + echo "✓ Whisper fp16 composed ($COMPOSED_MODEL)" else - echo "▶ Whisper not fully cached; downloading (${WHISPER_REPO} + companions)..." + echo "▶ Whisper fp16 runtime directory incomplete; composing it..." "$ROOT_DIR/scripts/download-model.sh" fi else diff --git a/tests/e2e_full_pipeline.rs b/tests/e2e_full_pipeline.rs index c557665c..d49ac688 100644 --- a/tests/e2e_full_pipeline.rs +++ b/tests/e2e_full_pipeline.rs @@ -100,39 +100,9 @@ fn find_model_path() -> Option { let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string()); - let direct = [ - PathBuf::from(&home).join(".codescribe/models/whisper-large-v3-turbo-mlx-q8"), - PathBuf::from(&home).join(".codescribe/models/whisper-large-v3-mlx-q8"), - ]; - if let Some(p) = direct.iter().find(|p| p.join("tokenizer.json").exists()) { - return Some(p.clone()); - } - - let hf_cache = PathBuf::from(&home).join(".cache/huggingface/hub"); - let hf_repos = [ - "models--LibraxisAI--whisper-large-v3-turbo-mlx-q8", - "models--libraxisai--whisper-large-v3-mlx-q8", - ]; - for repo_dir in &hf_repos { - let snapshots = hf_cache.join(repo_dir).join("snapshots"); - if let Ok(entries) = std::fs::read_dir(&snapshots) { - let mut best: Option<(std::time::SystemTime, PathBuf)> = None; - for entry in entries.flatten() { - let path = entry.path(); - if path.is_dir() && path.join("tokenizer.json").exists() { - let mtime = entry - .metadata() - .and_then(|m| m.modified()) - .unwrap_or(std::time::SystemTime::UNIX_EPOCH); - if best.as_ref().is_none_or(|(t, _)| mtime > *t) { - best = Some((mtime, path)); - } - } - } - if let Some((_, path)) = best { - return Some(path); - } - } + let fp16 = PathBuf::from(&home).join(".codescribe/models/whisper-large-v3-turbo"); + if fp16.join("tokenizer.json").exists() { + return Some(fp16); } None diff --git a/tests/e2e_stt_transcription.rs b/tests/e2e_stt_transcription.rs index 46692996..51acaeb6 100644 --- a/tests/e2e_stt_transcription.rs +++ b/tests/e2e_stt_transcription.rs @@ -17,9 +17,9 @@ use tempfile::TempDir; mod e2e_stt_matrix; use e2e_stt_matrix::{ - ModelDiscovery, ModelSource, STT_OPT_IN_ENV, WHISPER_LARGE_MODEL, WHISPER_TURBO_MODEL, - discover_local_whisper_model, discover_local_whisper_model_for, model_discovery_hint, - parse_opt_in, skip_unless_opt_in, test_audio_path, whisper_model_missing_parts, + ModelDiscovery, ModelSource, STT_OPT_IN_ENV, WHISPER_FP16_MODEL, discover_local_whisper_model, + discover_local_whisper_model_for, model_discovery_hint, parse_opt_in, skip_unless_opt_in, + test_audio_path, whisper_model_missing_parts, }; fn home_dir() -> PathBuf { @@ -223,10 +223,10 @@ fn deterministic_gate_parser_requires_explicit_opt_in_values() { fn deterministic_model_discovery_prefers_complete_env_override() { let (_tmp, home) = temp_home(); let models_root = home.join(".codescribe/models"); - let turbo = models_root.join(WHISPER_TURBO_MODEL); + let fp16 = models_root.join(WHISPER_FP16_MODEL); let env_model = home.join("custom/whisper-model"); - create_complete_model(&turbo); + create_complete_model(&fp16); create_complete_model(&env_model); let hf_bases = Vec::::new(); @@ -245,30 +245,20 @@ fn deterministic_model_discovery_prefers_complete_env_override() { } #[test] -fn deterministic_model_discovery_skips_incomplete_turbo_and_falls_back_to_large() { +fn deterministic_model_discovery_refuses_incomplete_fp16_without_legacy_fallback() { let (_tmp, home) = temp_home(); let models_root = home.join(".codescribe/models"); - let turbo = models_root.join(WHISPER_TURBO_MODEL); - let large = models_root.join(WHISPER_LARGE_MODEL); + let fp16 = models_root.join(WHISPER_FP16_MODEL); - create_incomplete_model(&turbo); - create_complete_model(&large); + create_incomplete_model(&fp16); let hf_bases = Vec::::new(); - let found = discover_local_whisper_model_for(&home, None, &hf_bases) - .expect("expected fallback to large model"); - - assert_eq!( - found.source, - ModelSource::UserLarge, - "incomplete turbo model must not block fallback to complete large model" - ); - assert_eq!( - found.path, large, - "expected large model path to be selected" + assert!( + discover_local_whisper_model_for(&home, None, &hf_bases).is_none(), + "an incomplete fp16 model must not fall back to a quantized model" ); - let missing = whisper_model_missing_parts(&turbo); + let missing = whisper_model_missing_parts(&fp16); assert!( missing.contains(&"config.json"), "incomplete turbo should report missing artifacts for easier diagnosis" diff --git a/tests/support/e2e_stt_matrix.rs b/tests/support/e2e_stt_matrix.rs index b69a777e..0603481d 100644 --- a/tests/support/e2e_stt_matrix.rs +++ b/tests/support/e2e_stt_matrix.rs @@ -7,32 +7,17 @@ #![allow(dead_code)] use std::path::{Path, PathBuf}; -use std::time::SystemTime; pub const STT_OPT_IN_ENV: &str = "CODESCRIBE_E2E_STT"; pub const ROUNDTRIP_OPT_IN_ENV: &str = "CODESCRIBE_E2E_ROUNDTRIP"; -/// Default fp16 alias (composed dir: mlx-community weights + q8 companions). +/// Default composed fp16 alias. pub const WHISPER_FP16_MODEL: &str = "whisper-large-v3-turbo"; -pub const WHISPER_TURBO_MODEL: &str = "whisper-large-v3-turbo-mlx-q8"; -pub const WHISPER_LARGE_MODEL: &str = "whisper-large-v3-mlx-q8"; - -const HF_TURBO_REPO_DIRS: &[&str] = &[ - "models--LibraxisAI--whisper-large-v3-turbo-mlx-q8", - "models--libraxisai--whisper-large-v3-turbo-mlx-q8", -]; -const HF_LARGE_REPO_DIRS: &[&str] = &[ - "models--LibraxisAI--whisper-large-v3-mlx-q8", - "models--libraxisai--whisper-large-v3-mlx-q8", -]; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ModelSource { EnvOverride, - UserTurbo, - UserLarge, - HfTurboSnapshot, - HfLargeSnapshot, + UserFp16, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -145,96 +130,27 @@ pub fn discover_local_whisper_model_for( }); } - // The fp16 default and the legacy q8 dir are the same turbo model, so both - // report as UserTurbo; the fp16 alias wins, mirroring runtime precedence. let user_fp16 = home_dir.join(".codescribe/models").join(WHISPER_FP16_MODEL); if whisper_model_is_complete(&user_fp16) { return Some(ModelDiscovery { - source: ModelSource::UserTurbo, + source: ModelSource::UserFp16, path: user_fp16, }); } - let user_turbo = home_dir - .join(".codescribe/models") - .join(WHISPER_TURBO_MODEL); - if whisper_model_is_complete(&user_turbo) { - return Some(ModelDiscovery { - source: ModelSource::UserTurbo, - path: user_turbo, - }); - } - - let user_large = home_dir - .join(".codescribe/models") - .join(WHISPER_LARGE_MODEL); - if whisper_model_is_complete(&user_large) { - return Some(ModelDiscovery { - source: ModelSource::UserLarge, - path: user_large, - }); - } - - if let Some(path) = find_latest_hf_snapshot(hf_cache_bases, HF_TURBO_REPO_DIRS) { - return Some(ModelDiscovery { - source: ModelSource::HfTurboSnapshot, - path, - }); - } - - if let Some(path) = find_latest_hf_snapshot(hf_cache_bases, HF_LARGE_REPO_DIRS) { - return Some(ModelDiscovery { - source: ModelSource::HfLargeSnapshot, - path, - }); - } + let _ = hf_cache_bases; None } pub fn model_discovery_hint(home_dir: &Path) -> String { format!( - "Looked for complete Whisper model in CODESCRIBE_MODEL_PATH, {home}/.codescribe/models/{fp16}, {home}/.codescribe/models/{turbo}, {home}/.codescribe/models/{large}, and HF cache snapshots. Required files: config.json, tokenizer.json, mel_filters.npz, weights.safetensors or model.safetensors.", + "Looked for complete fp16 Whisper model in CODESCRIBE_MODEL_PATH and {home}/.codescribe/models/{fp16}. Required files: config.json, tokenizer.json, mel_filters.npz, weights.safetensors or model.safetensors.", home = home_dir.display(), - fp16 = WHISPER_FP16_MODEL, - turbo = WHISPER_TURBO_MODEL, - large = WHISPER_LARGE_MODEL + fp16 = WHISPER_FP16_MODEL ) } pub fn normalize_transcript(text: &str) -> String { text.split_whitespace().collect::>().join(" ") } - -fn find_latest_hf_snapshot(hf_cache_bases: &[PathBuf], repo_dir_names: &[&str]) -> Option { - let mut best: Option<(SystemTime, PathBuf)> = None; - - for base in hf_cache_bases { - for repo in repo_dir_names { - let snapshots = base.join(repo).join("snapshots"); - let entries = match std::fs::read_dir(&snapshots) { - Ok(entries) => entries, - Err(_) => continue, - }; - - for entry in entries.flatten() { - let path = entry.path(); - if !path.is_dir() || !whisper_model_is_complete(&path) { - continue; - } - - let modified = entry - .metadata() - .and_then(|m| m.modified()) - .unwrap_or(SystemTime::UNIX_EPOCH); - - match &best { - Some((best_time, _)) if *best_time >= modified => {} - _ => best = Some((modified, path)), - } - } - } - } - - best.map(|(_, path)| path) -} From 2bf97be8391f527f488291e72223ce784aac1f2c Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Fri, 21 Aug 2026 13:47:48 +0200 Subject: [PATCH 02/45] [codex/vc-workflow] fix(stt): satisfy newer Clippy PCM chunk lint --- core/stt/tail_provider.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/stt/tail_provider.rs b/core/stt/tail_provider.rs index 07eb80dd..c20010c3 100644 --- a/core/stt/tail_provider.rs +++ b/core/stt/tail_provider.rs @@ -782,9 +782,11 @@ fn decode_pcm_f32le(bytes: &[u8]) -> Result> { bail!("sidecar PCM frame has an invalid bounded length"); } bytes - .chunks_exact(4) + .as_chunks::<4>() + .0 + .iter() .map(|chunk| { - let sample = f32::from_le_bytes(chunk.try_into().expect("four-byte chunk")); + let sample = f32::from_le_bytes(*chunk); if sample.is_finite() { Ok(sample) } else { From be6b6c9d6b4097c5cda92a2593f08e83949cb64e Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Fri, 21 Aug 2026 14:02:24 +0200 Subject: [PATCH 03/45] [codex/vc-workflow] fix(stt): remove dead q8 dequantization path (PR #81 review) --- core/stt/whisper/engine.rs | 179 ++++--------------------------------- 1 file changed, 15 insertions(+), 164 deletions(-) diff --git a/core/stt/whisper/engine.rs b/core/stt/whisper/engine.rs index 4a287c6a..c1cd852d 100644 --- a/core/stt/whisper/engine.rs +++ b/core/stt/whisper/engine.rs @@ -436,15 +436,10 @@ impl LocalWhisperEngine { /// Missing config or weights, an unreadable tokenizer, a quantized payload, /// or tensor shapes the fp16 loader cannot reconcile. pub fn new(model_path: &Path) -> Result { - if !crate::config::models::is_unquantized_whisper_model_dir(model_path) { - anyhow::bail!( - "Quantized or malformed Whisper payload refused before tensor load; install the complete fp16 model" - ); - } - let device = process_device(); - tracing::debug!("LocalWhisperEngine using device: {:?}", device); - let config_path = model_path.join("config.json"); + if !config_path.is_file() { + anyhow::bail!("Whisper config not found at {}", config_path.display()); + } let weights_path = if model_path.join("weights.safetensors").exists() { model_path.join("weights.safetensors") } else { @@ -458,6 +453,13 @@ impl LocalWhisperEngine { } let tokenizer_path = model_path.join("tokenizer.json"); let mel_filters_path = model_path.join("mel_filters.npz"); + if !crate::config::models::is_unquantized_whisper_model_dir(model_path) { + anyhow::bail!( + "Quantized or malformed Whisper payload refused before tensor load; install the complete fp16 model" + ); + } + let device = process_device(); + tracing::debug!("LocalWhisperEngine using device: {:?}", device); let config_str = safe_path::safe_read_to_string(&config_path)?; @@ -512,13 +514,12 @@ impl LocalWhisperEngine { let load_started = std::time::Instant::now(); let read_secs; let plain_secs; - let dequant_secs; let vb = unsafe { let tensors = candle_core::safetensors::MmapedSafetensors::new(&weights_path)?; let mut raw_tensors: HashMap = HashMap::new(); - // Load everything on CPU first so we can dequantize packed weights. + // Load the verified unquantized tensors on CPU before device transfer. let read_started = std::time::Instant::now(); for (name, view) in tensors.tensors() { let loaded = view.load(&Device::Cpu)?; @@ -537,19 +538,9 @@ impl LocalWhisperEngine { let plain_started = std::time::Instant::now(); let mut tensor_map = HashMap::new(); - let mut quantized_weights: Vec = Vec::new(); - // First pass: handle non-quantized tensors and collect quantized weight names. + // The payload gate above makes every tensor in this pass unquantized. for (name, tensor) in raw_tensors.iter() { - if name.ends_with(".weight") && tensor.dtype() == DType::U32 { - quantized_weights.push(name.clone()); - continue; - } - - if name.ends_with(".scales") || name.ends_with(".biases") { - continue; - } - let mapped_name = map_tensor_name(name); let mut t = tensor.clone(); if t.dtype() != DType::F32 { @@ -570,47 +561,16 @@ impl LocalWhisperEngine { plain_secs = plain_started.elapsed().as_secs_f64(); - // Second pass: dequantize packed q8 weights. - let dequant_started = std::time::Instant::now(); - for weight_name in quantized_weights { - let base = weight_name.trim_end_matches(".weight"); - let packed = raw_tensors - .get(&weight_name) - .context(format!("Missing packed tensor for {}", weight_name))?; - let scales_key = format!("{}.scales", base); - let biases_key = format!("{}.biases", base); - let scales = raw_tensors - .get(&scales_key) - .context(format!("Missing scales tensor for {}", weight_name))?; - let biases = raw_tensors - .get(&biases_key) - .context(format!("Missing biases tensor for {}", weight_name))?; - - let mut dequant = dequantize_q8(packed, scales, biases, &device)?; - let mapped_name = map_tensor_name(&weight_name); - - if mapped_name.ends_with("conv1.weight") || mapped_name.ends_with("conv2.weight") { - let dims = dequant.dims(); - if dims.len() == 3 && dims[1] == 3 { - dequant = dequant.permute((0, 2, 1))?.contiguous()?; - } - } - - tensor_map.insert(mapped_name, dequant); - } - dequant_secs = dequant_started.elapsed().as_secs_f64(); - candle_nn::VarBuilder::from_tensors(tensor_map, DType::F32, &device) }; let build_started = std::time::Instant::now(); let model = Model::load(&vb, config.clone()).context("Failed to create Whisper Model")?; tracing::info!( - "whisper_cold_load_phases total={:.2}s read={:.2}s plain_tensors={:.2}s dequantize_q8={:.2}s build_model={:.2}s", + "whisper_cold_load_phases total={:.2}s read={:.2}s plain_tensors={:.2}s build_model={:.2}s", load_started.elapsed().as_secs_f64(), read_secs, plain_secs, - dequant_secs, build_started.elapsed().as_secs_f64() ); @@ -1972,78 +1932,7 @@ fn should_drop_for_quality_gate( low_logprob && high_compression } -/// Expand MLX-style q8 weights into `F32`. -/// -/// `packed` holds four uint8 weights per `u32`; each is scaled and offset by -/// the `scales` / `biases` entry for its 32-element group, producing an -/// `out_dim × in_dim` tensor. -/// -/// Cost note: this runs on every cold load and dominates it — measured at -/// roughly three quarters of Whisper cold-start time (commit `e9d8e5d9`). -/// Keeping weights resident is what avoids paying it again. -/// -/// # Errors -/// Non-`u32` packed input, non-2D tensors, or scale/bias dimensions that do not -/// match the packed shape. -fn dequantize_q8( - packed: &Tensor, - scales: &Tensor, - biases: &Tensor, - device: &Device, -) -> Result { - ensure!(packed.dtype() == DType::U32, "Packed tensor must be u32"); - - let packed_dims = packed.dims(); - ensure!(packed_dims.len() == 2, "Packed weight must be 2D"); - let out_dim = packed_dims[0]; - let packed_in = packed_dims[1]; - let in_dim = packed_in * 4; - - let scales_dims = scales.dims(); - let biases_dims = biases.dims(); - ensure!( - scales_dims.len() == 2 && biases_dims.len() == 2, - "Scales and biases must be 2D" - ); - ensure!( - scales_dims[0] == out_dim && biases_dims[0] == out_dim, - "Scales/biases out dimension mismatch" - ); - - let group_size = 32usize; - let expected_groups = in_dim / group_size; - ensure!( - scales_dims[1] == expected_groups && biases_dims[1] == expected_groups, - "Scales/biases group dimension mismatch" - ); - - let packed_data = packed.to_vec2::()?; - let scales_data = scales.to_dtype(DType::F32)?.to_vec2::()?; - let biases_data = biases.to_dtype(DType::F32)?.to_vec2::()?; - - let mut output: Vec = Vec::with_capacity(out_dim * in_dim); - - for (o, packed_row) in packed_data.iter().enumerate() { - for (p, &val) in packed_row.iter().enumerate() { - for b in 0..4 { - let idx = p * 4 + b; - let group = idx / group_size; - // Treat as uint8 - let w = ((val >> (8 * b)) & 0xff) as u8; - let scale = scales_data[o][group]; - let bias = biases_data[o][group]; - output.push((w as f32) * scale + bias); - } - } - } - - Ok(Tensor::from_vec(output, (out_dim, in_dim), device)?) -} - -/// Build VarBuilder from raw tensors with Q8 dequantization -/// -/// Handles MLX quantized weights (packed U32 + scales + biases) -/// and converts tensor names to Candle format. +/// Build a VarBuilder from verified unquantized tensors. fn build_varbuilder_from_tensors( raw_tensors: HashMap, device: &Device, @@ -2054,19 +1943,9 @@ fn build_varbuilder_from_tensors( anyhow::bail!("Quantized Whisper tensor payload refused; fp16 weights are required"); } let mut tensor_map = HashMap::new(); - let mut quantized_weights: Vec = Vec::new(); - // First pass: handle non-quantized tensors and collect quantized weight names + // The payload gate above makes every tensor in this pass unquantized. for (name, tensor) in raw_tensors.iter() { - if name.ends_with(".weight") && tensor.dtype() == DType::U32 { - quantized_weights.push(name.clone()); - continue; - } - - if name.ends_with(".scales") || name.ends_with(".biases") { - continue; - } - let mapped_name = map_tensor_name(name); let mut t = tensor.clone(); if t.dtype() != DType::F32 { @@ -2085,34 +1964,6 @@ fn build_varbuilder_from_tensors( tensor_map.insert(mapped_name, t); } - // Second pass: dequantize packed Q8 weights - for weight_name in quantized_weights { - let base = weight_name.trim_end_matches(".weight"); - let packed = raw_tensors - .get(&weight_name) - .context(format!("Missing packed tensor for {}", weight_name))?; - let scales_key = format!("{}.scales", base); - let biases_key = format!("{}.biases", base); - let scales = raw_tensors - .get(&scales_key) - .context(format!("Missing scales tensor for {}", weight_name))?; - let biases = raw_tensors - .get(&biases_key) - .context(format!("Missing biases tensor for {}", weight_name))?; - - let mut dequant = dequantize_q8(packed, scales, biases, device)?; - let mapped_name = map_tensor_name(&weight_name); - - if mapped_name.ends_with("conv1.weight") || mapped_name.ends_with("conv2.weight") { - let dims = dequant.dims(); - if dims.len() == 3 && dims[1] == 3 { - dequant = dequant.permute((0, 2, 1))?.contiguous()?; - } - } - - tensor_map.insert(mapped_name, dequant); - } - Ok(candle_nn::VarBuilder::from_tensors( tensor_map, DType::F32, From f3775483d1c4ff83a7f1ea725e4d505c0ce8b372 Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Fri, 21 Aug 2026 16:16:12 +0200 Subject: [PATCH 04/45] [codex/vc-workflow] fix(stt): make fp16 bundles repairable --- core/config/models.rs | 476 +++++++++++++++++---- core/stt/whisper/engine.rs | 90 +++- scripts/download-model.sh | 30 +- tests/e2e_stt_transcription.rs | 2 +- tests/fixtures/whisper_mel_filters.npz.hex | 143 +++++++ 5 files changed, 647 insertions(+), 94 deletions(-) create mode 100644 tests/fixtures/whisper_mel_filters.npz.hex diff --git a/core/config/models.rs b/core/config/models.rs index 74354fae..565d0d97 100644 --- a/core/config/models.rs +++ b/core/config/models.rs @@ -49,33 +49,57 @@ fn canonicalize_or_self(path: PathBuf) -> PathBuf { } } -/// Whether `path` holds a fully usable Whisper model. -/// -/// Requires every [`REQUIRED_MODEL_FILES`] entry plus at least one of -/// [`REQUIRED_MODEL_WEIGHTS`]. This is the gate that keeps half-downloaded -/// directories from being advertised or resolved as loadable models. +/// Whether `path` holds a fully usable, structurally valid Whisper model. fn is_complete_whisper_model_dir(path: &Path) -> bool { - let files_present = REQUIRED_MODEL_FILES - .iter() - .all(|name| path.join(name).exists()) - && REQUIRED_MODEL_WEIGHTS - .iter() - .any(|name| path.join(name).exists()); - files_present && is_unquantized_whisper_model_dir(path) + validate_whisper_model_bundle(path).is_ok() +} + +/// Validate every artifact required by the runtime loader. +/// +/// Safetensors validation is structural rather than cryptographic: the format +/// has no payload checksum. The validator checks the complete tensor table, +/// dtype allowlist, byte sizes, contiguous offsets, and final file length. +fn validate_whisper_model_bundle(path: &Path) -> Result<()> { + let config_path = path.join("config.json"); + validate_whisper_config(&config_path)?; + + let tokenizer_path = path.join("tokenizer.json"); + tokenizers::Tokenizer::from_file(&tokenizer_path).map_err(|err| { + anyhow!( + "invalid Whisper tokenizer {}: {err}", + tokenizer_path.display() + ) + })?; + + let mel_path = path.join("mel_filters.npz"); + verify_sha256(&mel_path, MEL_FILTERS_SHA256)?; + + let weights_path = resolve_weights_path(path)?; + validate_safetensors_file(&weights_path) } -/// Reject quantized or malformed weights before they can reach the expensive -/// engine loader. The config check catches normal MLX q8 exports; the -/// safetensors header check also catches a q8 payload hidden behind a renamed -/// directory or a config with its `quantization` field removed. +/// Reject unsupported or malformed weights before the expensive engine load. +/// This narrower payload gate is also used by `LocalWhisperEngine::new`, where +/// tokenizer and mel errors retain their own loader diagnostics. pub(crate) fn is_unquantized_whisper_model_dir(path: &Path) -> bool { - let config = match fs::read_to_string(path.join("config.json")) - .ok() - .and_then(|raw| serde_json::from_str::(&raw).ok()) - { - Some(config) => config, - None => return false, - }; + validate_whisper_config(&path.join("config.json")).is_ok() + && resolve_weights_path(path) + .and_then(|weights| validate_safetensors_file(&weights)) + .is_ok() +} + +fn validate_whisper_config(path: &Path) -> Result<()> { + // nosemgrep: rust.actix.path-traversal.tainted-path.tainted-path -- Read-only model inspection. `path` is an operator-selected local model config or an internally resolved bundle/cache child; no network/request path component reaches it. + let raw = fs::read_to_string(path) + .with_context(|| format!("read Whisper config {}", path.display()))?; + let config: serde_json::Value = serde_json::from_str(&raw) + .with_context(|| format!("parse Whisper config {}", path.display()))?; + if !config.is_object() { + return Err(anyhow!( + "Whisper config must be a JSON object: {}", + path.display() + )); + } if config .get("quantization") .is_some_and(|value| !value.is_null()) @@ -83,21 +107,21 @@ pub(crate) fn is_unquantized_whisper_model_dir(path: &Path) -> bool { .get("quantization_config") .is_some_and(|value| !value.is_null()) { - return false; + return Err(anyhow!("quantized Whisper config is unsupported")); } + Ok(()) +} - let Some(weights_path) = REQUIRED_MODEL_WEIGHTS +fn resolve_weights_path(path: &Path) -> Result { + REQUIRED_MODEL_WEIGHTS .iter() .map(|name| path.join(name)) - .find(|candidate| candidate.exists()) - else { - return false; - }; - safetensors_header_is_unquantized(&weights_path).unwrap_or(false) + .find(|candidate| candidate.is_file()) + .ok_or_else(|| anyhow!("Whisper weights are missing from {}", path.display())) } -/// Inspect only the bounded JSON header; model tensor data is never read. -fn safetensors_header_is_unquantized(path: &Path) -> Result { +/// Validate the complete safetensors structure without loading the tensor data. +fn validate_safetensors_file(path: &Path) -> Result<()> { const MAX_HEADER_BYTES: u64 = 16 * 1024 * 1024; // nosemgrep: rust.actix.path-traversal.tainted-path.tainted-path -- Read-only model inspection. `path` is an operator-selected local model file or an internally resolved bundle/cache child; no network/request path component reaches it. let mut file = fs::File::open(path).with_context(|| format!("open {}", path.display()))?; @@ -106,7 +130,10 @@ fn safetensors_header_is_unquantized(path: &Path) -> Result { .with_context(|| format!("read safetensors header length from {}", path.display()))?; let header_len = u64::from_le_bytes(len_bytes); if header_len == 0 || header_len > MAX_HEADER_BYTES { - return Ok(false); + return Err(anyhow!( + "invalid safetensors header length in {}", + path.display() + )); } let mut header = vec![0_u8; header_len as usize]; file.seek(SeekFrom::Start(8))?; @@ -115,14 +142,101 @@ fn safetensors_header_is_unquantized(path: &Path) -> Result { let metadata: serde_json::Value = serde_json::from_slice(&header) .with_context(|| format!("parse safetensors header from {}", path.display()))?; let Some(tensors) = metadata.as_object() else { - return Ok(false); + return Err(anyhow!( + "safetensors header is not an object: {}", + path.display() + )); }; - Ok(tensors.iter().all(|(name, tensor)| { - name == "__metadata__" - || (!name.ends_with(".scales") - && !name.ends_with(".biases") - && tensor.get("dtype").and_then(|value| value.as_str()) != Some("U32")) - })) + + let file_len = file.metadata()?.len(); + let data_start = 8_u64 + .checked_add(header_len) + .ok_or_else(|| anyhow!("safetensors header offset overflow"))?; + let data_len = file_len + .checked_sub(data_start) + .ok_or_else(|| anyhow!("truncated safetensors file: {}", path.display()))?; + let mut ranges = Vec::new(); + + for (name, tensor) in tensors.iter().filter(|(name, _)| *name != "__metadata__") { + let tensor = tensor + .as_object() + .ok_or_else(|| anyhow!("invalid tensor entry {name}"))?; + let dtype = tensor + .get("dtype") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| anyhow!("tensor {name} has no dtype"))?; + let bytes_per_element = match (name.as_str(), dtype) { + (_, "F16") => 2_u64, + (_, "F32") => 4_u64, + ("alignment_heads", "I64") => 8_u64, + _ => { + return Err(anyhow!( + "unsupported Whisper tensor dtype {dtype} for {name}" + )); + } + }; + if name.ends_with(".scales") || name.ends_with(".biases") { + return Err(anyhow!( + "quantized Whisper companion tensor refused: {name}" + )); + } + + let shape = tensor + .get("shape") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| anyhow!("tensor {name} has no shape"))?; + let element_count = shape.iter().try_fold(1_u64, |count, dim| { + let dim = dim + .as_u64() + .ok_or_else(|| anyhow!("tensor {name} has an invalid shape"))?; + count + .checked_mul(dim) + .ok_or_else(|| anyhow!("tensor {name} shape overflows")) + })?; + let expected_bytes = element_count + .checked_mul(bytes_per_element) + .ok_or_else(|| anyhow!("tensor {name} byte size overflows"))?; + + let offsets = tensor + .get("data_offsets") + .and_then(serde_json::Value::as_array) + .filter(|offsets| offsets.len() == 2) + .ok_or_else(|| anyhow!("tensor {name} has invalid data_offsets"))?; + let start = offsets[0] + .as_u64() + .ok_or_else(|| anyhow!("tensor {name} has invalid start offset"))?; + let end = offsets[1] + .as_u64() + .ok_or_else(|| anyhow!("tensor {name} has invalid end offset"))?; + if end.checked_sub(start) != Some(expected_bytes) { + return Err(anyhow!( + "tensor {name} byte range does not match its shape/dtype" + )); + } + ranges.push((start, end, name)); + } + + if ranges.is_empty() { + return Err(anyhow!( + "safetensors file contains no tensors: {}", + path.display() + )); + } + ranges.sort_by_key(|(start, _, _)| *start); + let mut cursor = 0_u64; + for (start, end, name) in ranges { + if start != cursor { + return Err(anyhow!("tensor {name} has a non-contiguous data offset")); + } + cursor = end; + } + if cursor != data_len { + return Err(anyhow!( + "safetensors data length mismatch in {}: header covers {cursor}, file has {data_len}", + path.display() + )); + } + Ok(()) } /// Whether a candidate models root owns at least one complete Whisper model. @@ -426,7 +540,10 @@ where fs::create_dir_all(&dest).with_context(|| format!("create {}", dest.display()))?; let client = reqwest::blocking::Client::builder() - .user_agent("codescribe-whisper-download/0.13") + .user_agent(format!( + "codescribe-whisper-download/{}", + env!("CARGO_PKG_VERSION") + )) .timeout(std::time::Duration::from_secs(60 * 30)) .build() .context("build HTTP client for Whisper download")?; @@ -453,11 +570,11 @@ where &dest.join("mel_filters.npz"), &mut on_progress, )?; - verify_sha256(&dest.join("mel_filters.npz"), MEL_FILTERS_SHA256)?; - let weights_dest = dest.join("weights.safetensors"); let weights_alt = dest.join("model.safetensors"); - if !weights_dest.exists() && !weights_alt.exists() { + if validate_model_file("weights.safetensors", &weights_dest).is_err() + && validate_model_file("model.safetensors", &weights_alt).is_err() + { // mlx-community ships weights.safetensors; fall back to model.safetensors if 404. match download_hf_file( &client, @@ -483,12 +600,12 @@ where } } - if !is_complete_whisper_model_dir(&dest) { - return Err(anyhow!( - "Whisper download finished but model dir is incomplete: {}", + validate_whisper_model_bundle(&dest).with_context(|| { + format!( + "Whisper download finished but bundle validation failed: {}", dest.display() - )); - } + ) + })?; Ok(canonicalize_or_self(dest)) } @@ -496,9 +613,9 @@ where /// Copy selected model files from a local source into the user models directory. /// /// Lets Settings → Download complete without network traffic when the pieces are -/// already on disk (warm official caches). A missing source is a -/// clean no-op and existing destination files are left alone, so an interrupted -/// composition resumes rather than restarting. +/// already on disk (warm official caches). Every copied file is validated in a +/// sibling `.partial` path before atomic promotion. Invalid destinations are +/// replaced; valid ones are preserved. fn copy_model_files(src: &Path, dest: &Path, names: &[&str]) -> Result<()> { if !src.is_dir() { return Ok(()); @@ -507,11 +624,22 @@ fn copy_model_files(src: &Path, dest: &Path, names: &[&str]) -> Result<()> { for name in names { let from = src.join(name); let to = dest.join(name); - if from.exists() && !to.exists() { - // nosemgrep: rust.actix.path-traversal.tainted-path.tainted-path -- Both ends are internal: `name` comes from compile-time model file constants, and callers pass HF cache snapshot dirs or ModelManager::get_model_path outputs. No caller-supplied path component reaches here. - fs::copy(&from, &to) - .with_context(|| format!("copy {} → {}", from.display(), to.display()))?; + if !from.is_file() || validate_model_file(name, &to).is_ok() { + continue; + } + if let Err(err) = validate_model_file(name, &from) { + tracing::warn!(source = %from.display(), error = %err, "ignoring invalid cached model artifact"); + continue; } + let partial = partial_path(&to); + // nosemgrep: rust.actix.path-traversal.tainted-path.tainted-path -- Both ends are internal: `name` comes from compile-time model file constants, and callers pass HF cache snapshot dirs or ModelManager::get_model_path outputs. No caller-supplied path component reaches here. + fs::copy(&from, &partial) + .with_context(|| format!("copy {} → {}", from.display(), partial.display()))?; + if let Err(err) = validate_model_file(name, &partial) { + let _ = fs::remove_file(&partial); + return Err(err).with_context(|| format!("validate copied {}", name)); + } + replace_file(&partial, &to)?; } Ok(()) } @@ -519,9 +647,9 @@ fn copy_model_files(src: &Path, dest: &Path, names: &[&str]) -> Result<()> { /// Fetch one file from the Hugging Face resolve endpoint into `dest`. /// /// Downloads to a sibling `.partial` file and renames on success, so an aborted -/// transfer can never leave a truncated file that passes the completeness check. -/// A non-empty `dest` is treated as already done. `HF_TOKEN` is sent as bearer -/// auth when set, for gated repos. +/// transfer can never leave a truncated file that passes bundle validation. +/// A valid `dest` is preserved; an invalid one is replaced. `HF_TOKEN` is sent +/// as bearer auth when set, for gated repos. fn download_hf_file( client: &reqwest::blocking::Client, repo: &str, @@ -532,15 +660,6 @@ fn download_hf_file( where F: FnMut(&str, u64, Option), { - if dest.exists() && dest.metadata().map(|m| m.len() > 0).unwrap_or(false) { - on_progress( - filename, - dest.metadata().map(|m| m.len()).unwrap_or(0), - None, - ); - return Ok(()); - } - let url = format!("https://huggingface.co/{repo}/resolve/main/{filename}"); download_url_file_authenticated(client, &url, filename, dest, on_progress, true) } @@ -569,6 +688,19 @@ fn download_url_file_authenticated( where F: FnMut(&str, u64, Option), { + if validate_model_file(filename, dest).is_ok() { + on_progress( + filename, + dest.metadata().map(|metadata| metadata.len()).unwrap_or(0), + None, + ); + return Ok(()); + } + if dest.exists() { + fs::remove_file(dest) + .with_context(|| format!("remove invalid model artifact {}", dest.display()))?; + } + let mut request = client.get(url); if use_hf_token && let Ok(token) = std::env::var("HF_TOKEN") { let token = token.trim(); @@ -584,12 +716,7 @@ where .with_context(|| format!("HTTP error for {url}"))?; let total = response.content_length(); - let partial = dest.with_file_name(format!( - "{}.partial", - dest.file_name() - .and_then(|n| n.to_str()) - .unwrap_or("download") - )); + let partial = partial_path(dest); if let Some(parent) = partial.parent() { fs::create_dir_all(parent)?; @@ -616,12 +743,45 @@ where file.flush()?; drop(file); - fs::rename(&partial, dest) - .with_context(|| format!("rename {} → {}", partial.display(), dest.display()))?; + if let Err(err) = validate_model_file(filename, &partial) { + let _ = fs::remove_file(&partial); + return Err(err).with_context(|| format!("validate downloaded {filename}")); + } + replace_file(&partial, dest)?; on_progress(filename, done, total.or(Some(done))); Ok(()) } +fn partial_path(dest: &Path) -> PathBuf { + dest.with_file_name(format!( + "{}.partial", + dest.file_name() + .and_then(|name| name.to_str()) + .unwrap_or("download") + )) +} + +fn replace_file(partial: &Path, dest: &Path) -> Result<()> { + if dest.exists() { + fs::remove_file(dest) + .with_context(|| format!("remove invalid model artifact {}", dest.display()))?; + } + fs::rename(partial, dest) + .with_context(|| format!("rename {} → {}", partial.display(), dest.display())) +} + +fn validate_model_file(filename: &str, path: &Path) -> Result<()> { + match filename { + "config.json" => validate_whisper_config(path), + "tokenizer.json" => tokenizers::Tokenizer::from_file(path) + .map(|_| ()) + .map_err(|err| anyhow!("invalid tokenizer {}: {err}", path.display())), + "mel_filters.npz" => verify_sha256(path, MEL_FILTERS_SHA256), + "weights.safetensors" | "model.safetensors" => validate_safetensors_file(path), + _ => Err(anyhow!("unsupported Whisper model artifact: {filename}")), + } +} + fn verify_sha256(path: &Path, expected: &str) -> Result<()> { // nosemgrep: rust.actix.path-traversal.tainted-path.tainted-path -- Read-only checksum of the fixed mel_filters.npz destination assembled under the internally resolved model directory. let bytes = fs::read(path).with_context(|| format!("read {} for checksum", path.display()))?; @@ -686,8 +846,16 @@ mod tests { fn create_complete_whisper_model(path: &Path) { fs::create_dir_all(path).unwrap(); fs::write(path.join("config.json"), "{}").unwrap(); - fs::write(path.join("tokenizer.json"), "{}").unwrap(); - fs::write(path.join("mel_filters.npz"), "npz").unwrap(); + tokenizers::Tokenizer::new(tokenizers::models::bpe::BPE::default()) + .save(path.join("tokenizer.json"), false) + .unwrap(); + fs::write( + path.join("mel_filters.npz"), + decode_hex(include_str!( + "../../tests/fixtures/whisper_mel_filters.npz.hex" + )), + ) + .unwrap(); let header = br#"{"model.weight":{"dtype":"F16","shape":[1],"data_offsets":[0,2]}}"#; let mut safetensors = (header.len() as u64).to_le_bytes().to_vec(); safetensors.extend_from_slice(header); @@ -695,6 +863,16 @@ mod tests { fs::write(path.join("model.safetensors"), safetensors).unwrap(); } + fn decode_hex(raw: &str) -> Vec { + let digits: String = raw.chars().filter(|ch| !ch.is_whitespace()).collect(); + assert!(digits.len().is_multiple_of(2)); + digits + .as_bytes() + .chunks_exact(2) + .map(|pair| u8::from_str_radix(std::str::from_utf8(pair).unwrap(), 16).unwrap()) + .collect() + } + fn create_q8_whisper_model(path: &Path) { create_complete_whisper_model(path); fs::write( @@ -819,6 +997,148 @@ mod tests { assert!(!is_complete_whisper_model_dir(&model)); } + /// A non-Q8 integer tensor is still outside the fp16/fp32 runtime contract. + #[test] + fn model_manager_rejects_non_allowlisted_integer_tensor() { + let temp_dir = TempDir::new().unwrap(); + let model = temp_dir.path().join("model"); + create_complete_whisper_model(&model); + let header = br#"{"encoder.weight":{"dtype":"I32","shape":[1],"data_offsets":[0,4]}}"#; + let mut safetensors = (header.len() as u64).to_le_bytes().to_vec(); + safetensors.extend_from_slice(header); + safetensors.extend_from_slice(&[0; 4]); + fs::write(model.join("model.safetensors"), safetensors).unwrap(); + + assert!(!is_complete_whisper_model_dir(&model)); + } + + /// Metadata alone is not a model and must not satisfy discovery. + #[test] + fn model_manager_rejects_safetensors_without_tensor_entries() { + let temp_dir = TempDir::new().unwrap(); + let model = temp_dir.path().join("model"); + create_complete_whisper_model(&model); + let header = br#"{"__metadata__":{"format":"mlx"}}"#; + let mut safetensors = (header.len() as u64).to_le_bytes().to_vec(); + safetensors.extend_from_slice(header); + fs::write(model.join("model.safetensors"), safetensors).unwrap(); + + assert!(!is_complete_whisper_model_dir(&model)); + } + + /// Header offsets must describe the actual payload, not a truncated file. + #[test] + fn model_manager_rejects_truncated_safetensors_payload() { + let temp_dir = TempDir::new().unwrap(); + let model = temp_dir.path().join("model"); + create_complete_whisper_model(&model); + let weights = model.join("model.safetensors"); + let len = fs::metadata(&weights).unwrap().len(); + fs::OpenOptions::new() + .write(true) + .open(&weights) + .unwrap() + .set_len(len - 1) + .unwrap(); + + assert!(!is_complete_whisper_model_dir(&model)); + } + + /// A checksum mismatch cannot leave a directory advertised as complete. + #[test] + fn model_manager_rejects_corrupt_mel_filters() { + let temp_dir = TempDir::new().unwrap(); + let model = temp_dir.path().join("model"); + create_complete_whisper_model(&model); + fs::write(model.join("mel_filters.npz"), b"corrupt").unwrap(); + + assert!(!is_complete_whisper_model_dir(&model)); + } + + /// A valid existing mel asset is reused without contacting the network. + #[test] + fn valid_existing_mel_skips_download() { + let temp_dir = TempDir::new().unwrap(); + let mel = temp_dir.path().join("mel_filters.npz"); + fs::write( + &mel, + decode_hex(include_str!( + "../../tests/fixtures/whisper_mel_filters.npz.hex" + )), + ) + .unwrap(); + let client = reqwest::blocking::Client::new(); + let mut progress = |_name: &str, _done: u64, _total: Option| {}; + + download_url_file( + &client, + "http://127.0.0.1:0/must-not-be-called", + "mel_filters.npz", + &mel, + &mut progress, + ) + .unwrap(); + } + + /// Invalid warm destinations are replaced from validated cache artifacts. + #[test] + fn cached_composition_repairs_invalid_destination_files() { + let temp_dir = TempDir::new().unwrap(); + let source = temp_dir.path().join("source"); + let destination = temp_dir.path().join("destination"); + create_complete_whisper_model(&source); + create_complete_whisper_model(&destination); + fs::write(destination.join("config.json"), b"not json").unwrap(); + fs::write(destination.join("tokenizer.json"), b"not json").unwrap(); + fs::write(destination.join("mel_filters.npz"), b"bad mel").unwrap(); + fs::write(destination.join("model.safetensors"), b"bad weights").unwrap(); + + copy_model_files(&source, &destination, &REQUIRED_MODEL_FILES).unwrap(); + copy_model_files(&source, &destination, &REQUIRED_MODEL_WEIGHTS).unwrap(); + + validate_whisper_model_bundle(&destination).unwrap(); + assert!(!destination.join("config.json.partial").exists()); + assert!(!destination.join("model.safetensors.partial").exists()); + } + + /// A downloaded checksum mismatch is never promoted to the final mel path. + #[test] + fn corrupt_download_is_removed_before_promotion() { + use std::io::{Read as _, Write as _}; + use std::net::TcpListener; + + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut request = [0_u8; 1024]; + let _ = stream.read(&mut request); + stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 7\r\n\r\ncorrupt") + .unwrap(); + }); + + let temp_dir = TempDir::new().unwrap(); + let mel = temp_dir.path().join("mel_filters.npz"); + let client = reqwest::blocking::Client::new(); + let mut progress = |_name: &str, _done: u64, _total: Option| {}; + let result = download_url_file( + &client, + &format!("http://{address}/mel_filters.npz"), + "mel_filters.npz", + &mel, + &mut progress, + ); + server.join().unwrap(); + + assert!(result.is_err()); + assert!(!mel.exists(), "corrupt download must not be promoted"); + assert!( + !partial_path(&mel).exists(), + "failed partial must be removed" + ); + } + /// Complete `CODESCRIBE_MODEL_PATH` wins over the bundled default tier. #[test] #[serial] diff --git a/core/stt/whisper/engine.rs b/core/stt/whisper/engine.rs index c1cd852d..a930f32c 100644 --- a/core/stt/whisper/engine.rs +++ b/core/stt/whisper/engine.rs @@ -525,13 +525,12 @@ impl LocalWhisperEngine { let loaded = view.load(&Device::Cpu)?; raw_tensors.insert(name.to_string(), loaded); } - if raw_tensors.iter().any(|(name, tensor)| { - tensor.dtype() == DType::U32 - || name.ends_with(".scales") - || name.ends_with(".biases") - }) { + if raw_tensors + .iter() + .any(|(name, tensor)| !is_supported_runtime_tensor(name, tensor)) + { anyhow::bail!( - "Quantized Whisper tensor payload refused; install the complete fp16 model" + "Unsupported Whisper tensor payload refused; install the complete fp16 model" ); } read_secs = read_started.elapsed().as_secs_f64(); @@ -1933,14 +1932,23 @@ fn should_drop_for_quality_gate( } /// Build a VarBuilder from verified unquantized tensors. +fn is_supported_runtime_tensor(name: &str, tensor: &Tensor) -> bool { + if name.ends_with(".scales") || name.ends_with(".biases") { + return false; + } + matches!(tensor.dtype(), DType::F16 | DType::F32) + || name == "alignment_heads" && tensor.dtype() == DType::I64 +} + fn build_varbuilder_from_tensors( raw_tensors: HashMap, device: &Device, ) -> Result> { - if raw_tensors.iter().any(|(name, tensor)| { - tensor.dtype() == DType::U32 || name.ends_with(".scales") || name.ends_with(".biases") - }) { - anyhow::bail!("Quantized Whisper tensor payload refused; fp16 weights are required"); + if raw_tensors + .iter() + .any(|(name, tensor)| !is_supported_runtime_tensor(name, tensor)) + { + anyhow::bail!("Unsupported Whisper tensor payload refused; fp16 weights are required"); } let mut tensor_map = HashMap::new(); @@ -1971,6 +1979,68 @@ fn build_varbuilder_from_tensors( )) } +#[cfg(test)] +mod model_payload_tests { + use super::*; + use std::fs; + use tempfile::TempDir; + + fn write_tiny_model(path: &Path, name: &str, dtype: &str, payload_bytes: usize) { + fs::create_dir_all(path).unwrap(); + fs::write(path.join("config.json"), "{}").unwrap(); + fs::write(path.join("tokenizer.json"), "{}").unwrap(); + fs::write(path.join("mel_filters.npz"), b"placeholder").unwrap(); + let header = serde_json::json!({ + name: { + "dtype": dtype, + "shape": [1], + "data_offsets": [0, payload_bytes] + } + }); + let header = serde_json::to_vec(&header).unwrap(); + let mut file = (header.len() as u64).to_le_bytes().to_vec(); + file.extend_from_slice(&header); + file.resize(file.len() + payload_bytes, 0); + fs::write(path.join("weights.safetensors"), file).unwrap(); + } + + #[test] + fn local_loader_refuses_tiny_u32_safetensors() { + let temp = TempDir::new().unwrap(); + write_tiny_model(temp.path(), "encoder.weight", "U32", 4); + + let err = LocalWhisperEngine::new(temp.path()) + .err() + .expect("U32 must be refused"); + assert!(format!("{err:#}").contains("refused")); + } + + #[test] + fn local_loader_refuses_non_allowlisted_integer_safetensors() { + let temp = TempDir::new().unwrap(); + write_tiny_model(temp.path(), "encoder.weight", "I32", 4); + + let err = LocalWhisperEngine::new(temp.path()) + .err() + .expect("I32 must be refused"); + assert!(format!("{err:#}").contains("refused")); + } + + #[test] + fn tensor_builder_refuses_u32_before_mapping() { + let mut tensors = HashMap::new(); + tensors.insert( + "encoder.weight".to_string(), + Tensor::from_vec(vec![0_u32], 1, &Device::Cpu).unwrap(), + ); + + let err = build_varbuilder_from_tensors(tensors, &Device::Cpu) + .err() + .expect("U32 must be refused by the builder gate"); + assert!(format!("{err:#}").contains("refused")); + } +} + // ═══════════════════════════════════════════════════════════════════════════════ // Repetition Deduplication (Word and Phrase Level) // ═══════════════════════════════════════════════════════════════════════════════ diff --git a/scripts/download-model.sh b/scripts/download-model.sh index bcc3ef50..7665eefe 100755 --- a/scripts/download-model.sh +++ b/scripts/download-model.sh @@ -16,6 +16,25 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)" +sha256_file() { + if command -v shasum >/dev/null 2>&1; then + shasum -a 256 "$1" | awk '{print $1}' + elif command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" | awk '{print $1}' + else + echo "ERROR: need shasum or sha256sum to verify model assets" >&2 + return 1 + fi +} + +atomic_copy() { + local source="$1" + local destination="$2" + local partial="${destination}.partial" + cp -fL "$source" "$partial" + mv -f "$partial" "$destination" +} + # Configuration DEFAULT_REPO="mlx-community/whisper-large-v3-turbo" TOKENIZER_REPO="openai/whisper-large-v3-turbo" @@ -95,19 +114,20 @@ if [[ "$MODEL_REPO" == "$DEFAULT_REPO" ]]; then TOKENIZER_PATH=$("$HF_BIN" download "$TOKENIZER_REPO" tokenizer.json --quiet) MODEL_DEST="${CODESCRIBE_MODELS_DIR:-$HOME/.codescribe/models}/whisper-large-v3-turbo" mkdir -p "$MODEL_DEST" - cp -fL "$MODEL_SNAPSHOT/config.json" "$MODEL_DEST/config.json" + atomic_copy "$MODEL_SNAPSHOT/config.json" "$MODEL_DEST/config.json" if [[ -f "$MODEL_SNAPSHOT/weights.safetensors" ]]; then - cp -fL "$MODEL_SNAPSHOT/weights.safetensors" "$MODEL_DEST/weights.safetensors" + atomic_copy "$MODEL_SNAPSHOT/weights.safetensors" "$MODEL_DEST/weights.safetensors" elif [[ -f "$MODEL_SNAPSHOT/model.safetensors" ]]; then - cp -fL "$MODEL_SNAPSHOT/model.safetensors" "$MODEL_DEST/model.safetensors" + atomic_copy "$MODEL_SNAPSHOT/model.safetensors" "$MODEL_DEST/model.safetensors" else echo "ERROR: fp16 snapshot has no safetensors weights: $MODEL_SNAPSHOT" >&2 exit 1 fi - cp -fL "$TOKENIZER_PATH" "$MODEL_DEST/tokenizer.json" + atomic_copy "$TOKENIZER_PATH" "$MODEL_DEST/tokenizer.json" curl -fsSL "$MEL_FILTERS_URL" -o "$MODEL_DEST/mel_filters.npz.partial" - ACTUAL_MEL_SHA=$(shasum -a 256 "$MODEL_DEST/mel_filters.npz.partial" | awk '{print $1}') + ACTUAL_MEL_SHA=$(sha256_file "$MODEL_DEST/mel_filters.npz.partial") if [[ "$ACTUAL_MEL_SHA" != "$MEL_FILTERS_SHA256" ]]; then + rm -f "$MODEL_DEST/mel_filters.npz.partial" echo "ERROR: mel_filters.npz checksum mismatch" >&2 exit 1 fi diff --git a/tests/e2e_stt_transcription.rs b/tests/e2e_stt_transcription.rs index 51acaeb6..78af1e5c 100644 --- a/tests/e2e_stt_transcription.rs +++ b/tests/e2e_stt_transcription.rs @@ -261,6 +261,6 @@ fn deterministic_model_discovery_refuses_incomplete_fp16_without_legacy_fallback let missing = whisper_model_missing_parts(&fp16); assert!( missing.contains(&"config.json"), - "incomplete turbo should report missing artifacts for easier diagnosis" + "incomplete fp16 should report missing artifacts for easier diagnosis" ); } diff --git a/tests/fixtures/whisper_mel_filters.npz.hex b/tests/fixtures/whisper_mel_filters.npz.hex new file mode 100644 index 00000000..fb5401df --- /dev/null +++ b/tests/fixtures/whisper_mel_filters.npz.hex @@ -0,0 +1,143 @@ +504b03041400000008000000210081b9f70d78070000c0fb00000a001400 +6d656c5f38302e6e707901001000c0fb0000000000007807000000000000 +eddd595054671ac6f1160da2820b8adb88a028606c151a17fa7b3e83010d +13a38208b8c4ad65915114a1dda69445848898800b681412174011cca082 +3ae282a282b8241243a1c1255154226262485c51864c796151f499aab9d1 +8be757d5177daececdbfdef73bd5757ac3389fb19e939ba916ab96d9f907 +e8fdc2ec84b59d0c1c6ca7b6b60b0c095b18a69b3f2324cc3fe0afeb1fe9 +82f5010dd7f541ba05010ddfed9d1dd4d61a07c77e6aeb70ebff4f6b954a +1595bee7bc54119141faa546489e76849d1029d02f8dc6aab6d9ec8448c1 +9ebc53f867c1167642a4a061ef927316c4b3132205899d6de421d785ec84 +48c19e3c37b9257d023b215270d6cb5f46183bb1132205faa5d1521764ce +4e8814ac285e2f5727ddc6dbbe0fa2775962e734e93169073b2152f07540 +ae543bbbb21322057bf24ec93baf22c4dbbe0fa277597ef31f788627fa1f +b6a45f6627440a1686c68acf7d4eb11322056ac7518830ce6527440a2efd +908690433bd8099102930e77a00b5acf4e88146cb7ed2827748b6627440a +8e9f7692ab93fcd80991821f6779cbe0c7aeec8448c113d522e931a9373b +215210ff5dbcd4e4376327440a36f74c916ae742fe069248c1b3fa2cf979 +400c3b2152f093fd01f974f9647642a4605887fdb22e23989d1029b8707f +971c609aca4e8814f8866e9295c657d8099182d8ca1572e315be6b8548c9 +c8839e7257dc087642a4e07a4217993245cf4e8814ac7c908b9aaa24d932 +d69967142203f439e7c4f233b1b26ddf5a7642a420df61983c737f24772f +2205570b8a1093bd4ae6dd49e04c213260dcf3cfc44a8c97f75a5b71a610 +29d03cac847e865e3a572672a6101970f6fa6191e124e5228fbeb25959a9 +f66ddf0fd1bb2acd631fc2e685caf88d659c29440a06ed68292fdb7d2ce7 +b588662b4406f8eadae1db6bb6f2bd3536b2e28a055b213260d2c670f8e6 +3bcbaa03ada5634a0effef81c880c4daaf60e3eb225dbd9ee117b718b642 +64c0911799485d2365af91bfc2bb3e85ad1019f060441a4aeb1c658f7e7f +a0e4c27db6426440f1a1442ceeda4b3a6518c911164e3cdb1319b0efa10e +9f58b59103fbb497022bd80a910155b1e630f1ba85ccb85e32bc28079f5c +f7e31e46d484719bbdc594f19938d3f26ff245e86d7c673194b385c88094 +c20928ea53833f6798c9e5f35331bf6a01670b51135aecdf2e36bdd88e89 +67cce4f17937e178d79bb385c880154bfb236d5c1174911da475d641d467 +75622f444d88b2d9a8ad77f343f1cf5731fe894a4e4cfd0a83ecef701723 +6a82ebcc30a1ff261c73743761b9fb21d69f5c87ee4197d90b51130e076c +16a52b22e1677f15436aeec2a4722d42e656b317a22694162508abe7c198 +fbf712141454e191f966b85f30e7f985a80985673f14d7cf7d8c33ebf6a1 +8bffef18959289caf4e1a81b63cf1943d4c8b4e71aed4bc76e787fcb06f8 +a6fc0875b3e3c899ac87d9ac32f642d4843ae3a3a2e3673af8c4ec47d7e7 +e5689efb252aca3538613b9ecd1035e2b9b34ceb1dd1090f6d5621cfe434 +9e5ccc47cce028f41eda06b7bbede5bb5f881aa99b1e238c9e6ae0fad33a +c48c3c879ab399e8dd2a0065aa47c27e990b9b216ae4fb5dbbc5a551c371 +ad623d2e752ac647d6db607cfc53bc58fc507c539ccc66881ad96eb947d4 +140f45cc8935e86879148b476cc581601d964f6a81ce1d2c799e216a6460 +d13a91b8a8375a9a2c87cbc52c8c2e4e47eba97a44f6b3c4bd9a2fd80c51 +23c32fbb88e7f78cd0a7c207f6b39350189183c1c9ab307aa20b3e35b929 +a2e35e723f237ac32ea7c9da1b67f3c48e217628375e08e3e06dd0ccda8c +ce57825052dd134f5fed16b51a477643f486b8a4fea2b27da528ecaf459c +5d141c0ea7615ded1ad4864ec0a9bab6b06f972c92c7143abfedfb247a97 +b46c3553f814d58a8ea78117e9e1f8bd702bcacfc76280660cb22f98c2eb +db2d22b0fb4cce1ba23748bfe9e2e0905f4455aa23fef87a3ea22e25c325 +271637323d31bcd60262d941f1a795119f0b10bd616f3b07916c5c22b28c +2cd12ada0779aa553812170fa3dce948f4b6c7b007b7c5d6e9f345aced2e +ee6a44af15edc8d7968ed9241efd562fe2460fc697a941f05e928080b085 +78d2d515ab0bda61c8857c91ab5173e610bd21c26190b811f96f712cd714 +0976c3312d651e2c6de25090a183db7b4e28f26f810d71d96268620fb643 +f45abcfa92737ad45851117e42ecb734c3923a2d644520dccf472321680a +d68a41586fda02b6c7f609afc2a1223ac39a3b1b5183892bcb9c63ef8e10 +9e09b9e25fdecd31e4961a47437ce1fa32124f8e4dc5b09383b1f4b429c2 +124bc49dfe7a713174279fb111bdb6f3597bf1382c5974ffc72db135a70b +f6bdff014e65f8a3fc5e20eedffc10afca7ba0efb65fc598b519a24a354c +fc1cd386fd1035509f8cd36eadf612de47f68a72d5533130cd0abf8d72c3 +feaa40585af8a0fd390d8c22dba37af635f17de926b16089adb839f93e77 +37a20663cb4bb4971f0489fac50745d292a7c2425a62a6fc00d97206da78 +8c85d91703d0f6bc197a9a5f135609a9c2ad93103d1d83397f881a48f713 +da59b3a6880d9e59e271c83d91ed6e8ea9e50e683bdb031137c662ed2207 +a49976c41cf76ab176d15e31c77d6ec3a7fabffdfc07504b030414000000 +080000002100f7f8321f37080000809201000b0014006d656c5f3132382e +6e70790100100080920100000000003708000000000000eddd5b50d5e51a +c7f16579d8924a42b03d0f382a68208808acfff3ea2807451132cd54148d +143c6b8029a59862a6e27127b83304c23c651c3d109e4649c943a6846099 +8aa5a26e35b64e829a66bbf655c3acf5ae996ef4e2fbb959b0aede9bdffc +9e67ad973f6b2386870f19d5c034db34d76d424cc2f87837c3c54dc5faba +79b8b8c5ce8c9f151f3d63dcccf809317fbedf3f7a5a42cc1fef274c8a7e +2be68fdfdd7bf80478b8f878f7e8eae192e4f2b7d8994ca60541fb072913 +00ab2a1e3e2023800d41fb4f9013406367fa5a3202d8e0d3bc55efa77d06 +e0591652572c7d932fc8d33e07f02cdb99ee4897003614cedfc06e026804 +9d2b2123800d936b4ccc5c8046485d80bae9b5ca78dae7009e65f3f676a3 +4b001bfa268f613701342a9def9011c0867e5967c809a0b1ab38858c0036 +14ce6fc7fe0e68844d4993f64d4cf409a0e1dbcd9e2e016c2888de4c9700 +1a2175c56404b0e13fe31a3173011a03bdddd44daf16fc2d16a0f138b40b +5d02d8a04267b09b001a677b5e2723800d7d932f901340a3a8722119016c +f0bee3c2fe0e6864ef1f2f6dbf69499f001abb8aede812c086fce03cba04 +d018e0584046001b267ddb94990bd0089be2a4aa060ce72e17a091f49d2b +5d02d8d0bec902761340e3fb21556404b0a14f4c353901346654c79111c0 +8682e8ceecef804678a37672dfdd8d3e01341acd6d409700367ced7c9c2e +013406bbcd5532f207fe2f16a0e11be7cccc05d890b371133317a0f1df1d +29ea5da30f771e018dc4a58d99b9001b3ae41e62e602340a2ef8aa09fbbc +c809a051d4762f19016cd8e9d790dd04d070aefc25c0f0df479f001a39e1 +cdd4f1e96f93134023aafb7cd536f90572026878dec851871a44f2fd22a0 +51beb58c2e016ce8e2f22339013432b23d0c8fd0c3e404d0a8ce8d96f697 +72c909a0b1332d5f4675cc22278046446985b41bbb8e9c001a9fba5c91c5 +b73f242780469cc745d9ee9b4a4e008d5aa3540a3f5b4f4e008d889919d2 +f4c4467202688c58f88a6477c8571ff4dbc4b388002bdc6aff6d8c9ab65d +95a5bccffd2e4063e3c9d5aa479b1be404d048d810ab0abdbbb2a3001a93 +977753a696d1e404d0c8317d2f29ab56aaa1530fb1cb035634eed05d96f6 +4a5551370eb2a3001a39e991eaf34477662f40a3ec8d66ea6caf38f5386a +29b31760c5736f25ca94392bd4c2c9c798bd000dbbed7d5587a97ecc5e80 +46d1afe5d2a968912ab99142a7005694846418cb1bbcaad6ba75a053008d +265bae8bcff1592a3262199d02583161e557464c49b09a17e94aa7001ae1 +b5476563c45c35ec40119d0258b1fba51273ebae1d55f6f3862a9bde98ac +0056985ddf936bab5e5369ee0d99bf008d8f1a5e9493f6e355e4eb85740a +6045c49a4063493f47757cbaa15af7e279f68035910dcda27eebaed6ae6a +a3ecda5fe7ee1760c5caf3abc5fc71a03a97fa9c6a3fc893ac0056a437db +25ed6f47a89a895798bf008d219f9e109fdbc3544ed429b2025891dcfcb0 +ff40fb4a49ab1da8ca728f9015c08a4693369bcb3fad90cba1212acab994 +ac0056fce49961fe72fd09b91811a2b6ce3825db1eb764b7072c38f748fc +035b164bd0657f95b9f992a8b042b2025831cf315342d674517e87eecb9e +03ad99c3002b06b9cf95cc1e0eeaee2d3b15df3096ac00565cadee2e9327 +d6c9fa15ed54df9a0cc93a5310f0b4cf043c8b1efc52682caa3d2ccf7fe1 +a9beab3e29df76bacace0258517164a59c6a6eaffaa9dfa5e8f309cc6180 +152b1fba4b69932ab9d7b58daab4cf93e65e59740b60c147f92186d3d174 +6937cd5e7df2f92d399b30826e01ac702ef1924ba515f2ea7027d5ea6081 +f8459ca75b000b3e9be56c24952f939e534d2a685a9d84f44996150d7f32 +3fed7301cfa2fe9b1f1af9e5f9d238f345b573eb37d2f96a6f6631c08adc +5f7bcb6efbaf65cd9866aac69c2fb3df7e81bc00161cc8bc698e089b2ae3 +b32fc89c8c2752e5f4b13898cfb2bb00169c755a6c5c9db84886a654cbc9 +163fcbcf812bc5efc26ef2025890702ccf28e8b54c82f65c93f39daec8b6 +f34be4ccc01de405b0e0ddfefb8c7fd42c9138bb9f6474d31f25b1d93289 +e8738cbc0016e495e51a496317c8e9d04aa9a8bb2c6f39ac119fe27be405 +b0202a7699717469ac787b1d9178ff5bd2c3d82097de69c3e7638005f3d7 +ba1a65e981f2c6e43c31757a2067ab726464cf2071d8e443c700f5fcbe68 +4880b79fb3b8a7a54a807d953c9a5422ddc626c8de8aa3e405b060c2a323 +c6d20f6648de8003f26df839997d2f55fcfd5cc5aedd11eec400f52c78e2 +6aacb9e329070bd64bc0e40b52747b87c4ac1b2f7bb77c49c70016dcde5d +66bcb368a23c19be5396743e232f3aa549cfa6ddc5397b009901eab9d5ba +d6fce2e97f4a5cd25289742c956087bd72ec7c923459db44fc87a6309701 +f58439a51877170648e6e334092e3e2dadedb6c9cb51e364f0e08b740c60 +c12b57f719050fc364832943fa352e17bba11be4330997c42d87c90c504f +cda35affe52995c68799c3e48e644af494e332aff73af174ed2f47561d24 +33403dc7b25a981b6c3d67343a304482877d2cc1774bc5cf334d22ca43e5 +cdd453c6fb81c53c8f0cf88befb20e07ccda74c298bd24584655fd4bea12 +4be4fd551f49b2c76bf2604e8d5130ba9ccf00807aaeecdf6eb46ae52d97 +262d961da53b64f40f5932bdd344e994da4266da8f643e03ead9f2708eb1 +28fe25c9b83645c6e47e22ab53b74bb73717c8f2711eb2b036dfd89314c3 +7c06fcc5572b4c46a25db5d1ff7ab06cdbb45c722b77c9ab4e6be4a583af +48b3ad4f8c4b5942d700f584bebecef0ca692d86e724e938225d0283374a +d8b539e218fcb214071d305c7f5bcd5e03fcc517b10fcd03ee9f31e24ef8 +4a9ff5ef4923dfcd62d7fc4359ed38fa8f57873f7ffe7fd7fc0f504b0102 +14031400000008000000210081b9f70d78070000c0fb00000a0000000000 +0000000000008001000000006d656c5f38302e6e7079504b010214031400 +0000080000002100f7f8321f37080000809201000b000000000000000000 +00008001b40700006d656c5f3132382e6e7079504b050600000000020002 +0071000000281000000000 From 4eba1ae48d76fe324473a937bb07e242cc2fba8b Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Fri, 21 Aug 2026 16:16:24 +0200 Subject: [PATCH 05/45] [codex/vc-workflow] docs(stt): document validated fp16 bundles --- README.md | 6 ++++++ docs/STT_CONTRACT.md | 2 +- docs/TEAM_SETUP.md | 3 +++ docs/TRANSCRIPT_LANES.md | 32 ++++++++++++++++---------------- docs/WHISPER_LIVE.md | 2 ++ 5 files changed, 28 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 23ceac54..c3059809 100644 --- a/README.md +++ b/README.md @@ -348,6 +348,12 @@ The mlx-community repo ships only `config.json` + `weights.safetensors`; the download paths compose `tokenizer.json` from the matching official OpenAI Transformers repo and `mel_filters.npz` from a checksum-pinned OpenAI Whisper asset. The resulting directory is validated as unquantized before resolution. +The shared bundle validator parses the config and tokenizer, verifies the pinned +mel SHA-256, and validates the complete safetensors tensor table, dtype +allowlist, offsets, and file length. Downloads and warm-cache copies are written +to `.partial` files and promoted only after per-file validation; an invalid +destination is repaired on the next Download action instead of being accepted +as complete. `CODESCRIBE_EMBED_EMBEDDER=1` is an explicit fat/debug path that compiles MiniLM into Rust artifacts. Normal builds resolve MiniLM from the signed app resource or HF cache. `CODESCRIBE_NO_EMBED=1` disables every optional binary embed; Silero remains embedded. diff --git a/docs/STT_CONTRACT.md b/docs/STT_CONTRACT.md index 5e989500..14b664bf 100644 --- a/docs/STT_CONTRACT.md +++ b/docs/STT_CONTRACT.md @@ -37,7 +37,7 @@ Recording stopped before a transcript was available. | Layer | Rule | | --------------------- | -------------------------------------------------------------------------------------------------------- | -| Empty `speech.engine` | Load pins **`stt_engine=apple`**; persist **`final_pass_mode=off`** explicitly on current installs | +| Empty `speech.engine` | Load pins **`stt_engine=apple`**; persist **`final_pass_mode=off`** explicitly on current installs | | Settings UI write | **Promoted** to `settings.json` + reconciles process env **and** `.env` (single brain) | | Record start | **`preflight_apple_live_ready()`** when engine is Apple — refuse before REC if Speech/bridge not ready | | Live vs final | Cloud/Apple-only live fails closed without local weights; explicit HQ/local Retranscribe may use Whisper | diff --git a/docs/TEAM_SETUP.md b/docs/TEAM_SETUP.md index fbc43807..0ca08ca3 100644 --- a/docs/TEAM_SETUP.md +++ b/docs/TEAM_SETUP.md @@ -67,6 +67,9 @@ Grant in: System Settings > Privacy & Security - Runtime fallback resolves Whisper from exactly one shared contract in `core/config/models.rs`: `CODESCRIBE_MODEL_PATH` → configured local model path/alias → configured HF repo snapshot → default local turbo model → default HF cache snapshot. +- A model is ready only after config/tokenizer parsing, pinned mel checksum, and + full safetensors structural/dtype validation. Invalid cached files are replaced + through validated `.partial` artifacts on the next download. - `make install-app` / `scripts/ensure-models.sh` are the easiest way to warm the expected cache paths. **Developer note:** diff --git a/docs/TRANSCRIPT_LANES.md b/docs/TRANSCRIPT_LANES.md index aed0704d..2672b6e9 100644 --- a/docs/TRANSCRIPT_LANES.md +++ b/docs/TRANSCRIPT_LANES.md @@ -80,14 +80,14 @@ mic ▶ recorder ▶ [J1] ▶ Silero VAD chunker ▶ utterance boundaries ↑ Refine lane: correction.rs partial passes (VAD-aligned windows) ``` -| # | station | code | what happens | since | -| --- | ------------- | ---------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------- | -| B1 | VAD filter | `core/audio/chunker.rs` (Silero, embedded, zero-I/O) | detects WORDS, not noise; silence edges close utterances — “fundament stabilności” | doctrine §3.5 | -| B2 | scheduling | `core/stt/scheduler.rs :: SttScheduler` | Fast lane = utterance decode; Refine lane = correction re-decodes; per-lane `initial_prompt_for_lane` (⚑ OFF, W13-6A) | — | -| B3 | decode | `core/stt/whisper/singleton.rs` | in-process Whisper (turbo fp16 only; official OpenAI tokenizer + pinned mel asset); TTL reaper unloads 30 min after last finished decode (`whisper_residency_reclaim`) | fp16 only | -| B4 | corrections | `streaming/correction.rs` | Phase-2 Refine: partial passes triggered by finals/speech-ms, **VAD-aligned windows** (`plan_vad_aligned_windows_with_config`) so windows never begin mid-phrase | W1-A | -| B5 | postprocess | `core/pipeline/stream_postprocess.rs` | lexicon rewrite table (compiled-in seed/programming/operator/protected), hallucination + SemanticGate + empty-drop gates | — | -| B6 | canvas + user | **[J2]** → overlay | same reducer/emitter contract as LINE A | — | +| # | station | code | what happens | since | +| --- | ------------- | ---------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------- | +| B1 | VAD filter | `core/audio/chunker.rs` (Silero, embedded, zero-I/O) | detects WORDS, not noise; silence edges close utterances — “fundament stabilności” | doctrine §3.5 | +| B2 | scheduling | `core/stt/scheduler.rs :: SttScheduler` | Fast lane = utterance decode; Refine lane = correction re-decodes; per-lane `initial_prompt_for_lane` (⚑ OFF, W13-6A) | — | +| B3 | decode | `core/stt/whisper/singleton.rs` | in-process Whisper (turbo fp16 only; official OpenAI tokenizer + pinned mel asset); TTL reaper unloads 30 min after last finished decode (`whisper_residency_reclaim`) | fp16 only | +| B4 | corrections | `streaming/correction.rs` | Phase-2 Refine: partial passes triggered by finals/speech-ms, **VAD-aligned windows** (`plan_vad_aligned_windows_with_config`) so windows never begin mid-phrase | W1-A | +| B5 | postprocess | `core/pipeline/stream_postprocess.rs` | lexicon rewrite table (compiled-in seed/programming/operator/protected), hallucination + SemanticGate + empty-drop gates | — | +| B6 | canvas + user | **[J2]** → overlay | same reducer/emitter contract as LINE A | — | ## 3. LINE L1 — Layer 1 tail-patch (rides on top of A **and** B) @@ -197,15 +197,15 @@ audio file ▶ `codescribe transcribe` CLI / cloud final pass ## 10. What the user sees, surface by surface -| surface | fed by | truth it shows | -| ------------- | ------------------------------ | ---------------------------------------------------------------------------------------------------- | -| overlay LIVE | LINE A/B via J2 | letters as spoken; live backspace corrections (L1); never a rewrite of committed text | -| overlay FINAL | LINE S→F via J7 | formatted draft + buttons (Copy / Insert / Revert / Format / To Agent); Auto Paste when guard allows | +| surface | fed by | truth it shows | +| ------------- | ------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------ | +| overlay LIVE | LINE A/B via J2 | letters as spoken; live backspace corrections (L1); never a rewrite of committed text | +| overlay FINAL | LINE S→F via J7 | formatted draft + buttons (Copy / Insert / Revert / Format / To Agent); Auto Paste when guard allows | | paste target | J7 | formatted text into the latched foreign app; refuse (`CopyTargetUnavailable` / mismatch) ⇒ Paste Here slot, user clipboard untouched | -| thread rail | LINE G | agent conversation, chained turn by turn | -| history dir | J6 | `~/.codescribe/transcriptions//` — raw, formatted, m4a, truth receipts | -| menu/tray | controller state | recording state; Audio truth section (W13-5: device, level, quality verdict) | -| warnings | `contracts.rs` warning classes | ONLY `transcription_failed` is terminal/UI; receipts (capture level, tail patch, seal) are log-only | +| thread rail | LINE G | agent conversation, chained turn by turn | +| history dir | J6 | `~/.codescribe/transcriptions//` — raw, formatted, m4a, truth receipts | +| menu/tray | controller state | recording state; Audio truth section (W13-5: device, level, quality verdict) | +| warnings | `contracts.rs` warning classes | ONLY `transcription_failed` is terminal/UI; receipts (capture level, tail patch, seal) are log-only | --- diff --git a/docs/WHISPER_LIVE.md b/docs/WHISPER_LIVE.md index cd2d9910..af76d656 100644 --- a/docs/WHISPER_LIVE.md +++ b/docs/WHISPER_LIVE.md @@ -35,6 +35,8 @@ See the ADR for the full contract. Codescribe’s Whisper layer power-ups: 1. **FP16-only Whisper model** (`whisper-large-v3-turbo`, mlx-community weights; q8 is rejected before load) + - readiness requires a parsed config and tokenizer, pinned mel SHA-256, and a + structurally complete safetensors file containing only supported runtime dtypes - build policy embeds Whisper whenever the model is available at build time - runtime lookup from `CODESCRIBE_MODEL_PATH`, configured model dirs, bundled app resources, or the Hugging Face cache is a fallback path for `CODESCRIBE_NO_EMBED=1` builds or recovery 2. **Live (streaming) transcription** while the user is recording From 38fb2dec832b24dfef1cf134e25ebc9f4594427d Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Fri, 21 Aug 2026 16:21:54 +0200 Subject: [PATCH 06/45] [codex/vc-workflow] test(stt): satisfy rust 1.98 clippy --- core/config/models.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/config/models.rs b/core/config/models.rs index 565d0d97..cfabd651 100644 --- a/core/config/models.rs +++ b/core/config/models.rs @@ -868,7 +868,9 @@ mod tests { assert!(digits.len().is_multiple_of(2)); digits .as_bytes() - .chunks_exact(2) + .as_chunks::<2>() + .0 + .iter() .map(|pair| u8::from_str_radix(std::str::from_utf8(pair).unwrap(), 16).unwrap()) .collect() } From e0063b7461f34350fb18b8fc638eb0747707d932 Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Fri, 21 Aug 2026 19:41:38 +0200 Subject: [PATCH 07/45] [codex/vc-workflow] fix(stt): select a valid weights alternative (PR #81 review) --- core/config/models.rs | 94 +++++++++++++++++++++++++++++++++----- core/stt/whisper/engine.rs | 33 ++++++++++--- 2 files changed, 110 insertions(+), 17 deletions(-) diff --git a/core/config/models.rs b/core/config/models.rs index cfabd651..cd45720a 100644 --- a/core/config/models.rs +++ b/core/config/models.rs @@ -74,8 +74,7 @@ fn validate_whisper_model_bundle(path: &Path) -> Result<()> { let mel_path = path.join("mel_filters.npz"); verify_sha256(&mel_path, MEL_FILTERS_SHA256)?; - let weights_path = resolve_weights_path(path)?; - validate_safetensors_file(&weights_path) + resolve_valid_whisper_weights_path(path).map(|_| ()) } /// Reject unsupported or malformed weights before the expensive engine load. @@ -83,9 +82,7 @@ fn validate_whisper_model_bundle(path: &Path) -> Result<()> { /// tokenizer and mel errors retain their own loader diagnostics. pub(crate) fn is_unquantized_whisper_model_dir(path: &Path) -> bool { validate_whisper_config(&path.join("config.json")).is_ok() - && resolve_weights_path(path) - .and_then(|weights| validate_safetensors_file(&weights)) - .is_ok() + && resolve_valid_whisper_weights_path(path).is_ok() } fn validate_whisper_config(path: &Path) -> Result<()> { @@ -112,12 +109,36 @@ fn validate_whisper_config(path: &Path) -> Result<()> { Ok(()) } -fn resolve_weights_path(path: &Path) -> Result { - REQUIRED_MODEL_WEIGHTS - .iter() - .map(|name| path.join(name)) - .find(|candidate| candidate.is_file()) - .ok_or_else(|| anyhow!("Whisper weights are missing from {}", path.display())) +/// Resolve the first structurally valid supported weight file. +/// +/// Upstream snapshots may contain either filename, and stale composition can +/// leave both behind. Preserve the documented filename priority, but never let +/// an invalid primary shadow a valid alternative that the runtime can load. +pub(crate) fn resolve_valid_whisper_weights_path(path: &Path) -> Result { + let mut failures = Vec::new(); + for name in REQUIRED_MODEL_WEIGHTS { + let candidate = path.join(name); + if !candidate.is_file() { + continue; + } + match validate_safetensors_file(&candidate) { + Ok(()) => return Ok(candidate), + Err(err) => failures.push(format!("{name}: {err:#}")), + } + } + + if failures.is_empty() { + Err(anyhow!( + "Whisper weights are missing from {}", + path.display() + )) + } else { + Err(anyhow!( + "no valid Whisper weights in {} ({})", + path.display(), + failures.join("; ") + )) + } } /// Validate the complete safetensors structure without loading the tensor data. @@ -1014,6 +1035,57 @@ mod tests { assert!(!is_complete_whisper_model_dir(&model)); } + /// A stale invalid primary filename must not shadow a valid alternative. + #[test] + fn model_manager_uses_valid_alternative_weights() { + let temp_dir = TempDir::new().unwrap(); + let model = temp_dir.path().join("model"); + create_complete_whisper_model(&model); + fs::write(model.join("weights.safetensors"), b"stale invalid weights").unwrap(); + + let resolved = resolve_valid_whisper_weights_path(&model).unwrap(); + assert_eq!( + resolved.file_name().and_then(|name| name.to_str()), + Some("model.safetensors") + ); + assert!(is_complete_whisper_model_dir(&model)); + } + + /// Filename priority remains deterministic when both alternatives validate. + #[test] + fn model_manager_prefers_valid_primary_weights() { + let temp_dir = TempDir::new().unwrap(); + let model = temp_dir.path().join("model"); + create_complete_whisper_model(&model); + fs::copy( + model.join("model.safetensors"), + model.join("weights.safetensors"), + ) + .unwrap(); + + let resolved = resolve_valid_whisper_weights_path(&model).unwrap(); + assert_eq!( + resolved.file_name().and_then(|name| name.to_str()), + Some("weights.safetensors") + ); + } + + /// Existing alternatives do not count when neither payload is valid. + #[test] + fn model_manager_rejects_all_invalid_weight_alternatives() { + let temp_dir = TempDir::new().unwrap(); + let model = temp_dir.path().join("model"); + create_complete_whisper_model(&model); + fs::write(model.join("weights.safetensors"), b"bad primary").unwrap(); + fs::write(model.join("model.safetensors"), b"bad alternative").unwrap(); + + let err = resolve_valid_whisper_weights_path(&model).unwrap_err(); + let message = format!("{err:#}"); + assert!(message.contains("weights.safetensors")); + assert!(message.contains("model.safetensors")); + assert!(!is_complete_whisper_model_dir(&model)); + } + /// Metadata alone is not a model and must not satisfy discovery. #[test] fn model_manager_rejects_safetensors_without_tensor_entries() { diff --git a/core/stt/whisper/engine.rs b/core/stt/whisper/engine.rs index a930f32c..2151012a 100644 --- a/core/stt/whisper/engine.rs +++ b/core/stt/whisper/engine.rs @@ -440,12 +440,9 @@ impl LocalWhisperEngine { if !config_path.is_file() { anyhow::bail!("Whisper config not found at {}", config_path.display()); } - let weights_path = if model_path.join("weights.safetensors").exists() { - model_path.join("weights.safetensors") - } else { - model_path.join("model.safetensors") - }; - if !weights_path.exists() { + if !model_path.join("weights.safetensors").is_file() + && !model_path.join("model.safetensors").is_file() + { anyhow::bail!( "Whisper weights not found (expected weights.safetensors or model.safetensors) in {}", model_path.display() @@ -458,6 +455,8 @@ impl LocalWhisperEngine { "Quantized or malformed Whisper payload refused before tensor load; install the complete fp16 model" ); } + let weights_path = crate::config::models::resolve_valid_whisper_weights_path(model_path) + .context("resolve validated Whisper weights")?; let device = process_device(); tracing::debug!("LocalWhisperEngine using device: {:?}", device); @@ -2039,6 +2038,28 @@ mod model_payload_tests { .expect("U32 must be refused by the builder gate"); assert!(format!("{err:#}").contains("refused")); } + + #[test] + fn local_loader_uses_valid_alternative_after_invalid_primary() { + let temp = TempDir::new().unwrap(); + write_tiny_model(temp.path(), "encoder.weight", "F16", 2); + fs::rename( + temp.path().join("weights.safetensors"), + temp.path().join("model.safetensors"), + ) + .unwrap(); + write_tiny_model(temp.path(), "encoder.weight", "U32", 4); + + let err = LocalWhisperEngine::new(temp.path()) + .err() + .expect("tiny valid alternative should reach model construction"); + let message = format!("{err:#}"); + assert!(!message.contains("payload refused"), "{message}"); + assert!( + message.contains("Failed to create Whisper Model"), + "{message}" + ); + } } // ═══════════════════════════════════════════════════════════════════════════════ From e8f0bf01c01d1229617dfec9f3be3b7605c5530d Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Fri, 21 Aug 2026 19:43:15 +0200 Subject: [PATCH 08/45] [codex/vc-workflow] fix(stt): skip invalid cached snapshots (PR #81 review) --- core/config/models.rs | 9 ++- core/hf_cache.rs | 147 +++++++++++++++++++++++++++++++++++++++--- 2 files changed, 143 insertions(+), 13 deletions(-) diff --git a/core/config/models.rs b/core/config/models.rs index cd45720a..a5ebfd4f 100644 --- a/core/config/models.rs +++ b/core/config/models.rs @@ -292,9 +292,12 @@ fn hf_snapshot_for_model(model_ref: &str) -> Option { } else { return None; }; - let snapshot = - hf_cache::find_snapshot_with_any(repo, &REQUIRED_MODEL_FILES, &REQUIRED_MODEL_WEIGHTS)?; - is_complete_whisper_model_dir(&snapshot).then_some(snapshot) + hf_cache::find_snapshot_with_any_matching( + repo, + &REQUIRED_MODEL_FILES, + &REQUIRED_MODEL_WEIGHTS, + is_complete_whisper_model_dir, + ) } /// Owner of the resolved runtime models directory. diff --git a/core/hf_cache.rs b/core/hf_cache.rs index 86ef6d74..55f0c125 100644 --- a/core/hf_cache.rs +++ b/core/hf_cache.rs @@ -5,6 +5,7 @@ use std::env; use std::fs; +use std::path::Path; use std::path::PathBuf; use std::time::SystemTime; @@ -26,8 +27,41 @@ pub fn find_snapshot_with_any( required_all: &[&str], required_any: &[&str], ) -> Option { - for base in cache_bases() { - if let Some(snapshot) = find_snapshot_in_base(&base, repo, required_all, required_any) { + find_snapshot_with_any_matching(repo, required_all, required_any, |_| true) +} + +/// Locate the first file-complete snapshot accepted by `predicate`. +/// +/// Cache-root precedence remains stable. Within each root, candidates are +/// examined newest first so an invalid fresh download can fall back to an +/// older usable revision without allowing a later cache root to jump ahead. +pub fn find_snapshot_with_any_matching( + repo: &str, + required_all: &[&str], + required_any: &[&str], + predicate: F, +) -> Option +where + F: Fn(&Path) -> bool, +{ + find_snapshot_in_bases_matching(cache_bases(), repo, required_all, required_any, &predicate) +} + +fn find_snapshot_in_bases_matching( + bases: I, + repo: &str, + required_all: &[&str], + required_any: &[&str], + predicate: &F, +) -> Option +where + I: IntoIterator, + F: Fn(&Path) -> bool, +{ + for base in bases { + if let Some(snapshot) = + find_snapshot_in_base_matching(&base, repo, required_all, required_any, predicate) + { return Some(snapshot); } } @@ -83,12 +117,16 @@ fn cache_bases() -> Vec { /// original casing, so a miss falls back to a case-insensitive scan rather than /// reporting the model absent. Among qualifying snapshots the most recently /// modified wins — the cache can hold several revisions at once. -fn find_snapshot_in_base( +fn find_snapshot_in_base_matching( base: &PathBuf, repo: &str, required_all: &[&str], required_any: &[&str], -) -> Option { + predicate: &F, +) -> Option +where + F: Fn(&Path) -> bool, +{ let repo_dir = base.join(format!("models--{}", repo.replace('/', "--"))); let snapshots_dir = repo_dir.join("snapshots"); @@ -120,7 +158,7 @@ fn find_snapshot_in_base( let entries = fs::read_dir(&snapshots_dir).ok()?; - let mut best: Option<(SystemTime, PathBuf)> = None; + let mut candidates = Vec::new(); for entry in entries.flatten() { let path = entry.path(); @@ -137,11 +175,100 @@ fn find_snapshot_in_base( .metadata() .and_then(|m| m.modified()) .unwrap_or(SystemTime::UNIX_EPOCH); - match &best { - Some((best_time, _)) if *best_time >= modified => {} - _ => best = Some((modified, path)), - } + candidates.push((modified, path)); } - best.map(|(_, p)| p) + candidates.sort_by(|(left_time, left_path), (right_time, right_path)| { + right_time + .cmp(left_time) + .then_with(|| left_path.cmp(right_path)) + }); + candidates + .into_iter() + .map(|(_, path)| path) + .find(|path| predicate(path)) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs::FileTimes; + use std::time::Duration; + use tempfile::TempDir; + + fn snapshot(base: &Path, repo: &str, name: &str, modified_secs: u64) -> PathBuf { + let path = base + .join(format!("models--{}", repo.replace('/', "--"))) + .join("snapshots") + .join(name); + fs::create_dir_all(&path).unwrap(); + fs::write(path.join("config.json"), "{}").unwrap(); + let times = FileTimes::new() + .set_modified(SystemTime::UNIX_EPOCH + Duration::from_secs(modified_secs)); + fs::File::open(&path).unwrap().set_times(times).unwrap(); + path + } + + #[test] + fn matching_snapshot_falls_back_from_invalid_newest_to_valid_older() { + let temp = TempDir::new().unwrap(); + let repo = "owner/model"; + let older = snapshot(temp.path(), repo, "older", 10); + let newer = snapshot(temp.path(), repo, "newer", 20); + fs::write(older.join("valid.marker"), "yes").unwrap(); + + let found = find_snapshot_in_bases_matching( + [temp.path().to_path_buf()], + repo, + &["config.json"], + &[], + &|path| path.join("valid.marker").is_file(), + ) + .unwrap(); + + assert_eq!(found, older); + assert_ne!(found, newer); + } + + #[test] + fn matching_snapshot_keeps_newest_valid_candidate() { + let temp = TempDir::new().unwrap(); + let repo = "owner/model"; + let older = snapshot(temp.path(), repo, "older", 10); + let newer = snapshot(temp.path(), repo, "newer", 20); + fs::write(older.join("valid.marker"), "yes").unwrap(); + fs::write(newer.join("valid.marker"), "yes").unwrap(); + + let found = find_snapshot_in_bases_matching( + [temp.path().to_path_buf()], + repo, + &["config.json"], + &[], + &|path| path.join("valid.marker").is_file(), + ) + .unwrap(); + + assert_eq!(found, newer); + } + + #[test] + fn invalid_earlier_cache_root_does_not_shadow_valid_later_root() { + let first = TempDir::new().unwrap(); + let second = TempDir::new().unwrap(); + let repo = "owner/model"; + snapshot(first.path(), repo, "invalid", 20); + let valid = snapshot(second.path(), repo, "valid", 10); + fs::write(valid.join("valid.marker"), "yes").unwrap(); + + let found = find_snapshot_in_bases_matching( + [first.path().to_path_buf(), second.path().to_path_buf()], + repo, + &["config.json"], + &[], + &|path| path.join("valid.marker").is_file(), + ) + .unwrap(); + + assert_eq!(found, valid); + } } From f064d5f903f498aa106060f452ad97fb17348106 Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Fri, 21 Aug 2026 19:45:37 +0200 Subject: [PATCH 09/45] [codex/vc-workflow] fix(stt): validate safetensors metadata (PR #81 review) --- core/config/models.rs | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/core/config/models.rs b/core/config/models.rs index a5ebfd4f..7c34a597 100644 --- a/core/config/models.rs +++ b/core/config/models.rs @@ -169,6 +169,19 @@ fn validate_safetensors_file(path: &Path) -> Result<()> { )); }; + if let Some(metadata) = tensors.get("__metadata__") { + let valid = metadata.is_null() + || metadata + .as_object() + .is_some_and(|entries| entries.values().all(serde_json::Value::is_string)); + if !valid { + return Err(anyhow!( + "invalid safetensors __metadata__ in {}", + path.display() + )); + } + } + let file_len = file.metadata()?.len(); let data_start = 8_u64 .checked_add(header_len) @@ -1103,6 +1116,36 @@ mod tests { assert!(!is_complete_whisper_model_dir(&model)); } + /// Discovery must enforce the metadata schema used by the runtime loader. + #[test] + fn model_manager_rejects_malformed_safetensors_metadata() { + let temp_dir = TempDir::new().unwrap(); + let model = temp_dir.path().join("model"); + create_complete_whisper_model(&model); + let header = br#"{"__metadata__":{"format":1},"model.weight":{"dtype":"F16","shape":[1],"data_offsets":[0,2]}}"#; + let mut safetensors = (header.len() as u64).to_le_bytes().to_vec(); + safetensors.extend_from_slice(header); + safetensors.extend_from_slice(&[0, 0]); + fs::write(model.join("model.safetensors"), safetensors).unwrap(); + + assert!(!is_complete_whisper_model_dir(&model)); + } + + /// String-valued safetensors metadata is compatible with the runtime loader. + #[test] + fn model_manager_accepts_string_safetensors_metadata() { + let temp_dir = TempDir::new().unwrap(); + let model = temp_dir.path().join("model"); + create_complete_whisper_model(&model); + let header = br#"{"__metadata__":{"format":"mlx"},"model.weight":{"dtype":"F16","shape":[1],"data_offsets":[0,2]}}"#; + let mut safetensors = (header.len() as u64).to_le_bytes().to_vec(); + safetensors.extend_from_slice(header); + safetensors.extend_from_slice(&[0, 0]); + fs::write(model.join("model.safetensors"), safetensors).unwrap(); + + assert!(is_complete_whisper_model_dir(&model)); + } + /// Header offsets must describe the actual payload, not a truncated file. #[test] fn model_manager_rejects_truncated_safetensors_payload() { From 1f2d0b329d663f0d47633aaef9e5234068ec924c Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Fri, 21 Aug 2026 19:46:19 +0200 Subject: [PATCH 10/45] [codex/vc-workflow] fix(stt): reject empty model tensors --- core/config/models.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/core/config/models.rs b/core/config/models.rs index 7c34a597..b1801ccd 100644 --- a/core/config/models.rs +++ b/core/config/models.rs @@ -227,6 +227,9 @@ fn validate_safetensors_file(path: &Path) -> Result<()> { .checked_mul(dim) .ok_or_else(|| anyhow!("tensor {name} shape overflows")) })?; + if element_count == 0 { + return Err(anyhow!("tensor {name} has an empty shape")); + } let expected_bytes = element_count .checked_mul(bytes_per_element) .ok_or_else(|| anyhow!("tensor {name} byte size overflows"))?; @@ -1131,6 +1134,20 @@ mod tests { assert!(!is_complete_whisper_model_dir(&model)); } + /// Structurally empty tensors cannot represent a loadable Whisper model. + #[test] + fn model_manager_rejects_zero_element_tensor() { + let temp_dir = TempDir::new().unwrap(); + let model = temp_dir.path().join("model"); + create_complete_whisper_model(&model); + let header = br#"{"model.weight":{"dtype":"F16","shape":[0],"data_offsets":[0,0]}}"#; + let mut safetensors = (header.len() as u64).to_le_bytes().to_vec(); + safetensors.extend_from_slice(header); + fs::write(model.join("model.safetensors"), safetensors).unwrap(); + + assert!(!is_complete_whisper_model_dir(&model)); + } + /// String-valued safetensors metadata is compatible with the runtime loader. #[test] fn model_manager_accepts_string_safetensors_metadata() { From 189b51f232d15fbded133f5a3405670fce80e801 Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Fri, 21 Aug 2026 19:46:55 +0200 Subject: [PATCH 11/45] [codex/vc-workflow] docs(stt): align final-pass default (PR #81 review) --- docs/STT_CONTRACT.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/STT_CONTRACT.md b/docs/STT_CONTRACT.md index 14b664bf..dbac8577 100644 --- a/docs/STT_CONTRACT.md +++ b/docs/STT_CONTRACT.md @@ -37,7 +37,7 @@ Recording stopped before a transcript was available. | Layer | Rule | | --------------------- | -------------------------------------------------------------------------------------------------------- | -| Empty `speech.engine` | Load pins **`stt_engine=apple`**; persist **`final_pass_mode=off`** explicitly on current installs | +| Empty `speech.engine` | Load defaults to **`stt_engine=apple`** and **`final_pass_mode=smart`**; explicit saved values still win | | Settings UI write | **Promoted** to `settings.json` + reconciles process env **and** `.env` (single brain) | | Record start | **`preflight_apple_live_ready()`** when engine is Apple — refuse before REC if Speech/bridge not ready | | Live vs final | Cloud/Apple-only live fails closed without local weights; explicit HQ/local Retranscribe may use Whisper | From 6de812820ddb972b65d3d8a8ad471f57ed92ef89 Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Fri, 21 Aug 2026 19:49:06 +0200 Subject: [PATCH 12/45] [codex/vc-workflow] fix(stt): validate release model bundles (PR #81 review) --- core/Cargo.toml | 4 ++++ core/bin/codescribe-whisper-validate.rs | 21 +++++++++++++++++++ core/config/models.rs | 2 +- docs/TEAM_SETUP.md | 2 ++ scripts/ensure-models.sh | 28 +++++++++++++++++-------- scripts/validate-whisper-model.sh | 15 +++++++++++++ 6 files changed, 62 insertions(+), 10 deletions(-) create mode 100644 core/bin/codescribe-whisper-validate.rs create mode 100755 scripts/validate-whisper-model.sh diff --git a/core/Cargo.toml b/core/Cargo.toml index c8e25fbb..13702803 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -18,6 +18,10 @@ path = "lib.rs" name = "codescribe-stt-sidecar" path = "bin/codescribe-stt-sidecar.rs" +[[bin]] +name = "codescribe-whisper-validate" +path = "bin/codescribe-whisper-validate.rs" + [features] default = [] offline_eval = [] diff --git a/core/bin/codescribe-whisper-validate.rs b/core/bin/codescribe-whisper-validate.rs new file mode 100644 index 00000000..7e045ca4 --- /dev/null +++ b/core/bin/codescribe-whisper-validate.rs @@ -0,0 +1,21 @@ +//! Validate a composed Whisper model with the runtime's canonical contract. + +use anyhow::{Context, Result, anyhow}; +use codescribe_core::config::models::validate_whisper_model_bundle; +use std::path::PathBuf; + +fn main() -> Result<()> { + let mut args = std::env::args_os().skip(1); + let path = args + .next() + .map(PathBuf::from) + .ok_or_else(|| anyhow!("usage: codescribe-whisper-validate "))?; + if args.next().is_some() { + return Err(anyhow!( + "usage: codescribe-whisper-validate " + )); + } + + validate_whisper_model_bundle(&path) + .with_context(|| format!("invalid Whisper model bundle: {}", path.display())) +} diff --git a/core/config/models.rs b/core/config/models.rs index b1801ccd..0640ca9a 100644 --- a/core/config/models.rs +++ b/core/config/models.rs @@ -59,7 +59,7 @@ fn is_complete_whisper_model_dir(path: &Path) -> bool { /// Safetensors validation is structural rather than cryptographic: the format /// has no payload checksum. The validator checks the complete tensor table, /// dtype allowlist, byte sizes, contiguous offsets, and final file length. -fn validate_whisper_model_bundle(path: &Path) -> Result<()> { +pub fn validate_whisper_model_bundle(path: &Path) -> Result<()> { let config_path = path.join("config.json"); validate_whisper_config(&config_path)?; diff --git a/docs/TEAM_SETUP.md b/docs/TEAM_SETUP.md index 0ca08ca3..46ddf82d 100644 --- a/docs/TEAM_SETUP.md +++ b/docs/TEAM_SETUP.md @@ -71,6 +71,8 @@ Grant in: System Settings > Privacy & Security full safetensors structural/dtype validation. Invalid cached files are replaced through validated `.partial` artifacts on the next download. - `make install-app` / `scripts/ensure-models.sh` are the easiest way to warm the expected cache paths. + Release/setup preflight calls the same runtime-owned bundle validator before it skips a download; + a directory with merely the expected filenames is repaired or rejected rather than embedded. **Developer note:** If runtime lookup cannot find the model, point `CODESCRIBE_MODEL_PATH` at a valid Whisper directory. diff --git a/scripts/ensure-models.sh b/scripts/ensure-models.sh index 8232ad16..13ebb471 100755 --- a/scripts/ensure-models.sh +++ b/scripts/ensure-models.sh @@ -6,20 +6,32 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)" +WHISPER_VALIDATOR="$ROOT_DIR/scripts/validate-whisper-model.sh" + +valid_whisper_bundle() { + "$WHISPER_VALIDATOR" "$1" +} # Prefer explicit path override for Whisper if [[ -n "${CODESCRIBE_MODEL_PATH:-}" ]]; then - if [[ -f "${CODESCRIBE_MODEL_PATH}/config.json" ]]; then + if valid_whisper_bundle "$CODESCRIBE_MODEL_PATH"; then echo "✓ Whisper model found via CODESCRIBE_MODEL_PATH (${CODESCRIBE_MODEL_PATH})" WHISPER_OK=1 + else + echo "⚠ CODESCRIBE_MODEL_PATH is not a valid Whisper bundle; continuing with product model" >&2 fi fi # If embed model points to a local directory, treat as satisfied. if [[ -z "${WHISPER_OK:-}" && -n "${CODESCRIBE_EMBED_MODEL:-}" ]]; then - if [[ -d "${CODESCRIBE_EMBED_MODEL}" && -f "${CODESCRIBE_EMBED_MODEL}/config.json" ]]; then - echo "✓ Whisper model found via CODESCRIBE_EMBED_MODEL (${CODESCRIBE_EMBED_MODEL})" - WHISPER_OK=1 + if [[ -d "${CODESCRIBE_EMBED_MODEL}" ]]; then + if valid_whisper_bundle "$CODESCRIBE_EMBED_MODEL"; then + echo "✓ Whisper model found via CODESCRIBE_EMBED_MODEL (${CODESCRIBE_EMBED_MODEL})" + WHISPER_OK=1 + else + echo "ERROR: CODESCRIBE_EMBED_MODEL is not a valid Whisper bundle: ${CODESCRIBE_EMBED_MODEL}" >&2 + exit 1 + fi fi fi @@ -90,14 +102,12 @@ EMBEDDER_REPO="${CODESCRIBE_EMBEDDER_REPO:-sentence-transformers/paraphrase-mult if [[ "${WHISPER_OK:-0}" -ne 1 ]]; then if [[ "$WHISPER_REPO" == "mlx-community/whisper-large-v3-turbo" ]]; then COMPOSED_MODEL="${CODESCRIBE_MODELS_DIR:-$HOME/.codescribe/models}/whisper-large-v3-turbo" - if [[ -f "$COMPOSED_MODEL/config.json" \ - && -f "$COMPOSED_MODEL/tokenizer.json" \ - && -f "$COMPOSED_MODEL/mel_filters.npz" \ - && ( -f "$COMPOSED_MODEL/weights.safetensors" || -f "$COMPOSED_MODEL/model.safetensors" ) ]]; then + if valid_whisper_bundle "$COMPOSED_MODEL"; then echo "✓ Whisper fp16 composed ($COMPOSED_MODEL)" else - echo "▶ Whisper fp16 runtime directory incomplete; composing it..." + echo "▶ Whisper fp16 runtime directory missing or invalid; composing it..." "$ROOT_DIR/scripts/download-model.sh" + valid_whisper_bundle "$COMPOSED_MODEL" fi else ensure_repo "Whisper" "$WHISPER_REPO" config.json tokenizer.json mel_filters.npz __ANY_SAFETENSORS__ diff --git a/scripts/validate-whisper-model.sh b/scripts/validate-whisper-model.sh new file mode 100755 index 00000000..26114228 --- /dev/null +++ b/scripts/validate-whisper-model.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# Run the runtime-owned Whisper bundle validator for release/setup scripts. + +set -euo pipefail + +if [[ "$#" -ne 1 ]]; then + echo "usage: $0 " >&2 + exit 2 +fi + +ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)" + +cd "$ROOT_DIR" +CODESCRIBE_NO_EMBED=1 cargo run --quiet -p codescribe-core \ + --bin codescribe-whisper-validate -- "$1" From 65df3fd32dc4cc28d6ca083ec2eb094f90171392 Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Fri, 21 Aug 2026 19:50:35 +0200 Subject: [PATCH 13/45] [codex/vc-workflow] fix(stt): fail closed on invalid model overrides --- docs/TEAM_SETUP.md | 2 ++ scripts/download-model.sh | 30 +++++++++++++++++++++++++++--- scripts/ensure-models.sh | 34 ++++++++++++++++++++++++++++++++-- 3 files changed, 61 insertions(+), 5 deletions(-) diff --git a/docs/TEAM_SETUP.md b/docs/TEAM_SETUP.md index 46ddf82d..c28c9c7f 100644 --- a/docs/TEAM_SETUP.md +++ b/docs/TEAM_SETUP.md @@ -73,6 +73,8 @@ Grant in: System Settings > Privacy & Security - `make install-app` / `scripts/ensure-models.sh` are the easiest way to warm the expected cache paths. Release/setup preflight calls the same runtime-owned bundle validator before it skips a download; a directory with merely the expected filenames is repaired or rejected rather than embedded. + An explicit local `CODESCRIBE_EMBED_MODEL` must exist and pass that validator; it is never + reinterpreted as a Hugging Face repository id. **Developer note:** If runtime lookup cannot find the model, point `CODESCRIBE_MODEL_PATH` at a valid Whisper directory. diff --git a/scripts/download-model.sh b/scripts/download-model.sh index 7665eefe..b089e55c 100755 --- a/scripts/download-model.sh +++ b/scripts/download-model.sh @@ -15,6 +15,15 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)" +WHISPER_VALIDATOR="$ROOT_DIR/scripts/validate-whisper-model.sh" + +is_hf_repo_id() { + [[ "$1" =~ ^[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$ ]] +} + +looks_like_local_path() { + [[ "$1" == /* || "$1" == ./* || "$1" == ../* || "$1" == ~/* ]] +} sha256_file() { if command -v shasum >/dev/null 2>&1; then @@ -44,14 +53,22 @@ MODEL_REPO="${CODESCRIBE_EMBED_MODEL:-$DEFAULT_REPO}" # If CODESCRIBE_EMBED_MODEL points to a local path, skip download. if [[ -n "${CODESCRIBE_EMBED_MODEL:-}" ]] && [[ -d "${CODESCRIBE_EMBED_MODEL}" ]]; then - if [[ -f "${CODESCRIBE_EMBED_MODEL}/config.json" ]]; then + if "$WHISPER_VALIDATOR" "$CODESCRIBE_EMBED_MODEL"; then echo "✓ Whisper model found at ${CODESCRIBE_EMBED_MODEL} (local path). Skipping download." exit 0 fi + echo "ERROR: CODESCRIBE_EMBED_MODEL is not a valid Whisper bundle: ${CODESCRIBE_EMBED_MODEL}" >&2 + exit 1 fi -# If override isn't an HF repo, fall back to default repo. -if [[ "$MODEL_REPO" != */* ]]; then +# An explicit path must never be passed to `hf download` as a repository id. +if [[ -n "${CODESCRIBE_EMBED_MODEL:-}" ]] && looks_like_local_path "$MODEL_REPO"; then + echo "ERROR: CODESCRIBE_EMBED_MODEL local path does not exist: $MODEL_REPO" >&2 + exit 1 +fi + +# A plain model alias keeps the historical default; only owner/repo selects HF. +if ! is_hf_repo_id "$MODEL_REPO"; then MODEL_REPO="$DEFAULT_REPO" fi @@ -103,6 +120,10 @@ echo " This may take a few minutes..." echo "" MODEL_SNAPSHOT=$("$HF_BIN" download "$MODEL_REPO" --quiet) +if [[ -z "$MODEL_SNAPSHOT" || "$MODEL_SNAPSHOT" == *$'\n'* || ! -d "$MODEL_SNAPSHOT" ]]; then + echo "ERROR: hf download did not return one snapshot directory for $MODEL_REPO" >&2 + exit 1 +fi # The default conversion ships only config + fp16 weights. Compose one # self-contained product directory using the matching official OpenAI @@ -133,6 +154,9 @@ if [[ "$MODEL_REPO" == "$DEFAULT_REPO" ]]; then fi mv "$MODEL_DEST/mel_filters.npz.partial" "$MODEL_DEST/mel_filters.npz" echo " Runtime directory: $MODEL_DEST" + "$WHISPER_VALIDATOR" "$MODEL_DEST" +else + "$WHISPER_VALIDATOR" "$MODEL_SNAPSHOT" fi echo "" diff --git a/scripts/ensure-models.sh b/scripts/ensure-models.sh index 13ebb471..375012fe 100755 --- a/scripts/ensure-models.sh +++ b/scripts/ensure-models.sh @@ -12,6 +12,14 @@ valid_whisper_bundle() { "$WHISPER_VALIDATOR" "$1" } +is_hf_repo_id() { + [[ "$1" =~ ^[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$ ]] +} + +looks_like_local_path() { + [[ "$1" == /* || "$1" == ./* || "$1" == ../* || "$1" == ~/* ]] +} + # Prefer explicit path override for Whisper if [[ -n "${CODESCRIBE_MODEL_PATH:-}" ]]; then if valid_whisper_bundle "$CODESCRIBE_MODEL_PATH"; then @@ -32,6 +40,9 @@ if [[ -z "${WHISPER_OK:-}" && -n "${CODESCRIBE_EMBED_MODEL:-}" ]]; then echo "ERROR: CODESCRIBE_EMBED_MODEL is not a valid Whisper bundle: ${CODESCRIBE_EMBED_MODEL}" >&2 exit 1 fi + elif looks_like_local_path "$CODESCRIBE_EMBED_MODEL"; then + echo "ERROR: CODESCRIBE_EMBED_MODEL local path does not exist: ${CODESCRIBE_EMBED_MODEL}" >&2 + exit 1 fi fi @@ -74,12 +85,31 @@ has_snapshot_with_files() { return 1 } +has_valid_whisper_snapshot() { + local repo="$1" + for base in "${CACHE_DIRS[@]}"; do + local dir + dir="$base/$(repo_dir "$repo")/snapshots" + [[ -d "$dir" ]] || continue + for snap in "$dir"/*; do + [[ -d "$snap" ]] || continue + if valid_whisper_bundle "$snap"; then + return 0 + fi + done + done + return 1 +} + ensure_repo() { local name="$1"; shift local repo="$1"; shift local required=("$@") - if has_snapshot_with_files "$repo" "${required[@]}"; then + if [[ "$name" == "Whisper" ]] && has_valid_whisper_snapshot "$repo"; then + echo "✓ ${name} cached (${repo})" + return 0 + elif [[ "$name" != "Whisper" ]] && has_snapshot_with_files "$repo" "${required[@]}"; then echo "✓ ${name} cached (${repo})" return 0 fi @@ -93,7 +123,7 @@ ensure_repo() { } WHISPER_REPO="mlx-community/whisper-large-v3-turbo" -if [[ -n "${CODESCRIBE_EMBED_MODEL:-}" && "${CODESCRIBE_EMBED_MODEL}" == */* ]]; then +if [[ -n "${CODESCRIBE_EMBED_MODEL:-}" ]] && is_hf_repo_id "$CODESCRIBE_EMBED_MODEL"; then WHISPER_REPO="$CODESCRIBE_EMBED_MODEL" fi EMBEDDER_REPO="${CODESCRIBE_EMBEDDER_REPO:-sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2}" From 7b85f718bfccfe6502c52cbe7d5e63cb8a2090b7 Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Fri, 21 Aug 2026 19:52:01 +0200 Subject: [PATCH 14/45] [codex/vc-workflow] test(stt): reuse canonical model discovery --- tests/e2e_full_pipeline.rs | 16 +--------------- tests/e2e_stt_transcription.rs | 6 ++---- tests/support/e2e_stt_matrix.rs | 28 +--------------------------- 3 files changed, 4 insertions(+), 46 deletions(-) diff --git a/tests/e2e_full_pipeline.rs b/tests/e2e_full_pipeline.rs index d49ac688..578fd3d8 100644 --- a/tests/e2e_full_pipeline.rs +++ b/tests/e2e_full_pipeline.rs @@ -91,21 +91,7 @@ const TEST_CASES: &[TestCase] = &[ ]; fn find_model_path() -> Option { - if let Ok(p) = std::env::var("CODESCRIBE_MODEL_PATH") { - let path = PathBuf::from(&p); - if path.join("tokenizer.json").exists() { - return Some(path); - } - } - - let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string()); - - let fp16 = PathBuf::from(&home).join(".codescribe/models/whisper-large-v3-turbo"); - if fp16.join("tokenizer.json").exists() { - return Some(fp16); - } - - None + codescribe_core::config::models::resolve_runtime_whisper_model_path(None).ok() } fn is_e2e_stt_enabled() -> bool { diff --git a/tests/e2e_stt_transcription.rs b/tests/e2e_stt_transcription.rs index 78af1e5c..1210e209 100644 --- a/tests/e2e_stt_transcription.rs +++ b/tests/e2e_stt_transcription.rs @@ -229,8 +229,7 @@ fn deterministic_model_discovery_prefers_complete_env_override() { create_complete_model(&fp16); create_complete_model(&env_model); - let hf_bases = Vec::::new(); - let found = discover_local_whisper_model_for(&home, Some(&env_model), &hf_bases) + let found = discover_local_whisper_model_for(&home, Some(&env_model)) .expect("expected env override to be discovered"); assert_eq!( @@ -252,9 +251,8 @@ fn deterministic_model_discovery_refuses_incomplete_fp16_without_legacy_fallback create_incomplete_model(&fp16); - let hf_bases = Vec::::new(); assert!( - discover_local_whisper_model_for(&home, None, &hf_bases).is_none(), + discover_local_whisper_model_for(&home, None).is_none(), "an incomplete fp16 model must not fall back to a quantized model" ); diff --git a/tests/support/e2e_stt_matrix.rs b/tests/support/e2e_stt_matrix.rs index 0603481d..89548c96 100644 --- a/tests/support/e2e_stt_matrix.rs +++ b/tests/support/e2e_stt_matrix.rs @@ -83,28 +83,6 @@ pub fn whisper_model_missing_parts(path: &Path) -> Vec<&'static str> { missing } -pub fn default_hf_cache_bases(home_dir: &Path) -> Vec { - let mut out = Vec::new(); - - if let Ok(path) = std::env::var("CODESCRIBE_HF_CACHE") { - out.push(PathBuf::from(path)); - } - if let Ok(path) = std::env::var("HUGGINGFACE_HUB_CACHE") { - out.push(PathBuf::from(path)); - } - if let Ok(path) = std::env::var("HF_HUB_CACHE") { - out.push(PathBuf::from(path)); - } - if let Ok(path) = std::env::var("HF_HOME") { - out.push(PathBuf::from(path).join("hub")); - } - - out.push(home_dir.join(".cache/huggingface/hub")); - out.sort(); - out.dedup(); - out -} - pub fn discover_local_whisper_model() -> Option { let home_dir = std::env::var("HOME") .map(PathBuf::from) @@ -112,14 +90,12 @@ pub fn discover_local_whisper_model() -> Option { let env_override = std::env::var("CODESCRIBE_MODEL_PATH") .ok() .map(PathBuf::from); - let hf_bases = default_hf_cache_bases(&home_dir); - discover_local_whisper_model_for(&home_dir, env_override.as_deref(), &hf_bases) + discover_local_whisper_model_for(&home_dir, env_override.as_deref()) } pub fn discover_local_whisper_model_for( home_dir: &Path, env_override: Option<&Path>, - hf_cache_bases: &[PathBuf], ) -> Option { if let Some(path) = env_override && whisper_model_is_complete(path) @@ -138,8 +114,6 @@ pub fn discover_local_whisper_model_for( }); } - let _ = hf_cache_bases; - None } From 2d34de520e8685bef1a030c39b7de870cdda7892 Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Fri, 21 Aug 2026 20:14:36 +0200 Subject: [PATCH 15/45] [codex/vc-workflow] fix(stt): expand tilde model overrides (PR #81 review) --- scripts/download-model.sh | 19 ++++++++++++------- scripts/ensure-models.sh | 25 +++++++++++++++---------- 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/scripts/download-model.sh b/scripts/download-model.sh index b089e55c..56e8ed28 100755 --- a/scripts/download-model.sh +++ b/scripts/download-model.sh @@ -16,13 +16,18 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)" WHISPER_VALIDATOR="$ROOT_DIR/scripts/validate-whisper-model.sh" +TILDE_PREFIX="$(printf '\176/')" +EMBED_MODEL_VALUE="${CODESCRIBE_EMBED_MODEL:-}" +if [[ "$EMBED_MODEL_VALUE" == "$TILDE_PREFIX"* ]]; then + EMBED_MODEL_VALUE="$HOME/${EMBED_MODEL_VALUE:2}" +fi is_hf_repo_id() { [[ "$1" =~ ^[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$ ]] } looks_like_local_path() { - [[ "$1" == /* || "$1" == ./* || "$1" == ../* || "$1" == ~/* ]] + [[ "$1" == /* || "$1" == ./* || "$1" == ../* ]] } sha256_file() { @@ -49,20 +54,20 @@ DEFAULT_REPO="mlx-community/whisper-large-v3-turbo" TOKENIZER_REPO="openai/whisper-large-v3-turbo" MEL_FILTERS_URL="https://raw.githubusercontent.com/openai/whisper/5f86d1d86363843179951550570367b37c5d6f78/whisper/assets/mel_filters.npz" MEL_FILTERS_SHA256="7450ae70723a5ef9d341e3cee628c7cb0177f36ce42c44b7ed2bf3325f0f6d4c" -MODEL_REPO="${CODESCRIBE_EMBED_MODEL:-$DEFAULT_REPO}" +MODEL_REPO="${EMBED_MODEL_VALUE:-$DEFAULT_REPO}" # If CODESCRIBE_EMBED_MODEL points to a local path, skip download. -if [[ -n "${CODESCRIBE_EMBED_MODEL:-}" ]] && [[ -d "${CODESCRIBE_EMBED_MODEL}" ]]; then - if "$WHISPER_VALIDATOR" "$CODESCRIBE_EMBED_MODEL"; then - echo "✓ Whisper model found at ${CODESCRIBE_EMBED_MODEL} (local path). Skipping download." +if [[ -n "$EMBED_MODEL_VALUE" ]] && [[ -d "$EMBED_MODEL_VALUE" ]]; then + if "$WHISPER_VALIDATOR" "$EMBED_MODEL_VALUE"; then + echo "✓ Whisper model found at $EMBED_MODEL_VALUE (local path). Skipping download." exit 0 fi - echo "ERROR: CODESCRIBE_EMBED_MODEL is not a valid Whisper bundle: ${CODESCRIBE_EMBED_MODEL}" >&2 + echo "ERROR: CODESCRIBE_EMBED_MODEL is not a valid Whisper bundle: $EMBED_MODEL_VALUE" >&2 exit 1 fi # An explicit path must never be passed to `hf download` as a repository id. -if [[ -n "${CODESCRIBE_EMBED_MODEL:-}" ]] && looks_like_local_path "$MODEL_REPO"; then +if [[ -n "$EMBED_MODEL_VALUE" ]] && looks_like_local_path "$MODEL_REPO"; then echo "ERROR: CODESCRIBE_EMBED_MODEL local path does not exist: $MODEL_REPO" >&2 exit 1 fi diff --git a/scripts/ensure-models.sh b/scripts/ensure-models.sh index 375012fe..19899bca 100755 --- a/scripts/ensure-models.sh +++ b/scripts/ensure-models.sh @@ -7,6 +7,11 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)" WHISPER_VALIDATOR="$ROOT_DIR/scripts/validate-whisper-model.sh" +TILDE_PREFIX="$(printf '\176/')" +EMBED_MODEL_VALUE="${CODESCRIBE_EMBED_MODEL:-}" +if [[ "$EMBED_MODEL_VALUE" == "$TILDE_PREFIX"* ]]; then + EMBED_MODEL_VALUE="$HOME/${EMBED_MODEL_VALUE:2}" +fi valid_whisper_bundle() { "$WHISPER_VALIDATOR" "$1" @@ -17,7 +22,7 @@ is_hf_repo_id() { } looks_like_local_path() { - [[ "$1" == /* || "$1" == ./* || "$1" == ../* || "$1" == ~/* ]] + [[ "$1" == /* || "$1" == ./* || "$1" == ../* ]] } # Prefer explicit path override for Whisper @@ -31,17 +36,17 @@ if [[ -n "${CODESCRIBE_MODEL_PATH:-}" ]]; then fi # If embed model points to a local directory, treat as satisfied. -if [[ -z "${WHISPER_OK:-}" && -n "${CODESCRIBE_EMBED_MODEL:-}" ]]; then - if [[ -d "${CODESCRIBE_EMBED_MODEL}" ]]; then - if valid_whisper_bundle "$CODESCRIBE_EMBED_MODEL"; then - echo "✓ Whisper model found via CODESCRIBE_EMBED_MODEL (${CODESCRIBE_EMBED_MODEL})" +if [[ -z "${WHISPER_OK:-}" && -n "$EMBED_MODEL_VALUE" ]]; then + if [[ -d "$EMBED_MODEL_VALUE" ]]; then + if valid_whisper_bundle "$EMBED_MODEL_VALUE"; then + echo "✓ Whisper model found via CODESCRIBE_EMBED_MODEL ($EMBED_MODEL_VALUE)" WHISPER_OK=1 else - echo "ERROR: CODESCRIBE_EMBED_MODEL is not a valid Whisper bundle: ${CODESCRIBE_EMBED_MODEL}" >&2 + echo "ERROR: CODESCRIBE_EMBED_MODEL is not a valid Whisper bundle: $EMBED_MODEL_VALUE" >&2 exit 1 fi - elif looks_like_local_path "$CODESCRIBE_EMBED_MODEL"; then - echo "ERROR: CODESCRIBE_EMBED_MODEL local path does not exist: ${CODESCRIBE_EMBED_MODEL}" >&2 + elif looks_like_local_path "$EMBED_MODEL_VALUE"; then + echo "ERROR: CODESCRIBE_EMBED_MODEL local path does not exist: $EMBED_MODEL_VALUE" >&2 exit 1 fi fi @@ -123,8 +128,8 @@ ensure_repo() { } WHISPER_REPO="mlx-community/whisper-large-v3-turbo" -if [[ -n "${CODESCRIBE_EMBED_MODEL:-}" ]] && is_hf_repo_id "$CODESCRIBE_EMBED_MODEL"; then - WHISPER_REPO="$CODESCRIBE_EMBED_MODEL" +if [[ -n "$EMBED_MODEL_VALUE" ]] && is_hf_repo_id "$EMBED_MODEL_VALUE"; then + WHISPER_REPO="$EMBED_MODEL_VALUE" fi EMBEDDER_REPO="${CODESCRIBE_EMBEDDER_REPO:-sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2}" From d3b82850d65d47a82812efa94dbf008a336ebafb Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Fri, 21 Aug 2026 20:19:22 +0200 Subject: [PATCH 16/45] [codex/vc-workflow] fix(stt): validate fat-build weight selection (PR #81 review) --- core/Cargo.toml | 2 + core/build.rs | 20 ++--- core/config/models.rs | 167 +------------------------------------- core/lib.rs | 1 + core/whisper_weights.rs | 172 ++++++++++++++++++++++++++++++++++++++++ docs/TEAM_SETUP.md | 2 + 6 files changed, 189 insertions(+), 175 deletions(-) create mode 100644 core/whisper_weights.rs diff --git a/core/Cargo.toml b/core/Cargo.toml index 13702803..bc52b7fb 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -106,7 +106,9 @@ mockito = "1" serial_test = "3" [build-dependencies] +anyhow = "1" dirs = "6" +serde_json = "1" sha2 = "0.10" [lints.rust] diff --git a/core/build.rs b/core/build.rs index a9210c3b..d5b3d682 100644 --- a/core/build.rs +++ b/core/build.rs @@ -24,6 +24,9 @@ use std::path::{Path, PathBuf}; use sha2::{Digest, Sha256}; +#[path = "whisper_weights.rs"] +mod whisper_weights; + /// The license key contract, included by path so the build script and the /// crate it builds share one definition of the dev key and its fingerprint /// instead of two copies that can drift apart. @@ -94,16 +97,13 @@ fn main() { .unwrap_or_else(|| DEFAULT_MODEL_NAME.to_string()); let model_path = resolve_whisper_embed_model_path(&manifest_dir, &embed_model, DEFAULT_WHISPER_REPO); - let weights_path = if model_path.join("weights.safetensors").exists() { - model_path.join("weights.safetensors") - } else { - model_path.join("model.safetensors") - }; + let weights_path = whisper_weights::resolve_valid_whisper_weights_path(&model_path).ok(); let model_exists = model_path.join("config.json").exists() && model_path.join("tokenizer.json").exists() && model_path.join("mel_filters.npz").exists() - && weights_path.exists(); + && weights_path.is_some(); if model_exists { + let weights_path = weights_path.as_ref().expect("validated Whisper weights"); println!( "cargo:rerun-if-changed={}", model_path.join("config.json").display() @@ -124,6 +124,7 @@ fn main() { let whisper_dest_path = Path::new(&out_dir).join("embedded_model_data.rs"); let whisper_embedded = embed_whisper_requested && !no_embed && model_exists; if whisper_embedded { + let weights_path = weights_path.as_ref().expect("validated Whisper weights"); println!( "cargo:warning=Embedding Whisper model from: {}", model_path.display() @@ -431,15 +432,10 @@ fn resolve_embed_model_path(manifest_dir: &str, embed_model: &str) -> PathBuf { /// tokenizer + mel into `~/.codescribe/models/`. Incomplete snapshots /// must not win over that composed tree. fn whisper_dir_complete(path: &Path) -> bool { - let weights = if path.join("weights.safetensors").exists() { - path.join("weights.safetensors") - } else { - path.join("model.safetensors") - }; path.join("config.json").exists() && path.join("tokenizer.json").exists() && path.join("mel_filters.npz").exists() - && weights.exists() + && whisper_weights::resolve_valid_whisper_weights_path(path).is_ok() } /// Locate the Whisper snapshot to embed. diff --git a/core/config/models.rs b/core/config/models.rs index 0640ca9a..657381bc 100644 --- a/core/config/models.rs +++ b/core/config/models.rs @@ -7,7 +7,6 @@ use anyhow::{Context, Result, anyhow}; use sha2::{Digest, Sha256}; use std::fs; -use std::io::{Read, Seek, SeekFrom}; use std::path::{Path, PathBuf}; use crate::hf_cache; @@ -29,7 +28,7 @@ pub const MEL_FILTERS_SHA256: &str = const REQUIRED_MODEL_FILES: [&str; 3] = ["config.json", "tokenizer.json", "mel_filters.npz"]; /// Weight file names, of which **any one** satisfies the completeness check — /// upstream repos ship either `model.safetensors` or `weights.safetensors`. -const REQUIRED_MODEL_WEIGHTS: [&str; 2] = ["weights.safetensors", "model.safetensors"]; +const REQUIRED_MODEL_WEIGHTS: [&str; 2] = crate::whisper_weights::SUPPORTED_NAMES; /// Canonicalize a path, falling back to the original on failure. /// @@ -114,167 +113,9 @@ fn validate_whisper_config(path: &Path) -> Result<()> { /// Upstream snapshots may contain either filename, and stale composition can /// leave both behind. Preserve the documented filename priority, but never let /// an invalid primary shadow a valid alternative that the runtime can load. -pub(crate) fn resolve_valid_whisper_weights_path(path: &Path) -> Result { - let mut failures = Vec::new(); - for name in REQUIRED_MODEL_WEIGHTS { - let candidate = path.join(name); - if !candidate.is_file() { - continue; - } - match validate_safetensors_file(&candidate) { - Ok(()) => return Ok(candidate), - Err(err) => failures.push(format!("{name}: {err:#}")), - } - } - - if failures.is_empty() { - Err(anyhow!( - "Whisper weights are missing from {}", - path.display() - )) - } else { - Err(anyhow!( - "no valid Whisper weights in {} ({})", - path.display(), - failures.join("; ") - )) - } -} - -/// Validate the complete safetensors structure without loading the tensor data. -fn validate_safetensors_file(path: &Path) -> Result<()> { - const MAX_HEADER_BYTES: u64 = 16 * 1024 * 1024; - // nosemgrep: rust.actix.path-traversal.tainted-path.tainted-path -- Read-only model inspection. `path` is an operator-selected local model file or an internally resolved bundle/cache child; no network/request path component reaches it. - let mut file = fs::File::open(path).with_context(|| format!("open {}", path.display()))?; - let mut len_bytes = [0_u8; 8]; - file.read_exact(&mut len_bytes) - .with_context(|| format!("read safetensors header length from {}", path.display()))?; - let header_len = u64::from_le_bytes(len_bytes); - if header_len == 0 || header_len > MAX_HEADER_BYTES { - return Err(anyhow!( - "invalid safetensors header length in {}", - path.display() - )); - } - let mut header = vec![0_u8; header_len as usize]; - file.seek(SeekFrom::Start(8))?; - file.read_exact(&mut header) - .with_context(|| format!("read safetensors header from {}", path.display()))?; - let metadata: serde_json::Value = serde_json::from_slice(&header) - .with_context(|| format!("parse safetensors header from {}", path.display()))?; - let Some(tensors) = metadata.as_object() else { - return Err(anyhow!( - "safetensors header is not an object: {}", - path.display() - )); - }; - - if let Some(metadata) = tensors.get("__metadata__") { - let valid = metadata.is_null() - || metadata - .as_object() - .is_some_and(|entries| entries.values().all(serde_json::Value::is_string)); - if !valid { - return Err(anyhow!( - "invalid safetensors __metadata__ in {}", - path.display() - )); - } - } - - let file_len = file.metadata()?.len(); - let data_start = 8_u64 - .checked_add(header_len) - .ok_or_else(|| anyhow!("safetensors header offset overflow"))?; - let data_len = file_len - .checked_sub(data_start) - .ok_or_else(|| anyhow!("truncated safetensors file: {}", path.display()))?; - let mut ranges = Vec::new(); - - for (name, tensor) in tensors.iter().filter(|(name, _)| *name != "__metadata__") { - let tensor = tensor - .as_object() - .ok_or_else(|| anyhow!("invalid tensor entry {name}"))?; - let dtype = tensor - .get("dtype") - .and_then(serde_json::Value::as_str) - .ok_or_else(|| anyhow!("tensor {name} has no dtype"))?; - let bytes_per_element = match (name.as_str(), dtype) { - (_, "F16") => 2_u64, - (_, "F32") => 4_u64, - ("alignment_heads", "I64") => 8_u64, - _ => { - return Err(anyhow!( - "unsupported Whisper tensor dtype {dtype} for {name}" - )); - } - }; - if name.ends_with(".scales") || name.ends_with(".biases") { - return Err(anyhow!( - "quantized Whisper companion tensor refused: {name}" - )); - } - - let shape = tensor - .get("shape") - .and_then(serde_json::Value::as_array) - .ok_or_else(|| anyhow!("tensor {name} has no shape"))?; - let element_count = shape.iter().try_fold(1_u64, |count, dim| { - let dim = dim - .as_u64() - .ok_or_else(|| anyhow!("tensor {name} has an invalid shape"))?; - count - .checked_mul(dim) - .ok_or_else(|| anyhow!("tensor {name} shape overflows")) - })?; - if element_count == 0 { - return Err(anyhow!("tensor {name} has an empty shape")); - } - let expected_bytes = element_count - .checked_mul(bytes_per_element) - .ok_or_else(|| anyhow!("tensor {name} byte size overflows"))?; - - let offsets = tensor - .get("data_offsets") - .and_then(serde_json::Value::as_array) - .filter(|offsets| offsets.len() == 2) - .ok_or_else(|| anyhow!("tensor {name} has invalid data_offsets"))?; - let start = offsets[0] - .as_u64() - .ok_or_else(|| anyhow!("tensor {name} has invalid start offset"))?; - let end = offsets[1] - .as_u64() - .ok_or_else(|| anyhow!("tensor {name} has invalid end offset"))?; - if end.checked_sub(start) != Some(expected_bytes) { - return Err(anyhow!( - "tensor {name} byte range does not match its shape/dtype" - )); - } - ranges.push((start, end, name)); - } - - if ranges.is_empty() { - return Err(anyhow!( - "safetensors file contains no tensors: {}", - path.display() - )); - } - ranges.sort_by_key(|(start, _, _)| *start); - let mut cursor = 0_u64; - for (start, end, name) in ranges { - if start != cursor { - return Err(anyhow!("tensor {name} has a non-contiguous data offset")); - } - cursor = end; - } - if cursor != data_len { - return Err(anyhow!( - "safetensors data length mismatch in {}: header covers {cursor}, file has {data_len}", - path.display() - )); - } - Ok(()) -} +pub(crate) use crate::whisper_weights::{ + resolve_valid_whisper_weights_path, validate_safetensors_file, +}; /// Whether a candidate models root owns at least one complete Whisper model. /// diff --git a/core/lib.rs b/core/lib.rs index 8f0aae03..f471e6cb 100644 --- a/core/lib.rs +++ b/core/lib.rs @@ -90,6 +90,7 @@ pub mod tts; pub mod util; /// Silero neural voice-activity detection. pub mod vad; +pub(crate) mod whisper_weights; pub use stt::whisper; // ═══════════════════════════════════════════════════════════ diff --git a/core/whisper_weights.rs b/core/whisper_weights.rs new file mode 100644 index 00000000..d3f1cb37 --- /dev/null +++ b/core/whisper_weights.rs @@ -0,0 +1,172 @@ +//! Shared Whisper safetensors validation for runtime and fat-build selection. + +use anyhow::{Context, Result, anyhow}; +use std::fs; +use std::io::{Read, Seek, SeekFrom}; +use std::path::{Path, PathBuf}; + +/// Supported weight filenames in deterministic preference order. +pub const SUPPORTED_NAMES: [&str; 2] = ["weights.safetensors", "model.safetensors"]; + +/// Resolve the first structurally valid supported weight file. +pub fn resolve_valid_whisper_weights_path(path: &Path) -> Result { + let mut failures = Vec::new(); + for name in SUPPORTED_NAMES { + let candidate = path.join(name); + if !candidate.is_file() { + continue; + } + match validate_safetensors_file(&candidate) { + Ok(()) => return Ok(candidate), + Err(err) => failures.push(format!("{name}: {err:#}")), + } + } + + if failures.is_empty() { + Err(anyhow!( + "Whisper weights are missing from {}", + path.display() + )) + } else { + Err(anyhow!( + "no valid Whisper weights in {} ({})", + path.display(), + failures.join("; ") + )) + } +} + +/// Validate the complete safetensors structure without loading tensor data. +pub(crate) fn validate_safetensors_file(path: &Path) -> Result<()> { + const MAX_HEADER_BYTES: u64 = 16 * 1024 * 1024; + // nosemgrep: rust.actix.path-traversal.tainted-path.tainted-path -- Read-only model inspection. `path` is an operator-selected local model file or an internally resolved bundle/cache child; no network/request path component reaches it. + let mut file = fs::File::open(path).with_context(|| format!("open {}", path.display()))?; + let mut len_bytes = [0_u8; 8]; + file.read_exact(&mut len_bytes) + .with_context(|| format!("read safetensors header length from {}", path.display()))?; + let header_len = u64::from_le_bytes(len_bytes); + if header_len == 0 || header_len > MAX_HEADER_BYTES { + return Err(anyhow!( + "invalid safetensors header length in {}", + path.display() + )); + } + let mut header = vec![0_u8; header_len as usize]; + file.seek(SeekFrom::Start(8))?; + file.read_exact(&mut header) + .with_context(|| format!("read safetensors header from {}", path.display()))?; + let metadata: serde_json::Value = serde_json::from_slice(&header) + .with_context(|| format!("parse safetensors header from {}", path.display()))?; + let Some(tensors) = metadata.as_object() else { + return Err(anyhow!( + "safetensors header is not an object: {}", + path.display() + )); + }; + + if let Some(metadata) = tensors.get("__metadata__") { + let valid = metadata.is_null() + || metadata + .as_object() + .is_some_and(|entries| entries.values().all(serde_json::Value::is_string)); + if !valid { + return Err(anyhow!( + "invalid safetensors __metadata__ in {}", + path.display() + )); + } + } + + let file_len = file.metadata()?.len(); + let data_start = 8_u64 + .checked_add(header_len) + .ok_or_else(|| anyhow!("safetensors header offset overflow"))?; + let data_len = file_len + .checked_sub(data_start) + .ok_or_else(|| anyhow!("truncated safetensors file: {}", path.display()))?; + let mut ranges = Vec::new(); + + for (name, tensor) in tensors.iter().filter(|(name, _)| *name != "__metadata__") { + let tensor = tensor + .as_object() + .ok_or_else(|| anyhow!("invalid tensor entry {name}"))?; + let dtype = tensor + .get("dtype") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| anyhow!("tensor {name} has no dtype"))?; + let bytes_per_element = match (name.as_str(), dtype) { + (_, "F16") => 2_u64, + (_, "F32") => 4_u64, + ("alignment_heads", "I64") => 8_u64, + _ => { + return Err(anyhow!( + "unsupported Whisper tensor dtype {dtype} for {name}" + )); + } + }; + if name.ends_with(".scales") || name.ends_with(".biases") { + return Err(anyhow!( + "quantized Whisper companion tensor refused: {name}" + )); + } + + let shape = tensor + .get("shape") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| anyhow!("tensor {name} has no shape"))?; + let element_count = shape.iter().try_fold(1_u64, |count, dim| { + let dim = dim + .as_u64() + .ok_or_else(|| anyhow!("tensor {name} has an invalid shape"))?; + count + .checked_mul(dim) + .ok_or_else(|| anyhow!("tensor {name} shape overflows")) + })?; + if element_count == 0 { + return Err(anyhow!("tensor {name} has an empty shape")); + } + let expected_bytes = element_count + .checked_mul(bytes_per_element) + .ok_or_else(|| anyhow!("tensor {name} byte size overflows"))?; + + let offsets = tensor + .get("data_offsets") + .and_then(serde_json::Value::as_array) + .filter(|offsets| offsets.len() == 2) + .ok_or_else(|| anyhow!("tensor {name} has invalid data_offsets"))?; + let start = offsets[0] + .as_u64() + .ok_or_else(|| anyhow!("tensor {name} has invalid start offset"))?; + let end = offsets[1] + .as_u64() + .ok_or_else(|| anyhow!("tensor {name} has invalid end offset"))?; + if end.checked_sub(start) != Some(expected_bytes) { + return Err(anyhow!( + "tensor {name} byte range does not match its shape/dtype" + )); + } + ranges.push((start, end, name)); + } + + if ranges.is_empty() { + return Err(anyhow!( + "safetensors file contains no tensors: {}", + path.display() + )); + } + ranges.sort_by_key(|(start, _, _)| *start); + let mut cursor = 0_u64; + for (start, end, name) in ranges { + if start != cursor { + return Err(anyhow!("tensor {name} has a non-contiguous data offset")); + } + cursor = end; + } + if cursor != data_len { + return Err(anyhow!( + "safetensors data length mismatch in {}: header covers {cursor}, file has {data_len}", + path.display() + )); + } + Ok(()) +} diff --git a/docs/TEAM_SETUP.md b/docs/TEAM_SETUP.md index c28c9c7f..3ac631d9 100644 --- a/docs/TEAM_SETUP.md +++ b/docs/TEAM_SETUP.md @@ -73,6 +73,8 @@ Grant in: System Settings > Privacy & Security - `make install-app` / `scripts/ensure-models.sh` are the easiest way to warm the expected cache paths. Release/setup preflight calls the same runtime-owned bundle validator before it skips a download; a directory with merely the expected filenames is repaired or rejected rather than embedded. + The fat-build selector shares the safetensors validator and embeds the first valid supported + weights filename, so a stale invalid primary cannot shadow a valid alternative. An explicit local `CODESCRIBE_EMBED_MODEL` must exist and pass that validator; it is never reinterpreted as a Hugging Face repository id. From 18a04a572792f4ca979a8b20583a1c16e003b2c2 Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Fri, 21 Aug 2026 20:41:00 +0200 Subject: [PATCH 17/45] [codex/vc-workflow] fix(stt): expand build-time tilde model paths (PR #81 review) --- core/build.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/core/build.rs b/core/build.rs index d5b3d682..ec735e54 100644 --- a/core/build.rs +++ b/core/build.rs @@ -414,7 +414,7 @@ fn decode_license_public_key(value: &str) -> [u8; 32] { /// resolved against the manifest dir, and a bare name is looked up under /// `/models/`. fn resolve_embed_model_path(manifest_dir: &str, embed_model: &str) -> PathBuf { - let candidate = PathBuf::from(embed_model); + let candidate = expand_tilde_path(embed_model); if candidate.is_absolute() { return candidate; } @@ -426,6 +426,16 @@ fn resolve_embed_model_path(manifest_dir: &str, embed_model: &str) -> PathBuf { Path::new(manifest_dir).join("models").join(embed_model) } +/// Expand a literal `~/` path because Cargo passes env values without shell expansion. +fn expand_tilde_path(value: &str) -> PathBuf { + if let Some(relative) = value.strip_prefix("~/") + && let Some(home) = dirs::home_dir() + { + return home.join(relative); + } + PathBuf::from(value) +} + /// True when a directory can be baked into the fat SKU. /// /// The default HF Whisper repo is weights-only. `make download-model` composes @@ -449,7 +459,7 @@ fn resolve_whisper_embed_model_path( default_repo: &str, ) -> PathBuf { if let Ok(model_path) = env::var("CODESCRIBE_MODEL_PATH") { - let p = PathBuf::from(model_path.trim()); + let p = expand_tilde_path(model_path.trim()); if whisper_dir_complete(&p) { return p; } From 677d9f1471e6f4f7cd81c7f9bd0fb7f0c3e5d78a Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Fri, 21 Aug 2026 20:41:30 +0200 Subject: [PATCH 18/45] [codex/vc-workflow] fix(stt): validate benchmark model discovery (PR #81 review) --- scripts/bench-stt.sh | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/scripts/bench-stt.sh b/scripts/bench-stt.sh index 32a4df50..2388c540 100755 --- a/scripts/bench-stt.sh +++ b/scripts/bench-stt.sh @@ -32,6 +32,7 @@ else repo_root="$(cd -- "$script_dir/.." && pwd)" fi home_dir="${HOME:-}" +model_validator="$repo_root/scripts/validate-whisper-model.sh" fixture_mode="${BENCH_STT_FIXTURES:-repo}" fixture_limit="${BENCH_STT_LIMIT:-10}" @@ -136,11 +137,7 @@ sha256_file() { model_is_complete() { local dir="$1" - [[ -d "$dir" ]] || return 1 - [[ -f "$dir/config.json" ]] || return 1 - [[ -f "$dir/tokenizer.json" ]] || return 1 - [[ -f "$dir/mel_filters.npz" ]] || return 1 - [[ -f "$dir/weights.safetensors" || -f "$dir/model.safetensors" ]] || return 1 + [[ -d "$dir" ]] && "$model_validator" "$dir" >/dev/null 2>&1 } discover_model() { From ba4887d1b18b4c893b17c24d10d21235761c5924 Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Fri, 21 Aug 2026 20:42:18 +0200 Subject: [PATCH 19/45] [codex/vc-workflow] fix(stt): stage model bundle before promotion (PR #81 review) --- docs/TEAM_SETUP.md | 4 ++-- scripts/download-model.sh | 45 +++++++++++++++++++++++++++++++-------- 2 files changed, 38 insertions(+), 11 deletions(-) diff --git a/docs/TEAM_SETUP.md b/docs/TEAM_SETUP.md index 3ac631d9..c13c8db9 100644 --- a/docs/TEAM_SETUP.md +++ b/docs/TEAM_SETUP.md @@ -68,8 +68,8 @@ Grant in: System Settings > Privacy & Security `CODESCRIBE_MODEL_PATH` → configured local model path/alias → configured HF repo snapshot → default local turbo model → default HF cache snapshot. - A model is ready only after config/tokenizer parsing, pinned mel checksum, and - full safetensors structural/dtype validation. Invalid cached files are replaced - through validated `.partial` artifacts on the next download. + full safetensors structural/dtype validation. Downloads compose and validate the + complete replacement in staging before any artifact is promoted over an installed model. - `make install-app` / `scripts/ensure-models.sh` are the easiest way to warm the expected cache paths. Release/setup preflight calls the same runtime-owned bundle validator before it skips a download; a directory with merely the expected filenames is repaired or rejected rather than embedded. diff --git a/scripts/download-model.sh b/scripts/download-model.sh index 56e8ed28..ae9933fb 100755 --- a/scripts/download-model.sh +++ b/scripts/download-model.sh @@ -139,27 +139,54 @@ if [[ "$MODEL_REPO" == "$DEFAULT_REPO" ]]; then echo "▶ Composing verified fp16 runtime directory..." TOKENIZER_PATH=$("$HF_BIN" download "$TOKENIZER_REPO" tokenizer.json --quiet) MODEL_DEST="${CODESCRIBE_MODELS_DIR:-$HOME/.codescribe/models}/whisper-large-v3-turbo" - mkdir -p "$MODEL_DEST" - atomic_copy "$MODEL_SNAPSHOT/config.json" "$MODEL_DEST/config.json" + MODEL_STAGE=$(mktemp -d "${TMPDIR:-/tmp}/codescribe-whisper-model.XXXXXX") + cleanup_model_stage() { + rm -f \ + "$MODEL_STAGE/config.json" \ + "$MODEL_STAGE/tokenizer.json" \ + "$MODEL_STAGE/mel_filters.npz" \ + "$MODEL_STAGE/mel_filters.npz.partial" \ + "$MODEL_STAGE/weights.safetensors" \ + "$MODEL_STAGE/model.safetensors" + rmdir "$MODEL_STAGE" 2>/dev/null || true + } + trap cleanup_model_stage EXIT + + atomic_copy "$MODEL_SNAPSHOT/config.json" "$MODEL_STAGE/config.json" if [[ -f "$MODEL_SNAPSHOT/weights.safetensors" ]]; then - atomic_copy "$MODEL_SNAPSHOT/weights.safetensors" "$MODEL_DEST/weights.safetensors" + atomic_copy "$MODEL_SNAPSHOT/weights.safetensors" "$MODEL_STAGE/weights.safetensors" elif [[ -f "$MODEL_SNAPSHOT/model.safetensors" ]]; then - atomic_copy "$MODEL_SNAPSHOT/model.safetensors" "$MODEL_DEST/model.safetensors" + atomic_copy "$MODEL_SNAPSHOT/model.safetensors" "$MODEL_STAGE/model.safetensors" else echo "ERROR: fp16 snapshot has no safetensors weights: $MODEL_SNAPSHOT" >&2 exit 1 fi - atomic_copy "$TOKENIZER_PATH" "$MODEL_DEST/tokenizer.json" - curl -fsSL "$MEL_FILTERS_URL" -o "$MODEL_DEST/mel_filters.npz.partial" - ACTUAL_MEL_SHA=$(sha256_file "$MODEL_DEST/mel_filters.npz.partial") + atomic_copy "$TOKENIZER_PATH" "$MODEL_STAGE/tokenizer.json" + curl -fsSL "$MEL_FILTERS_URL" -o "$MODEL_STAGE/mel_filters.npz.partial" + ACTUAL_MEL_SHA=$(sha256_file "$MODEL_STAGE/mel_filters.npz.partial") if [[ "$ACTUAL_MEL_SHA" != "$MEL_FILTERS_SHA256" ]]; then - rm -f "$MODEL_DEST/mel_filters.npz.partial" echo "ERROR: mel_filters.npz checksum mismatch" >&2 exit 1 fi - mv "$MODEL_DEST/mel_filters.npz.partial" "$MODEL_DEST/mel_filters.npz" + mv "$MODEL_STAGE/mel_filters.npz.partial" "$MODEL_STAGE/mel_filters.npz" + + # Do not touch a working installation until every cached/downloaded + # replacement passes the exact runtime-owned bundle contract. + "$WHISPER_VALIDATOR" "$MODEL_STAGE" + + mkdir -p "$MODEL_DEST" + atomic_copy "$MODEL_STAGE/config.json" "$MODEL_DEST/config.json" + atomic_copy "$MODEL_STAGE/tokenizer.json" "$MODEL_DEST/tokenizer.json" + atomic_copy "$MODEL_STAGE/mel_filters.npz" "$MODEL_DEST/mel_filters.npz" + if [[ -f "$MODEL_STAGE/weights.safetensors" ]]; then + atomic_copy "$MODEL_STAGE/weights.safetensors" "$MODEL_DEST/weights.safetensors" + else + atomic_copy "$MODEL_STAGE/model.safetensors" "$MODEL_DEST/model.safetensors" + fi echo " Runtime directory: $MODEL_DEST" "$WHISPER_VALIDATOR" "$MODEL_DEST" + cleanup_model_stage + trap - EXIT else "$WHISPER_VALIDATOR" "$MODEL_SNAPSHOT" fi From 671785fe33cacf6a73cd222559a123282aeb5977 Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Fri, 21 Aug 2026 21:09:51 +0200 Subject: [PATCH 20/45] [codex/vc-workflow] fix(stt): validate complete fat-build bundles (PR #81 review) --- core/Cargo.toml | 1 + core/build.rs | 14 ++++----- core/config/models.rs | 65 ++++------------------------------------- core/whisper_weights.rs | 58 ++++++++++++++++++++++++++++++++++++ docs/TEAM_SETUP.md | 5 ++-- 5 files changed, 72 insertions(+), 71 deletions(-) diff --git a/core/Cargo.toml b/core/Cargo.toml index bc52b7fb..5e3ce67d 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -110,6 +110,7 @@ anyhow = "1" dirs = "6" serde_json = "1" sha2 = "0.10" +tokenizers = "0.22" [lints.rust] # Allow unexpected_cfgs from objc crate's msg_send! macro (uses cargo-clippy cfg internally) diff --git a/core/build.rs b/core/build.rs index ec735e54..d894a5bc 100644 --- a/core/build.rs +++ b/core/build.rs @@ -97,11 +97,10 @@ fn main() { .unwrap_or_else(|| DEFAULT_MODEL_NAME.to_string()); let model_path = resolve_whisper_embed_model_path(&manifest_dir, &embed_model, DEFAULT_WHISPER_REPO); - let weights_path = whisper_weights::resolve_valid_whisper_weights_path(&model_path).ok(); - let model_exists = model_path.join("config.json").exists() - && model_path.join("tokenizer.json").exists() - && model_path.join("mel_filters.npz").exists() - && weights_path.is_some(); + let model_exists = whisper_weights::validate_whisper_model_bundle(&model_path).is_ok(); + let weights_path = model_exists + .then(|| whisper_weights::resolve_valid_whisper_weights_path(&model_path).ok()) + .flatten(); if model_exists { let weights_path = weights_path.as_ref().expect("validated Whisper weights"); println!( @@ -442,10 +441,7 @@ fn expand_tilde_path(value: &str) -> PathBuf { /// tokenizer + mel into `~/.codescribe/models/`. Incomplete snapshots /// must not win over that composed tree. fn whisper_dir_complete(path: &Path) -> bool { - path.join("config.json").exists() - && path.join("tokenizer.json").exists() - && path.join("mel_filters.npz").exists() - && whisper_weights::resolve_valid_whisper_weights_path(path).is_ok() + whisper_weights::validate_whisper_model_bundle(path).is_ok() } /// Locate the Whisper snapshot to embed. diff --git a/core/config/models.rs b/core/config/models.rs index 657381bc..d3e9488d 100644 --- a/core/config/models.rs +++ b/core/config/models.rs @@ -5,7 +5,6 @@ //! model from here instead of re-implementing its own precedence rules. use anyhow::{Context, Result, anyhow}; -use sha2::{Digest, Sha256}; use std::fs; use std::path::{Path, PathBuf}; @@ -22,8 +21,7 @@ pub const TOKENIZER_WHISPER_REPO: &str = "openai/whisper-large-v3-turbo"; /// Pinned OpenAI Whisper asset. The checksum is asserted by the installer. pub const MEL_FILTERS_URL: &str = "https://raw.githubusercontent.com/openai/whisper/5f86d1d86363843179951550570367b37c5d6f78/whisper/assets/mel_filters.npz"; /// SHA-256 of [`MEL_FILTERS_URL`]. -pub const MEL_FILTERS_SHA256: &str = - "7450ae70723a5ef9d341e3cee628c7cb0177f36ce42c44b7ed2bf3325f0f6d4c"; +pub const MEL_FILTERS_SHA256: &str = crate::whisper_weights::MEL_FILTERS_SHA256; /// Files that must all be present for a directory to count as a usable model. const REQUIRED_MODEL_FILES: [&str; 3] = ["config.json", "tokenizer.json", "mel_filters.npz"]; /// Weight file names, of which **any one** satisfies the completeness check — @@ -59,55 +57,17 @@ fn is_complete_whisper_model_dir(path: &Path) -> bool { /// has no payload checksum. The validator checks the complete tensor table, /// dtype allowlist, byte sizes, contiguous offsets, and final file length. pub fn validate_whisper_model_bundle(path: &Path) -> Result<()> { - let config_path = path.join("config.json"); - validate_whisper_config(&config_path)?; - - let tokenizer_path = path.join("tokenizer.json"); - tokenizers::Tokenizer::from_file(&tokenizer_path).map_err(|err| { - anyhow!( - "invalid Whisper tokenizer {}: {err}", - tokenizer_path.display() - ) - })?; - - let mel_path = path.join("mel_filters.npz"); - verify_sha256(&mel_path, MEL_FILTERS_SHA256)?; - - resolve_valid_whisper_weights_path(path).map(|_| ()) + crate::whisper_weights::validate_whisper_model_bundle(path) } /// Reject unsupported or malformed weights before the expensive engine load. /// This narrower payload gate is also used by `LocalWhisperEngine::new`, where /// tokenizer and mel errors retain their own loader diagnostics. pub(crate) fn is_unquantized_whisper_model_dir(path: &Path) -> bool { - validate_whisper_config(&path.join("config.json")).is_ok() + crate::whisper_weights::validate_whisper_config(&path.join("config.json")).is_ok() && resolve_valid_whisper_weights_path(path).is_ok() } -fn validate_whisper_config(path: &Path) -> Result<()> { - // nosemgrep: rust.actix.path-traversal.tainted-path.tainted-path -- Read-only model inspection. `path` is an operator-selected local model config or an internally resolved bundle/cache child; no network/request path component reaches it. - let raw = fs::read_to_string(path) - .with_context(|| format!("read Whisper config {}", path.display()))?; - let config: serde_json::Value = serde_json::from_str(&raw) - .with_context(|| format!("parse Whisper config {}", path.display()))?; - if !config.is_object() { - return Err(anyhow!( - "Whisper config must be a JSON object: {}", - path.display() - )); - } - if config - .get("quantization") - .is_some_and(|value| !value.is_null()) - || config - .get("quantization_config") - .is_some_and(|value| !value.is_null()) - { - return Err(anyhow!("quantized Whisper config is unsupported")); - } - Ok(()) -} - /// Resolve the first structurally valid supported weight file. /// /// Upstream snapshots may contain either filename, and stale composition can @@ -653,31 +613,16 @@ fn replace_file(partial: &Path, dest: &Path) -> Result<()> { fn validate_model_file(filename: &str, path: &Path) -> Result<()> { match filename { - "config.json" => validate_whisper_config(path), + "config.json" => crate::whisper_weights::validate_whisper_config(path), "tokenizer.json" => tokenizers::Tokenizer::from_file(path) .map(|_| ()) .map_err(|err| anyhow!("invalid tokenizer {}: {err}", path.display())), - "mel_filters.npz" => verify_sha256(path, MEL_FILTERS_SHA256), + "mel_filters.npz" => crate::whisper_weights::verify_mel_filters(path), "weights.safetensors" | "model.safetensors" => validate_safetensors_file(path), _ => Err(anyhow!("unsupported Whisper model artifact: {filename}")), } } -fn verify_sha256(path: &Path, expected: &str) -> Result<()> { - // nosemgrep: rust.actix.path-traversal.tainted-path.tainted-path -- Read-only checksum of the fixed mel_filters.npz destination assembled under the internally resolved model directory. - let bytes = fs::read(path).with_context(|| format!("read {} for checksum", path.display()))?; - let actual = format!("{:x}", Sha256::digest(bytes)); - if actual != expected { - return Err(anyhow!( - "SHA-256 mismatch for {}: expected {}, got {}", - path.display(), - expected, - actual - )); - } - Ok(()) -} - /// ModelManager resolution, completeness gates, and env-override isolation tests. #[cfg(test)] mod tests { diff --git a/core/whisper_weights.rs b/core/whisper_weights.rs index d3f1cb37..b17bbebf 100644 --- a/core/whisper_weights.rs +++ b/core/whisper_weights.rs @@ -1,12 +1,70 @@ //! Shared Whisper safetensors validation for runtime and fat-build selection. use anyhow::{Context, Result, anyhow}; +use sha2::{Digest, Sha256}; use std::fs; use std::io::{Read, Seek, SeekFrom}; use std::path::{Path, PathBuf}; /// Supported weight filenames in deterministic preference order. pub const SUPPORTED_NAMES: [&str; 2] = ["weights.safetensors", "model.safetensors"]; +/// SHA-256 of the pinned official OpenAI mel filterbank. +pub const MEL_FILTERS_SHA256: &str = + "7450ae70723a5ef9d341e3cee628c7cb0177f36ce42c44b7ed2bf3325f0f6d4c"; + +/// Validate every artifact required by runtime and embedded Whisper loaders. +pub fn validate_whisper_model_bundle(path: &Path) -> Result<()> { + validate_whisper_config(&path.join("config.json"))?; + let tokenizer_path = path.join("tokenizer.json"); + tokenizers::Tokenizer::from_file(&tokenizer_path).map_err(|err| { + anyhow!( + "invalid Whisper tokenizer {}: {err}", + tokenizer_path.display() + ) + })?; + verify_mel_filters(&path.join("mel_filters.npz"))?; + resolve_valid_whisper_weights_path(path).map(|_| ()) +} + +/// Validate the config schema and reject every declared quantization mode. +pub(crate) fn validate_whisper_config(path: &Path) -> Result<()> { + let raw = fs::read_to_string(path) + .with_context(|| format!("read Whisper config {}", path.display()))?; + let config: serde_json::Value = serde_json::from_str(&raw) + .with_context(|| format!("parse Whisper config {}", path.display()))?; + if !config.is_object() { + return Err(anyhow!( + "Whisper config must be a JSON object: {}", + path.display() + )); + } + if config + .get("quantization") + .is_some_and(|value| !value.is_null()) + || config + .get("quantization_config") + .is_some_and(|value| !value.is_null()) + { + return Err(anyhow!("quantized Whisper config is unsupported")); + } + Ok(()) +} + +/// Verify the pinned mel filterbank used by Whisper. +pub(crate) fn verify_mel_filters(path: &Path) -> Result<()> { + // nosemgrep: rust.actix.path-traversal.tainted-path.tainted-path -- Read-only checksum of an operator-selected local model artifact or internally resolved download destination. + let bytes = fs::read(path).with_context(|| format!("read {} for checksum", path.display()))?; + let actual = format!("{:x}", Sha256::digest(bytes)); + if actual != MEL_FILTERS_SHA256 { + return Err(anyhow!( + "SHA-256 mismatch for {}: expected {}, got {}", + path.display(), + MEL_FILTERS_SHA256, + actual + )); + } + Ok(()) +} /// Resolve the first structurally valid supported weight file. pub fn resolve_valid_whisper_weights_path(path: &Path) -> Result { diff --git a/docs/TEAM_SETUP.md b/docs/TEAM_SETUP.md index c13c8db9..b673fb85 100644 --- a/docs/TEAM_SETUP.md +++ b/docs/TEAM_SETUP.md @@ -73,8 +73,9 @@ Grant in: System Settings > Privacy & Security - `make install-app` / `scripts/ensure-models.sh` are the easiest way to warm the expected cache paths. Release/setup preflight calls the same runtime-owned bundle validator before it skips a download; a directory with merely the expected filenames is repaired or rejected rather than embedded. - The fat-build selector shares the safetensors validator and embeds the first valid supported - weights filename, so a stale invalid primary cannot shadow a valid alternative. + The fat-build selector shares the complete runtime bundle validator and embeds the first valid + supported weights filename, so malformed config/tokenizer/mel assets or a stale invalid primary + cannot shadow a valid alternative. An explicit local `CODESCRIBE_EMBED_MODEL` must exist and pass that validator; it is never reinterpreted as a Hugging Face repository id. From 5142c3aa5f93504731d95b49091e7423570b56ff Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Fri, 21 Aug 2026 21:10:52 +0200 Subject: [PATCH 21/45] [codex/vc-workflow] test(stt): validate E2E model discovery (PR #81 review) --- Cargo.lock | 1 + Cargo.toml | 1 + tests/e2e_stt_transcription.rs | 50 +++++++++++++++++++++++++++++++-- tests/support/e2e_stt_matrix.rs | 7 +---- 4 files changed, 50 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 80406866..fae415b0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -762,6 +762,7 @@ dependencies = [ "sha2", "tao", "tempfile", + "tokenizers", "tokio", "tokio-tungstenite", "tracing", diff --git a/Cargo.toml b/Cargo.toml index c850c3fc..d3933989 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -141,6 +141,7 @@ tempfile = "3" mockito = "1" serial_test = "3" hound = "3.5" +tokenizers = "0.22" [lints.rust] # Allow unexpected_cfgs from objc crate's msg_send! macro (uses cargo-clippy cfg internally) diff --git a/tests/e2e_stt_transcription.rs b/tests/e2e_stt_transcription.rs index 1210e209..a8f7e5c8 100644 --- a/tests/e2e_stt_transcription.rs +++ b/tests/e2e_stt_transcription.rs @@ -175,9 +175,31 @@ fn e2e_stt_model_init_stable() { fn create_complete_model(path: &Path) { std::fs::create_dir_all(path).expect("create model dir"); std::fs::write(path.join("config.json"), "{}").expect("write config"); - std::fs::write(path.join("tokenizer.json"), "{}").expect("write tokenizer"); - std::fs::write(path.join("mel_filters.npz"), []).expect("write mel filters"); - std::fs::write(path.join("weights.safetensors"), []).expect("write weights"); + tokenizers::Tokenizer::new(tokenizers::models::bpe::BPE::default()) + .save(path.join("tokenizer.json"), false) + .expect("write tokenizer"); + std::fs::write( + path.join("mel_filters.npz"), + decode_hex(include_str!("fixtures/whisper_mel_filters.npz.hex")), + ) + .expect("write mel filters"); + let header = br#"{"model.weight":{"dtype":"F16","shape":[1],"data_offsets":[0,2]}}"#; + let mut safetensors = (header.len() as u64).to_le_bytes().to_vec(); + safetensors.extend_from_slice(header); + safetensors.extend_from_slice(&[0, 0]); + std::fs::write(path.join("weights.safetensors"), safetensors).expect("write weights"); +} + +fn decode_hex(raw: &str) -> Vec { + let digits: String = raw.chars().filter(|ch| !ch.is_whitespace()).collect(); + assert!(digits.len().is_multiple_of(2)); + digits + .as_bytes() + .as_chunks::<2>() + .0 + .iter() + .map(|pair| u8::from_str_radix(std::str::from_utf8(pair).unwrap(), 16).unwrap()) + .collect() } fn create_incomplete_model(path: &Path) { @@ -243,6 +265,28 @@ fn deterministic_model_discovery_prefers_complete_env_override() { ); } +#[test] +fn deterministic_model_discovery_skips_invalid_env_override() { + let (_tmp, home) = temp_home(); + let models_root = home.join(".codescribe/models"); + let fp16 = models_root.join(WHISPER_FP16_MODEL); + let env_model = home.join("custom/quantized-whisper-model"); + + create_complete_model(&fp16); + create_complete_model(&env_model); + std::fs::write( + env_model.join("config.json"), + r#"{"quantization":{"bits":8}}"#, + ) + .expect("mark env override as quantized"); + + let found = discover_local_whisper_model_for(&home, Some(&env_model)) + .expect("expected valid standard fp16 model to be discovered"); + + assert_eq!(found.source, ModelSource::UserFp16); + assert_eq!(found.path, fp16); +} + #[test] fn deterministic_model_discovery_refuses_incomplete_fp16_without_legacy_fallback() { let (_tmp, home) = temp_home(); diff --git a/tests/support/e2e_stt_matrix.rs b/tests/support/e2e_stt_matrix.rs index 89548c96..785afb07 100644 --- a/tests/support/e2e_stt_matrix.rs +++ b/tests/support/e2e_stt_matrix.rs @@ -54,12 +54,7 @@ pub fn test_audio_path() -> PathBuf { } pub fn whisper_model_is_complete(path: &Path) -> bool { - let has_weights = - path.join("weights.safetensors").exists() || path.join("model.safetensors").exists(); - path.join("config.json").exists() - && path.join("tokenizer.json").exists() - && path.join("mel_filters.npz").exists() - && has_weights + codescribe_core::config::models::validate_whisper_model_bundle(path).is_ok() } pub fn whisper_model_missing_parts(path: &Path) -> Vec<&'static str> { From 94554f5ee1f96d6ba4382bd28f8e02246e44359b Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Fri, 21 Aug 2026 21:17:39 +0200 Subject: [PATCH 22/45] [codex/vc-workflow] chore(stt): preserve validator trust boundary --- core/whisper_weights.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/core/whisper_weights.rs b/core/whisper_weights.rs index b17bbebf..618ff291 100644 --- a/core/whisper_weights.rs +++ b/core/whisper_weights.rs @@ -28,6 +28,7 @@ pub fn validate_whisper_model_bundle(path: &Path) -> Result<()> { /// Validate the config schema and reject every declared quantization mode. pub(crate) fn validate_whisper_config(path: &Path) -> Result<()> { + // nosemgrep: rust.actix.path-traversal.tainted-path.tainted-path -- Read-only model inspection. `path` is an operator-selected local model config or an internally resolved bundle/cache child; no network/request path component reaches it. let raw = fs::read_to_string(path) .with_context(|| format!("read Whisper config {}", path.display()))?; let config: serde_json::Value = serde_json::from_str(&raw) From 6b1506678c264bb0eea3055176220fb9b2f24db6 Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Fri, 21 Aug 2026 21:35:39 +0200 Subject: [PATCH 23/45] [codex/vc-workflow] test(stt): describe E2E bundle validation (PR #81 review) --- tests/e2e_stt_transcription.rs | 9 +++++++++ tests/support/e2e_stt_matrix.rs | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/e2e_stt_transcription.rs b/tests/e2e_stt_transcription.rs index a8f7e5c8..996b1915 100644 --- a/tests/e2e_stt_transcription.rs +++ b/tests/e2e_stt_transcription.rs @@ -241,6 +241,15 @@ fn deterministic_gate_parser_requires_explicit_opt_in_values() { ); } +#[test] +fn deterministic_model_discovery_hint_names_the_validation_contract() { + let hint = model_discovery_hint(Path::new("/tmp/test-home")); + assert!(hint.contains("parseable config and tokenizer")); + assert!(hint.contains("pinned mel_filters.npz checksum")); + assert!(hint.contains("structurally valid F16/F32 safetensors")); + assert!(hint.contains("no quantization declaration")); +} + #[test] fn deterministic_model_discovery_prefers_complete_env_override() { let (_tmp, home) = temp_home(); diff --git a/tests/support/e2e_stt_matrix.rs b/tests/support/e2e_stt_matrix.rs index 785afb07..0f2c30d7 100644 --- a/tests/support/e2e_stt_matrix.rs +++ b/tests/support/e2e_stt_matrix.rs @@ -114,7 +114,7 @@ pub fn discover_local_whisper_model_for( pub fn model_discovery_hint(home_dir: &Path) -> String { format!( - "Looked for complete fp16 Whisper model in CODESCRIBE_MODEL_PATH and {home}/.codescribe/models/{fp16}. Required files: config.json, tokenizer.json, mel_filters.npz, weights.safetensors or model.safetensors.", + "Looked for a valid fp16 Whisper model in CODESCRIBE_MODEL_PATH and {home}/.codescribe/models/{fp16}. The bundle must have parseable config and tokenizer files, the pinned mel_filters.npz checksum, structurally valid F16/F32 safetensors, and no quantization declaration.", home = home_dir.display(), fp16 = WHISPER_FP16_MODEL ) From fa89b3c920cc2554d00478301990233bfdf40ab0 Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Fri, 21 Aug 2026 21:36:04 +0200 Subject: [PATCH 24/45] [codex/vc-workflow] fix(release): quiet invalid cache probes (PR #81 review) --- scripts/ensure-models.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/ensure-models.sh b/scripts/ensure-models.sh index 19899bca..a659e3b0 100755 --- a/scripts/ensure-models.sh +++ b/scripts/ensure-models.sh @@ -98,7 +98,9 @@ has_valid_whisper_snapshot() { [[ -d "$dir" ]] || continue for snap in "$dir"/*; do [[ -d "$snap" ]] || continue - if valid_whisper_bundle "$snap"; then + # Cache probing is intentionally quiet: invalid snapshots are expected + # candidates, and ensure_repo prints the actionable aggregate status. + if valid_whisper_bundle "$snap" >/dev/null 2>&1; then return 0 fi done From 1647a223ddbbcfc362590076b693952394ceee5e Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Fri, 21 Aug 2026 21:52:42 +0200 Subject: [PATCH 25/45] [codex/vc-workflow] fix(stt): require Whisper control tokens (PR #81 review) --- core/config/models.rs | 32 ++++++++++++++++++++++++++------ core/whisper_weights.rs | 24 +++++++++++++++++------- docs/TEAM_SETUP.md | 4 ++-- tests/e2e_stt_transcription.rs | 7 ++++++- 4 files changed, 51 insertions(+), 16 deletions(-) diff --git a/core/config/models.rs b/core/config/models.rs index d3e9488d..c6bf7636 100644 --- a/core/config/models.rs +++ b/core/config/models.rs @@ -614,9 +614,7 @@ fn replace_file(partial: &Path, dest: &Path) -> Result<()> { fn validate_model_file(filename: &str, path: &Path) -> Result<()> { match filename { "config.json" => crate::whisper_weights::validate_whisper_config(path), - "tokenizer.json" => tokenizers::Tokenizer::from_file(path) - .map(|_| ()) - .map_err(|err| anyhow!("invalid tokenizer {}: {err}", path.display())), + "tokenizer.json" => crate::whisper_weights::validate_whisper_tokenizer(path), "mel_filters.npz" => crate::whisper_weights::verify_mel_filters(path), "weights.safetensors" | "model.safetensors" => validate_safetensors_file(path), _ => Err(anyhow!("unsupported Whisper model artifact: {filename}")), @@ -672,9 +670,12 @@ mod tests { fn create_complete_whisper_model(path: &Path) { fs::create_dir_all(path).unwrap(); fs::write(path.join("config.json"), "{}").unwrap(); - tokenizers::Tokenizer::new(tokenizers::models::bpe::BPE::default()) - .save(path.join("tokenizer.json"), false) - .unwrap(); + let mut tokenizer = tokenizers::Tokenizer::new(tokenizers::models::bpe::BPE::default()); + tokenizer.add_special_tokens(&[ + tokenizers::AddedToken::from("<|startoftranscript|>", true), + tokenizers::AddedToken::from("<|endoftext|>", true), + ]); + tokenizer.save(path.join("tokenizer.json"), false).unwrap(); fs::write( path.join("mel_filters.npz"), decode_hex(include_str!( @@ -978,6 +979,25 @@ mod tests { assert!(!is_complete_whisper_model_dir(&model)); } + /// Parseable non-Whisper tokenizers must not be advertised as ready. + #[test] + fn model_manager_rejects_tokenizer_without_control_tokens() { + let temp_dir = TempDir::new().unwrap(); + let model = temp_dir.path().join("model"); + create_complete_whisper_model(&model); + tokenizers::Tokenizer::new(tokenizers::models::bpe::BPE::default()) + .save(model.join("tokenizer.json"), false) + .unwrap(); + + assert!(!is_complete_whisper_model_dir(&model)); + assert!( + validate_whisper_model_bundle(&model) + .unwrap_err() + .to_string() + .contains("missing required token") + ); + } + /// A valid existing mel asset is reused without contacting the network. #[test] fn valid_existing_mel_skips_download() { diff --git a/core/whisper_weights.rs b/core/whisper_weights.rs index 618ff291..cd17416c 100644 --- a/core/whisper_weights.rs +++ b/core/whisper_weights.rs @@ -11,21 +11,31 @@ pub const SUPPORTED_NAMES: [&str; 2] = ["weights.safetensors", "model.safetensor /// SHA-256 of the pinned official OpenAI mel filterbank. pub const MEL_FILTERS_SHA256: &str = "7450ae70723a5ef9d341e3cee628c7cb0177f36ce42c44b7ed2bf3325f0f6d4c"; +const REQUIRED_TOKENIZER_TOKENS: [&str; 2] = ["<|startoftranscript|>", "<|endoftext|>"]; /// Validate every artifact required by runtime and embedded Whisper loaders. pub fn validate_whisper_model_bundle(path: &Path) -> Result<()> { validate_whisper_config(&path.join("config.json"))?; - let tokenizer_path = path.join("tokenizer.json"); - tokenizers::Tokenizer::from_file(&tokenizer_path).map_err(|err| { - anyhow!( - "invalid Whisper tokenizer {}: {err}", - tokenizer_path.display() - ) - })?; + validate_whisper_tokenizer(&path.join("tokenizer.json"))?; verify_mel_filters(&path.join("mel_filters.npz"))?; resolve_valid_whisper_weights_path(path).map(|_| ()) } +/// Parse the tokenizer and require the control tokens used by every decode. +pub(crate) fn validate_whisper_tokenizer(path: &Path) -> Result<()> { + let tokenizer = tokenizers::Tokenizer::from_file(path) + .map_err(|err| anyhow!("invalid Whisper tokenizer {}: {err}", path.display()))?; + for token in REQUIRED_TOKENIZER_TOKENS { + if tokenizer.token_to_id(token).is_none() { + return Err(anyhow!( + "Whisper tokenizer {} is missing required token {token}", + path.display() + )); + } + } + Ok(()) +} + /// Validate the config schema and reject every declared quantization mode. pub(crate) fn validate_whisper_config(path: &Path) -> Result<()> { // nosemgrep: rust.actix.path-traversal.tainted-path.tainted-path -- Read-only model inspection. `path` is an operator-selected local model config or an internally resolved bundle/cache child; no network/request path component reaches it. diff --git a/docs/TEAM_SETUP.md b/docs/TEAM_SETUP.md index b673fb85..4eee325c 100644 --- a/docs/TEAM_SETUP.md +++ b/docs/TEAM_SETUP.md @@ -67,8 +67,8 @@ Grant in: System Settings > Privacy & Security - Runtime fallback resolves Whisper from exactly one shared contract in `core/config/models.rs`: `CODESCRIBE_MODEL_PATH` → configured local model path/alias → configured HF repo snapshot → default local turbo model → default HF cache snapshot. -- A model is ready only after config/tokenizer parsing, pinned mel checksum, and - full safetensors structural/dtype validation. Downloads compose and validate the +- A model is ready only after config parsing, tokenizer parsing plus required Whisper control + tokens, the pinned mel checksum, and full safetensors structural/dtype validation. Downloads compose and validate the complete replacement in staging before any artifact is promoted over an installed model. - `make install-app` / `scripts/ensure-models.sh` are the easiest way to warm the expected cache paths. Release/setup preflight calls the same runtime-owned bundle validator before it skips a download; diff --git a/tests/e2e_stt_transcription.rs b/tests/e2e_stt_transcription.rs index 996b1915..d63dabbb 100644 --- a/tests/e2e_stt_transcription.rs +++ b/tests/e2e_stt_transcription.rs @@ -175,7 +175,12 @@ fn e2e_stt_model_init_stable() { fn create_complete_model(path: &Path) { std::fs::create_dir_all(path).expect("create model dir"); std::fs::write(path.join("config.json"), "{}").expect("write config"); - tokenizers::Tokenizer::new(tokenizers::models::bpe::BPE::default()) + let mut tokenizer = tokenizers::Tokenizer::new(tokenizers::models::bpe::BPE::default()); + tokenizer.add_special_tokens(&[ + tokenizers::AddedToken::from("<|startoftranscript|>", true), + tokenizers::AddedToken::from("<|endoftext|>", true), + ]); + tokenizer .save(path.join("tokenizer.json"), false) .expect("write tokenizer"); std::fs::write( From f78056173dd0d032fa88679aaee7a1302000c1e0 Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Fri, 21 Aug 2026 21:56:25 +0200 Subject: [PATCH 26/45] [codex/vc-workflow] fix(stt): repair default model as paired assets (PR #81 review) --- core/config/models.rs | 154 ++++++++++++++++++++++++++++++------------ docs/TEAM_SETUP.md | 3 +- 2 files changed, 111 insertions(+), 46 deletions(-) diff --git a/core/config/models.rs b/core/config/models.rs index c6bf7636..ac97ec68 100644 --- a/core/config/models.rs +++ b/core/config/models.rs @@ -365,16 +365,16 @@ where // no-op when the pieces are already on disk. Config and weights come from // mlx-community's fp16 conversion; tokenizer comes from OpenAI's matching // Transformers repository. The pinned mel filterbank is fetched below. + let mut paired_default_model = false; if let Some(snapshot) = hf_cache::find_snapshot(DEFAULT_WHISPER_REPO, &["config.json"]) && snapshot != dest { - copy_model_files(&snapshot, &dest, &["config.json"])?; - copy_model_files(&snapshot, &dest, &REQUIRED_MODEL_WEIGHTS)?; + paired_default_model = copy_default_model_pair(&snapshot, &dest)?; } if let Some(snapshot) = hf_cache::find_snapshot(TOKENIZER_WHISPER_REPO, &["tokenizer.json"]) { - copy_model_files(&snapshot, &dest, &["tokenizer.json"])?; + copy_model_files(&snapshot, &dest, &["tokenizer.json"], true)?; } - if is_complete_whisper_model_dir(&dest) { + if paired_default_model && is_complete_whisper_model_dir(&dest) { return Ok(canonicalize_or_self(dest)); } @@ -396,6 +396,7 @@ where "config.json", &dest.join("config.json"), &mut on_progress, + true, )?; download_hf_file( &client, @@ -403,6 +404,7 @@ where "tokenizer.json", &dest.join("tokenizer.json"), &mut on_progress, + true, )?; download_url_file( &client, @@ -413,31 +415,33 @@ where )?; let weights_dest = dest.join("weights.safetensors"); let weights_alt = dest.join("model.safetensors"); - if validate_model_file("weights.safetensors", &weights_dest).is_err() - && validate_model_file("model.safetensors", &weights_alt).is_err() - { - // mlx-community ships weights.safetensors; fall back to model.safetensors if 404. - match download_hf_file( - &client, - DEFAULT_WHISPER_REPO, - "weights.safetensors", - &weights_dest, - &mut on_progress, - ) { - Ok(()) => {} - Err(err) => { - tracing::warn!( - error = %err, - "weights.safetensors missing; trying model.safetensors" - ); - download_hf_file( - &client, - DEFAULT_WHISPER_REPO, - "model.safetensors", - &weights_alt, - &mut on_progress, - )?; - } + // An incomplete default bundle is repaired as one model generation. Even + // structurally valid installed weights may belong to another architecture, + // so pair them with the freshly selected default config instead of reusing + // them independently. + match download_hf_file( + &client, + DEFAULT_WHISPER_REPO, + "weights.safetensors", + &weights_dest, + &mut on_progress, + true, + ) { + Ok(()) => remove_other_weight_file(&dest, "weights.safetensors")?, + Err(err) => { + tracing::warn!( + error = %err, + "weights.safetensors missing; trying model.safetensors" + ); + download_hf_file( + &client, + DEFAULT_WHISPER_REPO, + "model.safetensors", + &weights_alt, + &mut on_progress, + true, + )?; + remove_other_weight_file(&dest, "model.safetensors")?; } } @@ -455,9 +459,9 @@ where /// /// Lets Settings → Download complete without network traffic when the pieces are /// already on disk (warm official caches). Every copied file is validated in a -/// sibling `.partial` path before atomic promotion. Invalid destinations are -/// replaced; valid ones are preserved. -fn copy_model_files(src: &Path, dest: &Path, names: &[&str]) -> Result<()> { +/// sibling `.partial` path before atomic promotion. Callers may preserve valid +/// destinations or deliberately replace a config/weights generation as a pair. +fn copy_model_files(src: &Path, dest: &Path, names: &[&str], replace_valid: bool) -> Result<()> { if !src.is_dir() { return Ok(()); } @@ -465,7 +469,7 @@ fn copy_model_files(src: &Path, dest: &Path, names: &[&str]) -> Result<()> { for name in names { let from = src.join(name); let to = dest.join(name); - if !from.is_file() || validate_model_file(name, &to).is_ok() { + if !from.is_file() || (!replace_valid && validate_model_file(name, &to).is_ok()) { continue; } if let Err(err) = validate_model_file(name, &from) { @@ -485,24 +489,65 @@ fn copy_model_files(src: &Path, dest: &Path, names: &[&str]) -> Result<()> { Ok(()) } +/// Replace config and weights only when both come from one valid default snapshot. +fn copy_default_model_pair(src: &Path, dest: &Path) -> Result { + if validate_model_file("config.json", &src.join("config.json")).is_err() { + return Ok(false); + } + let Ok(weights) = resolve_valid_whisper_weights_path(src) else { + return Ok(false); + }; + let Some(weight_name) = weights.file_name().and_then(|name| name.to_str()) else { + return Ok(false); + }; + + copy_model_files(src, dest, &["config.json"], true)?; + copy_model_files(src, dest, &[weight_name], true)?; + remove_other_weight_file(dest, weight_name)?; + Ok(true) +} + +fn remove_other_weight_file(dest: &Path, selected: &str) -> Result<()> { + for name in REQUIRED_MODEL_WEIGHTS { + if name == selected { + continue; + } + let stale = dest.join(name); + if stale.exists() { + fs::remove_file(&stale) + .with_context(|| format!("remove stale Whisper weights {}", stale.display()))?; + } + } + Ok(()) +} + /// Fetch one file from the Hugging Face resolve endpoint into `dest`. /// /// Downloads to a sibling `.partial` file and renames on success, so an aborted /// transfer can never leave a truncated file that passes bundle validation. -/// A valid `dest` is preserved; an invalid one is replaced. `HF_TOKEN` is sent -/// as bearer auth when set, for gated repos. +/// A valid `dest` is preserved unless the caller is repairing a paired default +/// model generation. `HF_TOKEN` is sent as bearer auth when set, for gated repos. fn download_hf_file( client: &reqwest::blocking::Client, repo: &str, filename: &str, dest: &Path, on_progress: &mut F, + replace_valid: bool, ) -> Result<()> where F: FnMut(&str, u64, Option), { let url = format!("https://huggingface.co/{repo}/resolve/main/{filename}"); - download_url_file_authenticated(client, &url, filename, dest, on_progress, true) + download_url_file_authenticated( + client, + &url, + filename, + dest, + on_progress, + true, + replace_valid, + ) } fn download_url_file( @@ -515,7 +560,7 @@ fn download_url_file( where F: FnMut(&str, u64, Option), { - download_url_file_authenticated(client, url, filename, dest, on_progress, false) + download_url_file_authenticated(client, url, filename, dest, on_progress, false, false) } fn download_url_file_authenticated( @@ -525,11 +570,12 @@ fn download_url_file_authenticated( dest: &Path, on_progress: &mut F, use_hf_token: bool, + replace_valid: bool, ) -> Result<()> where F: FnMut(&str, u64, Option), { - if validate_model_file(filename, dest).is_ok() { + if !replace_valid && validate_model_file(filename, dest).is_ok() { on_progress( filename, dest.metadata().map(|metadata| metadata.len()).unwrap_or(0), @@ -537,11 +583,6 @@ where ); return Ok(()); } - if dest.exists() { - fs::remove_file(dest) - .with_context(|| format!("remove invalid model artifact {}", dest.display()))?; - } - let mut request = client.get(url); if use_hf_token && let Ok(token) = std::env::var("HF_TOKEN") { let token = token.trim(); @@ -1036,14 +1077,37 @@ mod tests { fs::write(destination.join("mel_filters.npz"), b"bad mel").unwrap(); fs::write(destination.join("model.safetensors"), b"bad weights").unwrap(); - copy_model_files(&source, &destination, &REQUIRED_MODEL_FILES).unwrap(); - copy_model_files(&source, &destination, &REQUIRED_MODEL_WEIGHTS).unwrap(); + copy_model_files(&source, &destination, &REQUIRED_MODEL_FILES, false).unwrap(); + copy_model_files(&source, &destination, &REQUIRED_MODEL_WEIGHTS, false).unwrap(); validate_whisper_model_bundle(&destination).unwrap(); assert!(!destination.join("config.json.partial").exists()); assert!(!destination.join("model.safetensors.partial").exists()); } + /// Default repair replaces individually valid weights from another bundle. + #[test] + fn cached_default_pair_replaces_stale_valid_weights() { + let temp_dir = TempDir::new().unwrap(); + let source = temp_dir.path().join("source"); + let destination = temp_dir.path().join("destination"); + create_complete_whisper_model(&source); + create_complete_whisper_model(&destination); + fs::rename( + destination.join("model.safetensors"), + destination.join("weights.safetensors"), + ) + .unwrap(); + + assert!(copy_default_model_pair(&source, &destination).unwrap()); + assert_eq!( + fs::read(destination.join("model.safetensors")).unwrap(), + fs::read(source.join("model.safetensors")).unwrap() + ); + assert!(!destination.join("weights.safetensors").exists()); + validate_whisper_model_bundle(&destination).unwrap(); + } + /// A downloaded checksum mismatch is never promoted to the final mel path. #[test] fn corrupt_download_is_removed_before_promotion() { diff --git a/docs/TEAM_SETUP.md b/docs/TEAM_SETUP.md index 4eee325c..1148f899 100644 --- a/docs/TEAM_SETUP.md +++ b/docs/TEAM_SETUP.md @@ -69,7 +69,8 @@ Grant in: System Settings > Privacy & Security default local turbo model → default HF cache snapshot. - A model is ready only after config parsing, tokenizer parsing plus required Whisper control tokens, the pinned mel checksum, and full safetensors structural/dtype validation. Downloads compose and validate the - complete replacement in staging before any artifact is promoted over an installed model. + complete replacement before promotion; repairing the default model replaces config and weights + as a paired generation instead of preserving independently valid artifacts from another architecture. - `make install-app` / `scripts/ensure-models.sh` are the easiest way to warm the expected cache paths. Release/setup preflight calls the same runtime-owned bundle validator before it skips a download; a directory with merely the expected filenames is repaired or rejected rather than embedded. From 40eefa4ce0af73233cf37699c96dfe0aca8d0239 Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Fri, 21 Aug 2026 23:58:17 +0200 Subject: [PATCH 27/45] [codex/vc-workflow] fix(stt): exclude alignment metadata from model weights --- core/stt/whisper/engine.rs | 59 ++++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 25 deletions(-) diff --git a/core/stt/whisper/engine.rs b/core/stt/whisper/engine.rs index 2151012a..41c65208 100644 --- a/core/stt/whisper/engine.rs +++ b/core/stt/whisper/engine.rs @@ -535,31 +535,9 @@ impl LocalWhisperEngine { read_secs = read_started.elapsed().as_secs_f64(); let plain_started = std::time::Instant::now(); - let mut tensor_map = HashMap::new(); - - // The payload gate above makes every tensor in this pass unquantized. - for (name, tensor) in raw_tensors.iter() { - let mapped_name = map_tensor_name(name); - let mut t = tensor.clone(); - if t.dtype() != DType::F32 { - t = t.to_dtype(DType::F32)?; - } - - // Fix shape for conv weights (MLX [out, kernel, in] -> Candle [out, in, kernel]) - if mapped_name.ends_with("conv1.weight") || mapped_name.ends_with("conv2.weight") { - let dims = t.dims(); - if dims.len() == 3 && dims[1] == 3 { - t = t.permute((0, 2, 1))?.contiguous()?; - } - } - - let t = t.to_device(&device)?; - tensor_map.insert(mapped_name, t); - } - + let vb = build_varbuilder_from_tensors(raw_tensors, &device)?; plain_secs = plain_started.elapsed().as_secs_f64(); - - candle_nn::VarBuilder::from_tensors(tensor_map, DType::F32, &device) + vb }; let build_started = std::time::Instant::now(); @@ -1951,8 +1929,12 @@ fn build_varbuilder_from_tensors( } let mut tensor_map = HashMap::new(); - // The payload gate above makes every tensor in this pass unquantized. + // alignment_heads is integer metadata used by upstream timestamp tooling, + // not a model weight consumed by Candle's Whisper loader. for (name, tensor) in raw_tensors.iter() { + if name == "alignment_heads" { + continue; + } let mapped_name = map_tensor_name(name); let mut t = tensor.clone(); if t.dtype() != DType::F32 { @@ -2039,6 +2021,33 @@ mod model_payload_tests { assert!(format!("{err:#}").contains("refused")); } + #[test] + fn tensor_builder_excludes_i64_alignment_metadata() { + let mut tensors = HashMap::new(); + tensors.insert( + "encoder.weight".to_string(), + Tensor::from_vec(vec![1.0_f32], 1, &Device::Cpu).unwrap(), + ); + tensors.insert( + "alignment_heads".to_string(), + Tensor::from_vec( + vec![2_i64, 4, 2, 11, 3, 3, 3, 6, 3, 11, 3, 14], + (6, 2), + &Device::Cpu, + ) + .unwrap(), + ); + + let vb = build_varbuilder_from_tensors(tensors, &Device::Cpu).unwrap(); + + assert!(vb.contains_tensor("model.encoder.weight")); + assert_eq!( + vb.get_unchecked("model.encoder.weight").unwrap().dtype(), + DType::F32 + ); + assert!(!vb.contains_tensor("model.alignment_heads")); + } + #[test] fn local_loader_uses_valid_alternative_after_invalid_primary() { let temp = TempDir::new().unwrap(); From 3723f9621935d3fbf7b49bdee2f1d47df0664596 Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Sat, 22 Aug 2026 00:00:25 +0200 Subject: [PATCH 28/45] [codex/vc-workflow] fix(release): remove shadowed Whisper weights --- scripts/download-model.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/download-model.sh b/scripts/download-model.sh index ae9933fb..4fcc0f59 100755 --- a/scripts/download-model.sh +++ b/scripts/download-model.sh @@ -180,8 +180,10 @@ if [[ "$MODEL_REPO" == "$DEFAULT_REPO" ]]; then atomic_copy "$MODEL_STAGE/mel_filters.npz" "$MODEL_DEST/mel_filters.npz" if [[ -f "$MODEL_STAGE/weights.safetensors" ]]; then atomic_copy "$MODEL_STAGE/weights.safetensors" "$MODEL_DEST/weights.safetensors" + rm -f "$MODEL_DEST/model.safetensors" else atomic_copy "$MODEL_STAGE/model.safetensors" "$MODEL_DEST/model.safetensors" + rm -f "$MODEL_DEST/weights.safetensors" fi echo " Runtime directory: $MODEL_DEST" "$WHISPER_VALIDATOR" "$MODEL_DEST" From 77e18bf33a90b6e49bb487227a0f4dc508268969 Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Sat, 22 Aug 2026 00:02:57 +0200 Subject: [PATCH 29/45] [codex/vc-workflow] fix(msrv): declare Rust 1.88 support floor --- .github/workflows/rust.yml | 14 ++++++++++++++ Cargo.toml | 2 ++ README.md | 2 +- bridge/Cargo.toml | 1 + core/Cargo.toml | 1 + examples/README.md | 2 +- 6 files changed, 20 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index d9638b06..4245b75e 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -34,6 +34,20 @@ env: CARGO_TERM_COLOR: always jobs: + msrv: + name: Rust 1.88 MSRV + runs-on: macos-latest + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Ensure MSRV toolchain on PATH + run: | + rustup toolchain install 1.88.0 --profile minimal --no-self-update + dirname "$(rustup which --toolchain 1.88.0 rustc)" >> "$GITHUB_PATH" + + - name: cargo check at MSRV + run: cargo check --workspace --all-targets + format: name: Format Check runs-on: macos-latest diff --git a/Cargo.toml b/Cargo.toml index d3933989..043490c6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,7 @@ members = [".", "core", "bridge"] [workspace.package] version = "0.14.1" edition = "2024" +rust-version = "1.88" authors = ["Vetcoders "] [workspace.dependencies] @@ -20,6 +21,7 @@ tracing = "0.1" name = "codescribe" version = "0.14.1" edition = "2024" +rust-version.workspace = true description = "Speech-to-text for macOS — SwiftUI front-end over a Rust engine (UniFFI bridge)" authors = ["Vetcoders "] license = "FSL-1.1-ALv2" diff --git a/README.md b/README.md index c3059809..db8cac74 100644 --- a/README.md +++ b/README.md @@ -139,7 +139,7 @@ Codescribe can load custom MCP servers from `~/.codescribe/mcp.json`. That keeps - **macOS 14+** (Sonoma or later) - **Apple Silicon** (M1, M2, M3, or later) -- **Rust Toolchain** (1.85+ with edition 2024 support) +- **Rust Toolchain** (1.88+; the workspace declares this MSRV) ### Install from Source diff --git a/bridge/Cargo.toml b/bridge/Cargo.toml index 21a5303e..caf31e74 100644 --- a/bridge/Cargo.toml +++ b/bridge/Cargo.toml @@ -2,6 +2,7 @@ name = "codescribe-ffi" version.workspace = true edition = "2024" +rust-version.workspace = true authors = ["Vetcoders "] description = "UniFFI bridge exposing the codescribe engine (agent streaming, STT, config) to Swift" diff --git a/core/Cargo.toml b/core/Cargo.toml index 5e3ce67d..07b46b4d 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -2,6 +2,7 @@ name = "codescribe-core" version.workspace = true edition.workspace = true +rust-version.workspace = true authors = ["Vetcoders "] license = "FSL-1.1-ALv2" description = "Core library for Codescribe (audio, STT, quality pipeline)" diff --git a/examples/README.md b/examples/README.md index b094cad4..c61baeb3 100644 --- a/examples/README.md +++ b/examples/README.md @@ -88,7 +88,7 @@ VAD internals are hardcoded in `core/vad/config.rs` (Silero defaults). - macOS (uses CoreAudio via cpal) - Microphone access permissions -- Rust 1.85+ (edition 2024) with tokio runtime +- Rust 1.88+ (the workspace MSRV) with tokio runtime --- From dfa9ec2c0579cf43662077d752377c86e41f8394 Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Sat, 22 Aug 2026 00:04:37 +0200 Subject: [PATCH 30/45] [codex/vc-workflow] fix(stt): align model discovery across build and tests --- core/build.rs | 59 ++++++++++++++++++++++++--------- scripts/bench-stt.sh | 6 +++- tests/e2e_stt_transcription.rs | 49 +++++++++++++++++++++++++-- tests/support/e2e_stt_matrix.rs | 37 +++++++++++++++++---- 4 files changed, 126 insertions(+), 25 deletions(-) diff --git a/core/build.rs b/core/build.rs index d894a5bc..f03ee7c0 100644 --- a/core/build.rs +++ b/core/build.rs @@ -70,6 +70,7 @@ fn main() { println!("cargo:rerun-if-changed=Cargo.toml"); println!("cargo:rerun-if-env-changed=CODESCRIBE_EMBED_MODEL"); println!("cargo:rerun-if-env-changed=CODESCRIBE_MODEL_PATH"); + println!("cargo:rerun-if-env-changed=CODESCRIBE_MODELS_DIR"); println!("cargo:rerun-if-env-changed=CODESCRIBE_NO_EMBED"); println!("cargo:rerun-if-env-changed=CODESCRIBE_EMBED_WHISPER"); println!("cargo:rerun-if-env-changed=CODESCRIBE_EMBED_EMBEDDER"); @@ -447,8 +448,9 @@ fn whisper_dir_complete(path: &Path) -> bool { /// Locate the Whisper snapshot to embed. /// /// `CODESCRIBE_MODEL_PATH` wins when it is a complete snapshot. The composed -/// `~/.codescribe/models` tree is next — that is what `make download-model` -/// writes. An HF cache hit is used only when it already has tokenizer + mel. +/// `CODESCRIBE_MODELS_DIR` or `~/.codescribe/models` tree is next — that is +/// what `make download-model` writes. An HF cache hit is used only when its +/// complete bundle passes the same validator used by runtime. fn resolve_whisper_embed_model_path( manifest_dir: &str, embed_model: &str, @@ -460,8 +462,22 @@ fn resolve_whisper_embed_model_path( return p; } } + let composed_name = if embed_model == default_repo || embed_model == DEFAULT_MODEL_NAME { + DEFAULT_MODEL_NAME + } else { + embed_model + }; + if let Ok(models_dir) = env::var("CODESCRIBE_MODELS_DIR") { + let models_dir = expand_tilde_path(models_dir.trim()); + if models_dir.exists() { + let composed = models_dir.join(composed_name); + if whisper_dir_complete(&composed) { + return composed; + } + } + } if let Some(home) = dirs::home_dir() { - let composed = home.join(".codescribe").join("models").join(embed_model); + let composed = home.join(".codescribe").join("models").join(composed_name); if whisper_dir_complete(&composed) { return composed; } @@ -475,14 +491,12 @@ fn resolve_whisper_embed_model_path( } } } - if embed_model.contains('/') - && let Some(snapshot) = find_hf_snapshot(embed_model) - && whisper_dir_complete(&snapshot) - { - return snapshot; + if embed_model.contains('/') { + if let Some(snapshot) = find_hf_snapshot_matching(embed_model, whisper_dir_complete) { + return snapshot; + } } else if embed_model == DEFAULT_MODEL_NAME - && let Some(snapshot) = find_hf_snapshot(default_repo) - && whisper_dir_complete(&snapshot) + && let Some(snapshot) = find_hf_snapshot_matching(default_repo, whisper_dir_complete) { return snapshot; } @@ -552,21 +566,36 @@ fn hf_cache_bases() -> Vec { /// First snapshot of `repo` found across the candidate cache bases. fn find_hf_snapshot(repo: &str) -> Option { + find_hf_snapshot_matching(repo, |_| true) +} + +/// First cache snapshot accepted by `predicate`, preserving cache-root order. +fn find_hf_snapshot_matching(repo: &str, predicate: F) -> Option +where + F: Fn(&Path) -> bool, +{ for base in hf_cache_bases() { - if let Some(snapshot) = find_hf_snapshot_in_base(&base, repo) { + if let Some(snapshot) = find_hf_snapshot_in_base_matching(&base, repo, &predicate) { return Some(snapshot); } } None } -/// Newest snapshot of `repo` under one cache base. +/// Newest accepted snapshot of `repo` under one cache base. /// /// The `models--owner--name` directory is tried first; failing that, the base /// is scanned case-insensitively, because HF repo ids differ in case between /// what a caller writes and what the cache recorded. Among the snapshots, the -/// most recently modified wins — that is the one a `hf download` just wrote. -fn find_hf_snapshot_in_base(base: &PathBuf, repo: &str) -> Option { +/// Newest snapshot under one cache base that satisfies `predicate`. +fn find_hf_snapshot_in_base_matching( + base: &PathBuf, + repo: &str, + predicate: &F, +) -> Option +where + F: Fn(&Path) -> bool, +{ let repo_dir = base.join(format!("models--{}", repo.replace('/', "--"))); let snapshots_dir = repo_dir.join("snapshots"); @@ -601,7 +630,7 @@ fn find_hf_snapshot_in_base(base: &PathBuf, repo: &str) -> Option { for entry in entries.flatten() { let path = entry.path(); - if !path.is_dir() { + if !path.is_dir() || !predicate(&path) { continue; } let modified = entry diff --git a/scripts/bench-stt.sh b/scripts/bench-stt.sh index 2388c540..6d9b3246 100755 --- a/scripts/bench-stt.sh +++ b/scripts/bench-stt.sh @@ -147,7 +147,11 @@ discover_model() { return 0 fi - candidate="$home_dir/.codescribe/models/whisper-large-v3-turbo" + local models_root="$home_dir/.codescribe/models" + if [[ -n "${CODESCRIBE_MODELS_DIR:-}" ]] && [[ -d "$CODESCRIBE_MODELS_DIR" ]]; then + models_root="$CODESCRIBE_MODELS_DIR" + fi + candidate="$models_root/whisper-large-v3-turbo" if model_is_complete "$candidate"; then printf '%s\n' "$candidate" return 0 diff --git a/tests/e2e_stt_transcription.rs b/tests/e2e_stt_transcription.rs index d63dabbb..010ae470 100644 --- a/tests/e2e_stt_transcription.rs +++ b/tests/e2e_stt_transcription.rs @@ -18,8 +18,9 @@ mod e2e_stt_matrix; use e2e_stt_matrix::{ ModelDiscovery, ModelSource, STT_OPT_IN_ENV, WHISPER_FP16_MODEL, discover_local_whisper_model, - discover_local_whisper_model_for, model_discovery_hint, parse_opt_in, skip_unless_opt_in, - test_audio_path, whisper_model_missing_parts, + discover_local_whisper_model_for, discover_local_whisper_model_for_with_root, + model_discovery_hint, parse_opt_in, skip_unless_opt_in, test_audio_path, + whisper_model_missing_parts, }; fn home_dir() -> PathBuf { @@ -297,10 +298,52 @@ fn deterministic_model_discovery_skips_invalid_env_override() { let found = discover_local_whisper_model_for(&home, Some(&env_model)) .expect("expected valid standard fp16 model to be discovered"); - assert_eq!(found.source, ModelSource::UserFp16); + assert_eq!(found.source, ModelSource::ModelsDir); assert_eq!(found.path, fp16); } +#[test] +fn deterministic_model_discovery_honors_existing_custom_models_root() { + let (_tmp, home) = temp_home(); + let custom_root = home.join("custom-models"); + let fp16 = custom_root.join(WHISPER_FP16_MODEL); + create_complete_model(&fp16); + + let found = discover_local_whisper_model_for_with_root(&home, None, Some(&custom_root)) + .expect("expected model under CODESCRIBE_MODELS_DIR"); + + assert_eq!(found.source, ModelSource::ModelsDir); + assert_eq!(found.path, fp16); +} + +#[test] +fn deterministic_existing_empty_models_root_shadows_home_fallback() { + let (_tmp, home) = temp_home(); + let home_fp16 = home.join(".codescribe/models").join(WHISPER_FP16_MODEL); + create_complete_model(&home_fp16); + let custom_root = home.join("empty-custom-models"); + std::fs::create_dir_all(&custom_root).unwrap(); + + assert!( + discover_local_whisper_model_for_with_root(&home, None, Some(&custom_root)).is_none(), + "an existing explicit models root must own discovery even when empty" + ); +} + +#[test] +fn deterministic_missing_models_root_falls_back_to_home() { + let (_tmp, home) = temp_home(); + let home_fp16 = home.join(".codescribe/models").join(WHISPER_FP16_MODEL); + create_complete_model(&home_fp16); + let missing_root = home.join("missing-custom-models"); + + let found = discover_local_whisper_model_for_with_root(&home, None, Some(&missing_root)) + .expect("missing override root should preserve runtime home fallback"); + + assert_eq!(found.source, ModelSource::ModelsDir); + assert_eq!(found.path, home_fp16); +} + #[test] fn deterministic_model_discovery_refuses_incomplete_fp16_without_legacy_fallback() { let (_tmp, home) = temp_home(); diff --git a/tests/support/e2e_stt_matrix.rs b/tests/support/e2e_stt_matrix.rs index 0f2c30d7..0631cc1e 100644 --- a/tests/support/e2e_stt_matrix.rs +++ b/tests/support/e2e_stt_matrix.rs @@ -17,7 +17,7 @@ pub const WHISPER_FP16_MODEL: &str = "whisper-large-v3-turbo"; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ModelSource { EnvOverride, - UserFp16, + ModelsDir, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -85,12 +85,27 @@ pub fn discover_local_whisper_model() -> Option { let env_override = std::env::var("CODESCRIBE_MODEL_PATH") .ok() .map(PathBuf::from); - discover_local_whisper_model_for(&home_dir, env_override.as_deref()) + let models_root = std::env::var("CODESCRIBE_MODELS_DIR") + .ok() + .map(PathBuf::from); + discover_local_whisper_model_for_with_root( + &home_dir, + env_override.as_deref(), + models_root.as_deref(), + ) } pub fn discover_local_whisper_model_for( home_dir: &Path, env_override: Option<&Path>, +) -> Option { + discover_local_whisper_model_for_with_root(home_dir, env_override, None) +} + +pub fn discover_local_whisper_model_for_with_root( + home_dir: &Path, + env_override: Option<&Path>, + models_root: Option<&Path>, ) -> Option { if let Some(path) = env_override && whisper_model_is_complete(path) @@ -101,10 +116,14 @@ pub fn discover_local_whisper_model_for( }); } - let user_fp16 = home_dir.join(".codescribe/models").join(WHISPER_FP16_MODEL); + let default_root = home_dir.join(".codescribe/models"); + let models_root = models_root + .filter(|path| path.exists()) + .unwrap_or(&default_root); + let user_fp16 = models_root.join(WHISPER_FP16_MODEL); if whisper_model_is_complete(&user_fp16) { return Some(ModelDiscovery { - source: ModelSource::UserFp16, + source: ModelSource::ModelsDir, path: user_fp16, }); } @@ -113,9 +132,15 @@ pub fn discover_local_whisper_model_for( } pub fn model_discovery_hint(home_dir: &Path) -> String { + let default_root = home_dir.join(".codescribe/models"); + let models_root = std::env::var("CODESCRIBE_MODELS_DIR") + .ok() + .map(PathBuf::from) + .filter(|path| path.exists()) + .unwrap_or(default_root); format!( - "Looked for a valid fp16 Whisper model in CODESCRIBE_MODEL_PATH and {home}/.codescribe/models/{fp16}. The bundle must have parseable config and tokenizer files, the pinned mel_filters.npz checksum, structurally valid F16/F32 safetensors, and no quantization declaration.", - home = home_dir.display(), + "Looked for a valid fp16 Whisper model in CODESCRIBE_MODEL_PATH and {root}/{fp16}. The bundle must have parseable config and tokenizer files, the pinned mel_filters.npz checksum, structurally valid F16/F32 safetensors, and no quantization declaration.", + root = models_root.display(), fp16 = WHISPER_FP16_MODEL ) } From c9562500ba6a83010d1fa6ab6645c17f6adcb423 Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Sat, 22 Aug 2026 00:06:51 +0200 Subject: [PATCH 31/45] [codex/vc-workflow] docs(stt): align FP16 compatibility contract --- CHANGELOG.md | 8 +++++++- core/config/models.rs | 12 ++++++++---- core/stt/whisper/engine.rs | 2 +- .../2026-05-26-LAYERED_INCREMENTAL_TRANSCRIPTION.md | 3 ++- docs/STT_CONTRACT.md | 8 ++------ 5 files changed, 20 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c443bc8e..358d6dfd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -68,6 +68,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- **Local Whisper is an explicitly validated FP16/F32 bundle.** Runtime, + Settings download, release scripts, E2E discovery, and the optional fat build + share the same config/tokenizer/mel/safetensors contract. Quantized payloads + and the legacy Q8 fallback are refused; the old public Q8 identifiers remain + deprecated source-compatibility constants only. Building from source now + declares Rust 1.88 as the minimum supported toolchain. + - **Supervisor findings own transcript-quality categories.** Engine catalog `codescribe-supervisor-findings/v1` (`core/quality/supervisor.rs`) names every issue class the tree already had — contract forbiddens, clock-lie, @@ -77,7 +84,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 Voice Lab three-judge emits those findings. WER stays a footnote of proposal agreement, not accuracy. - - **Layer 1 applies aligned same-utterance wording.** When live Apple and the Whisper window share most words, Layer 1 now substitutes those spans instead of discarding the repair at the 50% change cap. Unrelated diff --git a/core/config/models.rs b/core/config/models.rs index ac97ec68..29df87f9 100644 --- a/core/config/models.rs +++ b/core/config/models.rs @@ -16,12 +16,16 @@ pub const DEFAULT_MODEL: &str = "whisper-large-v3-turbo"; /// the Settings → Dictation download. fp16 weights: no q8→F32 dequantization /// on load, at the cost of a larger download than the q8 repo. pub const DEFAULT_WHISPER_REPO: &str = "mlx-community/whisper-large-v3-turbo"; +/// Former quantized model alias retained only for source compatibility. +#[deprecated(note = "quantized Whisper is unsupported; no runtime fallback uses this alias")] +pub const LEGACY_MODEL: &str = "whisper-large-v3-turbo-mlx-q8"; +/// Former quantized model repository retained only for source compatibility. +#[deprecated(note = "quantized Whisper is unsupported; no runtime fallback uses this repository")] +pub const LEGACY_WHISPER_REPO: &str = "LibraxisAI/whisper-large-v3-turbo-mlx-q8"; /// Official Transformers tokenizer paired with Whisper large-v3-turbo. -pub const TOKENIZER_WHISPER_REPO: &str = "openai/whisper-large-v3-turbo"; +pub(crate) const TOKENIZER_WHISPER_REPO: &str = "openai/whisper-large-v3-turbo"; /// Pinned OpenAI Whisper asset. The checksum is asserted by the installer. -pub const MEL_FILTERS_URL: &str = "https://raw.githubusercontent.com/openai/whisper/5f86d1d86363843179951550570367b37c5d6f78/whisper/assets/mel_filters.npz"; -/// SHA-256 of [`MEL_FILTERS_URL`]. -pub const MEL_FILTERS_SHA256: &str = crate::whisper_weights::MEL_FILTERS_SHA256; +pub(crate) const MEL_FILTERS_URL: &str = "https://raw.githubusercontent.com/openai/whisper/5f86d1d86363843179951550570367b37c5d6f78/whisper/assets/mel_filters.npz"; /// Files that must all be present for a directory to count as a usable model. const REQUIRED_MODEL_FILES: [&str; 3] = ["config.json", "tokenizer.json", "mel_filters.npz"]; /// Weight file names, of which **any one** satisfies the completeness check — diff --git a/core/stt/whisper/engine.rs b/core/stt/whisper/engine.rs index 41c65208..7d4ce144 100644 --- a/core/stt/whisper/engine.rs +++ b/core/stt/whisper/engine.rs @@ -508,7 +508,7 @@ impl LocalWhisperEngine { // median of 9.2 s (p90 21.9 s, worst 34.9 s, 805 s in total) because the // idle reaper drops the weights every 45 minutes and this function then // rebuilds them from scratch. A single aggregate number cannot say - // whether to attack the read, the dequantisation, or the GPU upload, so + // whether to attack the read, tensor conversion/mapping, or GPU upload, so // each phase reports its own cost. let load_started = std::time::Instant::now(); let read_secs; diff --git a/docs/ADR/2026-05-26-LAYERED_INCREMENTAL_TRANSCRIPTION.md b/docs/ADR/2026-05-26-LAYERED_INCREMENTAL_TRANSCRIPTION.md index 52289aad..33577328 100644 --- a/docs/ADR/2026-05-26-LAYERED_INCREMENTAL_TRANSCRIPTION.md +++ b/docs/ADR/2026-05-26-LAYERED_INCREMENTAL_TRANSCRIPTION.md @@ -31,7 +31,8 @@ describing for weeks: - `core/stt/apple_stt/mod.rs` — 522 LOC `AppleSpeechAnalyzerAdapter` implementing `TranscriptionAdapter`, defaulted through `CODESCRIBE_STT_ENGINE=auto`, with graceful fallback to Candle. -- `core/stt/whisper/*` — production Whisper path (embedded turbo-mlx-q8 + Silero VAD). +- At decision time, `core/stt/whisper/*` was the production Whisper path + (embedded turbo-mlx-q8 + Silero VAD). - `core/audio/streaming_recorder.rs` — chunker emits utterance events **and** the recorder always tees a full WAV to disk (`wav_path: PathBuf`, `recorder.rs:203/678`). Full audio is never lost, even when the chunker hands out short utterances. diff --git a/docs/STT_CONTRACT.md b/docs/STT_CONTRACT.md index dbac8577..e1184092 100644 --- a/docs/STT_CONTRACT.md +++ b/docs/STT_CONTRACT.md @@ -131,9 +131,7 @@ button. "engine": { "stt_engine": "apple", "whisper_model": "whisper-large-v3-turbo", - "final_pass_mode": "off", - "layered_transcription": "off", - "asr_mode": "cloud" + "final_pass_mode": "smart" }, "formatting": { "enabled": true, "level": "smart" } } @@ -273,9 +271,7 @@ Valid engine labels on verdict: `local_apple`, `local_whisper`, `streaming_whisp "engine": { "stt_engine": "apple", "whisper_model": "whisper-large-v3-turbo", - "final_pass_mode": "off", - "layered_transcription": "off", - "asr_mode": "cloud" + "final_pass_mode": "smart" } ``` 2. Full quit + relaunch. From e36890456edff386fbd1f7496a33229fa5800ff6 Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Sat, 22 Aug 2026 00:08:44 +0200 Subject: [PATCH 32/45] [codex/vc-workflow] test(release): cover alternate Whisper weight promotion --- scripts/tests/download-model-test.sh | 81 ++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100755 scripts/tests/download-model-test.sh diff --git a/scripts/tests/download-model-test.sh b/scripts/tests/download-model-test.sh new file mode 100755 index 00000000..ffac7e70 --- /dev/null +++ b/scripts/tests/download-model-test.sh @@ -0,0 +1,81 @@ +#!/bin/bash +# Hermetic regression for default Whisper bundle promotion. + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "$0")/../.." && pwd)" +TEST_ROOT=$(mktemp -d "${TMPDIR:-/tmp}/codescribe-download-model-test.XXXXXX") + +cleanup() { + find "$TEST_ROOT" -type f -delete 2>/dev/null || true + find "$TEST_ROOT" -type l -delete 2>/dev/null || true + find "$TEST_ROOT" -depth -type d -exec rmdir {} \; 2>/dev/null || true +} +trap cleanup EXIT + +make_tiny_weights() { + local destination="$1" + local header='{"model.weight":{"dtype":"F16","shape":[1],"data_offsets":[0,2]}}' + # The header is 65 bytes; safetensors prefixes it with little-endian u64. + printf '\x41\0\0\0\0\0\0\0%s\0\0' "$header" > "$destination" +} + +hf() { + if [[ "${1:-}" == "auth" ]]; then + return 1 + fi + if [[ "${2:-}" == "openai/whisper-large-v3-turbo" ]]; then + printf '%s\n' "$FAKE_TOKENIZER" + else + printf '%s\n' "$FAKE_MODEL_SNAPSHOT" + fi +} +export -f hf + +curl() { + local destination="" + while [[ "$#" -gt 0 ]]; do + if [[ "$1" == "-o" ]]; then + destination="$2" + shift 2 + else + shift + fi + done + cp "$FAKE_MEL_FILTERS" "$destination" +} +export -f curl + +export CI=true +ORIGINAL_HOME="$HOME" +export CARGO_HOME="${CARGO_HOME:-$ORIGINAL_HOME/.cargo}" +export RUSTUP_HOME="${RUSTUP_HOME:-$ORIGINAL_HOME/.rustup}" +export HOME="$TEST_ROOT/home" +export CODESCRIBE_MODELS_DIR="$TEST_ROOT/models" +export FAKE_TOKENIZER="$ROOT_DIR/core/models/whisper-large-v3-turbo-mlx-q8/tokenizer.json" +export FAKE_MEL_FILTERS="$TEST_ROOT/mel_filters.npz" +mkdir -p "$HOME" "$CODESCRIBE_MODELS_DIR" +xxd -r -p "$ROOT_DIR/tests/fixtures/whisper_mel_filters.npz.hex" > "$FAKE_MEL_FILTERS" + +run_promotion_case() { + local selected_name="$1" + local stale_name="$2" + local snapshot="$TEST_ROOT/snapshot-$selected_name" + local destination="$CODESCRIBE_MODELS_DIR/whisper-large-v3-turbo" + + mkdir -p "$snapshot" "$destination" + printf '{}\n' > "$snapshot/config.json" + make_tiny_weights "$snapshot/$selected_name" + make_tiny_weights "$destination/$stale_name" + export FAKE_MODEL_SNAPSHOT="$snapshot" + + "$ROOT_DIR/scripts/download-model.sh" >/dev/null + + [[ -f "$destination/$selected_name" ]] + [[ ! -e "$destination/$stale_name" ]] +} + +run_promotion_case model.safetensors weights.safetensors +run_promotion_case weights.safetensors model.safetensors + +echo "download-model alternate-weight promotion: PASS" From 8f41abd4a82e9057d305e1623c92e118efd146c5 Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Sat, 22 Aug 2026 00:09:37 +0200 Subject: [PATCH 33/45] [codex/vc-workflow] test(release): gate Whisper bundle promotion --- Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 7d8c610b..63a4a1dc 100644 --- a/Makefile +++ b/Makefile @@ -339,7 +339,7 @@ bump-major: # gate: check class=static ci=no -- cargo fmt, prettier, clippy, semgrep, validate-envs, validate-gates; executes ZERO tests # gate: lint class=static ci=no -- cargo fmt --check + clippy on the workspace + verify-swift-format; no tests # gate: semgrep class=static ci=no -- semgrep scan --config auto (semgrep.yml runs semgrep directly, not this target) -# gate: verify class=hermetic ci=yes -- the workspace test set + doctests + env registry + this ledger; the command rust.yml runs +# gate: verify class=hermetic ci=yes -- workspace tests, doctests, model-promotion regression, env registry + this ledger; rust.yml runs it # gate: verify-canaries class=hermetic ci=no -- claim-vs-execution canaries that read repo files only (scripts/canaries.sh); each row is born from a named incident # gate: verify-swift-format class=static ci=no -- swift-format lint --strict over macos/Codescribe + macos/CodescribeTests; skips the generated UniFFI binding; no Swift tests (that is test-swift) # gate: smoke-canaries class=operator ci=no -- verify-canaries + host rows: dist inputs, appcast feed, live-store purity, Sparkle key parity, keychain domain cleanliness (scripts/canaries.sh --host) @@ -1119,6 +1119,8 @@ verify: echo "=== Verify (hermetic: doctests) ==="; \ CODESCRIBE_NO_EMBED=1 CODESCRIBE_DISABLE_KEYCHAIN=1 \ cargo test --workspace --doc; \ + echo "=== Verify (Whisper model promotion) ==="; \ + bash scripts/tests/download-model-test.sh; \ echo "=== Verify (env registry) ==="; \ bash scripts/validate-envs.sh; \ echo "=== Verify (gate ledger) ==="; \ From 66b9ae7363c70befbb55d1ad7c83a30d2bd6db6b Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Sat, 22 Aug 2026 00:27:39 +0200 Subject: [PATCH 34/45] [codex/vc-workflow] fix(stt): expand models root home prefix --- core/config/models.rs | 37 +++++++++++++++++++++++++++- docs/ENV_REGISTRY.toml | 2 +- scripts/bench-stt.sh | 10 ++++++-- scripts/download-model.sh | 6 ++++- scripts/ensure-models.sh | 6 ++++- scripts/tests/download-model-test.sh | 7 +++--- tests/support/e2e_stt_matrix.rs | 8 +++--- 7 files changed, 64 insertions(+), 12 deletions(-) diff --git a/core/config/models.rs b/core/config/models.rs index 29df87f9..3a3d208c 100644 --- a/core/config/models.rs +++ b/core/config/models.rs @@ -50,6 +50,16 @@ fn canonicalize_or_self(path: PathBuf) -> PathBuf { } } +/// Expand a leading `~/` in operator-provided model paths. +fn expand_home_path(value: &str) -> PathBuf { + if let Some(relative) = value.strip_prefix("~/") + && let Some(home) = std::env::var_os("HOME") + { + return PathBuf::from(home).join(relative); + } + PathBuf::from(value) +} + /// Whether `path` holds a fully usable, structurally valid Whisper model. fn is_complete_whisper_model_dir(path: &Path) -> bool { validate_whisper_model_bundle(path).is_ok() @@ -151,7 +161,7 @@ impl ModelManager { fn resolve_models_dir() -> Result { // Environment override if let Ok(path) = std::env::var("CODESCRIBE_MODELS_DIR") { - let p = PathBuf::from(&path); + let p = expand_home_path(path.trim()); if p.exists() { return Ok(p); } @@ -689,6 +699,14 @@ mod tests { Self { key, prev } } + /// Set `key` to a literal string, including shell-like path syntax. + fn set_str(key: &'static str, value: &str) -> Self { + let prev = std::env::var(key).ok(); + // SAFETY: these tests run under `serial` and restore the prior env. + unsafe { std::env::set_var(key, value) }; + Self { key, prev } + } + /// Unset `key`, remembering the previous value for `Drop`. fn unset(key: &'static str) -> Self { let prev = std::env::var(key).ok(); @@ -820,6 +838,23 @@ mod tests { } } + /// A leading `~/` in the supported models-root override resolves via HOME. + #[test] + #[serial] + fn model_manager_expands_tilde_models_root() { + let temp_dir = TempDir::new().unwrap(); + let home = temp_dir.path().join("home"); + let models_dir = home.join("custom-models"); + create_complete_whisper_model(&models_dir.join(DEFAULT_MODEL)); + + let _home = EnvGuard::set("HOME", &home); + let _models_dir = EnvGuard::set_str("CODESCRIBE_MODELS_DIR", "~/custom-models"); + + let manager = ModelManager::new().unwrap(); + assert_eq!(manager.models_dir(), models_dir.as_path()); + assert!(manager.check_model_exists(DEFAULT_MODEL)); + } + /// Incomplete Whisper dirs are neither listed nor treated as existing. #[test] #[serial] diff --git a/docs/ENV_REGISTRY.toml b/docs/ENV_REGISTRY.toml index ad5a3549..7b600786 100644 --- a/docs/ENV_REGISTRY.toml +++ b/docs/ENV_REGISTRY.toml @@ -505,7 +505,7 @@ default = "" type = "string" reload = "restart" category = "stt" -description = "Directory containing all models" +description = "Directory containing all models; a leading ~/ expands to the user home directory" [vars.CODESCRIBE_EMBED_MODEL] default = "" diff --git a/scripts/bench-stt.sh b/scripts/bench-stt.sh index 6d9b3246..a218c2c5 100755 --- a/scripts/bench-stt.sh +++ b/scripts/bench-stt.sh @@ -148,8 +148,14 @@ discover_model() { fi local models_root="$home_dir/.codescribe/models" - if [[ -n "${CODESCRIBE_MODELS_DIR:-}" ]] && [[ -d "$CODESCRIBE_MODELS_DIR" ]]; then - models_root="$CODESCRIBE_MODELS_DIR" + local configured_models_root="${CODESCRIBE_MODELS_DIR:-}" + local tilde_prefix + printf -v tilde_prefix '%s/' '~' + if [[ "$configured_models_root" == "$tilde_prefix"* ]]; then + configured_models_root="$home_dir/${configured_models_root:2}" + fi + if [[ -n "$configured_models_root" ]] && [[ -d "$configured_models_root" ]]; then + models_root="$configured_models_root" fi candidate="$models_root/whisper-large-v3-turbo" if model_is_complete "$candidate"; then diff --git a/scripts/download-model.sh b/scripts/download-model.sh index 4fcc0f59..e32f1b13 100755 --- a/scripts/download-model.sh +++ b/scripts/download-model.sh @@ -21,6 +21,10 @@ EMBED_MODEL_VALUE="${CODESCRIBE_EMBED_MODEL:-}" if [[ "$EMBED_MODEL_VALUE" == "$TILDE_PREFIX"* ]]; then EMBED_MODEL_VALUE="$HOME/${EMBED_MODEL_VALUE:2}" fi +MODELS_DIR_VALUE="${CODESCRIBE_MODELS_DIR:-$HOME/.codescribe/models}" +if [[ "$MODELS_DIR_VALUE" == "$TILDE_PREFIX"* ]]; then + MODELS_DIR_VALUE="$HOME/${MODELS_DIR_VALUE:2}" +fi is_hf_repo_id() { [[ "$1" =~ ^[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$ ]] @@ -138,7 +142,7 @@ if [[ "$MODEL_REPO" == "$DEFAULT_REPO" ]]; then echo "" echo "▶ Composing verified fp16 runtime directory..." TOKENIZER_PATH=$("$HF_BIN" download "$TOKENIZER_REPO" tokenizer.json --quiet) - MODEL_DEST="${CODESCRIBE_MODELS_DIR:-$HOME/.codescribe/models}/whisper-large-v3-turbo" + MODEL_DEST="$MODELS_DIR_VALUE/whisper-large-v3-turbo" MODEL_STAGE=$(mktemp -d "${TMPDIR:-/tmp}/codescribe-whisper-model.XXXXXX") cleanup_model_stage() { rm -f \ diff --git a/scripts/ensure-models.sh b/scripts/ensure-models.sh index a659e3b0..a569371b 100755 --- a/scripts/ensure-models.sh +++ b/scripts/ensure-models.sh @@ -12,6 +12,10 @@ EMBED_MODEL_VALUE="${CODESCRIBE_EMBED_MODEL:-}" if [[ "$EMBED_MODEL_VALUE" == "$TILDE_PREFIX"* ]]; then EMBED_MODEL_VALUE="$HOME/${EMBED_MODEL_VALUE:2}" fi +MODELS_DIR_VALUE="${CODESCRIBE_MODELS_DIR:-$HOME/.codescribe/models}" +if [[ "$MODELS_DIR_VALUE" == "$TILDE_PREFIX"* ]]; then + MODELS_DIR_VALUE="$HOME/${MODELS_DIR_VALUE:2}" +fi valid_whisper_bundle() { "$WHISPER_VALIDATOR" "$1" @@ -138,7 +142,7 @@ EMBEDDER_REPO="${CODESCRIBE_EMBEDDER_REPO:-sentence-transformers/paraphrase-mult # If CODESCRIBE_MODEL_PATH already satisfied, skip Whisper cache check if [[ "${WHISPER_OK:-0}" -ne 1 ]]; then if [[ "$WHISPER_REPO" == "mlx-community/whisper-large-v3-turbo" ]]; then - COMPOSED_MODEL="${CODESCRIBE_MODELS_DIR:-$HOME/.codescribe/models}/whisper-large-v3-turbo" + COMPOSED_MODEL="$MODELS_DIR_VALUE/whisper-large-v3-turbo" if valid_whisper_bundle "$COMPOSED_MODEL"; then echo "✓ Whisper fp16 composed ($COMPOSED_MODEL)" else diff --git a/scripts/tests/download-model-test.sh b/scripts/tests/download-model-test.sh index ffac7e70..e1483ec2 100755 --- a/scripts/tests/download-model-test.sh +++ b/scripts/tests/download-model-test.sh @@ -51,17 +51,18 @@ ORIGINAL_HOME="$HOME" export CARGO_HOME="${CARGO_HOME:-$ORIGINAL_HOME/.cargo}" export RUSTUP_HOME="${RUSTUP_HOME:-$ORIGINAL_HOME/.rustup}" export HOME="$TEST_ROOT/home" -export CODESCRIBE_MODELS_DIR="$TEST_ROOT/models" +printf -v TILDE_PREFIX '%s/' '~' +export CODESCRIBE_MODELS_DIR="${TILDE_PREFIX}models" export FAKE_TOKENIZER="$ROOT_DIR/core/models/whisper-large-v3-turbo-mlx-q8/tokenizer.json" export FAKE_MEL_FILTERS="$TEST_ROOT/mel_filters.npz" -mkdir -p "$HOME" "$CODESCRIBE_MODELS_DIR" +mkdir -p "$HOME/models" xxd -r -p "$ROOT_DIR/tests/fixtures/whisper_mel_filters.npz.hex" > "$FAKE_MEL_FILTERS" run_promotion_case() { local selected_name="$1" local stale_name="$2" local snapshot="$TEST_ROOT/snapshot-$selected_name" - local destination="$CODESCRIBE_MODELS_DIR/whisper-large-v3-turbo" + local destination="$HOME/models/whisper-large-v3-turbo" mkdir -p "$snapshot" "$destination" printf '{}\n' > "$snapshot/config.json" diff --git a/tests/support/e2e_stt_matrix.rs b/tests/support/e2e_stt_matrix.rs index 0631cc1e..29e4461b 100644 --- a/tests/support/e2e_stt_matrix.rs +++ b/tests/support/e2e_stt_matrix.rs @@ -85,9 +85,11 @@ pub fn discover_local_whisper_model() -> Option { let env_override = std::env::var("CODESCRIBE_MODEL_PATH") .ok() .map(PathBuf::from); - let models_root = std::env::var("CODESCRIBE_MODELS_DIR") - .ok() - .map(PathBuf::from); + let models_root = std::env::var("CODESCRIBE_MODELS_DIR").ok().map(|value| { + value + .strip_prefix("~/") + .map_or_else(|| PathBuf::from(&value), |relative| home_dir.join(relative)) + }); discover_local_whisper_model_for_with_root( &home_dir, env_override.as_deref(), From 4f2db14587214e0714e7aadc9bf788349854c91b Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Sat, 22 Aug 2026 00:36:43 +0200 Subject: [PATCH 35/45] [codex/vc-workflow] test(release): make Whisper fixture hermetic --- scripts/tests/download-model-test.sh | 2 +- tests/fixtures/whisper_tokenizer.json | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 tests/fixtures/whisper_tokenizer.json diff --git a/scripts/tests/download-model-test.sh b/scripts/tests/download-model-test.sh index e1483ec2..ebf60c8e 100755 --- a/scripts/tests/download-model-test.sh +++ b/scripts/tests/download-model-test.sh @@ -53,7 +53,7 @@ export RUSTUP_HOME="${RUSTUP_HOME:-$ORIGINAL_HOME/.rustup}" export HOME="$TEST_ROOT/home" printf -v TILDE_PREFIX '%s/' '~' export CODESCRIBE_MODELS_DIR="${TILDE_PREFIX}models" -export FAKE_TOKENIZER="$ROOT_DIR/core/models/whisper-large-v3-turbo-mlx-q8/tokenizer.json" +export FAKE_TOKENIZER="$ROOT_DIR/tests/fixtures/whisper_tokenizer.json" export FAKE_MEL_FILTERS="$TEST_ROOT/mel_filters.npz" mkdir -p "$HOME/models" xxd -r -p "$ROOT_DIR/tests/fixtures/whisper_mel_filters.npz.hex" > "$FAKE_MEL_FILTERS" diff --git a/tests/fixtures/whisper_tokenizer.json b/tests/fixtures/whisper_tokenizer.json new file mode 100644 index 00000000..a7a39688 --- /dev/null +++ b/tests/fixtures/whisper_tokenizer.json @@ -0,0 +1,19 @@ +{ + "version": "1.0", + "truncation": null, + "padding": null, + "added_tokens": [], + "normalizer": null, + "pre_tokenizer": null, + "post_processor": null, + "decoder": null, + "model": { + "type": "WordLevel", + "vocab": { + "": 0, + "<|startoftranscript|>": 1, + "<|endoftext|>": 2 + }, + "unk_token": "" + } +} From 976b9ed2d315240e10965dcebfafcc7d1fb6a19f Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Sat, 22 Aug 2026 00:49:53 +0200 Subject: [PATCH 36/45] [codex/vc-workflow] fix(stt): validate model architecture and cache repair --- README.md | 6 +- core/config/models.rs | 99 +++++++++++++++- core/stt/whisper/engine.rs | 107 +++++------------ core/whisper_weights.rs | 168 +++++++++++++++++++++++++-- scripts/tests/download-model-test.sh | 3 +- tests/e2e_stt_transcription.rs | 6 +- tests/fixtures/whisper_config.json | 12 ++ 7 files changed, 310 insertions(+), 91 deletions(-) create mode 100644 tests/fixtures/whisper_config.json diff --git a/README.md b/README.md index db8cac74..2517bd7f 100644 --- a/README.md +++ b/README.md @@ -353,7 +353,11 @@ mel SHA-256, and validates the complete safetensors tensor table, dtype allowlist, offsets, and file length. Downloads and warm-cache copies are written to `.partial` files and promoted only after per-file validation; an invalid destination is repaired on the next Download action instead of being accepted -as complete. +as complete. Config validation requires the complete MLX Whisper architecture +used by the loader (including matching audio/text state widths and compatible +attention heads); missing dimensions are never replaced with runtime defaults. +Warm-cache repair checks older snapshots when the newest config, weights, or +tokenizer is invalid, preserving offline recovery from an earlier valid revision. `CODESCRIBE_EMBED_EMBEDDER=1` is an explicit fat/debug path that compiles MiniLM into Rust artifacts. Normal builds resolve MiniLM from the signed app resource or HF cache. `CODESCRIBE_NO_EMBED=1` disables every optional binary embed; Silero remains embedded. diff --git a/core/config/models.rs b/core/config/models.rs index 3a3d208c..dc0322b3 100644 --- a/core/config/models.rs +++ b/core/config/models.rs @@ -131,6 +131,29 @@ fn hf_snapshot_for_model(model_ref: &str) -> Option { ) } +/// Find a warm default-repo snapshot containing one valid config/weights pair. +fn find_cached_default_model_pair() -> Option { + hf_cache::find_snapshot_with_any_matching( + DEFAULT_WHISPER_REPO, + &["config.json"], + &REQUIRED_MODEL_WEIGHTS, + |snapshot| { + validate_model_file("config.json", &snapshot.join("config.json")).is_ok() + && resolve_valid_whisper_weights_path(snapshot).is_ok() + }, + ) +} + +/// Find a warm OpenAI snapshot containing a parseable Whisper tokenizer. +fn find_cached_whisper_tokenizer() -> Option { + hf_cache::find_snapshot_with_any_matching( + TOKENIZER_WHISPER_REPO, + &["tokenizer.json"], + &[], + |snapshot| validate_model_file("tokenizer.json", &snapshot.join("tokenizer.json")).is_ok(), + ) +} + /// Owner of the resolved runtime models directory. /// /// Scope is deliberately narrow: it locates and inspects model directories on @@ -380,12 +403,12 @@ where // mlx-community's fp16 conversion; tokenizer comes from OpenAI's matching // Transformers repository. The pinned mel filterbank is fetched below. let mut paired_default_model = false; - if let Some(snapshot) = hf_cache::find_snapshot(DEFAULT_WHISPER_REPO, &["config.json"]) + if let Some(snapshot) = find_cached_default_model_pair() && snapshot != dest { paired_default_model = copy_default_model_pair(&snapshot, &dest)?; } - if let Some(snapshot) = hf_cache::find_snapshot(TOKENIZER_WHISPER_REPO, &["tokenizer.json"]) { + if let Some(snapshot) = find_cached_whisper_tokenizer() { copy_model_files(&snapshot, &dest, &["tokenizer.json"], true)?; } if paired_default_model && is_complete_whisper_model_dir(&dest) { @@ -732,7 +755,11 @@ mod tests { /// Create a directory that passes `is_complete_whisper_model_dir`. fn create_complete_whisper_model(path: &Path) { fs::create_dir_all(path).unwrap(); - fs::write(path.join("config.json"), "{}").unwrap(); + fs::write( + path.join("config.json"), + include_str!("../../tests/fixtures/whisper_config.json"), + ) + .unwrap(); let mut tokenizer = tokenizers::Tokenizer::new(tokenizers::models::bpe::BPE::default()); tokenizer.add_special_tokens(&[ tokenizers::AddedToken::from("<|startoftranscript|>", true), @@ -891,6 +918,19 @@ mod tests { assert!(manager.list_models().unwrap().is_empty()); } + /// Presence alone is insufficient when the loader architecture is absent. + #[test] + fn model_manager_rejects_config_without_required_dimensions() { + let temp_dir = TempDir::new().unwrap(); + let model = temp_dir.path().join("model"); + create_complete_whisper_model(&model); + fs::write(model.join("config.json"), "{}").unwrap(); + + let err = validate_whisper_model_bundle(&model).unwrap_err(); + assert!(format!("{err:#}").contains("n_mels")); + assert!(!is_complete_whisper_model_dir(&model)); + } + /// Header-level detection catches packed q8 even if config metadata lies. #[test] fn model_manager_rejects_q8_tensor_header_without_quantization_config() { @@ -1147,6 +1187,59 @@ mod tests { validate_whisper_model_bundle(&destination).unwrap(); } + /// Offline repair skips invalid newest cache entries for both model pieces. + #[test] + #[serial] + fn cached_repair_falls_back_to_older_valid_snapshots() { + use std::fs::FileTimes; + use std::time::{Duration, SystemTime}; + + let temp_dir = TempDir::new().unwrap(); + let cache = temp_dir.path().join("cache"); + let home = temp_dir.path().join("home"); + fs::create_dir_all(&home).unwrap(); + + let _home = EnvGuard::set("HOME", &home); + let _cache = EnvGuard::set("CODESCRIBE_HF_CACHE", &cache); + let _hf_home = EnvGuard::unset("HF_HOME"); + let _hf_hub = EnvGuard::unset("HF_HUB_CACHE"); + let _huggingface_hub = EnvGuard::unset("HUGGINGFACE_HUB_CACHE"); + + let snapshot = |repo: &str, revision: &str| { + cache + .join(format!("models--{}", repo.replace('/', "--"))) + .join("snapshots") + .join(revision) + }; + let set_modified = |path: &Path, seconds: u64| { + fs::File::open(path) + .unwrap() + .set_times( + FileTimes::new() + .set_modified(SystemTime::UNIX_EPOCH + Duration::from_secs(seconds)), + ) + .unwrap(); + }; + + let older_model = snapshot(DEFAULT_WHISPER_REPO, "older"); + let newer_model = snapshot(DEFAULT_WHISPER_REPO, "newer"); + create_complete_whisper_model(&older_model); + create_complete_whisper_model(&newer_model); + fs::write(newer_model.join("model.safetensors"), b"corrupt").unwrap(); + set_modified(&older_model, 10); + set_modified(&newer_model, 20); + assert_eq!(find_cached_default_model_pair(), Some(older_model)); + + let older_tokenizer = snapshot(TOKENIZER_WHISPER_REPO, "older"); + let newer_tokenizer = snapshot(TOKENIZER_WHISPER_REPO, "newer"); + create_complete_whisper_model(&older_tokenizer); + create_complete_whisper_model(&newer_tokenizer); + fs::write(newer_tokenizer.join("tokenizer.json"), "{}").unwrap(); + set_modified(&older_tokenizer, 10); + set_modified(&newer_tokenizer, 20); + assert_eq!(find_cached_whisper_tokenizer(), Some(older_tokenizer)); + } + /// A downloaded checksum mismatch is never promoted to the final mel path. #[test] fn corrupt_download_is_removed_before_promotion() { diff --git a/core/stt/whisper/engine.rs b/core/stt/whisper/engine.rs index 7d4ce144..a8af9547 100644 --- a/core/stt/whisper/engine.rs +++ b/core/stt/whisper/engine.rs @@ -42,6 +42,21 @@ use crate::safe_path; use super::embedded::EmbeddedModel; use super::params::DecodingParams; +fn candle_config(architecture: crate::whisper_weights::WhisperArchitecture) -> Config { + Config { + num_mel_bins: architecture.n_mels, + max_source_positions: architecture.n_audio_ctx, + d_model: architecture.n_audio_state, + encoder_attention_heads: architecture.n_audio_head, + encoder_layers: architecture.n_audio_layer, + vocab_size: architecture.n_vocab, + max_target_positions: architecture.n_text_ctx, + decoder_attention_heads: architecture.n_text_head, + decoder_layers: architecture.n_text_layer, + suppress_tokens: Vec::new(), + } +} + /// Callback for streaming chunk results (called after each chunk is transcribed) pub type ChunkCallback<'a> = &'a dyn Fn(&str); @@ -462,46 +477,11 @@ impl LocalWhisperEngine { let config_str = safe_path::safe_read_to_string(&config_path)?; - // Parse MLX config and map to Candle Config - let mlx_config: serde_json::Value = - serde_json::from_str(&config_str).context("Failed to parse MLX config json")?; - if mlx_config - .get("quantization") - .is_some_and(|value| !value.is_null()) - || mlx_config - .get("quantization_config") - .is_some_and(|value| !value.is_null()) - { - anyhow::bail!( - "Quantized Whisper weights are not supported; install the complete fp16 model" - ); - } - - let n_mels = mlx_config["n_mels"].as_u64().unwrap_or(80); - let new_config_json = serde_json::json!({ - "num_mel_bins": n_mels, - "max_source_positions": mlx_config["n_audio_ctx"].as_u64().unwrap_or(1500), - "d_model": mlx_config["n_audio_state"].as_u64().unwrap_or(512), - "encoder_attention_heads": mlx_config["n_audio_head"].as_u64().unwrap_or(8), - "encoder_layers": mlx_config["n_audio_layer"].as_u64().unwrap_or(6), - "vocab_size": mlx_config["n_vocab"].as_u64().unwrap_or(51865), - "decoder_attention_heads": mlx_config["n_text_head"].as_u64().unwrap_or(8), - "decoder_layers": mlx_config["n_text_layer"].as_u64().unwrap_or(6), - "max_target_positions": mlx_config["n_text_ctx"].as_u64().unwrap_or(448), - "activation_function": "gelu", - // defaults - "dropout": 0.0, - "attention_dropout": 0.0, - "activation_dropout": 0.0, - "init_std": 0.02, - "encoder_layerdrop": 0.0, - "decoder_layerdrop": 0.0, - "use_cache": true, - "scale_embedding": false - }); - - let config: Config = serde_json::from_value(new_config_json) - .context("Failed to build Config from MLX values")?; + let architecture = crate::whisper_weights::parse_whisper_config( + &config_str, + &config_path.display().to_string(), + )?; + let config = candle_config(architecture); // Phase timings for the cold load. "Preloaded, zero latency" is the // product's claim, and the operator's logs show 56 cold loads costing a @@ -601,42 +581,9 @@ impl LocalWhisperEngine { // Parse config from bytes let config_str = std::str::from_utf8(embedded.config) .context("Invalid UTF-8 in embedded config.json")?; - let mlx_config: serde_json::Value = - serde_json::from_str(config_str).context("Failed to parse embedded config json")?; - if mlx_config - .get("quantization") - .is_some_and(|value| !value.is_null()) - || mlx_config - .get("quantization_config") - .is_some_and(|value| !value.is_null()) - { - anyhow::bail!("Embedded quantized Whisper payload refused; build with fp16 weights"); - } - - let n_mels = mlx_config["n_mels"].as_u64().unwrap_or(80); - let new_config_json = serde_json::json!({ - "num_mel_bins": n_mels, - "max_source_positions": mlx_config["n_audio_ctx"].as_u64().unwrap_or(1500), - "d_model": mlx_config["n_audio_state"].as_u64().unwrap_or(512), - "encoder_attention_heads": mlx_config["n_audio_head"].as_u64().unwrap_or(8), - "encoder_layers": mlx_config["n_audio_layer"].as_u64().unwrap_or(6), - "vocab_size": mlx_config["n_vocab"].as_u64().unwrap_or(51865), - "decoder_attention_heads": mlx_config["n_text_head"].as_u64().unwrap_or(8), - "decoder_layers": mlx_config["n_text_layer"].as_u64().unwrap_or(6), - "max_target_positions": mlx_config["n_text_ctx"].as_u64().unwrap_or(448), - "activation_function": "gelu", - "dropout": 0.0, - "attention_dropout": 0.0, - "activation_dropout": 0.0, - "init_std": 0.02, - "encoder_layerdrop": 0.0, - "decoder_layerdrop": 0.0, - "use_cache": true, - "scale_embedding": false - }); - - let config: Config = serde_json::from_value(new_config_json) - .context("Failed to build Config from embedded MLX values")?; + let architecture = + crate::whisper_weights::parse_whisper_config(config_str, "embedded config.json")?; + let config = candle_config(architecture); // Load weights directly from bytes - NO DISK I/O! let raw_tensors = candle_core::safetensors::load_buffer(embedded.weights, &Device::Cpu) @@ -650,7 +597,7 @@ impl LocalWhisperEngine { .map_err(|e| anyhow!("Failed to load embedded tokenizer: {}", e))?; // Load mel filters from bytes - let mel_filters = load_mel_filters_from_bytes(embedded.mel_filters, n_mels as usize) + let mel_filters = load_mel_filters_from_bytes(embedded.mel_filters, config.num_mel_bins) .context("Failed to load embedded mel filters")?; tracing::info!("Embedded Whisper model loaded successfully"); @@ -1968,7 +1915,11 @@ mod model_payload_tests { fn write_tiny_model(path: &Path, name: &str, dtype: &str, payload_bytes: usize) { fs::create_dir_all(path).unwrap(); - fs::write(path.join("config.json"), "{}").unwrap(); + fs::write( + path.join("config.json"), + include_str!("../../../tests/fixtures/whisper_config.json"), + ) + .unwrap(); fs::write(path.join("tokenizer.json"), "{}").unwrap(); fs::write(path.join("mel_filters.npz"), b"placeholder").unwrap(); let header = serde_json::json!({ diff --git a/core/whisper_weights.rs b/core/whisper_weights.rs index cd17416c..8ad74c94 100644 --- a/core/whisper_weights.rs +++ b/core/whisper_weights.rs @@ -12,6 +12,20 @@ pub const SUPPORTED_NAMES: [&str; 2] = ["weights.safetensors", "model.safetensor pub const MEL_FILTERS_SHA256: &str = "7450ae70723a5ef9d341e3cee628c7cb0177f36ce42c44b7ed2bf3325f0f6d4c"; const REQUIRED_TOKENIZER_TOKENS: [&str; 2] = ["<|startoftranscript|>", "<|endoftext|>"]; +/// MLX Whisper architecture shared by validation, disk loading, and embedding. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct WhisperArchitecture { + pub n_mels: usize, + pub n_audio_ctx: usize, + pub n_audio_state: usize, + pub n_audio_head: usize, + pub n_audio_layer: usize, + pub n_vocab: usize, + pub n_text_ctx: usize, + pub n_text_state: usize, + pub n_text_head: usize, + pub n_text_layer: usize, +} /// Validate every artifact required by runtime and embedded Whisper loaders. pub fn validate_whisper_model_bundle(path: &Path) -> Result<()> { @@ -41,13 +55,15 @@ pub(crate) fn validate_whisper_config(path: &Path) -> Result<()> { // nosemgrep: rust.actix.path-traversal.tainted-path.tainted-path -- Read-only model inspection. `path` is an operator-selected local model config or an internally resolved bundle/cache child; no network/request path component reaches it. let raw = fs::read_to_string(path) .with_context(|| format!("read Whisper config {}", path.display()))?; - let config: serde_json::Value = serde_json::from_str(&raw) - .with_context(|| format!("parse Whisper config {}", path.display()))?; + parse_whisper_config(&raw, &path.display().to_string()).map(|_| ()) +} + +/// Parse and validate the MLX architecture consumed by Candle's Whisper loader. +pub(crate) fn parse_whisper_config(raw: &str, source: &str) -> Result { + let config: serde_json::Value = + serde_json::from_str(raw).with_context(|| format!("parse Whisper config {source}"))?; if !config.is_object() { - return Err(anyhow!( - "Whisper config must be a JSON object: {}", - path.display() - )); + return Err(anyhow!("Whisper config must be a JSON object: {source}")); } if config .get("quantization") @@ -58,7 +74,64 @@ pub(crate) fn validate_whisper_config(path: &Path) -> Result<()> { { return Err(anyhow!("quantized Whisper config is unsupported")); } - Ok(()) + let dimension = |name: &str| -> Result { + let value = config + .get(name) + .and_then(serde_json::Value::as_u64) + .and_then(|value| usize::try_from(value).ok()) + .filter(|value| *value > 0) + .ok_or_else(|| anyhow!("Whisper config {source} requires a positive integer {name}"))?; + Ok(value) + }; + let architecture = WhisperArchitecture { + n_mels: dimension("n_mels")?, + n_audio_ctx: dimension("n_audio_ctx")?, + n_audio_state: dimension("n_audio_state")?, + n_audio_head: dimension("n_audio_head")?, + n_audio_layer: dimension("n_audio_layer")?, + n_vocab: dimension("n_vocab")?, + n_text_ctx: dimension("n_text_ctx")?, + n_text_state: dimension("n_text_state")?, + n_text_head: dimension("n_text_head")?, + n_text_layer: dimension("n_text_layer")?, + }; + if !matches!(architecture.n_mels, 80 | 128) { + return Err(anyhow!( + "Whisper config {source} requires n_mels to be 80 or 128" + )); + } + if architecture.n_audio_ctx > u32::MAX as usize { + return Err(anyhow!( + "Whisper config {source} requires n_audio_ctx to fit in u32" + )); + } + if architecture.n_audio_state != architecture.n_text_state { + return Err(anyhow!( + "Whisper config {source} requires n_audio_state to equal n_text_state" + )); + } + if architecture.n_audio_state < 4 || !architecture.n_audio_state.is_multiple_of(2) { + return Err(anyhow!( + "Whisper config {source} requires an even n_audio_state of at least 4" + )); + } + if !architecture + .n_audio_state + .is_multiple_of(architecture.n_audio_head) + { + return Err(anyhow!( + "Whisper config {source} requires n_audio_state divisible by n_audio_head" + )); + } + if !architecture + .n_text_state + .is_multiple_of(architecture.n_text_head) + { + return Err(anyhow!( + "Whisper config {source} requires n_text_state divisible by n_text_head" + )); + } + Ok(architecture) } /// Verify the pinned mel filterbank used by Whisper. @@ -239,3 +312,84 @@ pub(crate) fn validate_safetensors_file(path: &Path) -> Result<()> { } Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + fn valid_config() -> serde_json::Value { + serde_json::from_str(include_str!("../tests/fixtures/whisper_config.json")).unwrap() + } + + #[test] + fn official_whisper_architecture_is_accepted() { + let raw = include_str!("../tests/fixtures/whisper_config.json"); + let architecture = parse_whisper_config(raw, "fixture").unwrap(); + assert_eq!(architecture.n_mels, 128); + assert_eq!(architecture.n_audio_state, 1280); + assert_eq!(architecture.n_text_state, 1280); + } + + #[test] + fn every_loader_dimension_is_required() { + let fields = [ + "n_mels", + "n_audio_ctx", + "n_audio_state", + "n_audio_head", + "n_audio_layer", + "n_vocab", + "n_text_ctx", + "n_text_state", + "n_text_head", + "n_text_layer", + ]; + for field in fields { + let mut config = valid_config(); + config.as_object_mut().unwrap().remove(field); + let err = parse_whisper_config(&config.to_string(), "fixture").unwrap_err(); + assert!(err.to_string().contains(field), "{field}: {err:#}"); + } + } + + #[test] + fn non_positive_or_non_integer_dimensions_are_rejected() { + for value in [ + serde_json::Value::Null, + serde_json::json!("128"), + serde_json::json!(-1), + serde_json::json!(0), + serde_json::json!(80.5), + ] { + let mut config = valid_config(); + config["n_mels"] = value; + assert!(parse_whisper_config(&config.to_string(), "fixture").is_err()); + } + } + + #[test] + fn incompatible_architecture_relationships_are_rejected() { + for (field, value, expected) in [ + ("n_mels", 81, "n_mels"), + ("n_text_state", 640, "equal"), + ("n_audio_head", 3, "n_audio_head"), + ("n_text_head", 3, "n_text_head"), + ("n_audio_state", 3, "equal"), + ] { + let mut config = valid_config(); + config[field] = serde_json::json!(value); + let err = parse_whisper_config(&config.to_string(), "fixture").unwrap_err(); + assert!(err.to_string().contains(expected), "{field}: {err:#}"); + } + + let mut odd_state = valid_config(); + odd_state["n_audio_state"] = serde_json::json!(3); + odd_state["n_text_state"] = serde_json::json!(3); + assert!( + parse_whisper_config(&odd_state.to_string(), "fixture") + .unwrap_err() + .to_string() + .contains("even") + ); + } +} diff --git a/scripts/tests/download-model-test.sh b/scripts/tests/download-model-test.sh index ebf60c8e..94812e24 100755 --- a/scripts/tests/download-model-test.sh +++ b/scripts/tests/download-model-test.sh @@ -53,6 +53,7 @@ export RUSTUP_HOME="${RUSTUP_HOME:-$ORIGINAL_HOME/.rustup}" export HOME="$TEST_ROOT/home" printf -v TILDE_PREFIX '%s/' '~' export CODESCRIBE_MODELS_DIR="${TILDE_PREFIX}models" +export FAKE_CONFIG="$ROOT_DIR/tests/fixtures/whisper_config.json" export FAKE_TOKENIZER="$ROOT_DIR/tests/fixtures/whisper_tokenizer.json" export FAKE_MEL_FILTERS="$TEST_ROOT/mel_filters.npz" mkdir -p "$HOME/models" @@ -65,7 +66,7 @@ run_promotion_case() { local destination="$HOME/models/whisper-large-v3-turbo" mkdir -p "$snapshot" "$destination" - printf '{}\n' > "$snapshot/config.json" + cp "$FAKE_CONFIG" "$snapshot/config.json" make_tiny_weights "$snapshot/$selected_name" make_tiny_weights "$destination/$stale_name" export FAKE_MODEL_SNAPSHOT="$snapshot" diff --git a/tests/e2e_stt_transcription.rs b/tests/e2e_stt_transcription.rs index 010ae470..9af8a47c 100644 --- a/tests/e2e_stt_transcription.rs +++ b/tests/e2e_stt_transcription.rs @@ -175,7 +175,11 @@ fn e2e_stt_model_init_stable() { fn create_complete_model(path: &Path) { std::fs::create_dir_all(path).expect("create model dir"); - std::fs::write(path.join("config.json"), "{}").expect("write config"); + std::fs::write( + path.join("config.json"), + include_str!("fixtures/whisper_config.json"), + ) + .expect("write config"); let mut tokenizer = tokenizers::Tokenizer::new(tokenizers::models::bpe::BPE::default()); tokenizer.add_special_tokens(&[ tokenizers::AddedToken::from("<|startoftranscript|>", true), diff --git a/tests/fixtures/whisper_config.json b/tests/fixtures/whisper_config.json new file mode 100644 index 00000000..6baf7a68 --- /dev/null +++ b/tests/fixtures/whisper_config.json @@ -0,0 +1,12 @@ +{ + "n_mels": 128, + "n_audio_ctx": 1500, + "n_audio_state": 1280, + "n_audio_head": 20, + "n_audio_layer": 32, + "n_vocab": 51866, + "n_text_state": 1280, + "n_text_head": 20, + "n_text_layer": 4, + "n_text_ctx": 448 +} From 12a870bc1472d24a36a524d13da7edc5f3972bca Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Sat, 22 Aug 2026 01:32:46 +0200 Subject: [PATCH 37/45] [codex/vc-workflow] fix(stt): validate complete model compatibility --- CHANGELOG.md | 9 +- README.md | 17 +- core/build.rs | 12 +- core/config/models.rs | 77 +++++-- core/stt/whisper/engine.rs | 45 +++-- core/whisper_weights.rs | 258 +++++++++++++++++++++++- scripts/tests/download-model-test.sh | 9 +- tests/e2e_stt_transcription.rs | 86 +++++++- tests/fixtures/whisper_test_config.json | 12 ++ tests/support/e2e_stt_matrix.rs | 16 +- 10 files changed, 471 insertions(+), 70 deletions(-) create mode 100644 tests/fixtures/whisper_test_config.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 358d6dfd..b0d45c4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -70,10 +70,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Local Whisper is an explicitly validated FP16/F32 bundle.** Runtime, Settings download, release scripts, E2E discovery, and the optional fat build - share the same config/tokenizer/mel/safetensors contract. Quantized payloads - and the legacy Q8 fallback are refused; the old public Q8 identifiers remain - deprecated source-compatibility constants only. Building from source now - declares Rust 1.88 as the minimum supported toolchain. + share the same architecture, tokenizer-vocabulary, pinned-mel, and required + tensor-name/shape contract. Quantized payloads and the legacy Q8 fallback are + refused; the old public Q8 identifiers remain deprecated source-compatibility + constants only. Building from source now declares Rust 1.88 as the minimum + supported toolchain. - **Supervisor findings own transcript-quality categories.** Engine catalog `codescribe-supervisor-findings/v1` (`core/quality/supervisor.rs`) names diff --git a/README.md b/README.md index 2517bd7f..01f4f7a5 100644 --- a/README.md +++ b/README.md @@ -347,13 +347,16 @@ Runtime resolution when Whisper is not embedded: The mlx-community repo ships only `config.json` + `weights.safetensors`; the download paths compose `tokenizer.json` from the matching official OpenAI Transformers repo and `mel_filters.npz` from a checksum-pinned OpenAI Whisper -asset. The resulting directory is validated as unquantized before resolution. -The shared bundle validator parses the config and tokenizer, verifies the pinned -mel SHA-256, and validates the complete safetensors tensor table, dtype -allowlist, offsets, and file length. Downloads and warm-cache copies are written -to `.partial` files and promoted only after per-file validation; an invalid -destination is repaired on the next Download action instead of being accepted -as complete. Config validation requires the complete MLX Whisper architecture +asset. The resulting directory is validated as loader-compatible fp16/fp32 +before resolution. +The shared bundle validator parses the config, requires the tokenizer to cover +the configured vocabulary, verifies the pinned mel SHA-256, and validates every +required Whisper tensor name and shape plus the complete safetensors tensor +table, dtype allowlist, offsets, and file length. Downloads and warm-cache +copies are written to `.partial` files and promoted only after per-file +validation; an invalid destination is repaired on the next Download action +instead of being accepted as complete. Config validation requires the complete +MLX Whisper architecture used by the loader (including matching audio/text state widths and compatible attention heads); missing dimensions are never replaced with runtime defaults. Warm-cache repair checks older snapshots when the newest config, weights, or diff --git a/core/build.rs b/core/build.rs index f03ee7c0..38829797 100644 --- a/core/build.rs +++ b/core/build.rs @@ -25,6 +25,7 @@ use std::path::{Path, PathBuf}; use sha2::{Digest, Sha256}; #[path = "whisper_weights.rs"] +#[allow(dead_code)] mod whisper_weights; /// The license key contract, included by path so the build script and the @@ -100,7 +101,16 @@ fn main() { resolve_whisper_embed_model_path(&manifest_dir, &embed_model, DEFAULT_WHISPER_REPO); let model_exists = whisper_weights::validate_whisper_model_bundle(&model_path).is_ok(); let weights_path = model_exists - .then(|| whisper_weights::resolve_valid_whisper_weights_path(&model_path).ok()) + .then(|| { + let config = std::fs::read_to_string(model_path.join("config.json")).ok()?; + let architecture = whisper_weights::parse_whisper_config( + &config, + &model_path.join("config.json").display().to_string(), + ) + .ok()?; + whisper_weights::resolve_compatible_whisper_weights_path(&model_path, architecture) + .ok() + }) .flatten(); if model_exists { let weights_path = weights_path.as_ref().expect("validated Whisper weights"); diff --git a/core/config/models.rs b/core/config/models.rs index dc0322b3..f943684c 100644 --- a/core/config/models.rs +++ b/core/config/models.rs @@ -78,17 +78,18 @@ pub fn validate_whisper_model_bundle(path: &Path) -> Result<()> { /// This narrower payload gate is also used by `LocalWhisperEngine::new`, where /// tokenizer and mel errors retain their own loader diagnostics. pub(crate) fn is_unquantized_whisper_model_dir(path: &Path) -> bool { - crate::whisper_weights::validate_whisper_config(&path.join("config.json")).is_ok() - && resolve_valid_whisper_weights_path(path).is_ok() + crate::whisper_weights::validate_whisper_model_pair(path).is_ok() } +#[cfg(test)] +use crate::whisper_weights::resolve_valid_whisper_weights_path; /// Resolve the first structurally valid supported weight file. /// /// Upstream snapshots may contain either filename, and stale composition can /// leave both behind. Preserve the documented filename priority, but never let /// an invalid primary shadow a valid alternative that the runtime can load. pub(crate) use crate::whisper_weights::{ - resolve_valid_whisper_weights_path, validate_safetensors_file, + resolve_compatible_whisper_weights_path, validate_safetensors_file, }; /// Whether a candidate models root owns at least one complete Whisper model. @@ -137,10 +138,7 @@ fn find_cached_default_model_pair() -> Option { DEFAULT_WHISPER_REPO, &["config.json"], &REQUIRED_MODEL_WEIGHTS, - |snapshot| { - validate_model_file("config.json", &snapshot.join("config.json")).is_ok() - && resolve_valid_whisper_weights_path(snapshot).is_ok() - }, + |snapshot| crate::whisper_weights::validate_whisper_model_pair(snapshot).is_ok(), ) } @@ -528,12 +526,15 @@ fn copy_model_files(src: &Path, dest: &Path, names: &[&str], replace_valid: bool /// Replace config and weights only when both come from one valid default snapshot. fn copy_default_model_pair(src: &Path, dest: &Path) -> Result { - if validate_model_file("config.json", &src.join("config.json")).is_err() { + if crate::whisper_weights::validate_whisper_model_pair(src).is_err() { return Ok(false); } - let Ok(weights) = resolve_valid_whisper_weights_path(src) else { - return Ok(false); - }; + let architecture = crate::whisper_weights::parse_whisper_config( + // nosemgrep: rust.actix.path-traversal.tainted-path.tainted-path -- `src` is a resolved child of an internal HF cache root selected by repository id; no request path component reaches this repair path. + &fs::read_to_string(src.join("config.json"))?, + &src.join("config.json").display().to_string(), + )?; + let weights = resolve_compatible_whisper_weights_path(src, architecture)?; let Some(weight_name) = weights.file_name().and_then(|name| name.to_str()) else { return Ok(false); }; @@ -757,13 +758,14 @@ mod tests { fs::create_dir_all(path).unwrap(); fs::write( path.join("config.json"), - include_str!("../../tests/fixtures/whisper_config.json"), + include_str!("../../tests/fixtures/whisper_test_config.json"), ) .unwrap(); let mut tokenizer = tokenizers::Tokenizer::new(tokenizers::models::bpe::BPE::default()); tokenizer.add_special_tokens(&[ tokenizers::AddedToken::from("<|startoftranscript|>", true), tokenizers::AddedToken::from("<|endoftext|>", true), + tokenizers::AddedToken::from("<|transcribe|>", true), ]); tokenizer.save(path.join("tokenizer.json"), false).unwrap(); fs::write( @@ -773,11 +775,16 @@ mod tests { )), ) .unwrap(); - let header = br#"{"model.weight":{"dtype":"F16","shape":[1],"data_offsets":[0,2]}}"#; - let mut safetensors = (header.len() as u64).to_le_bytes().to_vec(); - safetensors.extend_from_slice(header); - safetensors.extend_from_slice(&[0, 0]); - fs::write(path.join("model.safetensors"), safetensors).unwrap(); + let architecture = crate::whisper_weights::parse_whisper_config( + include_str!("../../tests/fixtures/whisper_test_config.json"), + "test fixture", + ) + .unwrap(); + crate::whisper_weights::write_test_whisper_weights( + &path.join("model.safetensors"), + architecture, + ) + .unwrap(); } fn decode_hex(raw: &str) -> Vec { @@ -931,6 +938,40 @@ mod tests { assert!(!is_complete_whisper_model_dir(&model)); } + /// A syntactically valid tokenizer must cover every configured vocabulary id. + #[test] + fn model_manager_rejects_tokenizer_smaller_than_configured_vocabulary() { + let temp_dir = TempDir::new().unwrap(); + let model = temp_dir.path().join("model"); + create_complete_whisper_model(&model); + fs::write( + model.join("config.json"), + include_str!("../../tests/fixtures/whisper_config.json"), + ) + .unwrap(); + + let err = validate_whisper_model_bundle(&model).unwrap_err(); + assert!(format!("{err:#}").contains("does not cover configured vocabulary")); + assert!(!is_complete_whisper_model_dir(&model)); + } + + /// A structurally valid safetensors file is not a Whisper model without its required tensors. + #[test] + fn model_manager_rejects_weights_missing_required_whisper_tensors() { + let temp_dir = TempDir::new().unwrap(); + let model = temp_dir.path().join("model"); + create_complete_whisper_model(&model); + let header = br#"{"model.weight":{"dtype":"F16","shape":[1],"data_offsets":[0,2]}}"#; + let mut safetensors = (header.len() as u64).to_le_bytes().to_vec(); + safetensors.extend_from_slice(header); + safetensors.extend_from_slice(&[0, 0]); + fs::write(model.join("model.safetensors"), safetensors).unwrap(); + + let err = validate_whisper_model_bundle(&model).unwrap_err(); + assert!(format!("{err:#}").contains("missing tensor")); + assert!(!is_complete_whisper_model_dir(&model)); + } + /// Header-level detection catches packed q8 even if config metadata lies. #[test] fn model_manager_rejects_q8_tensor_header_without_quantization_config() { @@ -1067,7 +1108,7 @@ mod tests { safetensors.extend_from_slice(&[0, 0]); fs::write(model.join("model.safetensors"), safetensors).unwrap(); - assert!(is_complete_whisper_model_dir(&model)); + assert!(validate_safetensors_file(&model.join("model.safetensors")).is_ok()); } /// Header offsets must describe the actual payload, not a truncated file. diff --git a/core/stt/whisper/engine.rs b/core/stt/whisper/engine.rs index a8af9547..e9e65eee 100644 --- a/core/stt/whisper/engine.rs +++ b/core/stt/whisper/engine.rs @@ -470,17 +470,18 @@ impl LocalWhisperEngine { "Quantized or malformed Whisper payload refused before tensor load; install the complete fp16 model" ); } - let weights_path = crate::config::models::resolve_valid_whisper_weights_path(model_path) - .context("resolve validated Whisper weights")?; - let device = process_device(); - tracing::debug!("LocalWhisperEngine using device: {:?}", device); - - let config_str = safe_path::safe_read_to_string(&config_path)?; - let architecture = crate::whisper_weights::parse_whisper_config( - &config_str, + &safe_path::safe_read_to_string(&config_path)?, &config_path.display().to_string(), )?; + let weights_path = crate::config::models::resolve_compatible_whisper_weights_path( + model_path, + architecture, + ) + .context("resolve architecture-compatible Whisper weights")?; + let device = process_device(); + tracing::debug!("LocalWhisperEngine using device: {:?}", device); + let config = candle_config(architecture); // Phase timings for the cold load. "Preloaded, zero latency" is the @@ -1917,7 +1918,7 @@ mod model_payload_tests { fs::create_dir_all(path).unwrap(); fs::write( path.join("config.json"), - include_str!("../../../tests/fixtures/whisper_config.json"), + include_str!("../../../tests/fixtures/whisper_test_config.json"), ) .unwrap(); fs::write(path.join("tokenizer.json"), "{}").unwrap(); @@ -2002,23 +2003,31 @@ mod model_payload_tests { #[test] fn local_loader_uses_valid_alternative_after_invalid_primary() { let temp = TempDir::new().unwrap(); - write_tiny_model(temp.path(), "encoder.weight", "F16", 2); - fs::rename( - temp.path().join("weights.safetensors"), - temp.path().join("model.safetensors"), + fs::create_dir_all(temp.path()).unwrap(); + fs::write( + temp.path().join("config.json"), + include_str!("../../../tests/fixtures/whisper_test_config.json"), + ) + .unwrap(); + fs::write(temp.path().join("tokenizer.json"), "{}").unwrap(); + fs::write(temp.path().join("mel_filters.npz"), b"placeholder").unwrap(); + let architecture = crate::whisper_weights::parse_whisper_config( + include_str!("../../../tests/fixtures/whisper_test_config.json"), + "test fixture", + ) + .unwrap(); + crate::whisper_weights::write_test_whisper_weights( + &temp.path().join("model.safetensors"), + architecture, ) .unwrap(); write_tiny_model(temp.path(), "encoder.weight", "U32", 4); let err = LocalWhisperEngine::new(temp.path()) .err() - .expect("tiny valid alternative should reach model construction"); + .expect("compatible alternative should pass the payload gate"); let message = format!("{err:#}"); assert!(!message.contains("payload refused"), "{message}"); - assert!( - message.contains("Failed to create Whisper Model"), - "{message}" - ); } } diff --git a/core/whisper_weights.rs b/core/whisper_weights.rs index 8ad74c94..958802fe 100644 --- a/core/whisper_weights.rs +++ b/core/whisper_weights.rs @@ -2,6 +2,7 @@ use anyhow::{Context, Result, anyhow}; use sha2::{Digest, Sha256}; +use std::collections::BTreeMap; use std::fs; use std::io::{Read, Seek, SeekFrom}; use std::path::{Path, PathBuf}; @@ -29,10 +30,10 @@ pub(crate) struct WhisperArchitecture { /// Validate every artifact required by runtime and embedded Whisper loaders. pub fn validate_whisper_model_bundle(path: &Path) -> Result<()> { - validate_whisper_config(&path.join("config.json"))?; - validate_whisper_tokenizer(&path.join("tokenizer.json"))?; + let architecture = load_whisper_architecture(&path.join("config.json"))?; + validate_whisper_tokenizer_for_architecture(&path.join("tokenizer.json"), architecture)?; verify_mel_filters(&path.join("mel_filters.npz"))?; - resolve_valid_whisper_weights_path(path).map(|_| ()) + resolve_compatible_whisper_weights_path(path, architecture).map(|_| ()) } /// Parse the tokenizer and require the control tokens used by every decode. @@ -50,12 +51,39 @@ pub(crate) fn validate_whisper_tokenizer(path: &Path) -> Result<()> { Ok(()) } +fn validate_whisper_tokenizer_for_architecture( + path: &Path, + architecture: WhisperArchitecture, +) -> Result<()> { + validate_whisper_tokenizer(path)?; + let tokenizer = tokenizers::Tokenizer::from_file(path) + .map_err(|err| anyhow!("invalid Whisper tokenizer {}: {err}", path.display()))?; + let mut covered = vec![false; architecture.n_vocab]; + for id in tokenizer.get_vocab(true).into_values() { + if let Some(slot) = covered.get_mut(id as usize) { + *slot = true; + } + } + if covered.iter().any(|present| !present) { + return Err(anyhow!( + "Whisper tokenizer {} does not cover configured vocabulary 0..{}", + path.display(), + architecture.n_vocab + )); + } + Ok(()) +} + /// Validate the config schema and reject every declared quantization mode. pub(crate) fn validate_whisper_config(path: &Path) -> Result<()> { + load_whisper_architecture(path).map(|_| ()) +} + +fn load_whisper_architecture(path: &Path) -> Result { // nosemgrep: rust.actix.path-traversal.tainted-path.tainted-path -- Read-only model inspection. `path` is an operator-selected local model config or an internally resolved bundle/cache child; no network/request path component reaches it. let raw = fs::read_to_string(path) .with_context(|| format!("read Whisper config {}", path.display()))?; - parse_whisper_config(&raw, &path.display().to_string()).map(|_| ()) + parse_whisper_config(&raw, &path.display().to_string()) } /// Parse and validate the MLX architecture consumed by Candle's Whisper loader. @@ -95,6 +123,11 @@ pub(crate) fn parse_whisper_config(raw: &str, source: &str) -> Result u32::MAX as usize { + return Err(anyhow!( + "Whisper config {source} requires n_vocab to fit tokenizer u32 IDs" + )); + } if !matches!(architecture.n_mels, 80 | 128) { return Err(anyhow!( "Whisper config {source} requires n_mels to be 80 or 128" @@ -151,6 +184,7 @@ pub(crate) fn verify_mel_filters(path: &Path) -> Result<()> { } /// Resolve the first structurally valid supported weight file. +#[cfg(test)] pub fn resolve_valid_whisper_weights_path(path: &Path) -> Result { let mut failures = Vec::new(); for name in SUPPORTED_NAMES { @@ -178,8 +212,48 @@ pub fn resolve_valid_whisper_weights_path(path: &Path) -> Result { } } +/// Resolve the first supported weight file matching the configured architecture. +pub(crate) fn resolve_compatible_whisper_weights_path( + path: &Path, + architecture: WhisperArchitecture, +) -> Result { + let mut failures = Vec::new(); + for name in SUPPORTED_NAMES { + let candidate = path.join(name); + if !candidate.is_file() { + continue; + } + match validate_whisper_weights_for_architecture(&candidate, architecture) { + Ok(()) => return Ok(candidate), + Err(err) => failures.push(format!("{name}: {err:#}")), + } + } + if failures.is_empty() { + Err(anyhow!( + "Whisper weights are missing from {}", + path.display() + )) + } else { + Err(anyhow!( + "no architecture-compatible Whisper weights in {} ({})", + path.display(), + failures.join("; ") + )) + } +} + +/// Validate the config/weights generation used by warm-cache composition. +pub(crate) fn validate_whisper_model_pair(path: &Path) -> Result<()> { + let architecture = load_whisper_architecture(&path.join("config.json"))?; + resolve_compatible_whisper_weights_path(path, architecture).map(|_| ()) +} + /// Validate the complete safetensors structure without loading tensor data. pub(crate) fn validate_safetensors_file(path: &Path) -> Result<()> { + read_validated_tensor_shapes(path).map(|_| ()) +} + +fn read_validated_tensor_shapes(path: &Path) -> Result>> { const MAX_HEADER_BYTES: u64 = 16 * 1024 * 1024; // nosemgrep: rust.actix.path-traversal.tainted-path.tainted-path -- Read-only model inspection. `path` is an operator-selected local model file or an internally resolved bundle/cache child; no network/request path component reaches it. let mut file = fs::File::open(path).with_context(|| format!("open {}", path.display()))?; @@ -227,6 +301,7 @@ pub(crate) fn validate_safetensors_file(path: &Path) -> Result<()> { .checked_sub(data_start) .ok_or_else(|| anyhow!("truncated safetensors file: {}", path.display()))?; let mut ranges = Vec::new(); + let mut tensor_shapes = BTreeMap::new(); for (name, tensor) in tensors.iter().filter(|(name, _)| *name != "__metadata__") { let tensor = tensor @@ -256,12 +331,17 @@ pub(crate) fn validate_safetensors_file(path: &Path) -> Result<()> { .get("shape") .and_then(serde_json::Value::as_array) .ok_or_else(|| anyhow!("tensor {name} has no shape"))?; - let element_count = shape.iter().try_fold(1_u64, |count, dim| { - let dim = dim - .as_u64() - .ok_or_else(|| anyhow!("tensor {name} has an invalid shape"))?; + let dimensions = shape + .iter() + .map(|dim| { + dim.as_u64() + .and_then(|value| usize::try_from(value).ok()) + .ok_or_else(|| anyhow!("tensor {name} has an invalid shape")) + }) + .collect::>>()?; + let element_count = dimensions.iter().try_fold(1_u64, |count, dim| { count - .checked_mul(dim) + .checked_mul(*dim as u64) .ok_or_else(|| anyhow!("tensor {name} shape overflows")) })?; if element_count == 0 { @@ -288,6 +368,7 @@ pub(crate) fn validate_safetensors_file(path: &Path) -> Result<()> { )); } ranges.push((start, end, name)); + tensor_shapes.insert(name.clone(), dimensions); } if ranges.is_empty() { @@ -310,6 +391,165 @@ pub(crate) fn validate_safetensors_file(path: &Path) -> Result<()> { path.display() )); } + Ok(tensor_shapes) +} + +fn validate_whisper_weights_for_architecture( + path: &Path, + architecture: WhisperArchitecture, +) -> Result<()> { + let tensors = read_validated_tensor_shapes(path)?; + for (name, expected) in expected_whisper_tensor_shapes(architecture)? { + let actual = tensors.get(&name).ok_or_else(|| { + anyhow!( + "Whisper weights {} are missing tensor {name}", + path.display() + ) + })?; + if actual != &expected { + return Err(anyhow!( + "Whisper tensor {name} in {} has shape {:?}, expected {:?}", + path.display(), + actual, + expected + )); + } + } + Ok(()) +} + +fn expected_whisper_tensor_shapes( + architecture: WhisperArchitecture, +) -> Result>> { + fn add(out: &mut BTreeMap>, name: String, shape: &[usize]) { + out.insert(name, shape.to_vec()); + } + + fn add_attention(out: &mut BTreeMap>, prefix: &str, model_width: usize) { + add( + out, + format!("{prefix}.key.weight"), + &[model_width, model_width], + ); + add( + out, + format!("{prefix}.query.weight"), + &[model_width, model_width], + ); + add(out, format!("{prefix}.query.bias"), &[model_width]); + add( + out, + format!("{prefix}.value.weight"), + &[model_width, model_width], + ); + add(out, format!("{prefix}.value.bias"), &[model_width]); + add( + out, + format!("{prefix}.out.weight"), + &[model_width, model_width], + ); + add(out, format!("{prefix}.out.bias"), &[model_width]); + } + + fn add_block_tail( + out: &mut BTreeMap>, + prefix: &str, + model_width: usize, + feed_forward_width: usize, + ) { + add(out, format!("{prefix}.attn_ln.weight"), &[model_width]); + add(out, format!("{prefix}.attn_ln.bias"), &[model_width]); + add( + out, + format!("{prefix}.mlp1.weight"), + &[feed_forward_width, model_width], + ); + add(out, format!("{prefix}.mlp1.bias"), &[feed_forward_width]); + add( + out, + format!("{prefix}.mlp2.weight"), + &[model_width, feed_forward_width], + ); + add(out, format!("{prefix}.mlp2.bias"), &[model_width]); + add(out, format!("{prefix}.mlp_ln.weight"), &[model_width]); + add(out, format!("{prefix}.mlp_ln.bias"), &[model_width]); + } + + let mut out = BTreeMap::new(); + let d = architecture.n_audio_state; + let ff = d + .checked_mul(4) + .ok_or_else(|| anyhow!("Whisper feed-forward width overflows"))?; + add( + &mut out, + "encoder.conv1.weight".into(), + &[d, 3, architecture.n_mels], + ); + add(&mut out, "encoder.conv1.bias".into(), &[d]); + add(&mut out, "encoder.conv2.weight".into(), &[d, 3, d]); + add(&mut out, "encoder.conv2.bias".into(), &[d]); + add(&mut out, "encoder.ln_post.weight".into(), &[d]); + add(&mut out, "encoder.ln_post.bias".into(), &[d]); + add( + &mut out, + "decoder.token_embedding.weight".into(), + &[architecture.n_vocab, d], + ); + add( + &mut out, + "decoder.positional_embedding".into(), + &[architecture.n_text_ctx, d], + ); + add(&mut out, "decoder.ln.weight".into(), &[d]); + add(&mut out, "decoder.ln.bias".into(), &[d]); + + for layer in 0..architecture.n_audio_layer { + let prefix = format!("encoder.blocks.{layer}"); + add_attention(&mut out, &format!("{prefix}.attn"), d); + add_block_tail(&mut out, &prefix, d, ff); + } + for layer in 0..architecture.n_text_layer { + let prefix = format!("decoder.blocks.{layer}"); + add_attention(&mut out, &format!("{prefix}.attn"), d); + add_attention(&mut out, &format!("{prefix}.cross_attn"), d); + add(&mut out, format!("{prefix}.cross_attn_ln.weight"), &[d]); + add(&mut out, format!("{prefix}.cross_attn_ln.bias"), &[d]); + add_block_tail(&mut out, &prefix, d, ff); + } + Ok(out) +} + +#[cfg(test)] +pub(crate) fn write_test_whisper_weights( + path: &Path, + architecture: WhisperArchitecture, +) -> Result<()> { + let mut offset = 0_u64; + let mut header = serde_json::Map::new(); + for (name, shape) in expected_whisper_tensor_shapes(architecture)? { + let elements = shape.iter().try_fold(1_u64, |count, dim| { + count + .checked_mul(*dim as u64) + .ok_or_else(|| anyhow!("test tensor shape overflows")) + })?; + let end = offset + .checked_add(elements * 2) + .ok_or_else(|| anyhow!("test tensor payload overflows"))?; + header.insert( + name, + serde_json::json!({ + "dtype": "F16", + "shape": shape, + "data_offsets": [offset, end] + }), + ); + offset = end; + } + let header = serde_json::to_vec(&header)?; + let mut bytes = (header.len() as u64).to_le_bytes().to_vec(); + bytes.extend_from_slice(&header); + bytes.resize(bytes.len() + offset as usize, 0); + fs::write(path, bytes)?; Ok(()) } diff --git a/scripts/tests/download-model-test.sh b/scripts/tests/download-model-test.sh index 94812e24..bf5e16bf 100755 --- a/scripts/tests/download-model-test.sh +++ b/scripts/tests/download-model-test.sh @@ -51,9 +51,16 @@ ORIGINAL_HOME="$HOME" export CARGO_HOME="${CARGO_HOME:-$ORIGINAL_HOME/.cargo}" export RUSTUP_HOME="${RUSTUP_HOME:-$ORIGINAL_HOME/.rustup}" export HOME="$TEST_ROOT/home" +FAKE_BIN="$TEST_ROOT/bin" +mkdir -p "$FAKE_BIN" +# This shell regression owns destination promotion mechanics; Rust tests own +# the canonical bundle validator exercised by the real script. +printf '#!/bin/bash\nexit 0\n' > "$FAKE_BIN/cargo" +chmod +x "$FAKE_BIN/cargo" +export PATH="$FAKE_BIN:$PATH" printf -v TILDE_PREFIX '%s/' '~' export CODESCRIBE_MODELS_DIR="${TILDE_PREFIX}models" -export FAKE_CONFIG="$ROOT_DIR/tests/fixtures/whisper_config.json" +export FAKE_CONFIG="$ROOT_DIR/tests/fixtures/whisper_test_config.json" export FAKE_TOKENIZER="$ROOT_DIR/tests/fixtures/whisper_tokenizer.json" export FAKE_MEL_FILTERS="$TEST_ROOT/mel_filters.npz" mkdir -p "$HOME/models" diff --git a/tests/e2e_stt_transcription.rs b/tests/e2e_stt_transcription.rs index 9af8a47c..e042c8db 100644 --- a/tests/e2e_stt_transcription.rs +++ b/tests/e2e_stt_transcription.rs @@ -19,7 +19,7 @@ mod e2e_stt_matrix; use e2e_stt_matrix::{ ModelDiscovery, ModelSource, STT_OPT_IN_ENV, WHISPER_FP16_MODEL, discover_local_whisper_model, discover_local_whisper_model_for, discover_local_whisper_model_for_with_root, - model_discovery_hint, parse_opt_in, skip_unless_opt_in, test_audio_path, + expand_models_root, model_discovery_hint, parse_opt_in, skip_unless_opt_in, test_audio_path, whisper_model_missing_parts, }; @@ -177,13 +177,14 @@ fn create_complete_model(path: &Path) { std::fs::create_dir_all(path).expect("create model dir"); std::fs::write( path.join("config.json"), - include_str!("fixtures/whisper_config.json"), + include_str!("fixtures/whisper_test_config.json"), ) .expect("write config"); let mut tokenizer = tokenizers::Tokenizer::new(tokenizers::models::bpe::BPE::default()); tokenizer.add_special_tokens(&[ tokenizers::AddedToken::from("<|startoftranscript|>", true), tokenizers::AddedToken::from("<|endoftext|>", true), + tokenizers::AddedToken::from("<|transcribe|>", true), ]); tokenizer .save(path.join("tokenizer.json"), false) @@ -193,11 +194,72 @@ fn create_complete_model(path: &Path) { decode_hex(include_str!("fixtures/whisper_mel_filters.npz.hex")), ) .expect("write mel filters"); - let header = br#"{"model.weight":{"dtype":"F16","shape":[1],"data_offsets":[0,2]}}"#; + write_tiny_complete_weights(&path.join("weights.safetensors")); +} + +fn write_tiny_complete_weights(path: &Path) { + const D: usize = 4; + const FF: usize = 16; + let mut shapes = std::collections::BTreeMap::new(); + let mut add = |name: String, shape: &[usize]| { + shapes.insert(name, shape.to_vec()); + }; + add("encoder.conv1.weight".into(), &[D, 3, 80]); + add("encoder.conv1.bias".into(), &[D]); + add("encoder.conv2.weight".into(), &[D, 3, D]); + add("encoder.conv2.bias".into(), &[D]); + add("encoder.ln_post.weight".into(), &[D]); + add("encoder.ln_post.bias".into(), &[D]); + add("decoder.token_embedding.weight".into(), &[3, D]); + add("decoder.positional_embedding".into(), &[2, D]); + add("decoder.ln.weight".into(), &[D]); + add("decoder.ln.bias".into(), &[D]); + for prefix in [ + "encoder.blocks.0.attn", + "decoder.blocks.0.attn", + "decoder.blocks.0.cross_attn", + ] { + add(format!("{prefix}.key.weight"), &[D, D]); + add(format!("{prefix}.query.weight"), &[D, D]); + add(format!("{prefix}.query.bias"), &[D]); + add(format!("{prefix}.value.weight"), &[D, D]); + add(format!("{prefix}.value.bias"), &[D]); + add(format!("{prefix}.out.weight"), &[D, D]); + add(format!("{prefix}.out.bias"), &[D]); + } + for prefix in ["encoder.blocks.0", "decoder.blocks.0"] { + add(format!("{prefix}.attn_ln.weight"), &[D]); + add(format!("{prefix}.attn_ln.bias"), &[D]); + add(format!("{prefix}.mlp1.weight"), &[FF, D]); + add(format!("{prefix}.mlp1.bias"), &[FF]); + add(format!("{prefix}.mlp2.weight"), &[D, FF]); + add(format!("{prefix}.mlp2.bias"), &[D]); + add(format!("{prefix}.mlp_ln.weight"), &[D]); + add(format!("{prefix}.mlp_ln.bias"), &[D]); + } + add("decoder.blocks.0.cross_attn_ln.weight".into(), &[D]); + add("decoder.blocks.0.cross_attn_ln.bias".into(), &[D]); + + let mut offset = 0_u64; + let mut header = serde_json::Map::new(); + for (name, shape) in shapes { + let elements = shape.iter().product::() as u64; + let end = offset + elements * 2; + header.insert( + name, + serde_json::json!({ + "dtype": "F16", + "shape": shape, + "data_offsets": [offset, end] + }), + ); + offset = end; + } + let header = serde_json::to_vec(&header).expect("serialize weights header"); let mut safetensors = (header.len() as u64).to_le_bytes().to_vec(); - safetensors.extend_from_slice(header); - safetensors.extend_from_slice(&[0, 0]); - std::fs::write(path.join("weights.safetensors"), safetensors).expect("write weights"); + safetensors.extend_from_slice(&header); + safetensors.resize(safetensors.len() + offset as usize, 0); + std::fs::write(path, safetensors).expect("write weights"); } fn decode_hex(raw: &str) -> Vec { @@ -260,6 +322,18 @@ fn deterministic_model_discovery_hint_names_the_validation_contract() { assert!(hint.contains("no quantization declaration")); } +#[test] +fn deterministic_models_root_expands_home_relative_override() { + assert_eq!( + expand_models_root(Path::new("/tmp/test-home"), "~/models"), + PathBuf::from("/tmp/test-home/models") + ); + assert_eq!( + expand_models_root(Path::new("/tmp/test-home"), "/opt/models"), + PathBuf::from("/opt/models") + ); +} + #[test] fn deterministic_model_discovery_prefers_complete_env_override() { let (_tmp, home) = temp_home(); diff --git a/tests/fixtures/whisper_test_config.json b/tests/fixtures/whisper_test_config.json new file mode 100644 index 00000000..8c487270 --- /dev/null +++ b/tests/fixtures/whisper_test_config.json @@ -0,0 +1,12 @@ +{ + "n_mels": 80, + "n_audio_ctx": 2, + "n_audio_state": 4, + "n_audio_head": 1, + "n_audio_layer": 1, + "n_vocab": 3, + "n_text_ctx": 2, + "n_text_state": 4, + "n_text_head": 1, + "n_text_layer": 1 +} diff --git a/tests/support/e2e_stt_matrix.rs b/tests/support/e2e_stt_matrix.rs index 29e4461b..ac71956b 100644 --- a/tests/support/e2e_stt_matrix.rs +++ b/tests/support/e2e_stt_matrix.rs @@ -85,11 +85,9 @@ pub fn discover_local_whisper_model() -> Option { let env_override = std::env::var("CODESCRIBE_MODEL_PATH") .ok() .map(PathBuf::from); - let models_root = std::env::var("CODESCRIBE_MODELS_DIR").ok().map(|value| { - value - .strip_prefix("~/") - .map_or_else(|| PathBuf::from(&value), |relative| home_dir.join(relative)) - }); + let models_root = std::env::var("CODESCRIBE_MODELS_DIR") + .ok() + .map(|value| expand_models_root(&home_dir, &value)); discover_local_whisper_model_for_with_root( &home_dir, env_override.as_deref(), @@ -97,6 +95,12 @@ pub fn discover_local_whisper_model() -> Option { ) } +pub fn expand_models_root(home_dir: &Path, value: &str) -> PathBuf { + value + .strip_prefix("~/") + .map_or_else(|| PathBuf::from(value), |relative| home_dir.join(relative)) +} + pub fn discover_local_whisper_model_for( home_dir: &Path, env_override: Option<&Path>, @@ -137,7 +141,7 @@ pub fn model_discovery_hint(home_dir: &Path) -> String { let default_root = home_dir.join(".codescribe/models"); let models_root = std::env::var("CODESCRIBE_MODELS_DIR") .ok() - .map(PathBuf::from) + .map(|value| expand_models_root(home_dir, &value)) .filter(|path| path.exists()) .unwrap_or(default_root); format!( From 33928710eacb18e1ba02d6d1c5e8559a8bd81291 Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Sat, 22 Aug 2026 01:59:48 +0200 Subject: [PATCH 38/45] [codex/vc-workflow] fix(stt): harden tokenizer compatibility --- CHANGELOG.md | 4 +- README.md | 3 +- core/config/models.rs | 45 ++++++++++---- core/stt/whisper/engine.rs | 11 ++-- core/whisper_weights.rs | 81 ++++++++++++++++++++++--- tests/e2e_stt_transcription.rs | 3 +- tests/fixtures/whisper_test_config.json | 2 +- tests/fixtures/whisper_tokenizer.json | 3 +- 8 files changed, 116 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b0d45c4c..077d3d90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -70,8 +70,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Local Whisper is an explicitly validated FP16/F32 bundle.** Runtime, Settings download, release scripts, E2E discovery, and the optional fat build - share the same architecture, tokenizer-vocabulary, pinned-mel, and required - tensor-name/shape contract. Quantized payloads and the legacy Q8 fallback are + share the same architecture, tokenizer-vocabulary/language, pinned-mel, and + required tensor-name/shape contract. Quantized payloads and the legacy Q8 fallback are refused; the old public Q8 identifiers remain deprecated source-compatibility constants only. Building from source now declares Rust 1.88 as the minimum supported toolchain. diff --git a/README.md b/README.md index 01f4f7a5..ee33f965 100644 --- a/README.md +++ b/README.md @@ -350,7 +350,8 @@ Transformers repo and `mel_filters.npz` from a checksum-pinned OpenAI Whisper asset. The resulting directory is validated as loader-compatible fp16/fp32 before resolution. The shared bundle validator parses the config, requires the tokenizer to cover -the configured vocabulary, verifies the pinned mel SHA-256, and validates every +the configured vocabulary and provide automatic-language tokens, verifies the +pinned mel SHA-256, and validates every required Whisper tensor name and shape plus the complete safetensors tensor table, dtype allowlist, offsets, and file length. Downloads and warm-cache copies are written to `.partial` files and promoted only after per-file diff --git a/core/config/models.rs b/core/config/models.rs index f943684c..31367362 100644 --- a/core/config/models.rs +++ b/core/config/models.rs @@ -74,13 +74,6 @@ pub fn validate_whisper_model_bundle(path: &Path) -> Result<()> { crate::whisper_weights::validate_whisper_model_bundle(path) } -/// Reject unsupported or malformed weights before the expensive engine load. -/// This narrower payload gate is also used by `LocalWhisperEngine::new`, where -/// tokenizer and mel errors retain their own loader diagnostics. -pub(crate) fn is_unquantized_whisper_model_dir(path: &Path) -> bool { - crate::whisper_weights::validate_whisper_model_pair(path).is_ok() -} - #[cfg(test)] use crate::whisper_weights::resolve_valid_whisper_weights_path; /// Resolve the first structurally valid supported weight file. @@ -143,12 +136,20 @@ fn find_cached_default_model_pair() -> Option { } /// Find a warm OpenAI snapshot containing a parseable Whisper tokenizer. -fn find_cached_whisper_tokenizer() -> Option { +fn find_cached_whisper_tokenizer( + architecture: crate::whisper_weights::WhisperArchitecture, +) -> Option { hf_cache::find_snapshot_with_any_matching( TOKENIZER_WHISPER_REPO, &["tokenizer.json"], &[], - |snapshot| validate_model_file("tokenizer.json", &snapshot.join("tokenizer.json")).is_ok(), + |snapshot| { + crate::whisper_weights::validate_whisper_tokenizer_for_architecture( + &snapshot.join("tokenizer.json"), + architecture, + ) + .is_ok() + }, ) } @@ -406,7 +407,10 @@ where { paired_default_model = copy_default_model_pair(&snapshot, &dest)?; } - if let Some(snapshot) = find_cached_whisper_tokenizer() { + if let Ok(architecture) = + crate::whisper_weights::load_whisper_architecture(&dest.join("config.json")) + && let Some(snapshot) = find_cached_whisper_tokenizer(architecture) + { copy_model_files(&snapshot, &dest, &["tokenizer.json"], true)?; } if paired_default_model && is_complete_whisper_model_dir(&dest) { @@ -766,6 +770,7 @@ mod tests { tokenizers::AddedToken::from("<|startoftranscript|>", true), tokenizers::AddedToken::from("<|endoftext|>", true), tokenizers::AddedToken::from("<|transcribe|>", true), + tokenizers::AddedToken::from("<|pl|>", true), ]); tokenizer.save(path.join("tokenizer.json"), false).unwrap(); fs::write( @@ -1269,16 +1274,30 @@ mod tests { fs::write(newer_model.join("model.safetensors"), b"corrupt").unwrap(); set_modified(&older_model, 10); set_modified(&newer_model, 20); - assert_eq!(find_cached_default_model_pair(), Some(older_model)); + assert_eq!(find_cached_default_model_pair(), Some(older_model.clone())); let older_tokenizer = snapshot(TOKENIZER_WHISPER_REPO, "older"); let newer_tokenizer = snapshot(TOKENIZER_WHISPER_REPO, "newer"); create_complete_whisper_model(&older_tokenizer); create_complete_whisper_model(&newer_tokenizer); - fs::write(newer_tokenizer.join("tokenizer.json"), "{}").unwrap(); + let mut incomplete_tokenizer = + tokenizers::Tokenizer::new(tokenizers::models::bpe::BPE::default()); + incomplete_tokenizer.add_special_tokens(&[ + tokenizers::AddedToken::from("<|startoftranscript|>", true), + tokenizers::AddedToken::from("<|endoftext|>", true), + ]); + incomplete_tokenizer + .save(newer_tokenizer.join("tokenizer.json"), false) + .unwrap(); set_modified(&older_tokenizer, 10); set_modified(&newer_tokenizer, 20); - assert_eq!(find_cached_whisper_tokenizer(), Some(older_tokenizer)); + let architecture = + crate::whisper_weights::load_whisper_architecture(&older_model.join("config.json")) + .unwrap(); + assert_eq!( + find_cached_whisper_tokenizer(architecture), + Some(older_tokenizer) + ); } /// A downloaded checksum mismatch is never promoted to the final mel path. diff --git a/core/stt/whisper/engine.rs b/core/stt/whisper/engine.rs index e9e65eee..dab1ba90 100644 --- a/core/stt/whisper/engine.rs +++ b/core/stt/whisper/engine.rs @@ -465,11 +465,8 @@ impl LocalWhisperEngine { } let tokenizer_path = model_path.join("tokenizer.json"); let mel_filters_path = model_path.join("mel_filters.npz"); - if !crate::config::models::is_unquantized_whisper_model_dir(model_path) { - anyhow::bail!( - "Quantized or malformed Whisper payload refused before tensor load; install the complete fp16 model" - ); - } + crate::whisper_weights::validate_whisper_model_pair(model_path) + .context("validate Whisper config and architecture-compatible weights")?; let architecture = crate::whisper_weights::parse_whisper_config( &safe_path::safe_read_to_string(&config_path)?, &config_path.display().to_string(), @@ -1945,7 +1942,7 @@ mod model_payload_tests { let err = LocalWhisperEngine::new(temp.path()) .err() .expect("U32 must be refused"); - assert!(format!("{err:#}").contains("refused")); + assert!(format!("{err:#}").contains("unsupported Whisper tensor dtype U32")); } #[test] @@ -1956,7 +1953,7 @@ mod model_payload_tests { let err = LocalWhisperEngine::new(temp.path()) .err() .expect("I32 must be refused"); - assert!(format!("{err:#}").contains("refused")); + assert!(format!("{err:#}").contains("unsupported Whisper tensor dtype I32")); } #[test] diff --git a/core/whisper_weights.rs b/core/whisper_weights.rs index 958802fe..90eb8963 100644 --- a/core/whisper_weights.rs +++ b/core/whisper_weights.rs @@ -2,7 +2,7 @@ use anyhow::{Context, Result, anyhow}; use sha2::{Digest, Sha256}; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashSet}; use std::fs; use std::io::{Read, Seek, SeekFrom}; use std::path::{Path, PathBuf}; @@ -51,35 +51,53 @@ pub(crate) fn validate_whisper_tokenizer(path: &Path) -> Result<()> { Ok(()) } -fn validate_whisper_tokenizer_for_architecture( +pub(crate) fn validate_whisper_tokenizer_for_architecture( path: &Path, architecture: WhisperArchitecture, ) -> Result<()> { validate_whisper_tokenizer(path)?; let tokenizer = tokenizers::Tokenizer::from_file(path) .map_err(|err| anyhow!("invalid Whisper tokenizer {}: {err}", path.display()))?; - let mut covered = vec![false; architecture.n_vocab]; - for id in tokenizer.get_vocab(true).into_values() { - if let Some(slot) = covered.get_mut(id as usize) { - *slot = true; - } - } - if covered.iter().any(|present| !present) { + let vocab = tokenizer.get_vocab(true); + let covered: HashSet = vocab + .values() + .copied() + .filter(|id| (*id as usize) < architecture.n_vocab) + .collect(); + if covered.len() != architecture.n_vocab { return Err(anyhow!( "Whisper tokenizer {} does not cover configured vocabulary 0..{}", path.display(), architecture.n_vocab )); } + if !vocab + .iter() + .any(|(token, id)| (*id as usize) < architecture.n_vocab && is_language_token(token)) + { + return Err(anyhow!( + "Whisper tokenizer {} has no language token required for automatic detection", + path.display() + )); + } Ok(()) } +fn is_language_token(token: &str) -> bool { + token + .strip_prefix("<|") + .and_then(|inner| inner.strip_suffix("|>")) + .is_some_and(|inner| { + (2..=3).contains(&inner.len()) && inner.chars().all(|ch| ch.is_ascii_alphabetic()) + }) +} + /// Validate the config schema and reject every declared quantization mode. pub(crate) fn validate_whisper_config(path: &Path) -> Result<()> { load_whisper_architecture(path).map(|_| ()) } -fn load_whisper_architecture(path: &Path) -> Result { +pub(crate) fn load_whisper_architecture(path: &Path) -> Result { // nosemgrep: rust.actix.path-traversal.tainted-path.tainted-path -- Read-only model inspection. `path` is an operator-selected local model config or an internally resolved bundle/cache child; no network/request path component reaches it. let raw = fs::read_to_string(path) .with_context(|| format!("read Whisper config {}", path.display()))?; @@ -556,6 +574,7 @@ pub(crate) fn write_test_whisper_weights( #[cfg(test)] mod tests { use super::*; + use tempfile::TempDir; fn valid_config() -> serde_json::Value { serde_json::from_str(include_str!("../tests/fixtures/whisper_config.json")).unwrap() @@ -632,4 +651,46 @@ mod tests { .contains("even") ); } + + #[test] + fn tokenizer_without_language_tokens_is_rejected() { + let temp = TempDir::new().unwrap(); + let path = temp.path().join("tokenizer.json"); + let mut tokenizer = tokenizers::Tokenizer::new(tokenizers::models::bpe::BPE::default()); + tokenizer.add_special_tokens(&[ + tokenizers::AddedToken::from("<|startoftranscript|>", true), + tokenizers::AddedToken::from("<|endoftext|>", true), + tokenizers::AddedToken::from("<|transcribe|>", true), + tokenizers::AddedToken::from("<|notimestamps|>", true), + ]); + tokenizer.save(&path, false).unwrap(); + let architecture = parse_whisper_config( + include_str!("../tests/fixtures/whisper_test_config.json"), + "test fixture", + ) + .unwrap(); + + let err = validate_whisper_tokenizer_for_architecture(&path, architecture).unwrap_err(); + assert!(format!("{err:#}").contains("no language token")); + } + + #[test] + fn sparse_tokenizer_rejects_extreme_vocab_without_dense_allocation() { + let temp = TempDir::new().unwrap(); + let path = temp.path().join("tokenizer.json"); + let tokenizer = tokenizers::Tokenizer::from_file( + Path::new(env!("CARGO_MANIFEST_DIR")).join("../tests/fixtures/whisper_tokenizer.json"), + ) + .unwrap(); + tokenizer.save(&path, false).unwrap(); + let mut architecture = parse_whisper_config( + include_str!("../tests/fixtures/whisper_test_config.json"), + "test fixture", + ) + .unwrap(); + architecture.n_vocab = u32::MAX as usize; + + let err = validate_whisper_tokenizer_for_architecture(&path, architecture).unwrap_err(); + assert!(format!("{err:#}").contains("does not cover configured vocabulary")); + } } diff --git a/tests/e2e_stt_transcription.rs b/tests/e2e_stt_transcription.rs index e042c8db..bb4f9117 100644 --- a/tests/e2e_stt_transcription.rs +++ b/tests/e2e_stt_transcription.rs @@ -185,6 +185,7 @@ fn create_complete_model(path: &Path) { tokenizers::AddedToken::from("<|startoftranscript|>", true), tokenizers::AddedToken::from("<|endoftext|>", true), tokenizers::AddedToken::from("<|transcribe|>", true), + tokenizers::AddedToken::from("<|pl|>", true), ]); tokenizer .save(path.join("tokenizer.json"), false) @@ -210,7 +211,7 @@ fn write_tiny_complete_weights(path: &Path) { add("encoder.conv2.bias".into(), &[D]); add("encoder.ln_post.weight".into(), &[D]); add("encoder.ln_post.bias".into(), &[D]); - add("decoder.token_embedding.weight".into(), &[3, D]); + add("decoder.token_embedding.weight".into(), &[4, D]); add("decoder.positional_embedding".into(), &[2, D]); add("decoder.ln.weight".into(), &[D]); add("decoder.ln.bias".into(), &[D]); diff --git a/tests/fixtures/whisper_test_config.json b/tests/fixtures/whisper_test_config.json index 8c487270..9542179d 100644 --- a/tests/fixtures/whisper_test_config.json +++ b/tests/fixtures/whisper_test_config.json @@ -4,7 +4,7 @@ "n_audio_state": 4, "n_audio_head": 1, "n_audio_layer": 1, - "n_vocab": 3, + "n_vocab": 4, "n_text_ctx": 2, "n_text_state": 4, "n_text_head": 1, diff --git a/tests/fixtures/whisper_tokenizer.json b/tests/fixtures/whisper_tokenizer.json index a7a39688..89433feb 100644 --- a/tests/fixtures/whisper_tokenizer.json +++ b/tests/fixtures/whisper_tokenizer.json @@ -12,7 +12,8 @@ "vocab": { "": 0, "<|startoftranscript|>": 1, - "<|endoftext|>": 2 + "<|endoftext|>": 2, + "<|pl|>": 3 }, "unk_token": "" } From 04ff5f017940285a4b30284fa5bbaad41a98be8a Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Sat, 22 Aug 2026 02:35:04 +0200 Subject: [PATCH 39/45] [codex/vc-workflow] fix(stt): align bundle validation with runtime --- CHANGELOG.md | 11 +- README.md | 15 +- core/stt/whisper/engine.rs | 308 ++++++++++++++---------- core/whisper_weights.rs | 276 +++++++++++++++++++-- tests/e2e_stt_transcription.rs | 2 +- tests/fixtures/whisper_test_config.json | 2 +- 6 files changed, 455 insertions(+), 159 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 077d3d90..c0fe59d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -71,10 +71,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Local Whisper is an explicitly validated FP16/F32 bundle.** Runtime, Settings download, release scripts, E2E discovery, and the optional fat build share the same architecture, tokenizer-vocabulary/language, pinned-mel, and - required tensor-name/shape contract. Quantized payloads and the legacy Q8 fallback are - refused; the old public Q8 identifiers remain deprecated source-compatibility - constants only. Building from source now declares Rust 1.88 as the minimum - supported toolchain. + required tensor-name/shape contract. The loader rejects incomplete bundles + before cold model construction; prompt/control token IDs, automatic-language + candidates, layer/context resource bounds, and mapped tensor-name collisions + are validated by the same runtime-owned helpers. Quantized payloads and the + legacy Q8 fallback are refused; the old public Q8 identifiers remain + deprecated source-compatibility constants only. Building from source now + declares Rust 1.88 as the minimum supported toolchain. - **Supervisor findings own transcript-quality categories.** Engine catalog `codescribe-supervisor-findings/v1` (`core/quality/supervisor.rs`) names diff --git a/README.md b/README.md index ee33f965..4d39f5d9 100644 --- a/README.md +++ b/README.md @@ -349,17 +349,22 @@ the download paths compose `tokenizer.json` from the matching official OpenAI Transformers repo and `mel_filters.npz` from a checksum-pinned OpenAI Whisper asset. The resulting directory is validated as loader-compatible fp16/fp32 before resolution. -The shared bundle validator parses the config, requires the tokenizer to cover -the configured vocabulary and provide automatic-language tokens, verifies the -pinned mel SHA-256, and validates every +The shared bundle validator parses the config, applies bounded architecture +resource limits, requires every runtime prompt/control token to fit the +configured vocabulary, and uses the same automatic-language candidate logic as +the decoder. It verifies the pinned mel SHA-256 and validates every required Whisper tensor name and shape plus the complete safetensors tensor -table, dtype allowlist, offsets, and file length. Downloads and warm-cache +table, dtype allowlist, mapped-name uniqueness, offsets, and file length. The +disk loader applies this complete gate before mmap or model construction. +Downloads and warm-cache copies are written to `.partial` files and promoted only after per-file validation; an invalid destination is repaired on the next Download action instead of being accepted as complete. Config validation requires the complete MLX Whisper architecture used by the loader (including matching audio/text state widths and compatible -attention heads); missing dimensions are never replaced with runtime defaults. +attention heads, a decode context that leaves room for output, and broad layer +count safety fences); missing dimensions are never replaced with runtime +defaults. Warm-cache repair checks older snapshots when the newest config, weights, or tokenizer is invalid, preserving offline recovery from an earlier valid revision. diff --git a/core/stt/whisper/engine.rs b/core/stt/whisper/engine.rs index dab1ba90..17f46540 100644 --- a/core/stt/whisper/engine.rs +++ b/core/stt/whisper/engine.rs @@ -136,8 +136,13 @@ fn prepend_initial_prompt_tokens( tokens: &mut Vec, start_of_previous_token: u32, prompt_tokens: &[u32], + max_target_positions: usize, ) -> usize { - let keep = prompt_tokens.len().min(WHISPER_INITIAL_PROMPT_TOKEN_BUDGET); + let available = max_target_positions.saturating_sub(tokens.len() + 2); + let keep = prompt_tokens + .len() + .min(WHISPER_INITIAL_PROMPT_TOKEN_BUDGET) + .min(available); if keep == 0 { return 0; } @@ -465,8 +470,8 @@ impl LocalWhisperEngine { } let tokenizer_path = model_path.join("tokenizer.json"); let mel_filters_path = model_path.join("mel_filters.npz"); - crate::whisper_weights::validate_whisper_model_pair(model_path) - .context("validate Whisper config and architecture-compatible weights")?; + crate::whisper_weights::validate_whisper_model_bundle(model_path) + .context("validate complete Whisper model bundle")?; let architecture = crate::whisper_weights::parse_whisper_config( &safe_path::safe_read_to_string(&config_path)?, &config_path.display().to_string(), @@ -1077,7 +1082,8 @@ impl LocalWhisperEngine { let last_logits = logits.i((.., seq_len - 1, ..))?.squeeze(0)?; let logits_vec = last_logits.to_vec1::()?; - let candidates = self.language_token_candidates(logits_vec.len()); + let candidates = + crate::whisper_weights::language_token_candidates(&self.tokenizer, logits_vec.len()); ensure!( !candidates.is_empty(), "No language token candidates available in tokenizer" @@ -1100,50 +1106,6 @@ impl LocalWhisperEngine { Ok(best_lang) } - /// Enumerate `(token_id, language_code)` pairs to score during detection. - /// - /// Sweeps the conventional language-token range and keeps the ids this - /// tokenizer actually defines, bounded by `vocab_size`. Falls back to a - /// small common-language set when the sweep finds nothing, so detection - /// still works on a tokenizer that numbers its tokens differently. - fn language_token_candidates(&self, vocab_size: usize) -> Vec<(u32, String)> { - // Whisper language tokens are typically in this range. - /// First id of the conventional Whisper language-token block. - const LANG_TOKEN_START: u32 = 50_259; - /// Last id of that block (inclusive). - const LANG_TOKEN_END: u32 = 50_358; - - let mut out = Vec::new(); - for id in LANG_TOKEN_START..=LANG_TOKEN_END { - if (id as usize) >= vocab_size { - break; - } - if let Some(tok) = self.tokenizer.id_to_token(id) - && let Some(lang) = parse_language_token(&tok) - { - out.push((id, lang.to_string())); - } - } - - if !out.is_empty() { - return out; - } - - // Fallback: common languages only. - let fallback = [ - "en", "pl", "de", "fr", "es", "it", "pt", "nl", "ru", "uk", "cs", "sk", - ]; - for lang in fallback { - let tok = format!("<|{}|>", lang); - if let Some(id) = self.tokenizer.token_to_id(&tok) - && (id as usize) < vocab_size - { - out.push((id, lang.to_string())); - } - } - out - } - /// Text-only wrapper over [`Self::transcribe_samples_16k_raw`]. fn transcribe_samples_16k( &mut self, @@ -1208,15 +1170,22 @@ impl LocalWhisperEngine { let mut tokens = vec![start_token]; if let Some(lang) = language { let lang_tok = format!("<|{}|>", lang.to_lowercase()); - if let Some(t) = self.tokenizer.token_to_id(&lang_tok) { + if let Some(t) = self.tokenizer.token_to_id(&lang_tok) + && (t as usize) < self.config.vocab_size + { tokens.push(t); } } - if let Some(t) = self.tokenizer.token_to_id("<|transcribe|>") { + if let Some(t) = self.tokenizer.token_to_id("<|transcribe|>") + && (t as usize) < self.config.vocab_size + { tokens.push(t); } let timestamps_enabled = self.decoding_params.emit_timestamps && self.ts_range.is_some(); - if !timestamps_enabled && let Some(t) = self.tokenizer.token_to_id("<|notimestamps|>") { + if !timestamps_enabled + && let Some(t) = self.tokenizer.token_to_id("<|notimestamps|>") + && (t as usize) < self.config.vocab_size + { tokens.push(t); } @@ -1227,11 +1196,14 @@ impl LocalWhisperEngine { { let prompt_tokens: Vec = encoding.get_ids().to_vec(); if !prompt_tokens.is_empty() { - if let Some(start_of_previous_token) = start_of_previous_token { + if let Some(start_of_previous_token) = start_of_previous_token + && (start_of_previous_token as usize) < self.config.vocab_size + { let used = prepend_initial_prompt_tokens( &mut tokens, start_of_previous_token, &prompt_tokens, + self.config.max_target_positions, ); tracing::debug!("Initial prompt: {} ({} tokens)", prompt, used); } else { @@ -1523,26 +1495,6 @@ impl LocalWhisperEngine { } } -/// Extract the language code from a `<|xx|>` token, or `None` when the token is -/// not a language marker. -/// -/// Accepts only 2–3 ASCII letters between the delimiters, which is what -/// separates `<|pl|>` from control tokens like `<|notimestamps|>`. -fn parse_language_token(token: &str) -> Option<&str> { - if !token.starts_with("<|") || !token.ends_with("|>") { - return None; - } - let inner = &token[2..token.len() - 2]; - if inner.len() < 2 || inner.len() > 3 { - return None; - } - if inner.chars().all(|c| c.is_ascii_alphabetic()) { - Some(inner) - } else { - None - } -} - /// Normalize a word for overlap comparison: lowercase, alphanumerics only. /// /// Falls back to the lowercased original when stripping would leave nothing, so @@ -1707,53 +1659,6 @@ fn load_mel_filters_from_reader( Ok(data) } -/// Rewrite an MLX/OpenAI Whisper tensor name into the Candle naming scheme. -/// -/// Order is load-bearing and must not be "simplified": cross-attention names -/// are rewritten before the generic attention rules, otherwise `cross_attn` -/// would be mangled by the `attn` replacements and the weight would silently -/// land under the wrong module. -fn map_tensor_name(name: &str) -> String { - let mut new_name = name.to_string(); - - new_name = new_name.replace("blocks", "layers"); - new_name = new_name.replace("mlp1", "fc1"); - new_name = new_name.replace("mlp2", "fc2"); - new_name = new_name.replace("decoder.ln", "decoder.layer_norm"); - // Replace cross-attn layer norms before generic attn replacement to avoid mangling - new_name = new_name.replace("cross_attn_ln", "encoder_attn_layer_norm"); - new_name = new_name.replace("attn_ln", "self_attn_layer_norm"); - new_name = new_name.replace("mlp_ln", "final_layer_norm"); - new_name = new_name.replace("ln_post", "layer_norm"); - - // Important: handle cross_attn BEFORE attn - new_name = new_name.replace("cross_attn", "encoder_attn"); - - // Replace ".attn." segment with ".self_attn." - new_name = new_name.replace(".attn.", ".self_attn."); - - // Projections - new_name = new_name.replace("query", "q_proj"); - new_name = new_name.replace("key", "k_proj"); - new_name = new_name.replace("value", "v_proj"); - new_name = new_name.replace(".out.", ".out_proj."); - - // Embedding aliases - new_name = new_name.replace("decoder.token_embedding", "decoder.embed_tokens"); - - // Prefix - if !new_name.starts_with("model.") { - new_name = format!("model.{}", new_name); - } - - // Positional embedding key from MLX - if new_name == "model.decoder.positional_embedding" { - new_name = "model.decoder.embed_positions.weight".to_string(); - } - new_name = new_name.replace(".biases", ".bias"); - new_name -} - /// Incremental no-repeat n-gram blocker. /// /// Replaces the per-step O(n) full scan of `all_tokens` (which made the decode @@ -1855,11 +1760,13 @@ fn should_drop_for_quality_gate( /// Build a VarBuilder from verified unquantized tensors. fn is_supported_runtime_tensor(name: &str, tensor: &Tensor) -> bool { + if name == "alignment_heads" { + return tensor.dtype() == DType::I64; + } if name.ends_with(".scales") || name.ends_with(".biases") { return false; } matches!(tensor.dtype(), DType::F16 | DType::F32) - || name == "alignment_heads" && tensor.dtype() == DType::I64 } fn build_varbuilder_from_tensors( @@ -1872,6 +1779,9 @@ fn build_varbuilder_from_tensors( { anyhow::bail!("Unsupported Whisper tensor payload refused; fp16 weights are required"); } + crate::whisper_weights::validate_mapped_tensor_name_uniqueness( + raw_tensors.keys().map(String::as_str), + )?; let mut tensor_map = HashMap::new(); // alignment_heads is integer metadata used by upstream timestamp tooling, @@ -1880,7 +1790,7 @@ fn build_varbuilder_from_tensors( if name == "alignment_heads" { continue; } - let mapped_name = map_tensor_name(name); + let mapped_name = crate::whisper_weights::map_whisper_tensor_name(name); let mut t = tensor.clone(); if t.dtype() != DType::F32 { t = t.to_dtype(DType::F32)?; @@ -1911,15 +1821,41 @@ mod model_payload_tests { use std::fs; use tempfile::TempDir; - fn write_tiny_model(path: &Path, name: &str, dtype: &str, payload_bytes: usize) { + fn decode_hex(raw: &str) -> Vec { + let digits: String = raw.chars().filter(|ch| !ch.is_whitespace()).collect(); + assert!(digits.len().is_multiple_of(2)); + digits + .as_bytes() + .as_chunks::<2>() + .0 + .iter() + .map(|pair| u8::from_str_radix(std::str::from_utf8(pair).unwrap(), 16).unwrap()) + .collect() + } + + fn write_valid_bundle_artifacts(path: &Path) { fs::create_dir_all(path).unwrap(); fs::write( path.join("config.json"), include_str!("../../../tests/fixtures/whisper_test_config.json"), ) .unwrap(); - fs::write(path.join("tokenizer.json"), "{}").unwrap(); - fs::write(path.join("mel_filters.npz"), b"placeholder").unwrap(); + fs::write( + path.join("tokenizer.json"), + include_str!("../../../tests/fixtures/whisper_tokenizer.json"), + ) + .unwrap(); + fs::write( + path.join("mel_filters.npz"), + decode_hex(include_str!( + "../../../tests/fixtures/whisper_mel_filters.npz.hex" + )), + ) + .unwrap(); + } + + fn write_tiny_model(path: &Path, name: &str, dtype: &str, payload_bytes: usize) { + write_valid_bundle_artifacts(path); let header = serde_json::json!({ name: { "dtype": dtype, @@ -1997,6 +1933,97 @@ mod model_payload_tests { assert!(!vb.contains_tensor("model.alignment_heads")); } + #[test] + fn tensor_builder_rejects_mapped_name_collisions() { + let mut tensors = HashMap::new(); + tensors.insert( + "decoder.ln.weight".to_string(), + Tensor::from_vec(vec![1.0_f32], 1, &Device::Cpu).unwrap(), + ); + tensors.insert( + "decoder.layer_norm.weight".to_string(), + Tensor::from_vec(vec![2.0_f32], 1, &Device::Cpu).unwrap(), + ); + + let err = build_varbuilder_from_tensors(tensors, &Device::Cpu) + .err() + .expect("mapped collision must be rejected"); + let message = format!("{err:#}"); + assert!(message.contains("decoder.ln.weight"), "{message}"); + assert!(message.contains("decoder.layer_norm.weight"), "{message}"); + assert!( + message.contains("model.decoder.layer_norm.weight"), + "{message}" + ); + } + + #[test] + fn tensor_builder_rejects_float_alignment_metadata() { + let mut tensors = HashMap::new(); + tensors.insert( + "alignment_heads".to_string(), + Tensor::from_vec(vec![1.0_f32], 1, &Device::Cpu).unwrap(), + ); + let err = build_varbuilder_from_tensors(tensors, &Device::Cpu) + .err() + .expect("float alignment metadata must be rejected"); + assert!(format!("{err:#}").contains("refused")); + } + + #[test] + fn local_loader_rejects_invalid_tokenizer_before_model_load() { + let temp = TempDir::new().unwrap(); + write_valid_bundle_artifacts(temp.path()); + let architecture = crate::whisper_weights::parse_whisper_config( + include_str!("../../../tests/fixtures/whisper_test_config.json"), + "test fixture", + ) + .unwrap(); + crate::whisper_weights::write_test_whisper_weights( + &temp.path().join("weights.safetensors"), + architecture, + ) + .unwrap(); + fs::write(temp.path().join("tokenizer.json"), "{}").unwrap(); + + let err = LocalWhisperEngine::new(temp.path()) + .err() + .expect("invalid tokenizer must be rejected"); + let message = format!("{err:#}"); + assert!(message.contains("tokenizer"), "{message}"); + assert!( + !message.contains("Failed to create Whisper Model"), + "{message}" + ); + } + + #[test] + fn local_loader_rejects_unpinned_mel_before_model_load() { + let temp = TempDir::new().unwrap(); + write_valid_bundle_artifacts(temp.path()); + let architecture = crate::whisper_weights::parse_whisper_config( + include_str!("../../../tests/fixtures/whisper_test_config.json"), + "test fixture", + ) + .unwrap(); + crate::whisper_weights::write_test_whisper_weights( + &temp.path().join("weights.safetensors"), + architecture, + ) + .unwrap(); + fs::write(temp.path().join("mel_filters.npz"), b"wrong").unwrap(); + + let err = LocalWhisperEngine::new(temp.path()) + .err() + .expect("unpinned mel must be rejected"); + let message = format!("{err:#}"); + assert!(message.contains("SHA-256 mismatch"), "{message}"); + assert!( + !message.contains("Failed to create Whisper Model"), + "{message}" + ); + } + #[test] fn local_loader_uses_valid_alternative_after_invalid_primary() { let temp = TempDir::new().unwrap(); @@ -2006,8 +2033,7 @@ mod model_payload_tests { include_str!("../../../tests/fixtures/whisper_test_config.json"), ) .unwrap(); - fs::write(temp.path().join("tokenizer.json"), "{}").unwrap(); - fs::write(temp.path().join("mel_filters.npz"), b"placeholder").unwrap(); + write_valid_bundle_artifacts(temp.path()); let architecture = crate::whisper_weights::parse_whisper_config( include_str!("../../../tests/fixtures/whisper_test_config.json"), "test fixture", @@ -2020,11 +2046,10 @@ mod model_payload_tests { .unwrap(); write_tiny_model(temp.path(), "encoder.weight", "U32", 4); - let err = LocalWhisperEngine::new(temp.path()) - .err() - .expect("compatible alternative should pass the payload gate"); - let message = format!("{err:#}"); - assert!(!message.contains("payload refused"), "{message}"); + assert!( + LocalWhisperEngine::new(temp.path()).is_ok(), + "compatible alternative should load after invalid primary" + ); } } @@ -2259,7 +2284,7 @@ mod dedup_tests { let without_prompt = vec![1_u32, 2, 3]; let mut with_prompt = without_prompt.clone(); - let used = prepend_initial_prompt_tokens(&mut with_prompt, 99, &[10, 11, 12]); + let used = prepend_initial_prompt_tokens(&mut with_prompt, 99, &[10, 11, 12], 448); assert_eq!(used, 3); assert_ne!(with_prompt, without_prompt); @@ -2274,7 +2299,7 @@ mod dedup_tests { let prompt_tokens: Vec = (0..(WHISPER_INITIAL_PROMPT_TOKEN_BUDGET as u32 + 10)).collect(); - let used = prepend_initial_prompt_tokens(&mut tokens, 99, &prompt_tokens); + let used = prepend_initial_prompt_tokens(&mut tokens, 99, &prompt_tokens, 448); assert_eq!(used, WHISPER_INITIAL_PROMPT_TOKEN_BUDGET); assert_eq!(tokens.len(), 4 + WHISPER_INITIAL_PROMPT_TOKEN_BUDGET); @@ -2289,6 +2314,23 @@ mod dedup_tests { ); } + #[test] + fn initial_prompt_reserves_one_decode_position() { + let prompt = [10_u32, 11, 12, 13]; + + let mut minimum_context = vec![1_u32, 2, 3, 4]; + let used = prepend_initial_prompt_tokens(&mut minimum_context, 99, &prompt, 5); + assert_eq!(used, 0); + assert_eq!(minimum_context, vec![1, 2, 3, 4]); + assert!(minimum_context.len() < 5); + + let mut short_context = vec![1_u32, 2, 3, 4]; + let used = prepend_initial_prompt_tokens(&mut short_context, 99, &prompt, 8); + assert_eq!(used, 2); + assert_eq!(short_context, vec![99, 10, 11, 1, 2, 3, 4]); + assert_eq!(8 - short_context.len(), 1); + } + /// Incremental n-gram blocker matches full-scan blocks across sizes and edges. #[test] fn ngram_block_parity() { diff --git a/core/whisper_weights.rs b/core/whisper_weights.rs index 90eb8963..a6ac4804 100644 --- a/core/whisper_weights.rs +++ b/core/whisper_weights.rs @@ -13,6 +13,13 @@ pub const SUPPORTED_NAMES: [&str; 2] = ["weights.safetensors", "model.safetensor pub const MEL_FILTERS_SHA256: &str = "7450ae70723a5ef9d341e3cee628c7cb0177f36ce42c44b7ed2bf3325f0f6d4c"; const REQUIRED_TOKENIZER_TOKENS: [&str; 2] = ["<|startoftranscript|>", "<|endoftext|>"]; +const OPTIONAL_PROMPT_TOKENS: [&str; 3] = ["<|transcribe|>", "<|notimestamps|>", "<|startofprev|>"]; +const MAX_WHISPER_LAYERS: usize = 64; +const LANG_TOKEN_START: u32 = 50_259; +const LANG_TOKEN_END: u32 = 50_358; +const FALLBACK_LANGUAGES: [&str; 12] = [ + "en", "pl", "de", "fr", "es", "it", "pt", "nl", "ru", "uk", "cs", "sk", +]; /// MLX Whisper architecture shared by validation, disk loading, and embedding. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) struct WhisperArchitecture { @@ -55,10 +62,44 @@ pub(crate) fn validate_whisper_tokenizer_for_architecture( path: &Path, architecture: WhisperArchitecture, ) -> Result<()> { - validate_whisper_tokenizer(path)?; let tokenizer = tokenizers::Tokenizer::from_file(path) .map_err(|err| anyhow!("invalid Whisper tokenizer {}: {err}", path.display()))?; + for token in REQUIRED_TOKENIZER_TOKENS { + let id = tokenizer.token_to_id(token).ok_or_else(|| { + anyhow!( + "Whisper tokenizer {} is missing required token {token}", + path.display() + ) + })?; + if id as usize >= architecture.n_vocab { + return Err(anyhow!( + "Whisper tokenizer {} required token {token} has id {id} outside configured vocabulary 0..{}", + path.display(), + architecture.n_vocab + )); + } + } + for token in OPTIONAL_PROMPT_TOKENS { + if let Some(id) = tokenizer.token_to_id(token) + && id as usize >= architecture.n_vocab + { + return Err(anyhow!( + "Whisper tokenizer {} prompt token {token} has id {id} outside configured vocabulary 0..{}", + path.display(), + architecture.n_vocab + )); + } + } let vocab = tokenizer.get_vocab(true); + if let Some((token, id)) = vocab.iter().find(|(token, id)| { + parse_language_token(token).is_some() && (**id as usize) >= architecture.n_vocab + }) { + return Err(anyhow!( + "Whisper tokenizer {} language token {token} has id {id} outside configured vocabulary 0..{}", + path.display(), + architecture.n_vocab + )); + } let covered: HashSet = vocab .values() .copied() @@ -71,25 +112,48 @@ pub(crate) fn validate_whisper_tokenizer_for_architecture( architecture.n_vocab )); } - if !vocab - .iter() - .any(|(token, id)| (*id as usize) < architecture.n_vocab && is_language_token(token)) - { + if language_token_candidates(&tokenizer, architecture.n_vocab).is_empty() { return Err(anyhow!( - "Whisper tokenizer {} has no language token required for automatic detection", + "Whisper tokenizer {} has no runtime-discoverable language token required for automatic detection", path.display() )); } Ok(()) } -fn is_language_token(token: &str) -> bool { - token - .strip_prefix("<|") - .and_then(|inner| inner.strip_suffix("|>")) - .is_some_and(|inner| { - (2..=3).contains(&inner.len()) && inner.chars().all(|ch| ch.is_ascii_alphabetic()) - }) +pub(crate) fn language_token_candidates( + tokenizer: &tokenizers::Tokenizer, + vocab_size: usize, +) -> Vec<(u32, String)> { + let mut out = Vec::new(); + for id in LANG_TOKEN_START..=LANG_TOKEN_END { + if id as usize >= vocab_size { + break; + } + if let Some(token) = tokenizer.id_to_token(id) + && let Some(language) = parse_language_token(&token) + { + out.push((id, language.to_string())); + } + } + if !out.is_empty() { + return out; + } + for language in FALLBACK_LANGUAGES { + let token = format!("<|{language}|>"); + if let Some(id) = tokenizer.token_to_id(&token) + && (id as usize) < vocab_size + { + out.push((id, language.to_string())); + } + } + out +} + +fn parse_language_token(token: &str) -> Option<&str> { + let inner = token.strip_prefix("<|")?.strip_suffix("|>")?; + ((2..=3).contains(&inner.len()) && inner.chars().all(|ch| ch.is_ascii_alphabetic())) + .then_some(inner) } /// Validate the config schema and reject every declared quantization mode. @@ -146,6 +210,18 @@ pub(crate) fn parse_whisper_config(raw: &str, source: &str) -> Result MAX_WHISPER_LAYERS + || architecture.n_text_layer > MAX_WHISPER_LAYERS + { + return Err(anyhow!( + "Whisper config {source} exceeds the resource limit of {MAX_WHISPER_LAYERS} encoder or decoder layers" + )); + } + if architecture.n_text_ctx < 5 { + return Err(anyhow!( + "Whisper config {source} requires n_text_ctx of at least 5 for the decode prefix and one output token" + )); + } if !matches!(architecture.n_mels, 80 | 128) { return Err(anyhow!( "Whisper config {source} requires n_mels to be 80 or 128" @@ -330,9 +406,14 @@ fn read_validated_tensor_shapes(path: &Path) -> Result 8_u64, + ("alignment_heads", _) => { + return Err(anyhow!( + "unsupported Whisper tensor dtype {dtype} for {name}" + )); + } (_, "F16") => 2_u64, (_, "F32") => 4_u64, - ("alignment_heads", "I64") => 8_u64, _ => { return Err(anyhow!( "unsupported Whisper tensor dtype {dtype} for {name}" @@ -417,6 +498,7 @@ fn validate_whisper_weights_for_architecture( architecture: WhisperArchitecture, ) -> Result<()> { let tensors = read_validated_tensor_shapes(path)?; + validate_mapped_tensor_name_uniqueness(tensors.keys().map(String::as_str))?; for (name, expected) in expected_whisper_tensor_shapes(architecture)? { let actual = tensors.get(&name).ok_or_else(|| { anyhow!( @@ -436,6 +518,59 @@ fn validate_whisper_weights_for_architecture( Ok(()) } +pub(crate) fn validate_mapped_tensor_name_uniqueness<'a>( + names: impl IntoIterator, +) -> Result<()> { + let mut mapped_sources = BTreeMap::::new(); + for name in names { + if name == "alignment_heads" { + continue; + } + let mapped = map_whisper_tensor_name(name); + if let Some(previous) = mapped_sources.insert(mapped.clone(), name.to_string()) { + let mut sources = [previous, name.to_string()]; + sources.sort(); + return Err(anyhow!( + "Whisper tensors {} and {} collide after runtime mapping to {mapped}", + sources[0], + sources[1] + )); + } + } + Ok(()) +} + +/// Rewrite an MLX/OpenAI tensor name into Candle's Whisper namespace. +/// +/// Replacement order is load-bearing: cross-attention names must be handled +/// before the generic attention rules so aliases map exactly as the loader sees +/// them and the shared collision gate can reject ambiguous payloads. +pub(crate) fn map_whisper_tensor_name(name: &str) -> String { + let mut mapped = name.to_string(); + mapped = mapped.replace("blocks", "layers"); + mapped = mapped.replace("mlp1", "fc1"); + mapped = mapped.replace("mlp2", "fc2"); + mapped = mapped.replace("decoder.ln", "decoder.layer_norm"); + mapped = mapped.replace("cross_attn_ln", "encoder_attn_layer_norm"); + mapped = mapped.replace("attn_ln", "self_attn_layer_norm"); + mapped = mapped.replace("mlp_ln", "final_layer_norm"); + mapped = mapped.replace("ln_post", "layer_norm"); + mapped = mapped.replace("cross_attn", "encoder_attn"); + mapped = mapped.replace(".attn.", ".self_attn."); + mapped = mapped.replace("query", "q_proj"); + mapped = mapped.replace("key", "k_proj"); + mapped = mapped.replace("value", "v_proj"); + mapped = mapped.replace(".out.", ".out_proj."); + mapped = mapped.replace("decoder.token_embedding", "decoder.embed_tokens"); + if !mapped.starts_with("model.") { + mapped = format!("model.{mapped}"); + } + if mapped == "model.decoder.positional_embedding" { + mapped = "model.decoder.embed_positions.weight".to_string(); + } + mapped.replace(".biases", ".bias") +} + fn expected_whisper_tensor_shapes( architecture: WhisperArchitecture, ) -> Result>> { @@ -652,6 +787,39 @@ mod tests { ); } + #[test] + fn architecture_resource_limits_are_enforced_before_schema_expansion() { + for field in ["n_audio_layer", "n_text_layer"] { + let mut accepted = valid_config(); + accepted[field] = serde_json::json!(MAX_WHISPER_LAYERS); + parse_whisper_config(&accepted.to_string(), "fixture").unwrap(); + + let mut rejected = valid_config(); + rejected[field] = serde_json::json!(MAX_WHISPER_LAYERS + 1); + let err = parse_whisper_config(&rejected.to_string(), "fixture").unwrap_err(); + assert!( + format!("{err:#}").contains("resource limit"), + "{field}: {err:#}" + ); + } + } + + #[test] + fn text_context_reserves_decode_output() { + for value in 1..5 { + let mut config = valid_config(); + config["n_text_ctx"] = serde_json::json!(value); + let err = parse_whisper_config(&config.to_string(), "fixture").unwrap_err(); + assert!( + format!("{err:#}").contains("at least 5"), + "{value}: {err:#}" + ); + } + let mut config = valid_config(); + config["n_text_ctx"] = serde_json::json!(5); + parse_whisper_config(&config.to_string(), "fixture").unwrap(); + } + #[test] fn tokenizer_without_language_tokens_is_rejected() { let temp = TempDir::new().unwrap(); @@ -671,7 +839,7 @@ mod tests { .unwrap(); let err = validate_whisper_tokenizer_for_architecture(&path, architecture).unwrap_err(); - assert!(format!("{err:#}").contains("no language token")); + assert!(format!("{err:#}").contains("runtime-discoverable")); } #[test] @@ -693,4 +861,82 @@ mod tests { let err = validate_whisper_tokenizer_for_architecture(&path, architecture).unwrap_err(); assert!(format!("{err:#}").contains("does not cover configured vocabulary")); } + + fn write_wordlevel_tokenizer(path: &Path, vocab: &[(&str, u32)]) { + let mut tokenizer: serde_json::Value = + serde_json::from_str(include_str!("../tests/fixtures/whisper_tokenizer.json")).unwrap(); + tokenizer["model"]["vocab"] = serde_json::Value::Object( + vocab + .iter() + .map(|(token, id)| ((*token).to_string(), serde_json::json!(id))) + .collect(), + ); + fs::write(path, serde_json::to_vec(&tokenizer).unwrap()).unwrap(); + } + + #[test] + fn tokenizer_language_gate_matches_runtime_candidates() { + let temp = TempDir::new().unwrap(); + let path = temp.path().join("tokenizer.json"); + write_wordlevel_tokenizer( + &path, + &[ + ("", 0), + ("<|startoftranscript|>", 1), + ("<|endoftext|>", 2), + ("<|ja|>", 3), + ], + ); + let architecture = parse_whisper_config( + include_str!("../tests/fixtures/whisper_test_config.json"), + "test fixture", + ) + .unwrap(); + let tokenizer = tokenizers::Tokenizer::from_file(&path).unwrap(); + assert!(language_token_candidates(&tokenizer, architecture.n_vocab).is_empty()); + let err = validate_whisper_tokenizer_for_architecture(&path, architecture).unwrap_err(); + assert!(format!("{err:#}").contains("runtime-discoverable")); + } + + #[test] + fn tokenizer_control_tokens_must_fit_model_vocabulary() { + let temp = TempDir::new().unwrap(); + let path = temp.path().join("tokenizer.json"); + write_wordlevel_tokenizer( + &path, + &[ + ("", 0), + ("ordinary", 1), + ("other", 2), + ("<|pl|>", 3), + ("<|startoftranscript|>", 4), + ("<|endoftext|>", 5), + ], + ); + let architecture = parse_whisper_config( + include_str!("../tests/fixtures/whisper_test_config.json"), + "test fixture", + ) + .unwrap(); + let err = validate_whisper_tokenizer_for_architecture(&path, architecture).unwrap_err(); + let message = format!("{err:#}"); + assert!(message.contains("<|startoftranscript|>"), "{message}"); + assert!(message.contains("id 4"), "{message}"); + } + + #[test] + fn mapped_tensor_aliases_are_rejected_deterministically() { + let err = validate_mapped_tensor_name_uniqueness([ + "decoder.layer_norm.weight", + "decoder.ln.weight", + ]) + .unwrap_err(); + let message = format!("{err:#}"); + assert!(message.contains("decoder.layer_norm.weight"), "{message}"); + assert!(message.contains("decoder.ln.weight"), "{message}"); + assert!( + message.contains("model.decoder.layer_norm.weight"), + "{message}" + ); + } } diff --git a/tests/e2e_stt_transcription.rs b/tests/e2e_stt_transcription.rs index bb4f9117..29a2f58c 100644 --- a/tests/e2e_stt_transcription.rs +++ b/tests/e2e_stt_transcription.rs @@ -212,7 +212,7 @@ fn write_tiny_complete_weights(path: &Path) { add("encoder.ln_post.weight".into(), &[D]); add("encoder.ln_post.bias".into(), &[D]); add("decoder.token_embedding.weight".into(), &[4, D]); - add("decoder.positional_embedding".into(), &[2, D]); + add("decoder.positional_embedding".into(), &[5, D]); add("decoder.ln.weight".into(), &[D]); add("decoder.ln.bias".into(), &[D]); for prefix in [ diff --git a/tests/fixtures/whisper_test_config.json b/tests/fixtures/whisper_test_config.json index 9542179d..4344ca39 100644 --- a/tests/fixtures/whisper_test_config.json +++ b/tests/fixtures/whisper_test_config.json @@ -5,7 +5,7 @@ "n_audio_head": 1, "n_audio_layer": 1, "n_vocab": 4, - "n_text_ctx": 2, + "n_text_ctx": 5, "n_text_state": 4, "n_text_head": 1, "n_text_layer": 1 From fbf447f82839a55754ecdf5636a8171193d29e98 Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Sat, 22 Aug 2026 03:04:08 +0200 Subject: [PATCH 40/45] [codex/vc-workflow] fix(stt): close model resource boundaries --- CHANGELOG.md | 4 +- README.md | 8 +- core/stt/whisper/engine.rs | 30 ++++--- core/whisper_weights.rs | 113 ++++++++++++++++++++++++--- scripts/download-model.sh | 20 ++++- scripts/tests/download-model-test.sh | 58 +++++++++++++- 6 files changed, 201 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c0fe59d6..620c6fc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -74,7 +74,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 required tensor-name/shape contract. The loader rejects incomplete bundles before cold model construction; prompt/control token IDs, automatic-language candidates, layer/context resource bounds, and mapped tensor-name collisions - are validated by the same runtime-owned helpers. Quantized payloads and the + are validated by the same runtime-owned helpers. Tokenizers cannot emit IDs + without embedding rows; audio context is bounded to the supported 30-second + window; and surplus tensors are refused before allocation. Quantized payloads and the legacy Q8 fallback are refused; the old public Q8 identifiers remain deprecated source-compatibility constants only. Building from source now declares Rust 1.88 as the minimum supported toolchain. diff --git a/README.md b/README.md index 4d39f5d9..004d4c5f 100644 --- a/README.md +++ b/README.md @@ -354,7 +354,8 @@ resource limits, requires every runtime prompt/control token to fit the configured vocabulary, and uses the same automatic-language candidate logic as the decoder. It verifies the pinned mel SHA-256 and validates every required Whisper tensor name and shape plus the complete safetensors tensor -table, dtype allowlist, mapped-name uniqueness, offsets, and file length. The +table, exact consumed tensor set, bounded alignment metadata, dtype allowlist, +mapped-name uniqueness, offsets, and file length. The disk loader applies this complete gate before mmap or model construction. Downloads and warm-cache copies are written to `.partial` files and promoted only after per-file @@ -363,8 +364,9 @@ instead of being accepted as complete. Config validation requires the complete MLX Whisper architecture used by the loader (including matching audio/text state widths and compatible attention heads, a decode context that leaves room for output, and broad layer -count safety fences); missing dimensions are never replaced with runtime -defaults. +count safety fences). Audio context is capped at the 1500 positions consumed by +the supported 30-second Whisper window; missing dimensions are never replaced +with runtime defaults. Warm-cache repair checks older snapshots when the newest config, weights, or tokenizer is invalid, preserving offline recovery from an earlier valid revision. diff --git a/core/stt/whisper/engine.rs b/core/stt/whisper/engine.rs index 17f46540..11dc998f 100644 --- a/core/stt/whisper/engine.rs +++ b/core/stt/whisper/engine.rs @@ -155,6 +155,10 @@ fn prepend_initial_prompt_tokens( keep } +fn prompt_token_ids_fit_vocab(tokens: &[u32], vocab_size: usize) -> bool { + tokens.iter().all(|token| (*token as usize) < vocab_size) +} + /// Record that a requested final pass was skipped, with the reason. /// /// `None` when no final pass was requested — the caller must not fabricate a @@ -504,17 +508,12 @@ impl LocalWhisperEngine { // Load the verified unquantized tensors on CPU before device transfer. let read_started = std::time::Instant::now(); for (name, view) in tensors.tensors() { + if name == "alignment_heads" { + continue; + } let loaded = view.load(&Device::Cpu)?; raw_tensors.insert(name.to_string(), loaded); } - if raw_tensors - .iter() - .any(|(name, tensor)| !is_supported_runtime_tensor(name, tensor)) - { - anyhow::bail!( - "Unsupported Whisper tensor payload refused; install the complete fp16 model" - ); - } read_secs = read_started.elapsed().as_secs_f64(); let plain_started = std::time::Instant::now(); @@ -1194,15 +1193,17 @@ impl LocalWhisperEngine { if let Some(ref prompt) = self.decoding_params.initial_prompt && let Ok(encoding) = self.tokenizer.encode(prompt.as_str(), false) { - let prompt_tokens: Vec = encoding.get_ids().to_vec(); - if !prompt_tokens.is_empty() { + let prompt_tokens = encoding.get_ids(); + if !prompt_token_ids_fit_vocab(prompt_tokens, self.config.vocab_size) { + tracing::warn!("Ignoring Whisper initial prompt containing out-of-vocabulary IDs"); + } else if !prompt_tokens.is_empty() { if let Some(start_of_previous_token) = start_of_previous_token && (start_of_previous_token as usize) < self.config.vocab_size { let used = prepend_initial_prompt_tokens( &mut tokens, start_of_previous_token, - &prompt_tokens, + prompt_tokens, self.config.max_target_positions, ); tracing::debug!("Initial prompt: {} ({} tokens)", prompt, used); @@ -2331,6 +2332,13 @@ mod dedup_tests { assert_eq!(8 - short_context.len(), 1); } + #[test] + fn initial_prompt_rejects_any_out_of_vocabulary_id() { + assert!(prompt_token_ids_fit_vocab(&[0, 1, 3], 4)); + assert!(!prompt_token_ids_fit_vocab(&[0, 4], 4)); + assert!(!prompt_token_ids_fit_vocab(&[5], 4)); + } + /// Incremental n-gram blocker matches full-scan blocks across sizes and edges. #[test] fn ngram_block_parity() { diff --git a/core/whisper_weights.rs b/core/whisper_weights.rs index a6ac4804..8b71cbeb 100644 --- a/core/whisper_weights.rs +++ b/core/whisper_weights.rs @@ -15,6 +15,7 @@ pub const MEL_FILTERS_SHA256: &str = const REQUIRED_TOKENIZER_TOKENS: [&str; 2] = ["<|startoftranscript|>", "<|endoftext|>"]; const OPTIONAL_PROMPT_TOKENS: [&str; 3] = ["<|transcribe|>", "<|notimestamps|>", "<|startofprev|>"]; const MAX_WHISPER_LAYERS: usize = 64; +const MAX_WHISPER_AUDIO_CONTEXT: usize = 1_500; const LANG_TOKEN_START: u32 = 50_259; const LANG_TOKEN_END: u32 = 50_358; const FALLBACK_LANGUAGES: [&str; 12] = [ @@ -100,6 +101,16 @@ pub(crate) fn validate_whisper_tokenizer_for_architecture( architecture.n_vocab )); } + if let Some((token, id)) = vocab + .iter() + .find(|(_, id)| (**id as usize) >= architecture.n_vocab) + { + return Err(anyhow!( + "Whisper tokenizer {} token {token} has id {id} outside configured vocabulary 0..{}", + path.display(), + architecture.n_vocab + )); + } let covered: HashSet = vocab .values() .copied() @@ -227,9 +238,9 @@ pub(crate) fn parse_whisper_config(raw: &str, source: &str) -> Result u32::MAX as usize { + if architecture.n_audio_ctx > MAX_WHISPER_AUDIO_CONTEXT { return Err(anyhow!( - "Whisper config {source} requires n_audio_ctx to fit in u32" + "Whisper config {source} exceeds the resource limit of {MAX_WHISPER_AUDIO_CONTEXT} audio context positions" )); } if architecture.n_audio_state != architecture.n_text_state { @@ -499,22 +510,45 @@ fn validate_whisper_weights_for_architecture( ) -> Result<()> { let tensors = read_validated_tensor_shapes(path)?; validate_mapped_tensor_name_uniqueness(tensors.keys().map(String::as_str))?; - for (name, expected) in expected_whisper_tensor_shapes(architecture)? { - let actual = tensors.get(&name).ok_or_else(|| { - anyhow!( - "Whisper weights {} are missing tensor {name}", - path.display() - ) - })?; - if actual != &expected { + validate_whisper_tensor_shapes(&tensors, architecture) + .with_context(|| format!("validate Whisper tensor schema in {}", path.display())) +} + +fn validate_whisper_tensor_shapes( + tensors: &BTreeMap>, + architecture: WhisperArchitecture, +) -> Result<()> { + let expected_shapes = expected_whisper_tensor_shapes(architecture)?; + if let Some(shape) = tensors.get("alignment_heads") { + let maximum = architecture + .n_text_layer + .checked_mul(architecture.n_text_head) + .ok_or_else(|| anyhow!("Whisper alignment-head bound overflows"))?; + if shape.len() != 2 || shape[0] == 0 || shape[0] > maximum || shape[1] != 2 { return Err(anyhow!( - "Whisper tensor {name} in {} has shape {:?}, expected {:?}", - path.display(), + "Whisper alignment_heads has shape {:?}, expected [N, 2] with 1 <= N <= {maximum}", + shape + )); + } + } + for (name, expected) in &expected_shapes { + let actual = tensors + .get(name) + .ok_or_else(|| anyhow!("Whisper weights are missing tensor {name}"))?; + if actual != expected { + return Err(anyhow!( + "Whisper tensor {name} has shape {:?}, expected {:?}", actual, expected )); } } + if let Some(unexpected) = tensors + .keys() + .find(|name| name.as_str() != "alignment_heads" && !expected_shapes.contains_key(*name)) + { + return Err(anyhow!("unexpected Whisper tensor {unexpected}")); + } Ok(()) } @@ -802,6 +836,15 @@ mod tests { "{field}: {err:#}" ); } + + let mut accepted_context = valid_config(); + accepted_context["n_audio_ctx"] = serde_json::json!(MAX_WHISPER_AUDIO_CONTEXT); + parse_whisper_config(&accepted_context.to_string(), "fixture").unwrap(); + + let mut rejected_context = valid_config(); + rejected_context["n_audio_ctx"] = serde_json::json!(MAX_WHISPER_AUDIO_CONTEXT + 1); + let err = parse_whisper_config(&rejected_context.to_string(), "fixture").unwrap_err(); + assert!(format!("{err:#}").contains("audio context"), "{err:#}"); } #[test] @@ -924,6 +967,31 @@ mod tests { assert!(message.contains("id 4"), "{message}"); } + #[test] + fn tokenizer_cannot_encode_any_id_without_an_embedding_row() { + let temp = TempDir::new().unwrap(); + let path = temp.path().join("tokenizer.json"); + write_wordlevel_tokenizer( + &path, + &[ + ("", 0), + ("<|startoftranscript|>", 1), + ("<|endoftext|>", 2), + ("<|pl|>", 3), + ("surplus", 4), + ], + ); + let architecture = parse_whisper_config( + include_str!("../tests/fixtures/whisper_test_config.json"), + "test fixture", + ) + .unwrap(); + let err = validate_whisper_tokenizer_for_architecture(&path, architecture).unwrap_err(); + let message = format!("{err:#}"); + assert!(message.contains("surplus"), "{message}"); + assert!(message.contains("id 4"), "{message}"); + } + #[test] fn mapped_tensor_aliases_are_rejected_deterministically() { let err = validate_mapped_tensor_name_uniqueness([ @@ -939,4 +1007,25 @@ mod tests { "{message}" ); } + + #[test] + fn tensor_schema_rejects_surplus_but_allows_bounded_alignment_metadata() { + let architecture = parse_whisper_config( + include_str!("../tests/fixtures/whisper_test_config.json"), + "test fixture", + ) + .unwrap(); + let mut tensors = expected_whisper_tensor_shapes(architecture).unwrap(); + tensors.insert("surplus.weight".to_string(), vec![1]); + let err = validate_whisper_tensor_shapes(&tensors, architecture).unwrap_err(); + assert!(format!("{err:#}").contains("unexpected Whisper tensor surplus.weight")); + + tensors.remove("surplus.weight"); + tensors.insert("alignment_heads".to_string(), vec![1, 2]); + validate_whisper_tensor_shapes(&tensors, architecture).unwrap(); + + tensors.insert("alignment_heads".to_string(), vec![2, 2]); + let err = validate_whisper_tensor_shapes(&tensors, architecture).unwrap_err(); + assert!(format!("{err:#}").contains("alignment_heads")); + } } diff --git a/scripts/download-model.sh b/scripts/download-model.sh index e32f1b13..4c985db1 100755 --- a/scripts/download-model.sh +++ b/scripts/download-model.sh @@ -49,8 +49,14 @@ atomic_copy() { local source="$1" local destination="$2" local partial="${destination}.partial" - cp -fL "$source" "$partial" - mv -f "$partial" "$destination" + if ! cp -fL "$source" "$partial"; then + rm -f "$partial" + return 1 + fi + if ! mv -f "$partial" "$destination"; then + rm -f "$partial" + return 1 + fi } # Configuration @@ -142,16 +148,24 @@ if [[ "$MODEL_REPO" == "$DEFAULT_REPO" ]]; then echo "" echo "▶ Composing verified fp16 runtime directory..." TOKENIZER_PATH=$("$HF_BIN" download "$TOKENIZER_REPO" tokenizer.json --quiet) + if [[ -z "$TOKENIZER_PATH" || "$TOKENIZER_PATH" == *$'\n'* || ! -f "$TOKENIZER_PATH" ]]; then + echo "ERROR: hf download did not return one tokenizer file for $TOKENIZER_REPO" >&2 + exit 1 + fi MODEL_DEST="$MODELS_DIR_VALUE/whisper-large-v3-turbo" MODEL_STAGE=$(mktemp -d "${TMPDIR:-/tmp}/codescribe-whisper-model.XXXXXX") cleanup_model_stage() { rm -f \ "$MODEL_STAGE/config.json" \ + "$MODEL_STAGE/config.json.partial" \ "$MODEL_STAGE/tokenizer.json" \ + "$MODEL_STAGE/tokenizer.json.partial" \ "$MODEL_STAGE/mel_filters.npz" \ "$MODEL_STAGE/mel_filters.npz.partial" \ "$MODEL_STAGE/weights.safetensors" \ - "$MODEL_STAGE/model.safetensors" + "$MODEL_STAGE/weights.safetensors.partial" \ + "$MODEL_STAGE/model.safetensors" \ + "$MODEL_STAGE/model.safetensors.partial" rmdir "$MODEL_STAGE" 2>/dev/null || true } trap cleanup_model_stage EXIT diff --git a/scripts/tests/download-model-test.sh b/scripts/tests/download-model-test.sh index bf5e16bf..949b0f8d 100755 --- a/scripts/tests/download-model-test.sh +++ b/scripts/tests/download-model-test.sh @@ -5,6 +5,7 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "$0")/../.." && pwd)" TEST_ROOT=$(mktemp -d "${TMPDIR:-/tmp}/codescribe-download-model-test.XXXXXX") +export TEST_ROOT cleanup() { find "$TEST_ROOT" -type f -delete 2>/dev/null || true @@ -25,7 +26,12 @@ hf() { return 1 fi if [[ "${2:-}" == "openai/whisper-large-v3-turbo" ]]; then - printf '%s\n' "$FAKE_TOKENIZER" + case "${FAKE_TOKENIZER_OUTPUT_MODE:-valid}" in + empty) return 0 ;; + multi) printf '%s\n%s\n' "$FAKE_TOKENIZER" "$FAKE_TOKENIZER" ;; + directory) printf '%s\n' "$TEST_ROOT" ;; + *) printf '%s\n' "$FAKE_TOKENIZER" ;; + esac else printf '%s\n' "$FAKE_MODEL_SNAPSHOT" fi @@ -64,7 +70,55 @@ export FAKE_CONFIG="$ROOT_DIR/tests/fixtures/whisper_test_config.json" export FAKE_TOKENIZER="$ROOT_DIR/tests/fixtures/whisper_tokenizer.json" export FAKE_MEL_FILTERS="$TEST_ROOT/mel_filters.npz" mkdir -p "$HOME/models" -xxd -r -p "$ROOT_DIR/tests/fixtures/whisper_mel_filters.npz.hex" > "$FAKE_MEL_FILTERS" +command -v python3 >/dev/null 2>&1 || { + echo "python3 is required for download-model-test" >&2 + exit 1 +} +python3 -c 'import sys; sys.stdout.buffer.write(bytes.fromhex(sys.stdin.read()))' \ + < "$ROOT_DIR/tests/fixtures/whisper_mel_filters.npz.hex" \ + > "$FAKE_MEL_FILTERS" + +VALIDATION_SNAPSHOT="$TEST_ROOT/snapshot-tokenizer-validation" +mkdir -p "$VALIDATION_SNAPSHOT" +cp "$FAKE_CONFIG" "$VALIDATION_SNAPSHOT/config.json" +make_tiny_weights "$VALIDATION_SNAPSHOT/weights.safetensors" +export FAKE_MODEL_SNAPSHOT="$VALIDATION_SNAPSHOT" + +for mode in empty multi directory; do + export FAKE_TOKENIZER_OUTPUT_MODE="$mode" + if "$ROOT_DIR/scripts/download-model.sh" >"$TEST_ROOT/tokenizer-$mode.out" 2>"$TEST_ROOT/tokenizer-$mode.err"; then + echo "expected invalid tokenizer output mode to fail: $mode" >&2 + exit 1 + fi + grep -q "hf download did not return one tokenizer file" "$TEST_ROOT/tokenizer-$mode.err" +done +unset FAKE_TOKENIZER_OUTPUT_MODE + +FAILURE_TMP="$TEST_ROOT/failing-stage" +mkdir -p "$FAILURE_TMP" +export REAL_CP +REAL_CP=$(command -v cp) +# Literal child-script lines intentionally defer expansion until the fake cp runs. +# shellcheck disable=SC2016 +printf '%s\n' \ + '#!/bin/bash' \ + 'destination="${!#}"' \ + 'if [[ "$destination" == *"/weights.safetensors.partial" ]]; then' \ + ' printf partial > "$destination"' \ + ' exit 1' \ + 'fi' \ + 'exec "$REAL_CP" "$@"' \ + > "$FAKE_BIN/cp" +chmod +x "$FAKE_BIN/cp" +if TMPDIR="$FAILURE_TMP" "$ROOT_DIR/scripts/download-model.sh" >/dev/null 2>&1; then + echo "expected interrupted weight staging copy to fail" >&2 + exit 1 +fi +rm -f "$FAKE_BIN/cp" +if find "$FAILURE_TMP" -type d -name 'codescribe-whisper-model.*' | grep -q .; then + echo "failed staging cleanup left a model directory" >&2 + exit 1 +fi run_promotion_case() { local selected_name="$1" From 6dad4df0f47ef3043744557796ace1d3d68c910c Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Sat, 22 Aug 2026 03:31:52 +0200 Subject: [PATCH 41/45] [codex/vc-workflow] fix(stt): enforce runtime window bounds --- CHANGELOG.md | 5 +- README.md | 6 +- core/stt/whisper/engine.rs | 6 +- core/whisper_weights.rs | 88 +++++++++++++++++++++---- tests/fixtures/whisper_test_config.json | 2 +- 5 files changed, 88 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 620c6fc5..745839a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -75,8 +75,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 before cold model construction; prompt/control token IDs, automatic-language candidates, layer/context resource bounds, and mapped tensor-name collisions are validated by the same runtime-owned helpers. Tokenizers cannot emit IDs - without embedding rows; audio context is bounded to the supported 30-second - window; and surplus tensors are refused before allocation. Quantized payloads and the + without embedding rows; audio context must match the supported 30-second + window; mel verification is size-bounded before hashing; and surplus tensors + are refused before allocation. Quantized payloads and the legacy Q8 fallback are refused; the old public Q8 identifiers remain deprecated source-compatibility constants only. Building from source now declares Rust 1.88 as the minimum supported toolchain. diff --git a/README.md b/README.md index 004d4c5f..02efa9db 100644 --- a/README.md +++ b/README.md @@ -364,8 +364,10 @@ instead of being accepted as complete. Config validation requires the complete MLX Whisper architecture used by the loader (including matching audio/text state widths and compatible attention heads, a decode context that leaves room for output, and broad layer -count safety fences). Audio context is capped at the 1500 positions consumed by -the supported 30-second Whisper window; missing dimensions are never replaced +count safety fences). Audio context must equal the 1500 positions consumed by +the supported 30-second Whisper window; shorter contexts would silently truncate +audio. The pinned mel filterbank is size-checked and hashed through a bounded +stream before use; missing dimensions are never replaced with runtime defaults. Warm-cache repair checks older snapshots when the newest config, weights, or tokenizer is invalid, preserving offline recovery from an earlier valid revision. diff --git a/core/stt/whisper/engine.rs b/core/stt/whisper/engine.rs index 11dc998f..6c19518b 100644 --- a/core/stt/whisper/engine.rs +++ b/core/stt/whisper/engine.rs @@ -2012,7 +2012,11 @@ mod model_payload_tests { architecture, ) .unwrap(); - fs::write(temp.path().join("mel_filters.npz"), b"wrong").unwrap(); + fs::write( + temp.path().join("mel_filters.npz"), + vec![0_u8; crate::whisper_weights::MEL_FILTERS_SIZE_BYTES as usize], + ) + .unwrap(); let err = LocalWhisperEngine::new(temp.path()) .err() diff --git a/core/whisper_weights.rs b/core/whisper_weights.rs index 8b71cbeb..32fe5c1a 100644 --- a/core/whisper_weights.rs +++ b/core/whisper_weights.rs @@ -12,6 +12,8 @@ pub const SUPPORTED_NAMES: [&str; 2] = ["weights.safetensors", "model.safetensor /// SHA-256 of the pinned official OpenAI mel filterbank. pub const MEL_FILTERS_SHA256: &str = "7450ae70723a5ef9d341e3cee628c7cb0177f36ce42c44b7ed2bf3325f0f6d4c"; +/// Byte length of the pinned official OpenAI mel filterbank. +pub const MEL_FILTERS_SIZE_BYTES: u64 = 4_271; const REQUIRED_TOKENIZER_TOKENS: [&str; 2] = ["<|startoftranscript|>", "<|endoftext|>"]; const OPTIONAL_PROMPT_TOKENS: [&str; 3] = ["<|transcribe|>", "<|notimestamps|>", "<|startofprev|>"]; const MAX_WHISPER_LAYERS: usize = 64; @@ -238,9 +240,9 @@ pub(crate) fn parse_whisper_config(raw: &str, source: &str) -> Result MAX_WHISPER_AUDIO_CONTEXT { + if architecture.n_audio_ctx != MAX_WHISPER_AUDIO_CONTEXT { return Err(anyhow!( - "Whisper config {source} exceeds the resource limit of {MAX_WHISPER_AUDIO_CONTEXT} audio context positions" + "Whisper config {source} requires n_audio_ctx={MAX_WHISPER_AUDIO_CONTEXT} for the supported 30-second runtime window" )); } if architecture.n_audio_state != architecture.n_text_state { @@ -274,9 +276,48 @@ pub(crate) fn parse_whisper_config(raw: &str, source: &str) -> Result Result<()> { - // nosemgrep: rust.actix.path-traversal.tainted-path.tainted-path -- Read-only checksum of an operator-selected local model artifact or internally resolved download destination. - let bytes = fs::read(path).with_context(|| format!("read {} for checksum", path.display()))?; - let actual = format!("{:x}", Sha256::digest(bytes)); + let mut file = { + // nosemgrep: rust.actix.path-traversal.tainted-path.tainted-path -- Read-only checksum of an operator-selected local model artifact or internally resolved download destination. + fs::File::open(path) + } + .with_context(|| format!("open {} for checksum", path.display()))?; + let metadata_len = file + .metadata() + .with_context(|| format!("stat {} for checksum", path.display()))? + .len(); + if metadata_len != MEL_FILTERS_SIZE_BYTES { + return Err(anyhow!( + "size mismatch for {}: expected {} bytes, got {}", + path.display(), + MEL_FILTERS_SIZE_BYTES, + metadata_len + )); + } + + let mut hasher = Sha256::new(); + let mut total = 0_u64; + let mut buffer = [0_u8; 8 * 1024]; + let mut bounded = (&mut file).take(MEL_FILTERS_SIZE_BYTES + 1); + loop { + let read = bounded + .read(&mut buffer) + .with_context(|| format!("read {} for checksum", path.display()))?; + if read == 0 { + break; + } + total += read as u64; + hasher.update(&buffer[..read]); + } + if total != MEL_FILTERS_SIZE_BYTES { + return Err(anyhow!( + "size changed while reading {}: expected {} bytes, got {}", + path.display(), + MEL_FILTERS_SIZE_BYTES, + total + )); + } + + let actual = format!("{:x}", hasher.finalize()); if actual != MEL_FILTERS_SHA256 { return Err(anyhow!( "SHA-256 mismatch for {}: expected {}, got {}", @@ -837,14 +878,16 @@ mod tests { ); } - let mut accepted_context = valid_config(); - accepted_context["n_audio_ctx"] = serde_json::json!(MAX_WHISPER_AUDIO_CONTEXT); - parse_whisper_config(&accepted_context.to_string(), "fixture").unwrap(); - - let mut rejected_context = valid_config(); - rejected_context["n_audio_ctx"] = serde_json::json!(MAX_WHISPER_AUDIO_CONTEXT + 1); - let err = parse_whisper_config(&rejected_context.to_string(), "fixture").unwrap_err(); - assert!(format!("{err:#}").contains("audio context"), "{err:#}"); + for value in [ + 1, + MAX_WHISPER_AUDIO_CONTEXT - 1, + MAX_WHISPER_AUDIO_CONTEXT + 1, + ] { + let mut rejected_context = valid_config(); + rejected_context["n_audio_ctx"] = serde_json::json!(value); + let err = parse_whisper_config(&rejected_context.to_string(), "fixture").unwrap_err(); + assert!(format!("{err:#}").contains("30-second"), "{value}: {err:#}"); + } } #[test] @@ -885,6 +928,25 @@ mod tests { assert!(format!("{err:#}").contains("runtime-discoverable")); } + #[test] + fn mel_filter_verification_pins_size_before_checksum() { + let temp = TempDir::new().unwrap(); + let path = temp.path().join("mel_filters.npz"); + + fs::write(&path, vec![0_u8; MEL_FILTERS_SIZE_BYTES as usize]).unwrap(); + let checksum_err = verify_mel_filters(&path).unwrap_err(); + assert!(format!("{checksum_err:#}").contains("SHA-256 mismatch")); + + for size in [MEL_FILTERS_SIZE_BYTES - 1, MEL_FILTERS_SIZE_BYTES + 1] { + fs::File::create(&path).unwrap().set_len(size).unwrap(); + let err = verify_mel_filters(&path).unwrap_err(); + let message = format!("{err:#}"); + assert!(message.contains("size mismatch"), "{message}"); + assert!(message.contains("4271"), "{message}"); + assert!(message.contains(&size.to_string()), "{message}"); + } + } + #[test] fn sparse_tokenizer_rejects_extreme_vocab_without_dense_allocation() { let temp = TempDir::new().unwrap(); diff --git a/tests/fixtures/whisper_test_config.json b/tests/fixtures/whisper_test_config.json index 4344ca39..eaf3d000 100644 --- a/tests/fixtures/whisper_test_config.json +++ b/tests/fixtures/whisper_test_config.json @@ -1,6 +1,6 @@ { "n_mels": 80, - "n_audio_ctx": 2, + "n_audio_ctx": 1500, "n_audio_state": 4, "n_audio_head": 1, "n_audio_layer": 1, From cd8cbc94e96865d7fe37f01137f10b1948a41b76 Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Sat, 22 Aug 2026 04:00:03 +0200 Subject: [PATCH 42/45] [codex/vc-workflow] fix(stt): validate decode-time model contracts --- CHANGELOG.md | 7 ++- README.md | 7 ++- core/config/models.rs | 63 +++++++++++++++---- core/stt/onnx_adapter.rs | 3 +- core/stt/whisper/engine.rs | 4 +- core/stt/whisper/timestamps.rs | 41 +++++++----- core/whisper_weights.rs | 111 ++++++++++++++++++++++++++++++--- 7 files changed, 192 insertions(+), 44 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 745839a6..f623d312 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -76,11 +76,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 candidates, layer/context resource bounds, and mapped tensor-name collisions are validated by the same runtime-owned helpers. Tokenizers cannot emit IDs without embedding rows; audio context must match the supported 30-second - window; mel verification is size-bounded before hashing; and surplus tensors - are refused before allocation. Quantized payloads and the + window; decoder context is capped at the supported 448 positions; timestamp + token ranges are validated end to end; mel verification is size-bounded before + hashing; and surplus tensors are refused before allocation. Quantized payloads and the legacy Q8 fallback are refused; the old public Q8 identifiers remain deprecated source-compatibility constants only. Building from source now declares Rust 1.88 as the minimum supported toolchain. + Warm-cache tokenizer repair now returns immediately when it completes the + installed bundle instead of falling through to redundant network downloads. - **Supervisor findings own transcript-quality categories.** Engine catalog `codescribe-supervisor-findings/v1` (`core/quality/supervisor.rs`) names diff --git a/README.md b/README.md index 02efa9db..867c5917 100644 --- a/README.md +++ b/README.md @@ -364,13 +364,16 @@ instead of being accepted as complete. Config validation requires the complete MLX Whisper architecture used by the loader (including matching audio/text state widths and compatible attention heads, a decode context that leaves room for output, and broad layer -count safety fences). Audio context must equal the 1500 positions consumed by +count safety fences). Decoder context is bounded to the supported `5..=448` +range before its quadratic causal mask is allocated. Audio context must equal the 1500 positions consumed by the supported 30-second Whisper window; shorter contexts would silently truncate audio. The pinned mel filterbank is size-checked and hashed through a bounded stream before use; missing dimensions are never replaced with runtime defaults. Warm-cache repair checks older snapshots when the newest config, weights, or -tokenizer is invalid, preserving offline recovery from an earlier valid revision. +tokenizer is invalid, and returns as soon as the composed destination validates, +preserving offline recovery without redundant model downloads. Optional timestamp +tokens are accepted only as a complete contiguous 20 ms range from 0.00 to 30.00 seconds. `CODESCRIBE_EMBED_EMBEDDER=1` is an explicit fat/debug path that compiles MiniLM into Rust artifacts. Normal builds resolve MiniLM from the signed app resource or HF cache. `CODESCRIBE_NO_EMBED=1` disables every optional binary embed; Silero remains embedded. diff --git a/core/config/models.rs b/core/config/models.rs index 31367362..98326e0c 100644 --- a/core/config/models.rs +++ b/core/config/models.rs @@ -153,6 +153,22 @@ fn find_cached_whisper_tokenizer( ) } +/// Compose every available official warm-cache artifact and report final truth. +fn complete_default_model_from_warm_cache(dest: &Path) -> Result { + if let Some(snapshot) = find_cached_default_model_pair() + && snapshot != dest + { + copy_default_model_pair(&snapshot, dest)?; + } + if let Ok(architecture) = + crate::whisper_weights::load_whisper_architecture(&dest.join("config.json")) + && let Some(snapshot) = find_cached_whisper_tokenizer(architecture) + { + copy_model_files(&snapshot, dest, &["tokenizer.json"], true)?; + } + Ok(is_complete_whisper_model_dir(dest)) +} + /// Owner of the resolved runtime models directory. /// /// Scope is deliberately narrow: it locates and inspects model directories on @@ -401,19 +417,7 @@ where // no-op when the pieces are already on disk. Config and weights come from // mlx-community's fp16 conversion; tokenizer comes from OpenAI's matching // Transformers repository. The pinned mel filterbank is fetched below. - let mut paired_default_model = false; - if let Some(snapshot) = find_cached_default_model_pair() - && snapshot != dest - { - paired_default_model = copy_default_model_pair(&snapshot, &dest)?; - } - if let Ok(architecture) = - crate::whisper_weights::load_whisper_architecture(&dest.join("config.json")) - && let Some(snapshot) = find_cached_whisper_tokenizer(architecture) - { - copy_model_files(&snapshot, &dest, &["tokenizer.json"], true)?; - } - if paired_default_model && is_complete_whisper_model_dir(&dest) { + if complete_default_model_from_warm_cache(&dest)? { return Ok(canonicalize_or_self(dest)); } @@ -1300,6 +1304,39 @@ mod tests { ); } + /// A cached tokenizer can complete an installed pair without a model download. + #[test] + #[serial] + fn cached_tokenizer_completion_returns_final_bundle_truth() { + let temp_dir = TempDir::new().unwrap(); + let cache = temp_dir.path().join("cache"); + let home = temp_dir.path().join("home"); + let destination = temp_dir.path().join("destination"); + create_complete_whisper_model(&destination); + fs::remove_file(destination.join("tokenizer.json")).unwrap(); + + let tokenizer_snapshot = cache + .join("models--openai--whisper-large-v3-turbo") + .join("snapshots") + .join("tokenizer-only"); + fs::create_dir_all(&tokenizer_snapshot).unwrap(); + fs::write( + tokenizer_snapshot.join("tokenizer.json"), + include_bytes!("../../tests/fixtures/whisper_tokenizer.json"), + ) + .unwrap(); + + let _home = EnvGuard::set("HOME", &home); + let _cache = EnvGuard::set("CODESCRIBE_HF_CACHE", &cache); + let _hf_home = EnvGuard::unset("HF_HOME"); + let _hf_hub = EnvGuard::unset("HF_HUB_CACHE"); + let _huggingface_hub = EnvGuard::unset("HUGGINGFACE_HUB_CACHE"); + + assert!(find_cached_default_model_pair().is_none()); + assert!(complete_default_model_from_warm_cache(&destination).unwrap()); + validate_whisper_model_bundle(&destination).unwrap(); + } + /// A downloaded checksum mismatch is never promoted to the final mel path. #[test] fn corrupt_download_is_removed_before_promotion() { diff --git a/core/stt/onnx_adapter.rs b/core/stt/onnx_adapter.rs index a9ab1a7d..b8c721be 100644 --- a/core/stt/onnx_adapter.rs +++ b/core/stt/onnx_adapter.rs @@ -283,7 +283,8 @@ impl OnnxEngine { // Resolve special token IDs from tokenizer let tokens = ResolvedTokens::from_tokenizer(&tokenizer)?; - let ts_range = TimestampRange::from_tokenizer(&tokenizer); + let tokenizer_vocab_size = tokenizer.get_vocab_size(true); + let ts_range = TimestampRange::from_tokenizer(&tokenizer, tokenizer_vocab_size)?; let decoding_params = DecodingParams::default(); // Load mel filters from mel_filters.npz if available, otherwise compute diff --git a/core/stt/whisper/engine.rs b/core/stt/whisper/engine.rs index 6c19518b..f84830c0 100644 --- a/core/stt/whisper/engine.rs +++ b/core/stt/whisper/engine.rs @@ -552,7 +552,7 @@ impl LocalWhisperEngine { let mel_filters = load_mel_filters(&mel_filters_path, n_mels).context("Failed to load mel filters")?; - let ts_range = TimestampRange::from_tokenizer(&tokenizer); + let ts_range = TimestampRange::from_tokenizer(&tokenizer, config.vocab_size)?; Ok(Self { model, @@ -604,7 +604,7 @@ impl LocalWhisperEngine { tracing::info!("Embedded Whisper model loaded successfully"); - let ts_range = TimestampRange::from_tokenizer(&tokenizer); + let ts_range = TimestampRange::from_tokenizer(&tokenizer, config.vocab_size)?; Ok(Self { model, diff --git a/core/stt/whisper/timestamps.rs b/core/stt/whisper/timestamps.rs index a340926c..cf1b9630 100644 --- a/core/stt/whisper/timestamps.rs +++ b/core/stt/whisper/timestamps.rs @@ -18,13 +18,15 @@ pub struct TimestampRange { impl TimestampRange { /// Resolve the timestamp token range from tokenizer special tokens. - pub fn from_tokenizer(tokenizer: &Tokenizer) -> Option { - let begin = tokenizer.token_to_id("<|0.00|>")?; - let end_inclusive = tokenizer.token_to_id("<|30.00|>")?; - Some(Self { - begin, - end_inclusive, - }) + pub fn from_tokenizer(tokenizer: &Tokenizer, n_vocab: usize) -> anyhow::Result> { + Ok( + crate::whisper_weights::validated_timestamp_token_range(tokenizer, n_vocab)?.map( + |(begin, end_inclusive)| Self { + begin, + end_inclusive, + }, + ), + ) } /// Returns true when `tok` is a timestamp token. @@ -104,12 +106,15 @@ mod tests { ("hello".to_string(), 1_u32), ("world".to_string(), 2_u32), ("again".to_string(), 3_u32), - ("<|0.00|>".to_string(), 1000_u32), - ("<|0.02|>".to_string(), 1001_u32), - ("<|0.04|>".to_string(), 1002_u32), - ("<|30.00|>".to_string(), 1030_u32), ] .into_iter() + .chain((0..=1500).map(|step| { + let hundredths = step * 2; + ( + format!("<|{}.{:02}|>", hundredths / 100, hundredths % 100), + 1000_u32 + step, + ) + })) .collect(); let model = WordLevel::builder() @@ -125,9 +130,11 @@ mod tests { #[test] fn timestamp_range_resolves_from_tokenizer() { let tokenizer = test_tokenizer(); - let range = TimestampRange::from_tokenizer(&tokenizer).expect("timestamp range"); + let range = TimestampRange::from_tokenizer(&tokenizer, 2501) + .unwrap() + .expect("timestamp range"); assert_eq!(range.begin, 1000); - assert_eq!(range.end_inclusive, 1030); + assert_eq!(range.end_inclusive, 2500); assert!(range.is_timestamp(1005)); assert!(!range.is_timestamp(12)); } @@ -136,7 +143,9 @@ mod tests { #[test] fn extract_segments_parses_closed_spans() { let tokenizer = test_tokenizer(); - let range = TimestampRange::from_tokenizer(&tokenizer).expect("timestamp range"); + let range = TimestampRange::from_tokenizer(&tokenizer, 2501) + .unwrap() + .expect("timestamp range"); let tokens = vec![1000, 1, 2, 1002, 3, 1004]; let (text, segments) = extract_segments(&tokens, &tokenizer, &range); @@ -155,7 +164,9 @@ mod tests { #[test] fn extract_segments_ignores_unclosed_trailing_span() { let tokenizer = test_tokenizer(); - let range = TimestampRange::from_tokenizer(&tokenizer).expect("timestamp range"); + let range = TimestampRange::from_tokenizer(&tokenizer, 2501) + .unwrap() + .expect("timestamp range"); let tokens = vec![1000, 1, 1002, 2, 3]; let (text, segments) = extract_segments(&tokens, &tokenizer, &range); diff --git a/core/whisper_weights.rs b/core/whisper_weights.rs index 32fe5c1a..eaad9c4e 100644 --- a/core/whisper_weights.rs +++ b/core/whisper_weights.rs @@ -18,6 +18,8 @@ const REQUIRED_TOKENIZER_TOKENS: [&str; 2] = ["<|startoftranscript|>", "<|endoft const OPTIONAL_PROMPT_TOKENS: [&str; 3] = ["<|transcribe|>", "<|notimestamps|>", "<|startofprev|>"]; const MAX_WHISPER_LAYERS: usize = 64; const MAX_WHISPER_AUDIO_CONTEXT: usize = 1_500; +const MAX_WHISPER_TEXT_CONTEXT: usize = 448; +const WHISPER_TIMESTAMP_STEPS: u32 = 1_500; const LANG_TOKEN_START: u32 = 50_259; const LANG_TOKEN_END: u32 = 50_358; const FALLBACK_LANGUAGES: [&str; 12] = [ @@ -67,6 +69,8 @@ pub(crate) fn validate_whisper_tokenizer_for_architecture( ) -> Result<()> { let tokenizer = tokenizers::Tokenizer::from_file(path) .map_err(|err| anyhow!("invalid Whisper tokenizer {}: {err}", path.display()))?; + validated_timestamp_token_range(&tokenizer, architecture.n_vocab) + .with_context(|| format!("validate Whisper timestamp tokens in {}", path.display()))?; for token in REQUIRED_TOKENIZER_TOKENS { let id = tokenizer.token_to_id(token).ok_or_else(|| { anyhow!( @@ -134,6 +138,43 @@ pub(crate) fn validate_whisper_tokenizer_for_architecture( Ok(()) } +/// Resolve and validate the optional 20 ms timestamp-token block. +pub(crate) fn validated_timestamp_token_range( + tokenizer: &tokenizers::Tokenizer, + n_vocab: usize, +) -> Result> { + let begin = tokenizer.token_to_id("<|0.00|>"); + let end = tokenizer.token_to_id("<|30.00|>"); + let (begin, end) = match (begin, end) { + (None, None) => return Ok(None), + (Some(_), None) | (None, Some(_)) => { + return Err(anyhow!("incomplete Whisper timestamp token range")); + } + (Some(begin), Some(end)) => (begin, end), + }; + if begin.checked_add(WHISPER_TIMESTAMP_STEPS) != Some(end) { + return Err(anyhow!( + "invalid Whisper timestamp token span: begin={begin}, end={end}, expected delta={WHISPER_TIMESTAMP_STEPS}" + )); + } + if end as usize >= n_vocab { + return Err(anyhow!( + "Whisper timestamp endpoint id {end} is outside configured vocabulary 0..{n_vocab}" + )); + } + for step in 0..=WHISPER_TIMESTAMP_STEPS { + let hundredths = step * 2; + let expected = format!("<|{}.{:02}|>", hundredths / 100, hundredths % 100); + let id = begin + step; + if tokenizer.id_to_token(id).as_deref() != Some(expected.as_str()) { + return Err(anyhow!( + "Whisper timestamp token id {id} must be {expected}" + )); + } + } + Ok(Some((begin, end))) +} + pub(crate) fn language_token_candidates( tokenizer: &tokenizers::Tokenizer, vocab_size: usize, @@ -230,9 +271,9 @@ pub(crate) fn parse_whisper_config(raw: &str, source: &str) -> Result) -> tokenizers::Tokenizer { + let mut vocab = BTreeMap::from([ + ("".to_string(), 0_u32), + ("<|startoftranscript|>".to_string(), 1_u32), + ("<|endoftext|>".to_string(), 2_u32), + ("<|pl|>".to_string(), 3_u32), + ]); + for step in 0..=WHISPER_TIMESTAMP_STEPS { + let hundredths = step * 2; + let token = if malformed_step == Some(step) { + "lexical-collision".to_string() + } else { + format!("<|{}.{:02}|>", hundredths / 100, hundredths % 100) + }; + vocab.insert(token, begin + step); } - let mut config = valid_config(); - config["n_text_ctx"] = serde_json::json!(5); - parse_whisper_config(&config.to_string(), "fixture").unwrap(); + let model = tokenizers::models::wordlevel::WordLevel::builder() + .vocab(vocab.into_iter().collect()) + .unk_token("".to_string()) + .build() + .unwrap(); + tokenizers::Tokenizer::new(model) + } + + #[test] + fn timestamp_token_range_matches_runtime_semantics() { + let tokenizer = timestamp_tokenizer(100, None); + assert_eq!( + validated_timestamp_token_range(&tokenizer, 1_601).unwrap(), + Some((100, 1_600)) + ); + + let malformed = timestamp_tokenizer(100, Some(1)); + let err = validated_timestamp_token_range(&malformed, 1_601).unwrap_err(); + assert!(format!("{err:#}").contains("must be <|0.02|>")); + + let err = validated_timestamp_token_range(&tokenizer, 1_600).unwrap_err(); + assert!(format!("{err:#}").contains("outside configured vocabulary")); + + let no_timestamps = + tokenizers::Tokenizer::new(tokenizers::models::wordlevel::WordLevel::default()); + assert_eq!( + validated_timestamp_token_range(&no_timestamps, 1).unwrap(), + None + ); } #[test] From 83d13cc4fe44321aec45b436002855702ba052d9 Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Sat, 22 Aug 2026 04:26:45 +0200 Subject: [PATCH 43/45] [codex/vc-workflow] fix(stt): bound model width and align e2e discovery --- CHANGELOG.md | 3 +- README.md | 6 ++- core/whisper_weights.rs | 22 +++++++++++ tests/e2e_stt_transcription.rs | 65 +++++++++++++++++++++++++++++++++ tests/support/e2e_stt_matrix.rs | 21 ++++------- 5 files changed, 100 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f623d312..88d49a55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -76,7 +76,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 candidates, layer/context resource bounds, and mapped tensor-name collisions are validated by the same runtime-owned helpers. Tokenizers cannot emit IDs without embedding rows; audio context must match the supported 30-second - window; decoder context is capped at the supported 448 positions; timestamp + window; matching state widths are capped at the official Whisper maximum of + 1280; decoder context is capped at the supported 448 positions; timestamp token ranges are validated end to end; mel verification is size-bounded before hashing; and surplus tensors are refused before allocation. Quantized payloads and the legacy Q8 fallback are refused; the old public Q8 identifiers remain diff --git a/README.md b/README.md index 867c5917..bdade949 100644 --- a/README.md +++ b/README.md @@ -342,7 +342,8 @@ Runtime resolution when Whisper is not embedded: 1. `CODESCRIBE_MODEL_PATH` environment variable 2. `~/.codescribe/models/whisper-large-v3-turbo/` (fp16 default) -3. A complete Hugging Face snapshot explicitly configured by repo id +3. A complete Hugging Face snapshot configured by repo id, followed by the + default `mlx-community/whisper-large-v3-turbo` snapshot The mlx-community repo ships only `config.json` + `weights.safetensors`; the download paths compose `tokenizer.json` from the matching official OpenAI @@ -364,7 +365,8 @@ instead of being accepted as complete. Config validation requires the complete MLX Whisper architecture used by the loader (including matching audio/text state widths and compatible attention heads, a decode context that leaves room for output, and broad layer -count safety fences). Decoder context is bounded to the supported `5..=448` +count safety fences). Matching audio/text state widths are bounded to the +official Whisper range `4..=1280` before quadratic model allocations. Decoder context is bounded to the supported `5..=448` range before its quadratic causal mask is allocated. Audio context must equal the 1500 positions consumed by the supported 30-second Whisper window; shorter contexts would silently truncate audio. The pinned mel filterbank is size-checked and hashed through a bounded diff --git a/core/whisper_weights.rs b/core/whisper_weights.rs index eaad9c4e..662ca412 100644 --- a/core/whisper_weights.rs +++ b/core/whisper_weights.rs @@ -19,6 +19,7 @@ const OPTIONAL_PROMPT_TOKENS: [&str; 3] = ["<|transcribe|>", "<|notimestamps|>", const MAX_WHISPER_LAYERS: usize = 64; const MAX_WHISPER_AUDIO_CONTEXT: usize = 1_500; const MAX_WHISPER_TEXT_CONTEXT: usize = 448; +const MAX_WHISPER_STATE_WIDTH: usize = 1_280; const WHISPER_TIMESTAMP_STEPS: u32 = 1_500; const LANG_TOKEN_START: u32 = 50_259; const LANG_TOKEN_END: u32 = 50_358; @@ -291,6 +292,11 @@ pub(crate) fn parse_whisper_config(raw: &str, source: &str) -> Result MAX_WHISPER_STATE_WIDTH { + return Err(anyhow!( + "Whisper config {source} exceeds the supported Whisper state width of {MAX_WHISPER_STATE_WIDTH}" + )); + } if architecture.n_audio_state < 4 || !architecture.n_audio_state.is_multiple_of(2) { return Err(anyhow!( "Whisper config {source} requires an even n_audio_state of at least 4" @@ -929,6 +935,22 @@ mod tests { let err = parse_whisper_config(&rejected_context.to_string(), "fixture").unwrap_err(); assert!(format!("{err:#}").contains("30-second"), "{value}: {err:#}"); } + + for value in [MAX_WHISPER_STATE_WIDTH + 1, 100_000] { + let mut rejected_state = valid_config(); + rejected_state["n_audio_state"] = serde_json::json!(value); + rejected_state["n_text_state"] = serde_json::json!(value); + let err = parse_whisper_config(&rejected_state.to_string(), "fixture").unwrap_err(); + assert!( + format!("{err:#}").contains("state width of 1280"), + "{value}: {err:#}" + ); + } + + let mut maximum_state = valid_config(); + maximum_state["n_audio_state"] = serde_json::json!(MAX_WHISPER_STATE_WIDTH); + maximum_state["n_text_state"] = serde_json::json!(MAX_WHISPER_STATE_WIDTH); + parse_whisper_config(&maximum_state.to_string(), "fixture").unwrap(); } #[test] diff --git a/tests/e2e_stt_transcription.rs b/tests/e2e_stt_transcription.rs index 29a2f58c..66232322 100644 --- a/tests/e2e_stt_transcription.rs +++ b/tests/e2e_stt_transcription.rs @@ -11,6 +11,7 @@ use std::path::{Path, PathBuf}; use codescribe::whisper::LocalWhisperEngine; use codescribe_core::pipeline::contracts::FileTranscriptionOptions; +use serial_test::serial; use tempfile::TempDir; #[path = "support/e2e_stt_matrix.rs"] @@ -29,6 +30,40 @@ fn home_dir() -> PathBuf { .unwrap_or_else(|_| PathBuf::from(".")) } +/// Restores one process environment variable after a serialized test. +struct EnvGuard { + key: &'static str, + previous: Option, +} + +impl EnvGuard { + fn set(key: &'static str, value: &Path) -> Self { + let previous = std::env::var(key).ok(); + // SAFETY: callers use #[serial] and the guard restores the prior value. + unsafe { std::env::set_var(key, value) }; + Self { key, previous } + } + + fn unset(key: &'static str) -> Self { + let previous = std::env::var(key).ok(); + // SAFETY: callers use #[serial] and the guard restores the prior value. + unsafe { std::env::remove_var(key) }; + Self { key, previous } + } +} + +impl Drop for EnvGuard { + fn drop(&mut self) { + if let Some(previous) = &self.previous { + // SAFETY: callers use #[serial] and the guard restores the prior value. + unsafe { std::env::set_var(self.key, previous) }; + } else { + // SAFETY: callers use #[serial] and the guard restores the prior value. + unsafe { std::env::remove_var(self.key) }; + } + } +} + fn resolve_model_or_skip(suite: &str) -> Option { match discover_local_whisper_model() { Some(found) => Some(found), @@ -321,6 +356,36 @@ fn deterministic_model_discovery_hint_names_the_validation_contract() { assert!(hint.contains("pinned mel_filters.npz checksum")); assert!(hint.contains("structurally valid F16/F32 safetensors")); assert!(hint.contains("no quantization declaration")); + assert!(hint.contains("Hugging Face cache snapshot")); +} + +#[test] +#[serial] +fn live_model_discovery_uses_validated_default_hf_snapshot() { + let (_tmp, home) = temp_home(); + let models_root = home.join("empty-models"); + let hf_home = home.join("hf-home"); + let hf_cache = hf_home.join("hub"); + let snapshot = hf_cache + .join("models--mlx-community--whisper-large-v3-turbo") + .join("snapshots") + .join("revision"); + std::fs::create_dir_all(&models_root).unwrap(); + create_complete_model(&snapshot); + + let _home = EnvGuard::set("HOME", &home); + let _models_root = EnvGuard::set("CODESCRIBE_MODELS_DIR", &models_root); + let _model_path = EnvGuard::unset("CODESCRIBE_MODEL_PATH"); + let _codescribe_hf = EnvGuard::set("CODESCRIBE_HF_CACHE", &hf_cache); + let _huggingface_hub = EnvGuard::set("HUGGINGFACE_HUB_CACHE", &hf_cache); + let _hf_hub = EnvGuard::set("HF_HUB_CACHE", &hf_cache); + let _hf_home = EnvGuard::set("HF_HOME", &hf_home); + + let found = discover_local_whisper_model() + .expect("production E2E discovery should reuse the validated HF fallback"); + + assert_eq!(found.source, ModelSource::RuntimeResolver); + assert_eq!(found.path, snapshot); } #[test] diff --git a/tests/support/e2e_stt_matrix.rs b/tests/support/e2e_stt_matrix.rs index ac71956b..dea08f58 100644 --- a/tests/support/e2e_stt_matrix.rs +++ b/tests/support/e2e_stt_matrix.rs @@ -18,6 +18,7 @@ pub const WHISPER_FP16_MODEL: &str = "whisper-large-v3-turbo"; pub enum ModelSource { EnvOverride, ModelsDir, + RuntimeResolver, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -79,20 +80,12 @@ pub fn whisper_model_missing_parts(path: &Path) -> Vec<&'static str> { } pub fn discover_local_whisper_model() -> Option { - let home_dir = std::env::var("HOME") - .map(PathBuf::from) - .unwrap_or_else(|_| PathBuf::from(".")); - let env_override = std::env::var("CODESCRIBE_MODEL_PATH") + codescribe_core::config::models::resolve_runtime_whisper_model_path(None) .ok() - .map(PathBuf::from); - let models_root = std::env::var("CODESCRIBE_MODELS_DIR") - .ok() - .map(|value| expand_models_root(&home_dir, &value)); - discover_local_whisper_model_for_with_root( - &home_dir, - env_override.as_deref(), - models_root.as_deref(), - ) + .map(|path| ModelDiscovery { + source: ModelSource::RuntimeResolver, + path, + }) } pub fn expand_models_root(home_dir: &Path, value: &str) -> PathBuf { @@ -145,7 +138,7 @@ pub fn model_discovery_hint(home_dir: &Path) -> String { .filter(|path| path.exists()) .unwrap_or(default_root); format!( - "Looked for a valid fp16 Whisper model in CODESCRIBE_MODEL_PATH and {root}/{fp16}. The bundle must have parseable config and tokenizer files, the pinned mel_filters.npz checksum, structurally valid F16/F32 safetensors, and no quantization declaration.", + "The production resolver found no valid fp16 Whisper model in CODESCRIBE_MODEL_PATH, {root}/{fp16}, or a supported Hugging Face cache snapshot. The bundle must have parseable config and tokenizer files, the pinned mel_filters.npz checksum, structurally valid F16/F32 safetensors, and no quantization declaration.", root = models_root.display(), fp16 = WHISPER_FP16_MODEL ) From d95c1e70ba2174333f57b76e883336d7bafef1f1 Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Sat, 22 Aug 2026 05:00:21 +0200 Subject: [PATCH 44/45] [codex/vc-workflow] fix(stt): bound metadata and reuse runtime resolution --- CHANGELOG.md | 8 +- README.md | 5 +- core/bin/codescribe-whisper-validate.rs | 17 +++-- core/config/models.rs | 45 ++++++++++- core/stt/whisper/engine.rs | 45 ++++++++--- core/whisper_weights.rs | 99 ++++++++++++++++++++++--- scripts/bench-stt.sh | 33 +-------- scripts/validate-whisper-model.sh | 4 +- 8 files changed, 193 insertions(+), 63 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 88d49a55..b1be2175 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -79,12 +79,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 window; matching state widths are capped at the official Whisper maximum of 1280; decoder context is capped at the supported 448 positions; timestamp token ranges are validated end to end; mel verification is size-bounded before - hashing; and surplus tensors are refused before allocation. Quantized payloads and the + hashing; config/tokenizer JSON and vocabulary size are bounded before parsing + or allocation; and surplus tensors are refused before allocation. Quantized payloads and the legacy Q8 fallback are refused; the old public Q8 identifiers remain deprecated source-compatibility constants only. Building from source now declares Rust 1.88 as the minimum supported toolchain. Warm-cache tokenizer repair now returns immediately when it completes the - installed bundle instead of falling through to redundant network downloads. + installed bundle instead of falling through to redundant network downloads, + and it preserves a valid installed config/weights pair without creating a + weights-sized temporary copy. STT benchmarks now use the production model + resolver, including validated Hugging Face cache snapshots. - **Supervisor findings own transcript-quality categories.** Engine catalog `codescribe-supervisor-findings/v1` (`core/quality/supervisor.rs`) names diff --git a/README.md b/README.md index bdade949..96a58a8e 100644 --- a/README.md +++ b/README.md @@ -358,6 +358,8 @@ required Whisper tensor name and shape plus the complete safetensors tensor table, exact consumed tensor set, bounded alignment metadata, dtype allowlist, mapped-name uniqueness, offsets, and file length. The disk loader applies this complete gate before mmap or model construction. +Config and tokenizer JSON are size-bounded before parsing, and vocabulary size +is capped at the largest supported official Whisper vocabulary. Downloads and warm-cache copies are written to `.partial` files and promoted only after per-file validation; an invalid destination is repaired on the next Download action @@ -374,7 +376,8 @@ stream before use; missing dimensions are never replaced with runtime defaults. Warm-cache repair checks older snapshots when the newest config, weights, or tokenizer is invalid, and returns as soon as the composed destination validates, -preserving offline recovery without redundant model downloads. Optional timestamp +preserving an already-valid installed model pair when only a smaller artifact +needs repair and avoiding a weights-sized temporary copy. Optional timestamp tokens are accepted only as a complete contiguous 20 ms range from 0.00 to 30.00 seconds. `CODESCRIBE_EMBED_EMBEDDER=1` is an explicit fat/debug path that compiles MiniLM into Rust artifacts. Normal builds resolve MiniLM from the signed app resource or HF cache. `CODESCRIBE_NO_EMBED=1` disables every optional binary embed; Silero remains embedded. diff --git a/core/bin/codescribe-whisper-validate.rs b/core/bin/codescribe-whisper-validate.rs index 7e045ca4..a06647d1 100644 --- a/core/bin/codescribe-whisper-validate.rs +++ b/core/bin/codescribe-whisper-validate.rs @@ -1,21 +1,28 @@ //! Validate a composed Whisper model with the runtime's canonical contract. use anyhow::{Context, Result, anyhow}; -use codescribe_core::config::models::validate_whisper_model_bundle; +use codescribe_core::config::models::{ + resolve_runtime_whisper_model_path, validate_whisper_model_bundle, +}; use std::path::PathBuf; fn main() -> Result<()> { let mut args = std::env::args_os().skip(1); - let path = args + let command = args .next() - .map(PathBuf::from) - .ok_or_else(|| anyhow!("usage: codescribe-whisper-validate "))?; + .ok_or_else(|| anyhow!("usage: codescribe-whisper-validate |--resolve"))?; if args.next().is_some() { return Err(anyhow!( - "usage: codescribe-whisper-validate " + "usage: codescribe-whisper-validate |--resolve" )); } + if command == "--resolve" { + println!("{}", resolve_runtime_whisper_model_path(None)?.display()); + return Ok(()); + } + + let path = PathBuf::from(command); validate_whisper_model_bundle(&path) .with_context(|| format!("invalid Whisper model bundle: {}", path.display())) } diff --git a/core/config/models.rs b/core/config/models.rs index 98326e0c..3a056613 100644 --- a/core/config/models.rs +++ b/core/config/models.rs @@ -155,7 +155,8 @@ fn find_cached_whisper_tokenizer( /// Compose every available official warm-cache artifact and report final truth. fn complete_default_model_from_warm_cache(dest: &Path) -> Result { - if let Some(snapshot) = find_cached_default_model_pair() + if crate::whisper_weights::validate_whisper_model_pair(dest).is_err() + && let Some(snapshot) = find_cached_default_model_pair() && snapshot != dest { copy_default_model_pair(&snapshot, dest)?; @@ -1304,10 +1305,12 @@ mod tests { ); } - /// A cached tokenizer can complete an installed pair without a model download. + /// Cached tokenizer repair preserves an already-valid installed model pair. #[test] #[serial] - fn cached_tokenizer_completion_returns_final_bundle_truth() { + fn cached_tokenizer_completion_preserves_valid_installed_pair() { + use std::os::unix::fs::MetadataExt as _; + let temp_dir = TempDir::new().unwrap(); let cache = temp_dir.path().join("cache"); let home = temp_dir.path().join("home"); @@ -1315,6 +1318,12 @@ mod tests { create_complete_whisper_model(&destination); fs::remove_file(destination.join("tokenizer.json")).unwrap(); + let model_snapshot = cache + .join("models--mlx-community--whisper-large-v3-turbo") + .join("snapshots") + .join("model-pair"); + create_complete_whisper_model(&model_snapshot); + let tokenizer_snapshot = cache .join("models--openai--whisper-large-v3-turbo") .join("snapshots") @@ -1332,9 +1341,37 @@ mod tests { let _hf_hub = EnvGuard::unset("HF_HUB_CACHE"); let _huggingface_hub = EnvGuard::unset("HUGGINGFACE_HUB_CACHE"); - assert!(find_cached_default_model_pair().is_none()); + let config_before = fs::read(destination.join("config.json")).unwrap(); + let weights_before = fs::read(destination.join("model.safetensors")).unwrap(); + let config_inode = fs::metadata(destination.join("config.json")).unwrap().ino(); + let weights_inode = fs::metadata(destination.join("model.safetensors")) + .unwrap() + .ino(); + let weight_partial_sentinel = destination.join("model.safetensors.partial"); + fs::create_dir(&weight_partial_sentinel).unwrap(); + + assert_eq!(find_cached_default_model_pair(), Some(model_snapshot)); assert!(complete_default_model_from_warm_cache(&destination).unwrap()); validate_whisper_model_bundle(&destination).unwrap(); + assert_eq!( + fs::read(destination.join("config.json")).unwrap(), + config_before + ); + assert_eq!( + fs::read(destination.join("model.safetensors")).unwrap(), + weights_before + ); + assert_eq!( + fs::metadata(destination.join("config.json")).unwrap().ino(), + config_inode + ); + assert_eq!( + fs::metadata(destination.join("model.safetensors")) + .unwrap() + .ino(), + weights_inode + ); + assert!(weight_partial_sentinel.is_dir()); } /// A downloaded checksum mismatch is never promoted to the final mel path. diff --git a/core/stt/whisper/engine.rs b/core/stt/whisper/engine.rs index f84830c0..c381da4b 100644 --- a/core/stt/whisper/engine.rs +++ b/core/stt/whisper/engine.rs @@ -476,9 +476,10 @@ impl LocalWhisperEngine { let mel_filters_path = model_path.join("mel_filters.npz"); crate::whisper_weights::validate_whisper_model_bundle(model_path) .context("validate complete Whisper model bundle")?; - let architecture = crate::whisper_weights::parse_whisper_config( - &safe_path::safe_read_to_string(&config_path)?, - &config_path.display().to_string(), + let architecture = crate::whisper_weights::load_whisper_architecture(&config_path)?; + let tokenizer = crate::whisper_weights::load_validated_whisper_tokenizer_for_architecture( + &tokenizer_path, + architecture, )?; let weights_path = crate::config::models::resolve_compatible_whisper_weights_path( model_path, @@ -532,14 +533,6 @@ impl LocalWhisperEngine { build_started.elapsed().as_secs_f64() ); - let tokenizer = Tokenizer::from_file(&tokenizer_path).map_err(|e| { - anyhow!( - "Failed to load tokenizer from {}: {}", - tokenizer_path.display(), - e - ) - })?; - // Load mel filters if !mel_filters_path.exists() { return Err(anyhow!( @@ -1998,6 +1991,36 @@ mod model_payload_tests { ); } + #[test] + fn local_loader_rejects_oversized_tokenizer_before_model_load() { + let temp = TempDir::new().unwrap(); + write_valid_bundle_artifacts(temp.path()); + let architecture = crate::whisper_weights::parse_whisper_config( + include_str!("../../../tests/fixtures/whisper_test_config.json"), + "test fixture", + ) + .unwrap(); + crate::whisper_weights::write_test_whisper_weights( + &temp.path().join("weights.safetensors"), + architecture, + ) + .unwrap(); + fs::File::create(temp.path().join("tokenizer.json")) + .unwrap() + .set_len(crate::whisper_weights::MAX_WHISPER_TOKENIZER_BYTES + 1) + .unwrap(); + + let err = LocalWhisperEngine::new(temp.path()) + .err() + .expect("oversized tokenizer must be rejected"); + let message = format!("{err:#}"); + assert!(message.contains("16777216-byte limit"), "{message}"); + assert!( + !message.contains("Failed to create Whisper Model"), + "{message}" + ); + } + #[test] fn local_loader_rejects_unpinned_mel_before_model_load() { let temp = TempDir::new().unwrap(); diff --git a/core/whisper_weights.rs b/core/whisper_weights.rs index 662ca412..ae5e0e55 100644 --- a/core/whisper_weights.rs +++ b/core/whisper_weights.rs @@ -14,12 +14,15 @@ pub const MEL_FILTERS_SHA256: &str = "7450ae70723a5ef9d341e3cee628c7cb0177f36ce42c44b7ed2bf3325f0f6d4c"; /// Byte length of the pinned official OpenAI mel filterbank. pub const MEL_FILTERS_SIZE_BYTES: u64 = 4_271; +pub(crate) const MAX_WHISPER_CONFIG_BYTES: u64 = 1024 * 1024; +pub(crate) const MAX_WHISPER_TOKENIZER_BYTES: u64 = 16 * 1024 * 1024; const REQUIRED_TOKENIZER_TOKENS: [&str; 2] = ["<|startoftranscript|>", "<|endoftext|>"]; const OPTIONAL_PROMPT_TOKENS: [&str; 3] = ["<|transcribe|>", "<|notimestamps|>", "<|startofprev|>"]; const MAX_WHISPER_LAYERS: usize = 64; const MAX_WHISPER_AUDIO_CONTEXT: usize = 1_500; const MAX_WHISPER_TEXT_CONTEXT: usize = 448; const MAX_WHISPER_STATE_WIDTH: usize = 1_280; +const MAX_WHISPER_VOCAB: usize = 51_866; const WHISPER_TIMESTAMP_STEPS: u32 = 1_500; const LANG_TOKEN_START: u32 = 50_259; const LANG_TOKEN_END: u32 = 50_358; @@ -51,8 +54,7 @@ pub fn validate_whisper_model_bundle(path: &Path) -> Result<()> { /// Parse the tokenizer and require the control tokens used by every decode. pub(crate) fn validate_whisper_tokenizer(path: &Path) -> Result<()> { - let tokenizer = tokenizers::Tokenizer::from_file(path) - .map_err(|err| anyhow!("invalid Whisper tokenizer {}: {err}", path.display()))?; + let tokenizer = load_bounded_whisper_tokenizer(path)?; for token in REQUIRED_TOKENIZER_TOKENS { if tokenizer.token_to_id(token).is_none() { return Err(anyhow!( @@ -68,8 +70,14 @@ pub(crate) fn validate_whisper_tokenizer_for_architecture( path: &Path, architecture: WhisperArchitecture, ) -> Result<()> { - let tokenizer = tokenizers::Tokenizer::from_file(path) - .map_err(|err| anyhow!("invalid Whisper tokenizer {}: {err}", path.display()))?; + load_validated_whisper_tokenizer_for_architecture(path, architecture).map(|_| ()) +} + +pub(crate) fn load_validated_whisper_tokenizer_for_architecture( + path: &Path, + architecture: WhisperArchitecture, +) -> Result { + let tokenizer = load_bounded_whisper_tokenizer(path)?; validated_timestamp_token_range(&tokenizer, architecture.n_vocab) .with_context(|| format!("validate Whisper timestamp tokens in {}", path.display()))?; for token in REQUIRED_TOKENIZER_TOKENS { @@ -136,7 +144,13 @@ pub(crate) fn validate_whisper_tokenizer_for_architecture( path.display() )); } - Ok(()) + Ok(tokenizer) +} + +fn load_bounded_whisper_tokenizer(path: &Path) -> Result { + let bytes = read_bounded_metadata_file(path, MAX_WHISPER_TOKENIZER_BYTES, "tokenizer")?; + tokenizers::Tokenizer::from_bytes(bytes) + .map_err(|err| anyhow!("invalid Whisper tokenizer {}: {err}", path.display())) } /// Resolve and validate the optional 20 ms timestamp-token block. @@ -217,10 +231,47 @@ pub(crate) fn validate_whisper_config(path: &Path) -> Result<()> { } pub(crate) fn load_whisper_architecture(path: &Path) -> Result { - // nosemgrep: rust.actix.path-traversal.tainted-path.tainted-path -- Read-only model inspection. `path` is an operator-selected local model config or an internally resolved bundle/cache child; no network/request path component reaches it. - let raw = fs::read_to_string(path) - .with_context(|| format!("read Whisper config {}", path.display()))?; - parse_whisper_config(&raw, &path.display().to_string()) + let bytes = read_bounded_metadata_file(path, MAX_WHISPER_CONFIG_BYTES, "config")?; + let raw = std::str::from_utf8(&bytes) + .with_context(|| format!("Whisper config {} is not UTF-8", path.display()))?; + parse_whisper_config(raw, &path.display().to_string()) +} + +fn read_bounded_metadata_file(path: &Path, max_bytes: u64, label: &str) -> Result> { + let mut file = { + // nosemgrep: rust.actix.path-traversal.tainted-path.tainted-path -- Read-only validation of an operator-selected local model artifact or internally resolved cache child. + fs::File::open(path) + } + .with_context(|| format!("open Whisper {label} {}", path.display()))?; + let metadata = file + .metadata() + .with_context(|| format!("inspect Whisper {label} {}", path.display()))?; + if !metadata.is_file() { + return Err(anyhow!( + "Whisper {label} {} is not a regular file", + path.display() + )); + } + if metadata.len() > max_bytes { + return Err(anyhow!( + "Whisper {label} {} size {} exceeds the {max_bytes}-byte limit", + path.display(), + metadata.len() + )); + } + + let mut bytes = Vec::with_capacity(metadata.len() as usize); + (&mut file) + .take(max_bytes + 1) + .read_to_end(&mut bytes) + .with_context(|| format!("read Whisper {label} {}", path.display()))?; + if bytes.len() as u64 > max_bytes { + return Err(anyhow!( + "Whisper {label} {} grew beyond the {max_bytes}-byte limit while reading", + path.display() + )); + } + Ok(bytes) } /// Parse and validate the MLX architecture consumed by Candle's Whisper loader. @@ -265,6 +316,11 @@ pub(crate) fn parse_whisper_config(raw: &str, source: &str) -> Result MAX_WHISPER_VOCAB { + return Err(anyhow!( + "Whisper config {source} exceeds the supported Whisper vocabulary of {MAX_WHISPER_VOCAB} tokens" + )); + } if architecture.n_audio_layer > MAX_WHISPER_LAYERS || architecture.n_text_layer > MAX_WHISPER_LAYERS { @@ -951,6 +1007,11 @@ mod tests { maximum_state["n_audio_state"] = serde_json::json!(MAX_WHISPER_STATE_WIDTH); maximum_state["n_text_state"] = serde_json::json!(MAX_WHISPER_STATE_WIDTH); parse_whisper_config(&maximum_state.to_string(), "fixture").unwrap(); + + let mut oversized_vocab = valid_config(); + oversized_vocab["n_vocab"] = serde_json::json!(MAX_WHISPER_VOCAB + 1); + let err = parse_whisper_config(&oversized_vocab.to_string(), "fixture").unwrap_err(); + assert!(format!("{err:#}").contains("vocabulary of 51866")); } #[test] @@ -1062,6 +1123,26 @@ mod tests { } } + #[test] + fn config_and_tokenizer_are_bounded_before_json_parsing() { + let temp = TempDir::new().unwrap(); + let config = temp.path().join("config.json"); + let tokenizer = temp.path().join("tokenizer.json"); + fs::File::create(&config) + .unwrap() + .set_len(MAX_WHISPER_CONFIG_BYTES + 1) + .unwrap(); + fs::File::create(&tokenizer) + .unwrap() + .set_len(MAX_WHISPER_TOKENIZER_BYTES + 1) + .unwrap(); + + let config_err = load_whisper_architecture(&config).unwrap_err(); + let tokenizer_err = validate_whisper_tokenizer(&tokenizer).unwrap_err(); + assert!(format!("{config_err:#}").contains("1048576-byte limit")); + assert!(format!("{tokenizer_err:#}").contains("16777216-byte limit")); + } + #[test] fn sparse_tokenizer_rejects_extreme_vocab_without_dense_allocation() { let temp = TempDir::new().unwrap(); diff --git a/scripts/bench-stt.sh b/scripts/bench-stt.sh index a218c2c5..e22df4f5 100755 --- a/scripts/bench-stt.sh +++ b/scripts/bench-stt.sh @@ -18,6 +18,8 @@ Environment: trim active runtime prompt to first N deterministic terms (unset = full prompt) CODESCRIBE_BENCH_PROMPT_MAX_WER_DELTA_PP fail active-prompt probe above this WER regression threshold (default: 5.0) + Whisper model discovery follows the production resolver, including supported + Hugging Face cache snapshots. EOF } @@ -135,35 +137,8 @@ sha256_file() { fi } -model_is_complete() { - local dir="$1" - [[ -d "$dir" ]] && "$model_validator" "$dir" >/dev/null 2>&1 -} - discover_model() { - local candidate - if [[ -n "${CODESCRIBE_MODEL_PATH:-}" ]] && model_is_complete "$CODESCRIBE_MODEL_PATH"; then - printf '%s\n' "$CODESCRIBE_MODEL_PATH" - return 0 - fi - - local models_root="$home_dir/.codescribe/models" - local configured_models_root="${CODESCRIBE_MODELS_DIR:-}" - local tilde_prefix - printf -v tilde_prefix '%s/' '~' - if [[ "$configured_models_root" == "$tilde_prefix"* ]]; then - configured_models_root="$home_dir/${configured_models_root:2}" - fi - if [[ -n "$configured_models_root" ]] && [[ -d "$configured_models_root" ]]; then - models_root="$configured_models_root" - fi - candidate="$models_root/whisper-large-v3-turbo" - if model_is_complete "$candidate"; then - printf '%s\n' "$candidate" - return 0 - fi - - return 1 + "$model_validator" --resolve } fixture_source_label() { @@ -1225,7 +1200,7 @@ if [[ "$(($(count_lines "$manifest_tsv") - 1))" -le 0 ]]; then fi if ! model_path="$(discover_model)"; then - write_honest_report "No complete fp16 Whisper model found. Checked CODESCRIBE_MODEL_PATH and ~/.codescribe/models/whisper-large-v3-turbo." + write_honest_report "No complete fp16 Whisper model found by the production resolver, including supported Hugging Face cache snapshots." fi export CODESCRIBE_MODEL_PATH="$model_path" diff --git a/scripts/validate-whisper-model.sh b/scripts/validate-whisper-model.sh index 26114228..89af2603 100755 --- a/scripts/validate-whisper-model.sh +++ b/scripts/validate-whisper-model.sh @@ -1,10 +1,10 @@ #!/bin/bash -# Run the runtime-owned Whisper bundle validator for release/setup scripts. +# Run the runtime-owned Whisper bundle validator or resolver for shell clients. set -euo pipefail if [[ "$#" -ne 1 ]]; then - echo "usage: $0 " >&2 + echo "usage: $0 |--resolve" >&2 exit 2 fi From 13174b8ed1cd6a97f66ec214f628943d80d92a91 Mon Sep 17 00:00:00 2001 From: m-szymanska Date: Sat, 22 Aug 2026 05:21:24 +0200 Subject: [PATCH 45/45] [codex/vc-workflow] docs(setup): align Rust MSRV --- docs/TEAM_SETUP.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/TEAM_SETUP.md b/docs/TEAM_SETUP.md index 1148f899..5520bc90 100644 --- a/docs/TEAM_SETUP.md +++ b/docs/TEAM_SETUP.md @@ -5,7 +5,7 @@ ### 1. Prerequisites - macOS 14+ (Apple Silicon ARM64 only) -- Rust 1.83+ +- Rust 1.88+ (the workspace MSRV) ### 2. Build & Run (Native App)