From 1c4d60bb7e922370dcb2e1c0939a1f3f849ffba2 Mon Sep 17 00:00:00 2001 From: Nic Keating Date: Fri, 16 Oct 2026 21:20:24 +1300 Subject: [PATCH 1/3] feat: separate speakers in a meeting's microphone recording (D-007) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For meetings held in a room, where everyone reaches the microphone: a new "Separate speakers" action on the Audio tab works out who said what and labels each microphone line Speaker 1, Speaker 2 and so on. The labels reach copy, export, search, Ask and the notes model through combine(). Engine (src-tauri/src/diarise.rs), on Kōrero's own ort 2.0.0-rc.12 (one ONNX runtime, shared with transcribe-rs), CPU only, after the meeting: - pyannote segmentation-3.0 read for speaker changes (powerset), re-anchored every 10 s window; WeSpeaker ResNet34-LM embeddings; whole-meeting average-linkage clustering on segments >= 1.5 s, clusters under 20 s of speech folded in, the rest attached by voice. - Models download on first use (about 32 MB) and are checked by size and sha256 before use; a mismatch is discarded, never loaded. - Microphone lines take the speaker they overlap most. A line where the speaker changes mid-sentence takes whoever said most of it (word-timestamp splitting is a later step). Data: TranscriptSeg gains speaker (serde default, so every saved meeting loads unchanged); meetings gain speakersFound and speakerNames (naming UI is M3). One voice found: lines stay unlabelled. Editing a line is refused while that meeting's speakers are being separated, so the result cannot overwrite it. Evidence (legion, 16 Oct): - cargo test --lib: 262 passed (11 new), 5 ignored. bun test unit-tests: 82 passed. tsc: clean. eslint: 0 errors (3 pre-existing warnings elsewhere). clippy: no findings in the new code (64 pre-existing elsewhere; CI step is advisory). - diar_ami_der (ignored; local, 177 MB of AMI audio): mean DER 21.9% with a 0.25 s collar on four AMI far-field test meetings, under the 25% bar; the same output scores 22.0% in pyannote.metrics, which validates the check's own scorer. 92 minutes of audio separated in 65 s. --- src-tauri/Cargo.lock | 170 +++- src-tauri/Cargo.toml | 5 + .../resources/legal/THIRD_PARTY_NOTICES.md | 5 + src-tauri/src/diarise.rs | 940 ++++++++++++++++++ src-tauri/src/lib.rs | 2 + src-tauri/src/meeting.rs | 6 + src/bindings.ts | 31 + src/components/meetings/AudioPanel.tsx | 54 +- src/components/meetings/TranscriptPanel.tsx | 4 +- src/components/meetings/model.ts | 38 +- .../meetings/useMeetingsController.ts | 37 +- src/components/shell/searchSources.ts | 2 +- src/stores/meetingJobsStore.ts | 93 +- 13 files changed, 1342 insertions(+), 45 deletions(-) create mode 100644 src-tauri/src/diarise.rs diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index b1597e7574..4cadcac02f 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -184,7 +184,7 @@ dependencies = [ "objc2-foundation", "parking_lot", "percent-encoding", - "windows-sys 0.59.0", + "windows-sys 0.60.2", "wl-clipboard-rs", "x11rb", ] @@ -252,7 +252,7 @@ dependencies = [ "futures-lite", "parking", "polling", - "rustix", + "rustix 1.1.3", "slab", "windows-sys 0.61.2", ] @@ -283,7 +283,7 @@ dependencies = [ "cfg-if", "event-listener", "futures-lite", - "rustix", + "rustix 1.1.3", ] [[package]] @@ -309,7 +309,7 @@ dependencies = [ "cfg-if", "futures-core", "futures-io", - "rustix", + "rustix 1.1.3", "signal-hook-registry", "slab", "windows-sys 0.61.2", @@ -427,6 +427,29 @@ dependencies = [ "virtue", ] +[[package]] +name = "bindgen" +version = "0.69.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" +dependencies = [ + "bitflags 2.11.0", + "cexpr", + "clang-sys", + "itertools", + "lazy_static", + "lazycell", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash 1.1.0", + "shlex", + "syn 2.0.117", + "which", +] + [[package]] name = "bindgen" version = "0.72.1" @@ -442,7 +465,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "rustc-hash", + "rustc-hash 2.1.1", "shlex", "syn 2.0.117", ] @@ -1541,7 +1564,7 @@ dependencies = [ "libc", "option-ext", "redox_users 0.5.2", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -1811,7 +1834,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -2320,7 +2343,7 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bd49230192a3797a9a4d6abe9b3eed6f7fa4c8a8a4947977c6f80025f92cbd8" dependencies = [ - "rustix", + "rustix 1.1.3", "windows-link 0.2.1", ] @@ -2686,6 +2709,15 @@ version = "1.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec9d92d097f4749b64e8cc33d924d9f40a2d4eb91402b458014b781f5733d60f" +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "hound" version = "3.5.1" @@ -3098,7 +3130,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi 0.5.2", "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3230,6 +3262,27 @@ dependencies = [ "zeroize", ] +[[package]] +name = "knf-rs" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15ebe7cf9eae245a44ab76cabdd06c0b3769972fd06050ed9aea6a22ff0d682f" +dependencies = [ + "eyre", + "knf-rs-sys", + "ndarray 0.16.1", +] + +[[package]] +name = "knf-rs-sys" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9aa7679fdc0ecdaab1f4e7c9603413f303765f33c8e34e6ca4f26e7d61d6023" +dependencies = [ + "bindgen 0.69.5", + "cmake", +] + [[package]] name = "korero" version = "1.43.0" @@ -3248,10 +3301,13 @@ dependencies = [ "handy-keys", "hound", "keyring", + "knf-rs", "log", "natural", + "ndarray 0.17.2", "nnnoiseless", "once_cell", + "ort", "rdev 0.5.0-2", "regex", "reqwest 0.12.28", @@ -3314,6 +3370,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "leb128fmt" version = "0.1.0" @@ -3398,6 +3460,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + [[package]] name = "linux-raw-sys" version = "0.11.0" @@ -3651,6 +3719,21 @@ dependencies = [ "rust-stemmers", ] +[[package]] +name = "ndarray" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "882ed72dce9365842bf196bdeedf5055305f11fc8c03dee7bb0194a6cad34841" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + [[package]] name = "ndarray" version = "0.17.2" @@ -3761,7 +3844,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4273,7 +4356,7 @@ version = "2.0.0-rc.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7de3af33d24a745ffb8fab904b13478438d1cd52868e6f17735ef6e1f8bf133" dependencies = [ - "ndarray", + "ndarray 0.17.2", "ort-sys", "smallvec", "tracing", @@ -4314,7 +4397,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d8fae84b431384b68627d0f9b3b1245fcf9f46f6c0e3dc902e9dce64edd1967" dependencies = [ "libc", - "windows-sys 0.45.0", + "windows-sys 0.61.2", ] [[package]] @@ -4647,7 +4730,7 @@ dependencies = [ "concurrent-queue", "hermit-abi 0.5.2", "pin-project-lite", - "rustix", + "rustix 1.1.3", "windows-sys 0.61.2", ] @@ -5319,6 +5402,12 @@ dependencies = [ "serde_json", ] +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc-hash" version = "2.1.1" @@ -5348,6 +5437,19 @@ dependencies = [ "transpose", ] +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags 2.11.0", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + [[package]] name = "rustix" version = "1.1.3" @@ -5357,8 +5459,8 @@ dependencies = [ "bitflags 2.11.0", "errno", "libc", - "linux-raw-sys", - "windows-sys 0.59.0", + "linux-raw-sys 0.11.0", + "windows-sys 0.61.2", ] [[package]] @@ -5414,7 +5516,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -6898,8 +7000,8 @@ dependencies = [ "fastrand", "getrandom 0.4.1", "once_cell", - "rustix", - "windows-sys 0.59.0", + "rustix 1.1.3", + "windows-sys 0.61.2", ] [[package]] @@ -7282,7 +7384,7 @@ dependencies = [ "env_logger", "hound", "log", - "ndarray", + "ndarray 0.17.2", "once_cell", "ort", "regex", @@ -7533,7 +7635,7 @@ version = "0.1.6" source = "git+https://github.com/cjpais/vad-rs#2a412ed858695b9251f3f5a1a20d95b59fa7c498" dependencies = [ "eyre", - "ndarray", + "ndarray 0.17.2", "ort", "ringbuffer", ] @@ -7816,7 +7918,7 @@ checksum = "fee64194ccd96bf648f42a65a7e589547096dfa702f7cadef84347b66ad164f9" dependencies = [ "cc", "downcast-rs", - "rustix", + "rustix 1.1.3", "smallvec", "wayland-sys", ] @@ -7828,7 +7930,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8e6faa537fbb6c186cb9f1d41f2f811a4120d1b57ec61f50da451a0c5122bec" dependencies = [ "bitflags 2.11.0", - "rustix", + "rustix 1.1.3", "wayland-backend", "wayland-scanner", ] @@ -7983,6 +8085,18 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix 0.38.44", +] + [[package]] name = "whisper-rs" version = "0.16.0" @@ -7999,7 +8113,7 @@ version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6986c0fe081241d391f09b9a071fbcbb59720c3563628c3c829057cf69f2a56f" dependencies = [ - "bindgen", + "bindgen 0.72.1", "cfg-if", "cmake", "fs_extra", @@ -8034,7 +8148,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] @@ -8801,7 +8915,7 @@ dependencies = [ "libc", "log", "os_pipe", - "rustix", + "rustix 1.1.3", "thiserror 2.0.18", "tree_magic_mini", "wayland-backend", @@ -8898,7 +9012,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9993aa5be5a26815fe2c3eacfc1fde061fc1a1f094bf1ad2a18bf9c495dd7414" dependencies = [ "gethostname", - "rustix", + "rustix 1.1.3", "x11rb-protocol", ] @@ -8915,7 +9029,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" dependencies = [ "libc", - "rustix", + "rustix 1.1.3", ] [[package]] @@ -8980,7 +9094,7 @@ dependencies = [ "hex", "libc", "ordered-stream", - "rustix", + "rustix 1.1.3", "serde", "serde_repr", "tracing", diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 2d53ed1e61..052140d5cd 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -103,6 +103,11 @@ rodio = { git = "https://github.com/cjpais/rodio.git", features = [ ] } reqwest = { version = "0.12", features = ["json", "stream"] } futures-util = "0.3" +# Kōrero 1.45 (D-007): speaker separation. ort is pinned to the exact version +# transcribe-rs uses, so the app keeps one ONNX runtime (cargo tree -i ort-sys). +ort = { version = "=2.0.0-rc.12", features = ["ndarray"] } +ndarray = "0.17" +knf-rs = "=0.3.2" rustfft = "6.4.0" strsim = "0.11.0" natural = "0.5.0" diff --git a/src-tauri/resources/legal/THIRD_PARTY_NOTICES.md b/src-tauri/resources/legal/THIRD_PARTY_NOTICES.md index ba9e603d14..f8d0334f31 100644 --- a/src-tauri/resources/legal/THIRD_PARTY_NOTICES.md +++ b/src-tauri/resources/legal/THIRD_PARTY_NOTICES.md @@ -29,6 +29,8 @@ storage, and Windows build hardening. See the commit history for specifics. | `transcribe-rs` | _(verify)_ | crate on crates.io | | Silero VAD (via `vad-rs`, cjpais fork) | MIT _(verify the fork)_ | https://github.com/snakers4/silero-vad | | ONNX Runtime (`ort`) | MIT | https://github.com/microsoft/onnxruntime | +| `knf-rs` (filterbank features for speaker separation, 1.45) | MIT (verified on crates.io, 2026-10-16) | https://github.com/thewh1teagle/pyannote-rs | +| kaldi-native-fbank (C++, built through `knf-rs-sys`) | Apache-2.0 _(verify)_ | https://github.com/csukuangfj/kaldi-native-fbank | | React | MIT | https://react.dev | | Vite | MIT | https://vitejs.dev | | Tailwind CSS | MIT | https://tailwindcss.com | @@ -42,6 +44,9 @@ own licence at the point of download: - **Whisper** (ggml/`.bin`) — OpenAI Whisper weights, MIT _(verify per model)_. - **Parakeet V3** (`nvidia/parakeet-tdt-0.6b-v3`, redistributed as an int8 ONNX export) — **CC-BY-4.0** (verified 2026-05-30; "ready for commercial/non-commercial use"). Redistributable with attribution. The offline portable edition bundles this model and ships a `PARAKEET-MODEL-LICENCE.txt` crediting NVIDIA, linking CC-BY-4.0, and noting the int8 quantization as a modification. +- **Speaker separation (1.45), downloaded on first use of "Separate speakers" and checked by sha256:** + - **pyannote segmentation-3.0** (ONNX export, from the `pyannote-rs` v0.1.0 release) - MIT _(verify)_. https://huggingface.co/pyannote/segmentation-3.0 + - **WeSpeaker ResNet34-LM** (ONNX export, from the sherpa-onnx speaker-recognition-models release) - **CC-BY-4.0**, following the VoxCeleb dataset (verified 2026-10-16 on pyannote's model card, citing WeSpeaker's pretrained-models page). Credit: Wang et al., "Wespeaker: A research and production oriented speaker embedding learning toolkit", ICASSP 2023. ## Fonts diff --git a/src-tauri/src/diarise.rs b/src-tauri/src/diarise.rs new file mode 100644 index 0000000000..a89185eb55 --- /dev/null +++ b/src-tauri/src/diarise.rs @@ -0,0 +1,940 @@ +//! Kōrero 1.45 (D-007): in-room speaker separation ("who spoke when"). +//! +//! Runs after a meeting, on the CPU, on the microphone recording: +//! 1. pyannote segmentation-3.0 (MIT) on 10 s windows. Each frame's powerset class +//! is decoded into up to 3 window-local speakers, so a change of speaker with no +//! pause still splits a segment. Timing is re-anchored every window. +//! 2. WeSpeaker ResNet34-LM (CC BY 4.0, VoxCeleb) embeds each local segment, from +//! 80-bin Kaldi filterbanks at 16-bit scale (the model's metadata says +//! `normalize_samples: 0`), preferring frames where only that speaker talks. +//! 3. Whole-meeting clustering: average-linkage cosine on segments >= 1.5 s, clusters +//! with under 20 s of speech folded into the nearest kept one, and every other +//! segment attached to the nearest speaker by voice (or by time if too short). +//! +//! Measured in M0 (16 Oct 2026) on four AMI far-field test meetings: mean DER 23.1% +//! (0.25 s collar), 28.1% strict; about 80 s of CPU per hour of audio on 4 threads. +//! The packaged CAM++ model from `pyannote-rs` separated speakers at chance and must +//! not be used (LESSONS L-012). `diar_ami_der` below guards these numbers. + +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; + +use ndarray::Array3; +use ort::{ + session::{builder::GraphOptimizationLevel, Session}, + value::TensorRef, +}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use specta::Type; + +use crate::audio_toolkit::audio::FrameResampler; + +pub(crate) const SR: usize = 16_000; +const WIN: usize = SR * 10; // segmentation window +const STEP: usize = 270; // samples per segmentation frame (589 frames per window) +const F0: usize = 721; // first frame offset within a window +/// segmentation-3.0 powerset: 3 local speakers, at most 2 at once. +const POWERSET: [&[u8]; 7] = [&[], &[0], &[1], &[2], &[0, 1], &[0, 2], &[1, 2]]; +const MIN_SOLE: usize = SR / 2; // use sole-speaker frames for the embedding if >= 0.5 s +const MIN_EMBED: usize = SR / 4; // below 0.25 s, no embedding: attached by time +const CLUSTER_MIN_S: f64 = 1.5; // only these segments shape the clusters +const SPEAKER_MIN_S: f64 = 20.0; // a cluster needs this much speech to stay a speaker +const MAX_CLUSTERED: usize = 700; // longest N segments clustered; the rest attach by voice +/// Cosine distance at which clusters stop merging (M0 leave-one-out: 0.55-0.60). +pub(crate) const THRESHOLD: f32 = 0.6; +const PCM_SCALE: f32 = 32_768.0; + +/// A model file Kōrero downloads on first use and checks by size and sha256. +pub(crate) struct ModelFile { + pub name: &'static str, + pub url: &'static str, + pub sha256: &'static str, + pub bytes: u64, +} + +pub(crate) const SEGMENTATION: ModelFile = ModelFile { + name: "segmentation-3.0.onnx", + url: "https://github.com/thewh1teagle/pyannote-rs/releases/download/v0.1.0/segmentation-3.0.onnx", + sha256: "b78fc48113bb46fd247ae6a9aea737079550c647638db961df7e0e1e9f4ba62e", + bytes: 5_983_836, +}; + +pub(crate) const EMBEDDING: ModelFile = ModelFile { + name: "wespeaker_en_voxceleb_resnet34_LM.onnx", + url: "https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-recongition-models/wespeaker_en_voxceleb_resnet34_LM.onnx", + sha256: "e9848563da86f263117134dfd7ad63c92355b37de492b55e325400c9d9c39012", + bytes: 26_530_550, +}; + +// --------------------------------------------------------------------------- +// Models: downloaded on first use, never trusted without size and sha256. + +fn sha256_file(path: &Path) -> Result { + let mut f = std::fs::File::open(path).map_err(|e| format!("{}: {e}", path.display()))?; + let mut h = Sha256::new(); + std::io::copy(&mut f, &mut h).map_err(|e| format!("{}: {e}", path.display()))?; + Ok(format!("{:x}", h.finalize())) +} + +/// True when `path` holds exactly this model (size first, then the hash). +pub(crate) fn model_ok(path: &Path, m: &ModelFile) -> bool { + path.metadata().map(|md| md.len() == m.bytes).unwrap_or(false) + && sha256_file(path).map(|h| h == m.sha256).unwrap_or(false) +} + +/// Download one model into `dir` unless a verified copy is already there. Writes to +/// `.partial` and renames only after the size and hash match, so a cut-off or +/// substituted download never becomes a model. +pub(crate) async fn ensure_model( + dir: &Path, + m: &ModelFile, + mut progress: impl FnMut(u64, u64), +) -> Result { + let path = dir.join(m.name); + if model_ok(&path, m) { + return Ok(path); + } + std::fs::create_dir_all(dir).map_err(|e| format!("{}: {e}", dir.display()))?; + let partial = dir.join(format!("{}.partial", m.name)); + let _ = std::fs::remove_file(&partial); + let resp = reqwest::get(m.url) + .await + .map_err(|e| format!("Could not download {}: {e}", m.name))?; + if !resp.status().is_success() { + return Err(format!("Could not download {}: HTTP {}", m.name, resp.status())); + } + let mut file = std::fs::File::create(&partial).map_err(|e| format!("{}: {e}", partial.display()))?; + let mut hasher = Sha256::new(); + let (mut done, mut stream) = (0u64, resp.bytes_stream()); + use futures_util::StreamExt; + while let Some(chunk) = stream.next().await { + let chunk = chunk.map_err(|e| format!("Download of {} failed: {e}", m.name))?; + done += chunk.len() as u64; + if done > m.bytes { + drop(file); + let _ = std::fs::remove_file(&partial); + return Err(format!("{} is larger than expected; download refused.", m.name)); + } + hasher.update(&chunk); + file.write_all(&chunk).map_err(|e| format!("{}: {e}", partial.display()))?; + progress(done, m.bytes); + } + drop(file); + let got = format!("{:x}", hasher.finalize()); + if done != m.bytes || got != m.sha256 { + let _ = std::fs::remove_file(&partial); + return Err(format!( + "{} did not match its pinned size and checksum; it was discarded.", + m.name + )); + } + std::fs::rename(&partial, &path).map_err(|e| format!("{}: {e}", path.display()))?; + Ok(path) +} + +// --------------------------------------------------------------------------- +// Audio: any meeting WAV (device rate, any channels, int or float) to 16 kHz mono. + +pub(crate) fn decode_wav_16k(path: &str) -> Result, String> { + let reader = hound::WavReader::open(path).map_err(|e| format!("Could not open {path}: {e}"))?; + let spec = reader.spec(); + let channels = spec.channels.max(1) as usize; + let mut resampler = FrameResampler::new(spec.sample_rate as usize, SR, Duration::from_millis(30)); + let mut out: Vec = Vec::new(); + let mut interleave: Vec = Vec::with_capacity(channels); + let mut mono: Vec = Vec::with_capacity(8_192); + let mut push = |s: f32, out: &mut Vec, resampler: &mut FrameResampler| { + interleave.push(s); + if interleave.len() == channels { + mono.push(interleave.iter().sum::() / channels as f32); + interleave.clear(); + } + if mono.len() >= 8_192 { + resampler.push(&mono, |f| out.extend_from_slice(f)); + mono.clear(); + } + }; + match spec.sample_format { + hound::SampleFormat::Float => { + for s in reader.into_samples::() { + push(s.map_err(|e| format!("WAV read error: {e}"))?, &mut out, &mut resampler); + } + } + hound::SampleFormat::Int => { + let denom = (1i64 << (spec.bits_per_sample.clamp(1, 32) - 1)) as f32; + for s in reader.into_samples::() { + let v = s.map_err(|e| format!("WAV read error: {e}"))? as f32 / denom; + push(v, &mut out, &mut resampler); + } + } + } + resampler.push(&mono, |f| out.extend_from_slice(f)); + resampler.finish(|f| out.extend_from_slice(f)); + Ok(out) +} + +// --------------------------------------------------------------------------- +// Pure steps (unit-tested without models). + +/// One window-local speaker run, in seconds from the start of the file, with the sample +/// ranges used for its embedding. +#[derive(Debug, Clone)] +pub(crate) struct LocalSeg { + pub start: f64, + pub end: f64, + pub audio: Vec<(usize, usize)>, +} + +/// Turn per-window powerset classes into local speaker runs. `classes[w][f]` is the +/// argmax class of frame `f` in window `w`; `n` is the number of samples in the file. +pub(crate) fn local_segments(classes: &[Vec], n: usize) -> Vec { + let mut out = Vec::new(); + let secs = |s: usize| s.min(n) as f64 / SR as f64; + for (w, cls) in classes.iter().enumerate() { + let at = |f: usize| w * WIN + F0 + f * STEP; // re-anchored every window + for k in 0..3u8 { + let active = |f: usize| POWERSET[cls[f] as usize].contains(&k); + let mut i = 0; + while i < cls.len() { + if !active(i) { + i += 1; + continue; + } + let mut j = i; + while j < cls.len() && active(j) { + j += 1; + } + if at(i) < n { + let sole: Vec<(usize, usize)> = (i..j) + .filter(|&f| POWERSET[cls[f] as usize] == [k]) + .map(|f| (at(f).min(n), (at(f) + STEP).min(n))) + .collect(); + let sole_len: usize = sole.iter().map(|(a, b)| b - a).sum(); + let audio = if sole_len >= MIN_SOLE { + sole + } else { + vec![(at(i).min(n), at(j).min(n))] + }; + out.push(LocalSeg { start: secs(at(i)), end: secs(at(j)), audio }); + } + i = j; + } + } + } + out +} + +fn unit(v: &[f32]) -> Vec { + let norm = v.iter().map(|x| x * x).sum::().sqrt().max(1e-9); + v.iter().map(|x| x / norm).collect() +} + +fn dot(a: &[f32], b: &[f32]) -> f32 { + a.iter().zip(b).map(|(x, y)| x * y).sum() +} + +/// Average-linkage agglomerative clustering on unit vectors with cosine distance, +/// stopping when the closest pair of clusters is further apart than `threshold` +/// (the same cut as scipy's `fcluster(linkage(X, "average", "cosine"), t, "distance")`). +pub(crate) fn ahc_average(x: &[Vec], threshold: f32) -> Vec { + let n = x.len(); + let mut d = vec![0f32; n * n]; + for i in 0..n { + for j in (i + 1)..n { + let v = 1.0 - dot(&x[i], &x[j]); + d[i * n + j] = v; + d[j * n + i] = v; + } + } + let mut size = vec![1usize; n]; + let mut alive = vec![true; n]; + let mut label: Vec = (0..n).collect(); + loop { + let mut best = (f32::INFINITY, 0, 0); + for i in 0..n { + if !alive[i] { + continue; + } + for j in (i + 1)..n { + if alive[j] && d[i * n + j] < best.0 { + best = (d[i * n + j], i, j); + } + } + } + let (dist, a, b) = best; + if !dist.is_finite() || dist > threshold { + break; + } + // Lance-Williams update for average linkage. + let (sa, sb) = (size[a] as f32, size[b] as f32); + for k in 0..n { + if alive[k] && k != a && k != b { + let v = (sa * d[a * n + k] + sb * d[b * n + k]) / (sa + sb); + d[a * n + k] = v; + d[k * n + a] = v; + } + } + size[a] += size[b]; + alive[b] = false; + for l in label.iter_mut() { + if *l == b { + *l = a; + } + } + } + label +} + +/// Robust whole-meeting clustering (M0 "row 5"). `segs` are (start s, end s, embedding). +/// Returns one speaker index per segment, numbered 0.. in order of first speech, or +/// `None` only when nothing in the meeting could be embedded. +pub(crate) fn cluster(segs: &[(f64, f64, Option>)], threshold: f32) -> Vec> { + let dur = |i: usize| segs[i].1 - segs[i].0; + let mut long: Vec = (0..segs.len()) + .filter(|&i| segs[i].2.is_some() && dur(i) >= CLUSTER_MIN_S) + .collect(); + if long.len() > MAX_CLUSTERED { + long.sort_by(|&a, &b| dur(b).total_cmp(&dur(a))); + long.truncate(MAX_CLUSTERED); + long.sort_unstable(); + } + let embedded: Vec = (0..segs.len()).filter(|&i| segs[i].2.is_some()).collect(); + let mut labels: Vec> = vec![None; segs.len()]; + if long.len() < 2 { + // Too little speech to tell voices apart: everyone embedded is one speaker. + for &i in &embedded { + labels[i] = Some(0); + } + } else { + let x: Vec> = long.iter().map(|&i| unit(segs[i].2.as_ref().unwrap())).collect(); + let lab = ahc_average(&x, threshold); + let mut total: std::collections::BTreeMap = Default::default(); + for (p, &i) in long.iter().enumerate() { + *total.entry(lab[p]).or_default() += dur(i); + } + let mut keep: Vec = total.iter().filter(|(_, &t)| t >= SPEAKER_MIN_S).map(|(&c, _)| c).collect(); + if keep.is_empty() { + keep.push(*total.iter().max_by(|a, b| a.1.total_cmp(b.1)).unwrap().0); + } + let cents: Vec> = keep + .iter() + .map(|&c| { + let mut m = vec![0f32; x[0].len()]; + for (p, v) in x.iter().enumerate() { + if lab[p] == c { + m.iter_mut().zip(v).for_each(|(a, b)| *a += b); + } + } + unit(&m) + }) + .collect(); + for &i in &embedded { + let v = unit(segs[i].2.as_ref().unwrap()); + let best = (0..cents.len()) + .max_by(|&a, &b| dot(&v, ¢s[a]).total_cmp(&dot(&v, ¢s[b]))) + .unwrap(); + labels[i] = Some(best); + } + } + // Too short to embed: take the label of the nearest embedded segment in time. + let mids: Vec<(f64, usize)> = (0..segs.len()) + .filter_map(|i| labels[i].map(|l| ((segs[i].0 + segs[i].1) / 2.0, l))) + .collect(); + if !mids.is_empty() { + for i in 0..segs.len() { + if labels[i].is_none() { + let c = (segs[i].0 + segs[i].1) / 2.0; + let near = mids.iter().min_by(|a, b| (a.0 - c).abs().total_cmp(&(b.0 - c).abs())).unwrap(); + labels[i] = Some(near.1); + } + } + } + // Renumber 0.. in order of first speech, so "Speaker 1" is whoever spoke first. + let mut order: Vec = (0..segs.len()).collect(); + order.sort_by(|&a, &b| segs[a].0.total_cmp(&segs[b].0)); + let mut map: std::collections::HashMap = Default::default(); + for &i in &order { + if let Some(l) = labels[i] { + let next = map.len(); + map.entry(l).or_insert(next); + } + } + labels.iter().map(|l| l.map(|l| map[&l])).collect() +} + +/// A stretch of one person speaking, in milliseconds from the start of the recording. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Type)] +pub struct SpeakerTurn { + pub start_ms: u64, + pub end_ms: u64, + /// 0-based; shown to the user as "Speaker {speaker + 1}". + pub speaker: u32, +} + +/// Labelled segments to turns: sorted by start; a segment joins the previous turn only +/// when that turn is the same speaker's and they touch (within 50 ms, as at window +/// boundaries). Pauses are not filled. Turns of different speakers may overlap. +pub(crate) fn to_turns(segs: &[(f64, f64)], labels: &[Option]) -> Vec { + let mut v: Vec<(f64, f64, usize)> = segs + .iter() + .zip(labels) + .filter_map(|(&(s, e), l)| l.map(|l| (s, e, l))) + .collect(); + v.sort_by(|a, b| a.0.total_cmp(&b.0)); + let mut out: Vec = Vec::new(); + for (s, e, l) in v { + let (s_ms, e_ms) = ((s * 1000.0).round() as u64, (e * 1000.0).round() as u64); + if let Some(prev) = out.last_mut() { + if prev.speaker == l as u32 && s_ms <= prev.end_ms + 50 { + prev.end_ms = prev.end_ms.max(e_ms); + continue; + } + } + out.push(SpeakerTurn { start_ms: s_ms, end_ms: e_ms, speaker: l as u32 }); + } + out.sort_by_key(|t| (t.start_ms, t.speaker)); + out +} + +/// Give each microphone transcript segment the speaker it overlaps most. +/// +/// Live segments carry a start time but no end, so the end is estimated from the word +/// count (2.5 words a second, at least 1 s) and capped at the next segment's start. +/// A segment touching no turn takes the nearest turn within 2 s; otherwise `None`. +/// Lines where the speaker changes mid-sentence get the speaker who said most of it; +/// splitting them with word timestamps is a later step (plan M2, "straddle split"). +pub(crate) fn assign(starts_ms: &[u64], words: &[usize], turns: &[SpeakerTurn]) -> Vec> { + let mut order: Vec = (0..starts_ms.len()).collect(); + order.sort_by_key(|&i| starts_ms[i]); + let mut ends = vec![0u64; starts_ms.len()]; + for (pos, &i) in order.iter().enumerate() { + let est = ((words[i] as f64 / 2.5).max(1.0) * 1000.0) as u64; + let next = order.get(pos + 1).map(|&j| starts_ms[j]).unwrap_or(u64::MAX); + ends[i] = (starts_ms[i] + est).min(next.max(starts_ms[i] + 1)); + } + (0..starts_ms.len()) + .map(|i| { + let (s, e) = (starts_ms[i], ends[i]); + let mut overlap: std::collections::BTreeMap = Default::default(); + for t in turns { + let o = e.min(t.end_ms).saturating_sub(s.max(t.start_ms)); + if o > 0 { + *overlap.entry(t.speaker).or_default() += o; + } + } + if let Some((&spk, _)) = overlap.iter().max_by_key(|(_, &o)| o) { + return Some(spk); + } + turns + .iter() + .map(|t| (t.start_ms.saturating_sub(e).max(s.saturating_sub(t.end_ms)), t.speaker)) + .min() + .filter(|(gap, _)| *gap <= 2_000) + .map(|(_, spk)| spk) + }) + .collect() +} + +// --------------------------------------------------------------------------- +// The engine (two ONNX sessions on Kōrero's own `ort`, CPU only). + +pub(crate) struct Engine { + seg: Session, + emb: Session, +} + +fn e(x: X) -> String { + x.to_string() +} + +fn session(path: &Path, threads: usize) -> Result { + Session::builder() + .map_err(e)? + .with_optimization_level(GraphOptimizationLevel::Level3) + .map_err(e)? + .with_intra_threads(threads) + .map_err(e)? + .commit_from_file(path) + .map_err(|err| format!("Could not load {}: {err}", path.display())) +} + +/// The outcome of one run. +pub(crate) struct Diarisation { + pub turns: Vec, + pub speakers: u32, +} + +impl Engine { + pub fn load(seg_path: &Path, emb_path: &Path, threads: usize) -> Result { + Ok(Self { seg: session(seg_path, threads)?, emb: session(emb_path, threads)? }) + } + + fn frame_classes(&mut self, x: &[f32]) -> Result>, String> { + let mut out = Vec::with_capacity(x.len() / WIN + 1); + for start in (0..x.len()).step_by(WIN) { + let mut buf = vec![0f32; WIN]; + let end = (start + WIN).min(x.len()); + buf[..end - start].copy_from_slice(&x[start..end]); + let arr = Array3::from_shape_vec((1, 1, WIN), buf).map_err(e)?; + let outs = self + .seg + .run(ort::inputs![TensorRef::from_array_view(arr.view()).map_err(e)?]) + .map_err(e)?; + let v = outs.get("output").ok_or("segmentation model has no 'output'")?; + let (shape, data) = v.try_extract_tensor::().map_err(e)?; + let (f, c) = (shape[1] as usize, shape[2] as usize); + if c != POWERSET.len() { + return Err(format!("segmentation model gave {c} classes, expected 7")); + } + out.push( + (0..f) + .map(|i| { + let row = &data[i * c..(i + 1) * c]; + (0..c).max_by(|&a, &b| row[a].total_cmp(&row[b])).unwrap() as u8 + }) + .collect(), + ); + } + Ok(out) + } + + fn embed(&mut self, x: &[f32], audio: &[(usize, usize)]) -> Result>, String> { + let mut pcm: Vec = Vec::new(); + for &(a, b) in audio { + pcm.extend(x[a..b].iter().map(|s| s * PCM_SCALE)); + } + if pcm.len() < MIN_EMBED { + return Ok(None); + } + let fb = knf_rs::compute_fbank(&pcm).map_err(e)?; // mean-normalised, 80 bins + let (rows, cols) = fb.dim(); + let feats = Array3::from_shape_vec((1, rows, cols), fb.iter().copied().collect()).map_err(e)?; + let outs = self + .emb + .run(ort::inputs!["feats" => TensorRef::from_array_view(feats.view()).map_err(e)?]) + .map_err(e)?; + let v = outs.get("embs").ok_or("embedding model has no 'embs'")?; + let (_, data) = v.try_extract_tensor::().map_err(e)?; + Ok(Some(data.to_vec())) + } + + /// Separate speakers in 16 kHz mono audio. `progress(done, total)` counts segments. + pub fn run(&mut self, x: &[f32], mut progress: impl FnMut(usize, usize)) -> Result { + let classes = self.frame_classes(x)?; + let locals = local_segments(&classes, x.len()); + let mut segs: Vec<(f64, f64, Option>)> = Vec::with_capacity(locals.len()); + for (i, l) in locals.iter().enumerate() { + segs.push((l.start, l.end, self.embed(x, &l.audio)?)); + progress(i + 1, locals.len()); + } + let labels = cluster(&segs, THRESHOLD); + let spans: Vec<(f64, f64)> = segs.iter().map(|s| (s.0, s.1)).collect(); + let turns = to_turns(&spans, &labels); + let speakers = turns.iter().map(|t| t.speaker + 1).max().unwrap_or(0); + Ok(Diarisation { turns, speakers }) + } +} + +/// Threads for the engine: half the machine, between 1 and 4 (M0 measured 4). +pub(crate) fn engine_threads() -> usize { + std::thread::available_parallelism().map(|n| (n.get() / 2).clamp(1, 4)).unwrap_or(2) +} + + +// --------------------------------------------------------------------------- +// The command. + +/// Result of "Separate speakers" on a meeting. +#[derive(Debug, Clone, Serialize, Deserialize, Type)] +pub struct SpeakerSeparation { + pub speakers: u32, + pub turns: Vec, + /// One entry per input segment, in input order: the speaker of each microphone + /// ("you") line, `null` for system-audio lines and lines no turn reached. + pub segment_speakers: Vec>, + pub audio_s: f64, + pub took_s: f64, +} + +#[derive(Clone, Serialize)] +struct Progress { + /// "download" (models, first use) or "separate" + stage: &'static str, + done: u64, + total: u64, +} + +fn emit_progress(app: &tauri::AppHandle, stage: &'static str, done: u64, total: u64) { + use tauri::Emitter; + let _ = app.emit("meeting-speakers-progress", Progress { stage, done, total }); +} + +/// Kōrero 1.45 (D-007): separate the speakers in a meeting's microphone recording and +/// say which speaker each microphone line belongs to. Downloads the two models on first +/// use (about 32 MB, checked by sha256). Runs on the CPU; nothing leaves the computer +/// except those two downloads. +#[tauri::command] +#[specta::specta] +pub async fn meeting_separate_speakers( + app: tauri::AppHandle, + mic_path: String, + segments: Vec, +) -> Result { + let started = Instant::now(); + let dir = crate::portable::models_dir(&app) + .map_err(|err| format!("Could not find the models folder: {err}"))? + .join("diarise"); + let a = app.clone(); + let seg_path = ensure_model(&dir, &SEGMENTATION, |d, t| emit_progress(&a, "download", d, t)).await?; + let emb_path = ensure_model(&dir, &EMBEDDING, |d, t| emit_progress(&a, "download", d, t)).await?; + let a = app.clone(); + let (diar, audio_s) = tauri::async_runtime::spawn_blocking(move || -> Result<(Diarisation, f64), String> { + let x = decode_wav_16k(&mic_path)?; + if x.len() < SR * 5 { + return Err("The recording is too short to separate speakers.".into()); + } + let mut engine = Engine::load(&seg_path, &emb_path, engine_threads())?; + let d = engine.run(&x, |done, total| emit_progress(&a, "separate", done as u64, total as u64))?; + Ok((d, x.len() as f64 / SR as f64)) + }) + .await + .map_err(|err| format!("Speaker separation stopped unexpectedly: {err}"))??; + + let mic: Vec = (0..segments.len()).filter(|&i| segments[i].source == "you").collect(); + let starts: Vec = mic.iter().map(|&i| segments[i].start_ms).collect(); + let words: Vec = mic.iter().map(|&i| segments[i].text.split_whitespace().count()).collect(); + let mut segment_speakers = vec![None; segments.len()]; + for (k, spk) in assign(&starts, &words, &diar.turns).into_iter().enumerate() { + segment_speakers[mic[k]] = spk; + } + log::info!( + "speakers: {} found in {:.0} s of audio, {} turns, {:.1} s", + diar.speakers, + audio_s, + diar.turns.len(), + started.elapsed().as_secs_f64() + ); + Ok(SpeakerSeparation { + speakers: diar.speakers, + turns: diar.turns, + segment_speakers, + audio_s, + took_s: started.elapsed().as_secs_f64(), + }) +} + +// --------------------------------------------------------------------------- +// Scoring (used by the AMI regression check; kept here so the check and the engine +// cannot drift apart). + +/// Diarisation error rate on 10 ms frames, overlap scored, with `collar` seconds +/// removed around reference boundaries (half each side, as pyannote.metrics does). +/// `reference` is (start s, end s, speaker id). Returns (der, missed, false alarm, +/// confusion), each as a share of reference speech. +#[cfg(test)] +pub(crate) fn der(reference: &[(f64, f64, String)], hyp: &[SpeakerTurn], collar: f64) -> (f64, f64, f64, f64) { + let fr = |t: f64| (t * 100.0).round().max(0.0) as usize; + let ids: Vec<&String> = { + let mut v: Vec<&String> = reference.iter().map(|r| &r.2).collect(); + v.sort(); + v.dedup(); + v + }; + let nh = hyp.iter().map(|t| t.speaker as usize + 1).max().unwrap_or(0); + let end = reference + .iter() + .map(|r| fr(r.1)) + .chain(hyp.iter().map(|t| fr(t.end_ms as f64 / 1000.0))) + .max() + .unwrap_or(0) + + 1; + let mut r = vec![0u64; end]; + let mut h = vec![0u64; end]; + let mut scored = vec![true; end]; + for (s, e, id) in reference { + let k = ids.iter().position(|x| *x == id).unwrap(); + for f in fr(*s)..fr(*e) { + r[f] |= 1 << k; + } + let c = fr(collar / 2.0); + for b in [fr(*s), fr(*e)] { + for f in b.saturating_sub(c)..(b + c).min(end) { + scored[f] = false; + } + } + } + for t in hyp { + for f in fr(t.start_ms as f64 / 1000.0)..fr(t.end_ms as f64 / 1000.0) { + h[f] |= 1 << t.speaker; + } + } + // Overlap matrix, then the reference-to-hypothesis mapping that maximises it. + let nr = ids.len(); + let mut ov = vec![vec![0u64; nh]; nr]; + for f in 0..end { + if !scored[f] { + continue; + } + for (i, row) in ov.iter_mut().enumerate() { + if r[f] >> i & 1 == 1 { + for (j, cell) in row.iter_mut().enumerate() { + if h[f] >> j & 1 == 1 { + *cell += 1; + } + } + } + } + } + fn best(i: usize, used: u64, ov: &[Vec], map: &mut Vec>, cur: &mut Vec>, acc: u64, top: &mut u64) { + if i == ov.len() { + if acc > *top || map.is_empty() { + *top = acc; + *map = cur.clone(); + } + return; + } + cur.push(None); + best(i + 1, used, ov, map, cur, acc, top); + cur.pop(); + for j in 0..ov[i].len() { + if used >> j & 1 == 0 && ov[i][j] > 0 { + cur.push(Some(j)); + best(i + 1, used | 1 << j, ov, map, cur, acc + ov[i][j], top); + cur.pop(); + } + } + } + let (mut map, mut top) = (Vec::new(), 0u64); + best(0, 0, &ov, &mut map, &mut Vec::new(), 0, &mut top); + let (mut total, mut miss, mut fa, mut conf) = (0u64, 0u64, 0u64, 0u64); + for f in 0..end { + if !scored[f] { + continue; + } + let (nr_f, nh_f) = (r[f].count_ones() as u64, h[f].count_ones() as u64); + let correct = (0..nr) + .filter(|&i| r[f] >> i & 1 == 1 && map[i].map(|j| h[f] >> j & 1 == 1).unwrap_or(false)) + .count() as u64; + total += nr_f; + miss += nr_f.saturating_sub(nh_f); + fa += nh_f.saturating_sub(nr_f); + conf += nr_f.min(nh_f) - correct; + } + let t = total.max(1) as f64; + ((miss + fa + conf) as f64 / t, miss as f64 / t, fa as f64 / t, conf as f64 / t) +} + +#[cfg(test)] +mod tests { + use super::*; + + const FRAMES: usize = 589; // segmentation-3.0 frames per 10 s window (measured in M0) + + fn secs(samples: usize) -> f64 { + samples as f64 / SR as f64 + } + + #[test] + fn a_change_of_speaker_without_a_pause_splits_the_segment() { + let cls = vec![(0..FRAMES).map(|f| if f < 300 { 1 } else { 2 }).collect::>()]; + let s = local_segments(&cls, WIN); + assert_eq!(s.len(), 2, "pyannote-rs read this as one speech region"); + assert!((s[0].start - secs(F0)).abs() < 1e-9); + assert!((s[0].end - secs(F0 + 300 * STEP)).abs() < 1e-9); + assert!((s[1].start - s[0].end).abs() < 1e-9); + } + + #[test] + fn timing_is_reanchored_every_window_so_it_never_drifts() { + let cls = vec![vec![1u8; FRAMES]; 200]; // 2,000 s of one speaker + let s = local_segments(&cls, 200 * WIN); + let last = s.last().unwrap(); + assert!((last.start - secs(199 * WIN + F0)).abs() < 1e-9); + // pyannote-rs advanced 589 x 270 samples per window: 199 windows x 970 samples late. + assert!(secs(199 * (WIN - FRAMES * STEP)) > 12.0); + } + + #[test] + fn overlap_frames_count_for_both_speakers_but_embeddings_prefer_sole_frames() { + // A alone 0..20 (0.34 s), A+B 20..200, B alone 200..400. + let cls = vec![(0..FRAMES) + .map(|f| match f { 0..=19 => 1, 20..=199 => 4, 200..=399 => 2, _ => 0 }) + .collect::>()]; + let s = local_segments(&cls, WIN); + assert_eq!(s.len(), 2); + let (a, b) = (&s[0], &s[1]); + assert!((a.end - secs(F0 + 200 * STEP)).abs() < 1e-9, "A runs through the overlap"); + assert!((b.start - secs(F0 + 20 * STEP)).abs() < 1e-9, "B starts at the overlap"); + let sole_b: usize = b.audio.iter().map(|(x, y)| y - x).sum(); + assert_eq!(sole_b, 200 * STEP, "B is embedded from its 200 sole frames only"); + let a_len: usize = a.audio.iter().map(|(x, y)| y - x).sum(); + assert_eq!(a_len, 200 * STEP, "A has under 0.5 s alone, so its whole run is used"); + } + + #[test] + fn speech_past_the_end_of_the_file_is_clipped_or_dropped() { + let n = WIN / 2; // a 5 s file in a padded 10 s window + let cls = vec![vec![1u8; FRAMES]]; + let s = local_segments(&cls, n); + assert_eq!(s.len(), 1); + assert!((s[0].end - secs(n)).abs() < 1e-9); + assert!(s[0].audio.iter().all(|&(_, y)| y <= n)); + } + + fn v(angle: f32) -> Vec { + vec![angle.cos(), angle.sin(), 0.0] + } + + #[test] + fn average_linkage_finds_two_groups_and_stops_at_the_threshold() { + let x = vec![v(0.0), v(0.1), v(0.05), v(1.6), v(1.5)]; + let l = ahc_average(&x, 0.3); + assert_eq!(l[0], l[1]); + assert_eq!(l[0], l[2]); + assert_eq!(l[3], l[4]); + assert_ne!(l[0], l[3]); + assert!(ahc_average(&x, 2.0).iter().all(|&k| k == l[0]), "a loose threshold merges all"); + } + + #[test] + fn a_tiny_cluster_is_folded_and_speakers_are_numbered_by_first_speech() { + let mut segs = Vec::new(); + // Speaker X (angle 1.6) speaks first, 30 s; Y (angle 0.0) 30 s; a 3 s stray (angle 3.0). + for i in 0..10 { segs.push((i as f64 * 3.0, i as f64 * 3.0 + 3.0, Some(v(1.6)))); } + for i in 10..20 { segs.push((i as f64 * 3.0, i as f64 * 3.0 + 3.0, Some(v(0.0)))); } + segs.push((60.0, 63.0, Some(v(3.0)))); + segs.push((63.0, 63.2, None)); // too short to embed: takes its neighbour's label + let l = cluster(&segs, 0.6); + assert_eq!(l[0], Some(0), "first to speak is speaker 0"); + assert_eq!(l[10], Some(1)); + assert!(l[20] == Some(0) || l[20] == Some(1), "the stray joins a real speaker"); + assert_eq!(l[21], l[20]); + assert_eq!(l.iter().flatten().max(), Some(&1), "two speakers, not three"); + } + + #[test] + fn too_little_speech_is_one_speaker() { + let segs = vec![(0.0, 1.0, Some(v(0.0))), (1.0, 2.0, Some(v(1.6)))]; + assert_eq!(cluster(&segs, 0.6), vec![Some(0), Some(0)]); + assert_eq!(cluster(&[(0.0, 0.1, None)], 0.6), vec![None]); + } + + #[test] + fn turns_join_only_when_touching_and_keep_pauses() { + let spans = [(0.0, 2.0), (2.02, 4.0), (5.0, 6.0), (6.0, 7.0)]; + let labels = [Some(0), Some(0), Some(0), Some(1)]; + let t = to_turns(&spans, &labels); + assert_eq!( + t, + vec![ + SpeakerTurn { start_ms: 0, end_ms: 4000, speaker: 0 }, + SpeakerTurn { start_ms: 5000, end_ms: 6000, speaker: 0 }, + SpeakerTurn { start_ms: 6000, end_ms: 7000, speaker: 1 }, + ] + ); + } + + #[test] + fn lines_take_the_speaker_they_overlap_most_or_the_nearest_within_2_s() { + let turns = vec![ + SpeakerTurn { start_ms: 0, end_ms: 5000, speaker: 0 }, + SpeakerTurn { start_ms: 5000, end_ms: 9000, speaker: 1 }, + ]; + // 0 s, 10 words (4 s): speaker 0. 4.5 s, 10 words: mostly speaker 1. + // 10.5 s, 2 words: no overlap, 1.5 s after a turn: speaker 1. 30 s: nobody. + let a = assign(&[0, 4_500, 10_500, 30_000], &[10, 10, 2, 2], &turns); + assert_eq!(a, vec![Some(0), Some(1), Some(1), None]); + } + + #[test] + fn a_model_file_must_match_size_and_hash() { + const M: ModelFile = ModelFile { + name: "abc", + url: "", + sha256: "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", + bytes: 3, + }; + let dir = std::env::temp_dir().join(format!("korero-diarise-test-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let p = dir.join("abc"); + std::fs::write(&p, b"abc").unwrap(); + assert!(model_ok(&p, &M)); + std::fs::write(&p, b"abd").unwrap(); + assert!(!model_ok(&p, &M), "same size, wrong hash"); + std::fs::write(&p, b"abcd").unwrap(); + assert!(!model_ok(&p, &M), "wrong size"); + let _ = std::fs::remove_dir_all(&dir); + } + + /// Plan M2 `diar-ami-der` (LESSONS L-012): the shipped models on four AMI far-field + /// test meetings. Local and manual (177 MB of audio): + /// KORERO_AMI_DIR=.Array1-01.wav and rttm/.rttm> + /// KORERO_DIARISE_MODELS= + /// cargo test --release --lib diar_ami_der -- --ignored --nocapture + /// Fails when the mean DER (0.25 s collar) is above AMI_BAR. With KORERO_AMI_OUT set, + /// writes each meeting's turns as RTTM so this scorer can be checked against + /// pyannote.metrics. + const AMI_BAR: f64 = 0.25; + + #[test] + #[ignore] + fn diar_ami_der() { + let ami = std::path::PathBuf::from(std::env::var("KORERO_AMI_DIR").expect("set KORERO_AMI_DIR")); + let models = std::path::PathBuf::from(std::env::var("KORERO_DIARISE_MODELS").expect("set KORERO_DIARISE_MODELS")); + let (seg, emb) = (models.join(SEGMENTATION.name), models.join(EMBEDDING.name)); + assert!(model_ok(&seg, &SEGMENTATION) && model_ok(&emb, &EMBEDDING), "models missing or not the pinned files"); + let out = std::env::var("KORERO_AMI_OUT").ok().map(std::path::PathBuf::from); + let mut engine = Engine::load(&seg, &emb, 4).unwrap(); + let mut sum = 0.0; + let meetings = ["EN2002a", "ES2004a", "IS1009a", "TS3003a"]; + for m in meetings { + let reference: Vec<(f64, f64, String)> = std::fs::read_to_string(ami.join("rttm").join(format!("{m}.rttm"))) + .unwrap() + .lines() + .filter_map(|l| { + let p: Vec<&str> = l.split_whitespace().collect(); + (p.first() == Some(&"SPEAKER")).then(|| { + let s: f64 = p[3].parse().unwrap(); + (s, s + p[4].parse::().unwrap(), p[7].to_string()) + }) + }) + .collect(); + let t0 = Instant::now(); + let x = decode_wav_16k(ami.join("wav").join(format!("{m}.Array1-01.wav")).to_str().unwrap()).unwrap(); + let d = engine.run(&x, |_, _| {}).unwrap(); + let (c25, miss, fa, conf) = der(&reference, &d.turns, 0.25); + let c0 = der(&reference, &d.turns, 0.0).0; + println!( + "{m}: DER {:.1}% (strict {:.1}%) missed {:.0} / false {:.0} / wrong speaker {:.0}; {} speakers; {:.0} s audio in {:.1} s", + c25 * 100.0, c0 * 100.0, miss * 100.0, fa * 100.0, conf * 100.0, + d.speakers, x.len() as f64 / SR as f64, t0.elapsed().as_secs_f64() + ); + if let Some(dir) = &out { + std::fs::create_dir_all(dir).unwrap(); + let rttm: String = d.turns.iter().map(|t| format!( + "SPEAKER {m} 1 {:.3} {:.3} S{} \n", + t.start_ms as f64 / 1000.0, (t.end_ms - t.start_ms) as f64 / 1000.0, t.speaker + )).collect(); + std::fs::write(dir.join(format!("{m}.rttm")), rttm).unwrap(); + } + sum += c25; + } + let mean = sum / meetings.len() as f64; + println!("mean DER {:.1}% (bar {:.0}%)", mean * 100.0, AMI_BAR * 100.0); + assert!(mean <= AMI_BAR, "mean DER {:.1}% is above the agreed bar", mean * 100.0); + } + #[test] + fn der_is_zero_for_a_perfect_answer_whatever_the_labels() { + let r = vec![(0.0, 10.0, "A".to_string()), (10.0, 20.0, "B".to_string())]; + let h = vec![ + SpeakerTurn { start_ms: 0, end_ms: 10_000, speaker: 1 }, + SpeakerTurn { start_ms: 10_000, end_ms: 20_000, speaker: 0 }, + ]; + assert!(der(&r, &h, 0.0).0 < 1e-9); + let one = vec![SpeakerTurn { start_ms: 0, end_ms: 20_000, speaker: 0 }]; + assert!((der(&r, &one, 0.0).0 - 0.5).abs() < 1e-3, "one speaker for two: half is confusion"); + assert!((der(&r, &[], 0.0).1 - 1.0).abs() < 1e-9, "nothing found: all missed"); + } +} diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 2271ffef3d..1ca5214230 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -10,6 +10,7 @@ mod crash; // Kōrero (v1.12.0): global panic hook + crash report files mod corrections; // Kōrero (v1.15.0): user-taught transcription corrections mod update_check; // Kōrero (v1.16.0): notify-only update check (own repo only) mod window_info; // Kōrero (v1.18.1): contextual-routing spike (active window title) +mod diarise; // Kōrero 1.45 (D-007): in-room speaker separation mod meeting; // Kōrero (v1.13.0): meeting recorder (mic + system loopback) mod meeting_capture; // Kōrero (v1.13.2, Phase A): streaming-to-disk meeting capture #[cfg(windows)] @@ -618,6 +619,7 @@ pub(crate) fn specta_builder() -> Builder { meeting::meeting_stop_capture, meeting::meeting_transcribe_file, meeting::meeting_transcribe_merge, + diarise::meeting_separate_speakers, meeting::meeting_list_recordings, meeting::meeting_export_transcript, meeting::meeting_query, diff --git a/src-tauri/src/meeting.rs b/src-tauri/src/meeting.rs index 27349c4adb..093b70b544 100644 --- a/src-tauri/src/meeting.rs +++ b/src-tauri/src/meeting.rs @@ -131,6 +131,11 @@ pub struct TranscriptSeg { /// /// 0 for meetings recorded before v1.26.0 (the frontend defaults it). pub start_ms: u64, + /// Kōrero 1.45 (D-007): who said a microphone line, from "Separate speakers" + /// (0-based; shown as "Speaker {n + 1}"). `None` until speakers are separated, + /// for system-audio lines, and for every meeting saved before 1.45. + #[serde(default)] + pub speaker: Option, } /// A meeting WAV on disk, for the recovery list. @@ -252,6 +257,7 @@ fn to_transcript_segs(log: &[(u64, &'static str, String)]) -> Vec source: s.to_string(), text: t.clone(), start_ms: *ms, + speaker: None, }) .collect() } diff --git a/src/bindings.ts b/src/bindings.ts index 0fc4158793..4f6f1d3d02 100644 --- a/src/bindings.ts +++ b/src/bindings.ts @@ -714,6 +714,20 @@ async meetingTranscribeMerge(micPath: string | null, systemPath: string | null) else return { status: "error", error: e as any }; } }, +/** + * Kōrero 1.45 (D-007): separate the speakers in a meeting's microphone recording and + * say which speaker each microphone line belongs to. Downloads the two models on first + * use (about 32 MB, checked by sha256). Runs on the CPU; nothing leaves the computer + * except those two downloads. + */ +async meetingSeparateSpeakers(micPath: string, segments: { source: string; text: string; start_ms: number; speaker: number | null }[]) : Promise> { + try { + return { status: "ok", data: await TAURI_INVOKE("meeting_separate_speakers", { micPath, segments }) }; +} catch (e) { + if(e instanceof Error) throw e; + else return { status: "error", error: e as any }; +} +}, async meetingPrewarmPostProcess(transcriptChars: number | null) : Promise> { try { return { status: "ok", data: await TAURI_INVOKE("meeting_prewarm_post_process", { transcriptChars }) }; @@ -1256,6 +1270,23 @@ sample: string; */ gpu_share: number | null } export type AppSettings = { bindings: Partial<{ [key in string]: ShortcutBinding }>; push_to_talk: boolean; audio_feedback: boolean; audio_feedback_volume?: number; sound_theme?: SoundTheme; start_hidden?: boolean; autostart_enabled?: boolean; update_checks_enabled?: boolean; selected_model?: string; always_on_microphone?: boolean; selected_microphone?: string | null; clamshell_microphone?: string | null; selected_output_device?: string | null; translate_to_english?: boolean; selected_language?: string; overlay_position?: OverlayPosition; debug_mode?: boolean; log_level?: LogLevel; custom_words?: string[]; model_unload_timeout?: ModelUnloadTimeout; word_correction_threshold?: number; history_limit?: number; recording_retention_period?: RecordingRetentionPeriod; paste_method?: PasteMethod; clipboard_handling?: ClipboardHandling; auto_submit?: boolean; auto_submit_key?: AutoSubmitKey; post_process_enabled?: boolean; post_process_provider_id?: string; post_process_providers?: PostProcessProvider[]; post_process_api_keys?: SecretMap; post_process_models?: Partial<{ [key in string]: string }>; post_process_prompts?: LLMPrompt[]; post_process_selected_prompt_id?: string | null; mute_while_recording?: boolean; meeting_recording_dir?: string | null; meeting_export_dir?: string | null; denoise_enabled?: boolean; save_crash_reports?: boolean; transcript_corrections?: { wrong: string; right: string }[]; post_process_app_routes?: string[]; append_trailing_space?: boolean; app_language?: string; experimental_enabled?: boolean; lazy_stream_close?: boolean; keyboard_implementation?: KeyboardImplementation; show_tray_icon?: boolean; paste_delay_ms?: number; typing_tool?: TypingTool; external_script_path: string | null; custom_filler_words?: string[] | null; whisper_accelerator?: WhisperAcceleratorSetting; ort_accelerator?: OrtAcceleratorSetting; whisper_gpu_device?: number; extra_recording_buffer_ms?: number } +/** + * Result of "Separate speakers" on a meeting. + */ +export type SpeakerSeparation = { speakers: number; turns: SpeakerTurn[]; +/** + * One entry per input segment, in input order: the speaker of each microphone + * ("you") line, `null` for system-audio lines and lines no turn reached. + */ +segment_speakers: (number | null)[]; audio_s: number; took_s: number } +/** + * A stretch of one person speaking, in milliseconds from the start of the recording. + */ +export type SpeakerTurn = { start_ms: number; end_ms: number; +/** + * 0-based; shown to the user as "Speaker {speaker + 1}". + */ +speaker: number } export type AudioDevice = { index: string; name: string; is_default: boolean } export type AutoSubmitKey = "enter" | "ctrl_enter" | "cmd_enter" export type AvailableAccelerators = { whisper: string[]; ort: string[]; gpu_devices: GpuDeviceOption[] } diff --git a/src/components/meetings/AudioPanel.tsx b/src/components/meetings/AudioPanel.tsx index 679b3b84db..1b67e7697f 100644 --- a/src/components/meetings/AudioPanel.tsx +++ b/src/components/meetings/AudioPanel.tsx @@ -3,7 +3,7 @@ import React from "react"; import { convertFileSrc } from "@tauri-apps/api/core"; import { revealItemInDir } from "@tauri-apps/plugin-opener"; import { toast } from "sonner"; -import { FolderOpen, Loader2, RotateCcw, Wand2, GitMerge, Trash2 } from "lucide-react"; +import { FolderOpen, Loader2, RotateCcw, Wand2, GitMerge, Trash2, Users } from "lucide-react"; import { AudioPlayer } from "../ui/AudioPlayer"; import { Dropdown } from "../ui/Dropdown"; import type { MeetingsController } from "../settings/meetings/useMeetingsController"; @@ -79,7 +79,7 @@ export const AudioPanel: React.FC<{ c: MeetingsController }> = ({ c }) => { Re-transcribe + notes - {c.busy && c.busyMeetingId === m.id && c.task && ( + {c.busy && c.busy !== "speakers" && c.busyMeetingId === m.id && c.task && (
Transcribing @@ -105,6 +105,56 @@ export const AudioPanel: React.FC<{ c: MeetingsController }> = ({ c }) => { )} +
+

+ Separate speakers +

+

+ For a meeting in a room: works out who said what from your microphone recording and labels + each line Speaker 1, Speaker 2 and so on. It runs on this computer. The first time, it + downloads two small models (about 32 MB). + {m.speakersFound != null && + ` Last run found ${m.speakersFound} ${m.speakersFound === 1 ? "voice" : "voices"}.`} +

+
+ + {!c.activeCanSeparate && ( + Needs a recorded microphone file and a transcript with your lines. + )} +
+ {c.busy === "speakers" && c.busyMeetingId === m.id && c.task && ( +
+
+ + {c.task.stage === "download" ? "Downloading the speaker models" : "Separating speakers"} + + + {c.task.progress?.total + ? `${Math.min(100, Math.round((c.task.progress.window / c.task.progress.total) * 100))}%` + : `${c.busyElapsed}s`} + +
+
+ +
+
+ )} +
+ {c.meetings.length > 1 && (

diff --git a/src/components/meetings/TranscriptPanel.tsx b/src/components/meetings/TranscriptPanel.tsx index d27d97d708..9ffc996246 100644 --- a/src/components/meetings/TranscriptPanel.tsx +++ b/src/components/meetings/TranscriptPanel.tsx @@ -4,7 +4,7 @@ import { GraduationCap, Pencil, Scissors, TriangleAlert, X, Flag } from "lucide- import { AddCorrectionInline } from "../ui/Corrections"; import type { MeetingsController } from "../settings/meetings/useMeetingsController"; import { Highlighted, containsFolded } from "./Highlighted"; -import { fmtClock, fmtTrimMs, hiddenCount, isTrimmed, segInWindow, segMs } from "./model"; +import { fmtClock, fmtTrimMs, hiddenCount, isTrimmed, segInWindow, segMs, speakerLabel } from "./model"; /** * Kōrero 1.42: the transcript tab. The reading surface leads; speaker labels, @@ -206,7 +206,7 @@ export const TranscriptPanel: React.FC<{ c: MeetingsController }> = ({ c }) => {
{(m.transcript ?? []).map((s, i) => { if (!s.text.trim() && c.editingSegIdx !== i) return null; - const label = s.source === "you" ? m.youLabel : m.othersLabel; + const label = speakerLabel(m, s); const excluded = !segInWindow(m, s); const flagged = flags.some((f) => Math.abs(f.atSec * 1000 - segMs(s)) < 6000); const hit = !!c.highlight && containsFolded(s.text, c.highlight); diff --git a/src/components/meetings/model.ts b/src/components/meetings/model.ts index 85a3698bdd..c27ac9b912 100644 --- a/src/components/meetings/model.ts +++ b/src/components/meetings/model.ts @@ -13,6 +13,10 @@ export interface TranscriptSeg { // a trim window. Optional only because meetings recorded before v1.26.0 have // segments without it on disk; `normaliseMeetings` defaults those to 0. start_ms?: number; + /** Kōrero 1.45 (D-007): who said a microphone line, set by "Separate speakers" + * (0-based; shown as "Speaker n+1" or the name given). Absent or null before it + * runs, for call-audio lines, and on every meeting saved before 1.45. */ + speaker?: number | null; } /** A moment the user flagged while recording (1.42). */ @@ -50,6 +54,10 @@ export interface Meeting { durationSec?: number; /** The after-Stop wrap-up card was closed by the user. */ wrapUpDismissed?: boolean; + /** 1.45: how many people "Separate speakers" found. Absent until it has run. */ + speakersFound?: number; + /** 1.45: names for Speaker 1..N, by index; blank or missing = "Speaker n". */ + speakerNames?: string[]; } export interface RecordingFile { @@ -116,6 +124,20 @@ export const fmtDuration = (secs: number): string => { export const labelFor = (source: string, youLabel: string, othersLabel: string) => source === "you" ? youLabel : othersLabel; +/** + * Kōrero 1.45: the label for one transcript line. A microphone line that "Separate + * speakers" assigned shows its speaker ("Speaker 2", or the name given); every other + * line keeps the two channel labels. The transcript view and `combine` (which feeds + * the notes) both use this, so what you read is what the notes model reads. + */ +export const speakerLabel = ( + m: { youLabel: string; othersLabel: string; speakerNames?: string[] }, + s: { source: string; speaker?: number | null }, +): string => + s.source === "you" && s.speaker != null + ? m.speakerNames?.[s.speaker]?.trim() || `Speaker ${s.speaker + 1}` + : labelFor(s.source, m.youLabel, m.othersLabel); + /** * The transcript as text, in speaking order (`Label: line` per turn). Consecutive * segments from the same speaker merge into one turn. Falls back to the legacy @@ -127,19 +149,21 @@ export const combine = ( youLabel = "You", othersLabel = "Others", transcript?: TranscriptSeg[], + speakerNames?: string[], ): string => { if (transcript && transcript.length > 0) { - const merged: { source: string; text: string }[] = []; + // 1.45: lines are grouped by their label, so separated speakers become separate + // turns ("Speaker 1: …", "Speaker 2: …") in the text the notes model reads. + const merged: { label: string; text: string }[] = []; for (const s of transcript) { const text = s.text.trim(); if (!text) continue; + const label = speakerLabel({ youLabel, othersLabel, speakerNames }, s); const last = merged[merged.length - 1]; - if (last && last.source === s.source) last.text += ` ${text}`; - else merged.push({ source: s.source, text }); + if (last && last.label === label) last.text += ` ${text}`; + else merged.push({ label, text }); } - return merged - .map((m) => `${labelFor(m.source, youLabel, othersLabel)}: ${m.text}`) - .join("\n"); + return merged.map((m) => `${m.label}: ${m.text}`).join("\n"); } return [ you.trim() ? `${youLabel}:\n${you.trim()}` : "", @@ -236,7 +260,7 @@ export const sideWords = (m: Meeting): { you: number; others: number } => { * already applied). */ export const notesInput = (m: Meeting, segs: TranscriptSeg[]): string => { - const body = combine(m.you, m.others, m.youLabel, m.othersLabel, segs); + const body = combine(m.you, m.others, m.youLabel, m.othersLabel, segs, m.speakerNames); const flags = (m.flags ?? []).filter((f) => Number.isFinite(f.atSec)); if (!body.trim() || flags.length === 0) return body; const lines = flags diff --git a/src/components/settings/meetings/useMeetingsController.ts b/src/components/settings/meetings/useMeetingsController.ts index 7f110ad881..98acc43d65 100644 --- a/src/components/settings/meetings/useMeetingsController.ts +++ b/src/components/settings/meetings/useMeetingsController.ts @@ -106,12 +106,14 @@ export const useMeetingsController = () => { const jobRunningHere = job && job.meetingId === activeId ? job : null; // Keyed off ANY running work: the engine and the local model are shared, // and the store runs one thing at a time. - const busy: null | "transcribe" | "post" | "both" | "other" = + const busy: null | "transcribe" | "post" | "both" | "speakers" | "other" = task?.kind === "transcribe" ? "transcribe" : task?.kind === "both" ? "both" - : task + : task?.kind === "speakers" + ? "speakers" + : task ? "other" : job ? job.kind === "both" @@ -515,6 +517,7 @@ export const useMeetingsController = () => { active.youLabel, active.othersLabel, visibleSegs(active), + active.speakerNames, ); if (!transcript.trim()) { toast.message("There is no transcript to ask about yet."); @@ -554,6 +557,16 @@ export const useMeetingsController = () => { }); }; + const onSeparateSpeakers = async () => { + if (!active || busy || !active.micPath) return; + await useMeetingJobs.getState().separateSpeakers({ + meetingId: active.id, + title: titleOf(active), + micPath: active.micPath, + transcript: active.transcript ?? [], + }); + }; + const onPostProcess = async () => { if (!active || busy) return; // The trim is applied HERE, and flagged moments are added for the model. @@ -676,7 +689,7 @@ export const useMeetingsController = () => { // ---- copy / export / brief / notes ------------------------------------ const copyActive = async () => { if (!active) return; - const body = combine(active.you, active.others, active.youLabel, active.othersLabel, visibleSegs(active)); + const body = combine(active.you, active.others, active.youLabel, active.othersLabel, visibleSegs(active), active.speakerNames); const text = active.processed.trim() ? `${body}\n\n--- Processed ---\n${active.processed.trim()}` : body; @@ -760,6 +773,13 @@ export const useMeetingsController = () => { const saveSegment = (idx: number, text: string) => { if (!active) return; + // 1.45: "Separate speakers" will write the whole transcript back when it ends, + // so an edit made meanwhile would be overwritten. Refuse it, and say why. + if (task?.kind === "speakers" && task.meetingId === active.id) { + toast.message("Separating speakers — edit this line when it finishes."); + setEditingSegIdx(null); + return; + } const before = (active.transcript ?? [])[idx]?.text ?? ""; const next = (active.transcript ?? []).map((s, i) => (i === idx ? { ...s, text } : s)); patchMeeting(active.id, { transcript: next }); @@ -799,7 +819,7 @@ export const useMeetingsController = () => { const parts = [ `# ${titleOf(active)}`, "", - combine(active.you, active.others, active.youLabel, active.othersLabel, visibleSegs(active)) || + combine(active.you, active.others, active.youLabel, active.othersLabel, visibleSegs(active), active.speakerNames) || "(no transcript)", ]; if (isTrimmed(active)) { @@ -1013,6 +1033,13 @@ export const useMeetingsController = () => { })); const activeHasAudio = !!active && (!!active.micPath || !!active.systemPath); const activeHasTranscript = !!active && (!!active.you.trim() || !!active.others.trim()); + // 1.45: speakers can be separated on a recorded microphone WAV whose ordered + // transcript has microphone lines (imports and pre-1.26 meetings do not). + const activeCanSeparate = + !!active && + !!active.micPath && + /\.wav$/i.test(active.micPath) && + (active.transcript ?? []).some((s) => s.source === "you" && s.text.trim()); const selectMeeting = (id: string) => { activeIdRef.current = id; @@ -1108,6 +1135,8 @@ export const useMeetingsController = () => { setSearch, // actions onReTranscribe, + onSeparateSpeakers, + activeCanSeparate, onPostProcess, onBoth, mergeWithId, diff --git a/src/components/shell/searchSources.ts b/src/components/shell/searchSources.ts index 5921bf3491..dc4f9c3dad 100644 --- a/src/components/shell/searchSources.ts +++ b/src/components/shell/searchSources.ts @@ -76,7 +76,7 @@ export const loadSearchSources = async (): Promise => { kind: "meeting", title: titleOf(m), // Search honours the trim, as the Meetings search always has. - body: `${combine(m.you, m.others, m.youLabel, m.othersLabel, visibleSegs(m))}\n${m.processed}`, + body: `${combine(m.you, m.others, m.youLabel, m.othersLabel, visibleSegs(m), m.speakerNames)}\n${m.processed}`, meta: meetingMeta(m), at: m.createdAt, }); diff --git a/src/stores/meetingJobsStore.ts b/src/stores/meetingJobsStore.ts index 90e5ba28ad..cfbe85e7e8 100644 --- a/src/stores/meetingJobsStore.ts +++ b/src/stores/meetingJobsStore.ts @@ -83,7 +83,7 @@ export interface MeetingJob { status: "running"; } -export type MeetingTaskKind = "transcribe" | "both" | "import" | "recover"; +export type MeetingTaskKind = "transcribe" | "both" | "import" | "recover" | "speakers"; export interface MeetingTask { kind: MeetingTaskKind; @@ -96,12 +96,16 @@ export interface MeetingTask { path?: string; startedAt: number; progress: { window: number; total: number | null } | null; + /** 1.45 "speakers" task: fetching the models (first use) or separating. */ + stage?: "download" | "separate"; } export interface TranscriptSegLike { source: string; text: string; start_ms?: number; + /** 1.45: set by "Separate speakers" on microphone lines. */ + speaker?: number | null; } /** The transcript fields a (re-)transcription produces. */ @@ -149,6 +153,15 @@ interface MeetingJobsState { }; }) => Promise; + /** 1.45 (D-007): separate the speakers in a meeting's microphone recording and + * label each microphone line. Resolves true when the labels landed. */ + separateSpeakers: (args: { + meetingId: string; + title: string; + micPath: string; + transcript: TranscriptSegLike[]; + }) => Promise; + /** Transcribe an audio file into a new meeting; optionally generate notes. * `notesPrompt` null = transcript only. */ importFile: (args: { @@ -228,6 +241,25 @@ function attachGlobalListeners(set: SetState) { ).catch((err) => console.error("Could not attach the transcription progress listener:", err), ); + // 1.45: "Separate speakers" progress (model download on first use, then segments). + listen<{ stage: "download" | "separate"; done: number; total: number }>( + "meeting-speakers-progress", + (e) => { + set((s) => + s.task && s.task.kind === "speakers" + ? { + task: { + ...s.task, + stage: e.payload.stage, + progress: { window: e.payload.done, total: e.payload.total }, + }, + } + : {}, + ); + }, + ).catch((err) => + console.error("Could not attach the speakers progress listener:", err), + ); // Capture-health warnings raised DURING a meeting (silent microphone, no // system audio). A toast rather than Meetings-page state: the point is to // reach the user wherever they are in the app, while they can still fix it. @@ -301,6 +333,11 @@ const NOTES_MSGS = { saved: "Meeting notes are ready and saved — open Meetings to see them.", what: "Post-processing", }; +const SPEAKER_MSGS = { + done: "Speakers separated.", + saved: "Speakers separated and saved.", + what: "Separating speakers", +}; const TRANSCRIPT_MSGS = { done: "Transcription finished.", saved: "Transcription finished and was saved.", @@ -576,6 +613,60 @@ export const useMeetingJobs = create((set, get) => { } }, + separateSpeakers: async ({ meetingId, title, micPath, transcript }) => { + const claim = claimTask({ + kind: "speakers", + meetingId, + label: `separating speakers in “${title}”`, + }); + if (claim === null) return refuse(); + try { + const r = await commands.meetingSeparateSpeakers( + micPath, + transcript.map((s) => ({ + source: s.source, + text: s.text, + start_ms: s.start_ms ?? 0, + speaker: s.speaker ?? null, + })), + ); + if (r.status !== "ok") throw new Error(r.error); + const { segment_speakers, speakers } = r.data; + // Microphone lines get their speaker; call-audio lines are left as they were. + // With one voice there is nothing to separate: lines stay "You", unlabelled. + const updated = transcript.map((s, i) => + s.source === "you" + ? { ...s, speaker: speakers > 1 ? (segment_speakers[i] ?? null) : null } + : s, + ); + const outcome = await meetingsBridge.deliverPatch(meetingId, { + transcript: updated, + speakersFound: speakers, + }); + if (outcome === "missing" || outcome === "failed") { + reportLanding(outcome, SPEAKER_MSGS, meetingId, "transcript"); + return false; + } + if (speakers <= 1) { + toast.message( + "Only one voice was found in this recording, so the lines were not split between speakers.", + ); + } else reportLanding(outcome, SPEAKER_MSGS, meetingId, "transcript"); + return true; + } catch (e) { + toast.error(`Separating speakers failed: ${errText(e)}`); + logActivity({ + status: "failed", + title: "Separating speakers failed", + detail: errText(e), + target: { section: "meetings", meetingId, tab: "audio" }, + }); + return false; + } finally { + releaseTask(claim); + } + }, + importFile: async ({ path, label, makeMeeting, notesPrompt }) => { const claim = claimTask({ kind: "import", From 57915df1c5330e18a2c00ea4e6fa9a4de5b5002b Mon Sep 17 00:00:00 2001 From: Nic Keating Date: Fri, 16 Oct 2026 21:25:00 +1300 Subject: [PATCH 2/3] docs: D-008 model delivery, close L-012, add L-013 - D-008 (Accepted): the two speaker-separation models download on first use, pinned by size and sha256, instead of being bundled. - L-012 closes as check-added: diar_ami_der guards the shipped models. - L-013 (open): scripted edits normalised mixed line endings in src/bindings.ts and wrote garbled macrons from an ANSI-read script; both caught before any push. - Brief: B4's first slice recorded. product-check.py: 0 FAIL, 1 WARN (O3). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01M9niYsTjphhTSYCLhCwg5n --- LESSONS.md | 3 ++- docs/DECISIONS.md | 1 + docs/PRODUCT.md | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/LESSONS.md b/LESSONS.md index d0c60ccb2f..5858468ecf 100644 --- a/LESSONS.md +++ b/LESSONS.md @@ -38,4 +38,5 @@ the row is `open`, so several start past the 14-day limit and need a decision. | L-009 | 2026-10-08 | Audio brief failed with "voice engine not found" on the main laptop from the first build with PRIV-1 until 1.44: removing a developer path removed the only lookup that found the engine. | When a lookup path is removed, give its replacement (a setting) and a health check in the same change. | open | Fixed on `fix/1.44` (engine folder setting). | | L-010 | 2026-09-25 | On the main laptop, local `main` tracks `origin`, which is upstream Handy, not `korero`; a bare `git push` targets the wrong repo. | Always push with `git push korero `. | open | One-line fix: `git branch -u korero/main main` on each machine. | | L-011 | 2026-10-16 | The first in-room plan listed `pyannote-rs` as a fallback engine. The M0 probe found it does not compile against Kōrero's `ort` (29 errors), ignores the model's speaker-change output and drifts about 60 ms per 10 s. | Compile a dependency and read its source before naming it in a plan, even as a fallback. | advice-only | `app-dev-pm` Phase 1.5 already requires probing claims; this was a planning slip, caught by M0 before any app code. | -| L-012 | 2026-10-16 | M0's first scores were worse than calling everyone one speaker. The cause was the CAM++ embedding model packaged with `pyannote-rs`: it ran without error and gave stable output, but told AMI speakers apart no better than chance, even on headset audio. The harness was checked first (reference timing, feature scale, window type), then models were compared on clean labelled turns; WeSpeaker ResNet34-LM separated them 82–100% of the time. | A model that runs is not a model that works: score it on labelled data, and check the harness before trusting a bad number. | open | Plan M2 adds a check that scores the shipped models on the AMI test set and fails above the agreed error. | +| L-012 | 2026-10-16 | M0's first scores were worse than calling everyone one speaker. The cause was the CAM++ embedding model packaged with `pyannote-rs`: it ran without error and gave stable output, but told AMI speakers apart no better than chance, even on headset audio. The harness was checked first (reference timing, feature scale, window type), then models were compared on clean labelled turns; WeSpeaker ResNet34-LM separated them 82–100% of the time. | A model that runs is not a model that works: score it on labelled data, and check the harness before trusting a bad number. | check-added | `diar_ami_der` in `src-tauri/src/diarise.rs` scores the shipped models on four AMI far-field meetings and fails above a 25% mean error (manual: 177 MB of audio, not in CI). Its own scorer agrees with pyannote.metrics to within 0.2 points. | +| L-013 | 2026-10-16 | Editing `src/bindings.ts` with a script that normalised line endings turned a 31-line change into a 2,003-line diff: the file is stored in git with mixed CRLF and LF lines, and the rewrite made them uniform. The same session then wrote garbled macrons into a decision row, because Windows PowerShell 5.1 read a script saved without a byte-order mark as ANSI. Both were caught before any push. | Before committing a scripted edit, compare `git diff --numstat` with and without `--ignore-cr-at-eol`, and search the added lines for mojibake (Ã, Å); write text with macrons from Python or a UTF-8-with-BOM script. | open | A `.gitattributes` rule for `src/bindings.ts` (or regenerating it in CI) and a mojibake check on added lines would make these checks. | diff --git a/docs/DECISIONS.md b/docs/DECISIONS.md index 7c3458ce64..bd8b8c28b3 100644 --- a/docs/DECISIONS.md +++ b/docs/DECISIONS.md @@ -19,3 +19,4 @@ in the Kōrero claude.ai project), where some carried older labels (shown in bra | D-005 | 2026-10-08 | Accepted | **Notes for a meeting over 48,000 characters are written in parts and combined** (split at speaker turns, at most 8 parts); shorter meetings are unchanged. Nic: "go with reading in parts". | 8 of 35 meetings on the main laptop had notes from the start only, and lost every flagged moment. | A bigger context window: needs more video memory than the target machines have, and quality falls off on long inputs. | Ask needs whole-meeting answers (it still reads one part and says so). | | D-006 | 2026-10-08 | Accepted | **`wasapi` was upgraded from 0.23 to 0.25 to clear a security advisory**, rather than recording an exemption. Nic's choice during the 1.44 publish gates. | An exemption leaves the advisory in place; the upgrade's side effect (Windows manifest embedding in test binaries) was fixable. | Exempt the advisory in `cargo deny`. | — | | D-007 | 2026-10-16 | Accepted | **In-room speaker separation runs in Kōrero's own code on its existing ONNX runtime** (`ort 2.0.0-rc.12`), after Stop, on the CPU, never live: pyannote segmentation-3.0 (MIT) read for speaker changes, not only speech; WeSpeaker ResNet34-LM speaker embeddings (CC BY 4.0, attribution in the third-party notices); clustering of the whole meeting on segments of 1.5 s or more, clusters under 20 s of speech folded into the nearest one, short fragments attached by voice. Plan: `KORERO_1.45_IN-ROOM-SPEAKERS-PLAN_2026-10-16.md` (v2). (Approved by Nic 16 Oct 2026: "Execute D007".) | M0 probe, 16 Oct, on four AMI test meetings recorded by one far-field room microphone (92 minutes): mean error 23.1% with a 0.25 s collar, 28.1% strict, against 22.4% strict for pyannote's own pipeline on all 16 test meetings; right number of people in 2 of 4 meetings (one too many in one, two too few in the other). About 80 s of CPU time per hour of audio on 4 threads. It builds in the same binary as `transcribe-rs` with one `ort`. | `pyannote-rs` 0.3.4: does not compile against `ort` rc.12 (29 errors); reads the model only as a speech detector; its clock drifts 60.6 ms per 10 s; its packaged CAM++ embedding model told AMI speakers apart no better than chance, even on headset audio (57.5% error with the right model, 80.5% as published). `sherpa-onnx`: a second ONNX runtime beside `ort`. Telling the clusterer the number of people: worse (31.5%) than finding it. The notes model guessing speakers: invents attributions. | Results on Nic's own room recordings differ from AMI; `knf-rs` (C++ build, `ndarray` 0.16) is replaced by Rust feature code. | +| D-008 | 2026-10-16 | Accepted | **The speaker-separation models are downloaded on first use, not bundled.** "Separate speakers" fetches pyannote segmentation-3.0 (5,983,836 B) and WeSpeaker ResNet34-LM (26,530,550 B) into the models folder (`diarise/`), writes each as `.partial`, and keeps it only if its size and sha256 match the values pinned in `diarise.rs`; a mismatch is deleted and never loaded. Sources for now: the `pyannote-rs` v0.1.0 and sherpa-onnx speaker-recognition-models GitHub releases. Filterbank features come from `knf-rs` 0.3.2 (MIT; a C++ build through CMake and bindgen, and a second `ndarray`, 0.16, internal to it). | Bundling would roughly triple the installer (14 MB to about 46 MB) for every user, including those who never record in a room. Download-on-first-use matches how Kōrero already fetches speech models and stays offline afterwards. Pinning size and hash means a changed or substituted file at either URL fails closed. | Bundle in the installer: simplest offline story, but the size cost falls on everyone. Mirror to Kōrero's own GitHub release now: more control, but publishing a release is Nic's call. Rust filterbanks instead of `knf-rs`: removes the C++ build, but needs a parity check first (plan M2 step c). | Either upstream URL changes or disappears (the pinned hash makes that a clean failure, not a silent one): mirror both files to a Kōrero release. The Rust filterbank parity check passes (drop `knf-rs`). | diff --git a/docs/PRODUCT.md b/docs/PRODUCT.md index a1a65659ef..abddfde4c5 100644 --- a/docs/PRODUCT.md +++ b/docs/PRODUCT.md @@ -82,6 +82,7 @@ A build or enable bet enters Now only on research or a requirement. Keep Now to Dated one-liners for every reorder, drop or gate decision, newest first. Larger product decisions go to `docs/DECISIONS.md`. +- 2026-10-16: B4, first slice built on `feat/diarise`: the engine and a "Separate speakers" action on the Audio tab, with speaker labels reaching copy, export, search, Ask and the notes. Mean error 21.9% on the AMI check (bar 25%). Still to come in B4: naming, merge and split (M3), and splitting lines where the speaker changes mid-sentence. - 2026-10-16: Nic: "Execute D007". B4 moves to Now beside B1 and B3 (3 of 3), and is built on its own branch from `main`, ahead of B3 (M1); D-007 is Accepted. - 2026-10-16: First brief (dev-product Mode A). Proposed order, for Nic to confirm: B1 then B3 in Now; B6, B4 and B5 in Next. Nic sets the order; this is the session's proposal. From 5f0e7b7853237c8900145fdfa34989980330d7aa Mon Sep 17 00:00:00 2001 From: Nic Keating Date: Fri, 16 Oct 2026 22:04:39 +1300 Subject: [PATCH 3/3] fix: speaker separation review findings (M1-M4, L1-L4, L6, L8, L10, L12) An independent read-only review of the first commit found no Critical or High issues and no path that loses saved meetings. Fixed here: - M1, M2: meetings whose lines can't be matched by time to one recording are refused, in Rust (timing_problem) and in the UI (separationBlocker), with the reason shown: pre-1.26 meetings (no line timings: re-transcribe first) and combined meetings (marked mergedFrom from now on; older ones are caught because their times run backwards at the join). - M3, L6: model downloads have a 20 s connect and 10 min total timeout, so a stalled connection can't hold the one job slot; a failed download never leaves a .partial file behind. - M4: Activity, Today and the top bar name the job "Separating speakers" or "Downloading the speaker models" with a percentage, not "Re-transcribing, part 1048576 of 5983836". - L1: re-transcribing clears the stale speakersFound count. - L2: combine() groups by (source, speaker), so meetings without speakers give exactly the text they gave before (tested, including two channels renamed to the same label). - L3: turns join across the 61 ms gap between segmentation windows (tested). - L4: a non-finite embedding is treated as none. - L8: one separation at a time, enforced in Rust. - L10: finding no voice at all gets its own message. - L12: progress shows "Separating" once the download ends; download events are throttled to one per percent. Evidence (legion, 16 Oct): cargo test --lib 264 passed (2 new); bun test unit-tests 92 passed (10 new, unit-tests/speakers.test.ts); tsc clean; eslint: 0 warnings in the touched files. diar_ami_der: mean DER 21.5% (was 21.9%) with a 0.25 s collar, bar 25%. Not fixed here (follow-ups in the PR): L5 overlapping same-speaker turns, L7 hashing 32 MB on the async runtime per run, L9 the whole recording held in memory (about 230 MB per hour), L11 a line already open for editing when separation starts, L13 regenerate bindings.ts with tauri-specta, L14 CPU load if a recording starts during separation, L15 two licences to verify. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01M9niYsTjphhTSYCLhCwg5n --- docs/PRODUCT.md | 2 +- src-tauri/src/diarise.rs | 155 ++++++++++++++---- src/components/meetings/AudioPanel.tsx | 4 +- src/components/meetings/model.ts | 35 +++- .../meetings/useMeetingsController.ts | 14 +- src/components/shell/useRunning.ts | 14 +- src/stores/meetingJobsStore.ts | 6 +- unit-tests/speakers.test.ts | 89 ++++++++++ 8 files changed, 269 insertions(+), 50 deletions(-) create mode 100644 unit-tests/speakers.test.ts diff --git a/docs/PRODUCT.md b/docs/PRODUCT.md index abddfde4c5..ce389a10ff 100644 --- a/docs/PRODUCT.md +++ b/docs/PRODUCT.md @@ -82,7 +82,7 @@ A build or enable bet enters Now only on research or a requirement. Keep Now to Dated one-liners for every reorder, drop or gate decision, newest first. Larger product decisions go to `docs/DECISIONS.md`. -- 2026-10-16: B4, first slice built on `feat/diarise`: the engine and a "Separate speakers" action on the Audio tab, with speaker labels reaching copy, export, search, Ask and the notes. Mean error 21.9% on the AMI check (bar 25%). Still to come in B4: naming, merge and split (M3), and splitting lines where the speaker changes mid-sentence. +- 2026-10-16: B4, first slice built on `feat/diarise`: the engine and a "Separate speakers" action on the Audio tab, with speaker labels reaching copy, export, search, Ask and the notes. Mean error 21.5% on the AMI check (bar 25%). Still to come in B4: naming, merge and split (M3), and splitting lines where the speaker changes mid-sentence. - 2026-10-16: Nic: "Execute D007". B4 moves to Now beside B1 and B3 (3 of 3), and is built on its own branch from `main`, ahead of B3 (M1); D-007 is Accepted. - 2026-10-16: First brief (dev-product Mode A). Proposed order, for Nic to confirm: B1 then B3 in Now; B6, B4 and B5 in Next. Nic sets the order; this is the session's proposal. diff --git a/src-tauri/src/diarise.rs b/src-tauri/src/diarise.rs index a89185eb55..0607be8443 100644 --- a/src-tauri/src/diarise.rs +++ b/src-tauri/src/diarise.rs @@ -99,36 +99,55 @@ pub(crate) async fn ensure_model( std::fs::create_dir_all(dir).map_err(|e| format!("{}: {e}", dir.display()))?; let partial = dir.join(format!("{}.partial", m.name)); let _ = std::fs::remove_file(&partial); - let resp = reqwest::get(m.url) - .await - .map_err(|e| format!("Could not download {}: {e}", m.name))?; - if !resp.status().is_success() { - return Err(format!("Could not download {}: HTTP {}", m.name, resp.status())); - } - let mut file = std::fs::File::create(&partial).map_err(|e| format!("{}: {e}", partial.display()))?; - let mut hasher = Sha256::new(); - let (mut done, mut stream) = (0u64, resp.bytes_stream()); - use futures_util::StreamExt; - while let Some(chunk) = stream.next().await { - let chunk = chunk.map_err(|e| format!("Download of {} failed: {e}", m.name))?; - done += chunk.len() as u64; - if done > m.bytes { - drop(file); - let _ = std::fs::remove_file(&partial); - return Err(format!("{} is larger than expected; download refused.", m.name)); - } - hasher.update(&chunk); - file.write_all(&chunk).map_err(|e| format!("{}: {e}", partial.display()))?; - progress(done, m.bytes); - } - drop(file); - let got = format!("{:x}", hasher.finalize()); - if done != m.bytes || got != m.sha256 { + // Review M3: a stalled connection must not hold the one job slot until restart. + let client = reqwest::Client::builder() + .connect_timeout(Duration::from_secs(20)) + .timeout(Duration::from_secs(600)) + .build() + .map_err(|e| format!("Could not start the download of {}: {e}", m.name))?; + let fetched = async { + let resp = client + .get(m.url) + .send() + .await + .map_err(|e| format!("Could not download {}: {e}", m.name))?; + if !resp.status().is_success() { + return Err(format!("Could not download {}: HTTP {}", m.name, resp.status())); + } + let mut file = std::fs::File::create(&partial).map_err(|e| format!("{}: {e}", partial.display()))?; + let mut hasher = Sha256::new(); + let (mut done, mut last_pct, mut stream) = (0u64, u64::MAX, resp.bytes_stream()); + use futures_util::StreamExt; + while let Some(chunk) = stream.next().await { + let chunk = chunk.map_err(|e| format!("Download of {} failed: {e}", m.name))?; + done += chunk.len() as u64; + if done > m.bytes { + return Err(format!("{} is larger than expected; download refused.", m.name)); + } + hasher.update(&chunk); + file.write_all(&chunk).map_err(|e| format!("{}: {e}", partial.display()))?; + // Review L12: one event per percent, not one per network chunk. + let pct = done * 100 / m.bytes.max(1); + if pct != last_pct { + last_pct = pct; + progress(done, m.bytes); + } + } + file.flush().map_err(|e| format!("{}: {e}", partial.display()))?; + let got = format!("{:x}", hasher.finalize()); + if done != m.bytes || got != m.sha256 { + return Err(format!( + "{} did not match its pinned size and checksum; it was discarded.", + m.name + )); + } + Ok(()) + } + .await; + // Review L6: never leave a partial file behind, whatever failed. + if let Err(err) = fetched { let _ = std::fs::remove_file(&partial); - return Err(format!( - "{} did not match its pinned size and checksum; it was discarded.", - m.name - )); + return Err(err); } std::fs::rename(&partial, &path).map_err(|e| format!("{}: {e}", path.display()))?; Ok(path) @@ -374,8 +393,9 @@ pub struct SpeakerTurn { } /// Labelled segments to turns: sorted by start; a segment joins the previous turn only -/// when that turn is the same speaker's and they touch (within 50 ms, as at window -/// boundaries). Pauses are not filled. Turns of different speakers may overlap. +/// when that turn is the same speaker's and they nearly touch (within 100 ms: consecutive +/// windows leave a 61 ms gap, because 589 frames x 270 samples fall short of 10 s). +/// Pauses longer than that are not filled. Turns of different speakers may overlap. pub(crate) fn to_turns(segs: &[(f64, f64)], labels: &[Option]) -> Vec { let mut v: Vec<(f64, f64, usize)> = segs .iter() @@ -387,7 +407,7 @@ pub(crate) fn to_turns(segs: &[(f64, f64)], labels: &[Option]) -> Vec().map_err(e)?; + // Review L4: one NaN would make its cluster's centroid NaN and pull every line to it. + if data.iter().any(|x| !x.is_finite()) { + return Ok(None); + } Ok(Some(data.to_vec())) } @@ -546,6 +570,43 @@ pub(crate) fn engine_threads() -> usize { // --------------------------------------------------------------------------- // The command. +/// Why a transcript's lines cannot be matched by time to its microphone recording, if +/// they cannot (review M1, M2). `None` means it is safe to separate. +pub(crate) fn timing_problem(segs: &[crate::meeting::TranscriptSeg]) -> Option<&'static str> { + let mic: Vec = segs + .iter() + .filter(|s| s.source == "you" && !s.text.trim().is_empty()) + .map(|s| s.start_ms) + .collect(); + if mic.is_empty() { + return Some("This meeting has no microphone lines to label."); + } + if mic.len() >= 2 && mic.iter().all(|&ms| ms == mic[0]) { + return Some( + "This meeting's lines have no timings (it was recorded before Kōrero 1.26), so they \ + can't be matched to speakers. Re-transcribe it first.", + ); + } + if segs.windows(2).any(|w| w[1].start_ms < w[0].start_ms) { + return Some( + "This transcript is not in time order (a combined meeting, for example), so its \ + lines can't be matched to one recording.", + ); + } + None +} + +/// One separation at a time (review L8): two runs would write the same model files. +static RUNNING: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + +struct RunGuard; + +impl Drop for RunGuard { + fn drop(&mut self) { + RUNNING.store(false, std::sync::atomic::Ordering::SeqCst); + } +} + /// Result of "Separate speakers" on a meeting. #[derive(Debug, Clone, Serialize, Deserialize, Type)] pub struct SpeakerSeparation { @@ -582,6 +643,13 @@ pub async fn meeting_separate_speakers( mic_path: String, segments: Vec, ) -> Result { + if let Some(why) = timing_problem(&segments) { + return Err(why.to_string()); + } + if RUNNING.swap(true, std::sync::atomic::Ordering::SeqCst) { + return Err("Speakers are already being separated. Try again when that finishes.".into()); + } + let _running = RunGuard; let started = Instant::now(); let dir = crate::portable::models_dir(&app) .map_err(|err| format!("Could not find the models folder: {err}"))? @@ -591,6 +659,7 @@ pub async fn meeting_separate_speakers( let emb_path = ensure_model(&dir, &EMBEDDING, |d, t| emit_progress(&a, "download", d, t)).await?; let a = app.clone(); let (diar, audio_s) = tauri::async_runtime::spawn_blocking(move || -> Result<(Diarisation, f64), String> { + emit_progress(&a, "separate", 0, 1); let x = decode_wav_16k(&mic_path)?; if x.len() < SR * 5 { return Err("The recording is too short to separate speakers.".into()); @@ -836,6 +905,28 @@ mod tests { ); } + #[test] + fn one_speaker_across_window_boundaries_is_one_turn() { + let cls = vec![vec![1u8; FRAMES]; 3]; // 30 s, one speaker throughout + let s = local_segments(&cls, 3 * WIN); + let spans: Vec<(f64, f64)> = s.iter().map(|x| (x.start, x.end)).collect(); + let t = to_turns(&spans, &vec![Some(0); spans.len()]); + assert_eq!(t.len(), 1, "the 61 ms gap between windows must not split a turn"); + } + + fn seg(source: &str, ms: u64) -> crate::meeting::TranscriptSeg { + crate::meeting::TranscriptSeg { source: source.into(), text: "hello there".into(), start_ms: ms, speaker: None } + } + + #[test] + fn transcripts_that_cannot_be_matched_by_time_are_refused() { + assert!(timing_problem(&[seg("you", 0), seg("others", 1_000), seg("you", 2_000)]).is_none()); + assert!(timing_problem(&[seg("others", 0)]).is_some(), "no microphone lines"); + assert!(timing_problem(&[seg("you", 0), seg("you", 0), seg("you", 0)]).is_some(), "pre-1.26: no timings"); + assert!(timing_problem(&[seg("you", 0), seg("you", 9_000), seg("you", 1_000)]).is_some(), "a combined meeting"); + assert!(timing_problem(&[seg("you", 0)]).is_none(), "a single line at 0 s is a real timing"); + } + #[test] fn lines_take_the_speaker_they_overlap_most_or_the_nearest_within_2_s() { let turns = vec![ diff --git a/src/components/meetings/AudioPanel.tsx b/src/components/meetings/AudioPanel.tsx index 1b67e7697f..4807db7708 100644 --- a/src/components/meetings/AudioPanel.tsx +++ b/src/components/meetings/AudioPanel.tsx @@ -126,9 +126,7 @@ export const AudioPanel: React.FC<{ c: MeetingsController }> = ({ c }) => { {c.busy === "speakers" && c.busyMeetingId === m.id ? : } {m.speakersFound != null ? "Separate speakers again" : "Separate speakers"} - {!c.activeCanSeparate && ( - Needs a recorded microphone file and a transcript with your lines. - )} + {c.separateBlocker && {c.separateBlocker}}
{c.busy === "speakers" && c.busyMeetingId === m.id && c.task && (
diff --git a/src/components/meetings/model.ts b/src/components/meetings/model.ts index c27ac9b912..9b3f127398 100644 --- a/src/components/meetings/model.ts +++ b/src/components/meetings/model.ts @@ -58,6 +58,8 @@ export interface Meeting { speakersFound?: number; /** 1.45: names for Speaker 1..N, by index; blank or missing = "Speaker n". */ speakerNames?: string[]; + /** 1.45: the two meetings this one was combined from (set by Combine). */ + mergedFrom?: string[]; } export interface RecordingFile { @@ -124,6 +126,27 @@ export const fmtDuration = (secs: number): string => { export const labelFor = (source: string, youLabel: string, othersLabel: string) => source === "you" ? youLabel : othersLabel; +/** + * Kōrero 1.45: why "Separate speakers" cannot run on this meeting, or null if it can. + * Mirrors `timing_problem` in diarise.rs, which refuses the same cases. + */ +export const separationBlocker = (m: Meeting): string | null => { + if (!m.micPath || !/\.wav$/i.test(m.micPath)) + return "Needs a recorded microphone file (imports are not supported yet)."; + if (m.mergedFrom && m.mergedFrom.length > 0) + return "A combined meeting mixes two recordings, so its lines can't be matched to one."; + const segs = m.transcript ?? []; + const mic = segs.filter((s) => s.source === "you" && s.text.trim()).map((s) => s.start_ms ?? 0); + if (mic.length === 0) return "There are no microphone lines to label."; + if (mic.length >= 2 && mic.every((ms) => ms === mic[0])) + return "Its lines have no timings (recorded before Kōrero 1.26). Re-transcribe it first."; + for (let i = 1; i < segs.length; i++) { + if ((segs[i].start_ms ?? 0) < (segs[i - 1].start_ms ?? 0)) + return "Its transcript is not in time order (a combined meeting, for example), so its lines can't be matched to one recording."; + } + return null; +}; + /** * Kōrero 1.45: the label for one transcript line. A microphone line that "Separate * speakers" assigned shows its speaker ("Speaker 2", or the name given); every other @@ -154,16 +177,18 @@ export const combine = ( if (transcript && transcript.length > 0) { // 1.45: lines are grouped by their label, so separated speakers become separate // turns ("Speaker 1: …", "Speaker 2: …") in the text the notes model reads. - const merged: { label: string; text: string }[] = []; + const merged: { key: string; seg: TranscriptSeg; text: string }[] = []; for (const s of transcript) { const text = s.text.trim(); if (!text) continue; - const label = speakerLabel({ youLabel, othersLabel, speakerNames }, s); + const key = `${s.source}|${s.source === "you" && s.speaker != null ? s.speaker : ""}`; const last = merged[merged.length - 1]; - if (last && last.label === label) last.text += ` ${text}`; - else merged.push({ label, text }); + if (last && last.key === key) last.text += ` ${text}`; + else merged.push({ key, seg: s, text }); } - return merged.map((m) => `${m.label}: ${m.text}`).join("\n"); + return merged + .map((m) => `${speakerLabel({ youLabel, othersLabel, speakerNames }, m.seg)}: ${m.text}`) + .join("\n"); } return [ you.trim() ? `${youLabel}:\n${you.trim()}` : "", diff --git a/src/components/settings/meetings/useMeetingsController.ts b/src/components/settings/meetings/useMeetingsController.ts index 98acc43d65..3fe2d43936 100644 --- a/src/components/settings/meetings/useMeetingsController.ts +++ b/src/components/settings/meetings/useMeetingsController.ts @@ -40,6 +40,7 @@ import { notesAreStale, notesInput, parseTrimInput, + separationBlocker, titleOf, trimKeyOf, visibleSegs, @@ -492,6 +493,8 @@ export const useMeetingsController = () => { systemCaptured: first.systemCaptured || second.systemCaptured, micPath: first.micPath ?? second.micPath, systemPath: first.systemPath ?? second.systemPath, + // 1.45: lines from two recordings can't be matched to one for speakers. + mergedFrom: [first.id, second.id], }; setMeetings((prev) => [m, ...prev]); setActiveId(m.id); @@ -1033,13 +1036,9 @@ export const useMeetingsController = () => { })); const activeHasAudio = !!active && (!!active.micPath || !!active.systemPath); const activeHasTranscript = !!active && (!!active.you.trim() || !!active.others.trim()); - // 1.45: speakers can be separated on a recorded microphone WAV whose ordered - // transcript has microphone lines (imports and pre-1.26 meetings do not). - const activeCanSeparate = - !!active && - !!active.micPath && - /\.wav$/i.test(active.micPath) && - (active.transcript ?? []).some((s) => s.source === "you" && s.text.trim()); + // 1.45: why speakers can't be separated on this meeting (null = they can). + const separateBlocker = active ? separationBlocker(active) : "No meeting selected."; + const activeCanSeparate = separateBlocker === null; const selectMeeting = (id: string) => { activeIdRef.current = id; @@ -1137,6 +1136,7 @@ export const useMeetingsController = () => { onReTranscribe, onSeparateSpeakers, activeCanSeparate, + separateBlocker, onPostProcess, onBoth, mergeWithId, diff --git a/src/components/shell/useRunning.ts b/src/components/shell/useRunning.ts index 0624f5cab1..667f05c105 100644 --- a/src/components/shell/useRunning.ts +++ b/src/components/shell/useRunning.ts @@ -73,7 +73,19 @@ export const useRunning = (): RunningItem[] => { target: { section: "meetings" }, }); } - if (task) { + if (task && task.kind === "speakers") { + const p = task.progress; + const pct = p && p.total ? Math.min(100, Math.round((p.window / p.total) * 100)) : null; + items.push({ + id: "task", + kind: "transcribe", + title: task.stage === "download" ? "Downloading the speaker models" : "Separating speakers", + detail: task.label.replace(/^separating speakers in\s+/i, "") + (pct !== null ? ` · ${pct}%` : ""), + progress: pct, + startedAt: task.startedAt, + target: { section: "meetings", meetingId: task.meetingId ?? undefined, tab: "audio" }, + }); + } else if (task) { const p = task.progress; const label = task.kind === "import" diff --git a/src/stores/meetingJobsStore.ts b/src/stores/meetingJobsStore.ts index cfbe85e7e8..66b754373a 100644 --- a/src/stores/meetingJobsStore.ts +++ b/src/stores/meetingJobsStore.ts @@ -562,6 +562,8 @@ export const useMeetingJobs = create((set, get) => { const patch = await transcribeAudio(micPath, systemPath); const outcome = await meetingsBridge.deliverPatch(meetingId, { ...patch, + // 1.45: the new lines carry no speakers, so the last separation's count is stale. + speakersFound: undefined, }); if (outcome === "missing" || outcome === "failed") { reportLanding(outcome, TRANSCRIPT_MSGS, meetingId, "transcript"); @@ -647,7 +649,9 @@ export const useMeetingJobs = create((set, get) => { reportLanding(outcome, SPEAKER_MSGS, meetingId, "transcript"); return false; } - if (speakers <= 1) { + if (speakers === 0) { + toast.message("No speech long enough to recognise a voice was found in this recording."); + } else if (speakers === 1) { toast.message( "Only one voice was found in this recording, so the lines were not split between speakers.", ); diff --git a/unit-tests/speakers.test.ts b/unit-tests/speakers.test.ts new file mode 100644 index 0000000000..dce526233a --- /dev/null +++ b/unit-tests/speakers.test.ts @@ -0,0 +1,89 @@ +import { describe, expect, test } from "bun:test"; +import { + combine, + separationBlocker, + speakerLabel, + type Meeting, + type TranscriptSeg, +} from "../src/components/meetings/model"; + +// Kōrero 1.45 (D-007): speaker labels, and when "Separate speakers" is refused. + +const seg = (source: string, text: string, start_ms: number, speaker?: number | null): TranscriptSeg => ({ + source, + text, + start_ms, + speaker, +}); + +const meeting = (over: Partial = {}): Meeting => ({ + id: "m1", + title: "Room", + you: "", + others: "", + transcript: [seg("you", "kia ora", 0), seg("others", "hello", 1000), seg("you", "next", 2000)], + processed: "", + processPrompt: "", + createdAt: 0, + systemCaptured: false, + micPath: "C:/recordings/room-mic.wav", + systemPath: null, + youLabel: "You", + othersLabel: "Others", + imported: false, + ...over, +}); + +describe("combine without speakers is unchanged", () => { + test("alternating lines keep one turn per line", () => { + const t = [seg("you", "a", 0), seg("others", "b", 1), seg("you", "c", 2)]; + expect(combine("", "", "You", "Others", t)).toBe("You: a\nOthers: b\nYou: c"); + }); + + test("two channels renamed to the same label still alternate (review L2)", () => { + const t = [seg("you", "a", 0), seg("others", "b", 1)]; + expect(combine("", "", "Team", "Team", t)).toBe("Team: a\nTeam: b"); + }); + + test("consecutive lines from one channel merge", () => { + const t = [seg("you", "a", 0), seg("you", "b", 1)]; + expect(combine("", "", "You", "Others", t)).toBe("You: a b"); + }); +}); + +describe("combine with speakers", () => { + test("separated microphone lines become separate turns, names win over numbers", () => { + const t = [seg("you", "a", 0, 0), seg("you", "b", 1, 1), seg("you", "c", 2, 1), seg("others", "d", 3, null)]; + expect(combine("", "", "You", "Others", t, ["Aroha"])).toBe("Aroha: a\nSpeaker 2: b c\nOthers: d"); + }); + + test("a speaker on a call-audio line is ignored", () => { + expect(speakerLabel({ youLabel: "You", othersLabel: "Others" }, { source: "others", speaker: 2 })).toBe("Others"); + }); +}); + +describe("separationBlocker", () => { + test("a recorded meeting with timed microphone lines can be separated", () => { + expect(separationBlocker(meeting())).toBeNull(); + }); + + test("imports and missing recordings are refused", () => { + expect(separationBlocker(meeting({ micPath: null }))).not.toBeNull(); + expect(separationBlocker(meeting({ micPath: "C:/imports/phone.m4a" }))).not.toBeNull(); + }); + + test("combined meetings are refused, marked or not (review M2)", () => { + expect(separationBlocker(meeting({ mergedFrom: ["a", "b"] }))).not.toBeNull(); + const concatenated = [seg("you", "a", 0), seg("you", "b", 9000), seg("you", "c", 1000)]; + expect(separationBlocker(meeting({ transcript: concatenated }))).not.toBeNull(); + }); + + test("meetings without line timings are refused (review M1)", () => { + const untimed = [seg("you", "a", 0), seg("you", "b", 0), seg("you", "c", 0)]; + expect(separationBlocker(meeting({ transcript: untimed }))).toContain("Re-transcribe"); + }); + + test("no microphone lines is refused", () => { + expect(separationBlocker(meeting({ transcript: [seg("others", "hi", 0)] }))).not.toBeNull(); + }); +});