From 9926ed09e546e5562a2f19206c883b5206031182 Mon Sep 17 00:00:00 2001 From: Joel Carriere <104279162+JoelCarriere@users.noreply.github.com> Date: Fri, 24 Jul 2026 14:36:26 -0400 Subject: [PATCH] feat(desktop): add local voice dictation to message composer Signed-off-by: Joel Carriere <104279162+JoelCarriere@users.noreply.github.com> --- desktop/playwright.config.ts | 1 + desktop/src-tauri/Info.plist | 2 +- desktop/src-tauri/src/dictation.rs | 199 +++++++++++++ desktop/src-tauri/src/lib.rs | 5 + .../src/features/huddle/lib/audioWorklet.ts | 80 +----- .../messages/lib/dictationInsertion.test.mjs | 29 ++ .../messages/lib/dictationInsertion.ts | 17 ++ .../features/messages/ui/MessageComposer.tsx | 32 +-- .../messages/ui/MessageComposerToolbar.tsx | 62 +++- .../messages/ui/useComposerDictation.ts | 264 ++++++++++++++++++ desktop/src/shared/lib/pcmCapture.ts | 78 ++++++ desktop/src/testing/e2eBridge.ts | 5 + desktop/tests/e2e/composer-dictation.spec.ts | 115 ++++++++ 13 files changed, 801 insertions(+), 88 deletions(-) create mode 100644 desktop/src-tauri/src/dictation.rs create mode 100644 desktop/src/features/messages/lib/dictationInsertion.test.mjs create mode 100644 desktop/src/features/messages/lib/dictationInsertion.ts create mode 100644 desktop/src/features/messages/ui/useComposerDictation.ts create mode 100644 desktop/src/shared/lib/pcmCapture.ts create mode 100644 desktop/tests/e2e/composer-dictation.spec.ts diff --git a/desktop/playwright.config.ts b/desktop/playwright.config.ts index 5046b29688..68d94313fc 100644 --- a/desktop/playwright.config.ts +++ b/desktop/playwright.config.ts @@ -53,6 +53,7 @@ export default defineConfig({ "**/file-attachment.spec.ts", "**/image-attachment-gallery.spec.ts", "**/composer-image-draw.spec.ts", + "**/composer-dictation.spec.ts", "**/video-attachment.spec.ts", "**/spoiler.spec.ts", "**/composer-link-shortcut.spec.ts", diff --git a/desktop/src-tauri/Info.plist b/desktop/src-tauri/Info.plist index c1b7331ebc..9f2e76d726 100644 --- a/desktop/src-tauri/Info.plist +++ b/desktop/src-tauri/Info.plist @@ -7,7 +7,7 @@ CFBundleName Buzz NSMicrophoneUsageDescription - Buzz needs microphone access for voice huddles. + Buzz needs microphone access for voice dictation and huddles. NSCameraUsageDescription Buzz needs camera access to record animated avatars. diff --git a/desktop/src-tauri/src/dictation.rs b/desktop/src-tauri/src/dictation.rs new file mode 100644 index 0000000000..a6f4906f82 --- /dev/null +++ b/desktop/src-tauri/src/dictation.rs @@ -0,0 +1,199 @@ +//! Standalone composer dictation backed by the local huddle STT engine. +//! +//! Dictation deliberately owns a separate pipeline and PCM command from +//! huddles. Microphone audio captured for a draft must never be fanned out to +//! the huddle relay, even when a huddle is active in the same process. + +use std::sync::{atomic::AtomicBool, Arc, LazyLock, Mutex, MutexGuard}; + +use serde::Serialize; +use tauri::{AppHandle, Emitter, State}; + +use crate::{ + app_state::AppState, + huddle::{ + models::{self, ModelStatus}, + stt::SttPipeline, + }, +}; + +const MAX_AUDIO_BATCH_BYTES: usize = 100 * 1024; + +/// Runtime state for the one process-wide composer dictation session. +#[derive(Default)] +struct DictationState { + pipeline: Option>, + generation: u64, +} + +static DICTATION_STATE: LazyLock> = + LazyLock::new(|| Mutex::new(DictationState::default())); + +fn dictation() -> Result, String> { + DICTATION_STATE.lock().map_err(|error| error.to_string()) +} + +#[derive(Clone, Serialize)] +#[serde(rename_all = "camelCase")] +struct DictationTranscript { + session_id: u64, + text: String, +} + +fn model_not_ready_message(status: ModelStatus) -> String { + match status { + ModelStatus::NotDownloaded => { + "Voice input is getting ready. The speech model download has started; try again shortly." + .to_string() + } + ModelStatus::Downloading { progress_percent } => format!( + "Voice input is getting ready ({progress_percent}% downloaded). Try again shortly." + ), + ModelStatus::Ready => "Speech model is not available yet.".to_string(), + ModelStatus::Error(error) => format!("Speech model download failed: {error}"), + } +} + +/// Start an isolated, continuous-VAD dictation session. +/// +/// Returns a monotonically increasing session id. Transcript events include +/// the same id so composers can ignore results from replaced sessions. +#[tauri::command] +pub async fn start_dictation(app: AppHandle, state: State<'_, AppState>) -> Result { + let manager = models::global_model_manager() + .ok_or("model manager unavailable (home directory could not be resolved)")?; + manager.start_stt_download(state.http_client.clone()); + let model_dir = manager + .stt_model_dir() + .ok_or_else(|| model_not_ready_message(manager.stt_status()))?; + + let (session_id, old_pipeline) = { + let mut dictation = dictation()?; + dictation.generation = dictation.generation.wrapping_add(1); + let session_id = dictation.generation; + let old_pipeline = dictation.pipeline.take(); + (session_id, old_pipeline) + }; + + if let Some(ref pipeline) = old_pipeline { + pipeline.shutdown(); + } + // SttPipeline::drop joins its worker, so never drop it while holding state. + drop(old_pipeline); + + let constructed = tokio::task::spawn_blocking(move || { + SttPipeline::new(model_dir, Arc::new(AtomicBool::new(false)), None, None) + }) + .await; + let (pipeline, mut text_rx) = match constructed { + Ok(Ok(result)) => result, + Ok(Err(error)) => return Err(error), + Err(error) => return Err(format!("failed to start dictation worker: {error}")), + }; + let pipeline = Arc::new(pipeline); + + let installed = { + let mut dictation = dictation()?; + if dictation.generation != session_id { + false + } else { + dictation.pipeline = Some(Arc::clone(&pipeline)); + true + } + }; + if !installed { + pipeline.shutdown(); + drop(pipeline); + return Err("dictation start was superseded".to_string()); + } + + tauri::async_runtime::spawn(async move { + while let Some(text) = text_rx.recv().await { + if text.trim().is_empty() { + continue; + } + let is_active = dictation() + .map(|dictation| dictation.generation == session_id && dictation.pipeline.is_some()) + .unwrap_or(false); + if !is_active { + break; + } + let _ = app.emit( + "dictation-transcript", + DictationTranscript { session_id, text }, + ); + } + }); + + Ok(session_id) +} + +/// Stop a dictation session if it is still current. +/// +/// Stale callers are ignored so cleanup from an old composer cannot stop a +/// newer session started in another channel or thread. +#[tauri::command] +pub fn stop_dictation(session_id: u64) -> Result<(), String> { + let old_pipeline = { + let mut dictation = dictation()?; + if dictation.generation != session_id { + return Ok(()); + } + dictation.generation = dictation.generation.wrapping_add(1); + dictation.pipeline.take() + }; + + if let Some(ref pipeline) = old_pipeline { + pipeline.shutdown(); + } + drop(old_pipeline); + Ok(()) +} + +/// Feed raw microphone PCM to composer dictation only. +/// +/// Expects f32 little-endian samples at 48 kHz mono. This command intentionally +/// does not touch huddle state or the huddle relay encoder. +#[tauri::command] +pub fn push_dictation_audio_pcm(request: tauri::ipc::Request<'_>) -> Result<(), String> { + match request.body() { + tauri::ipc::InvokeBody::Raw(bytes) => { + if bytes.len() > MAX_AUDIO_BATCH_BYTES { + return Err(format!( + "audio batch too large: {} bytes (max {})", + bytes.len(), + MAX_AUDIO_BATCH_BYTES + )); + } + if let Some(ref pipeline) = dictation()?.pipeline { + pipeline.push_audio(bytes.to_vec())?; + } + Ok(()) + } + _ => Err("expected raw binary body".to_string()), + } +} + +#[cfg(test)] +mod tests { + use super::model_not_ready_message; + use crate::huddle::models::ModelStatus; + + #[test] + fn download_progress_is_actionable() { + assert_eq!( + model_not_ready_message(ModelStatus::Downloading { + progress_percent: 42 + }), + "Voice input is getting ready (42% downloaded). Try again shortly." + ); + } + + #[test] + fn download_errors_are_preserved() { + assert_eq!( + model_not_ready_message(ModelStatus::Error("disk full".to_string())), + "Speech model download failed: disk full" + ); + } +} diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index 3b47866ac4..16e01d3e36 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -4,6 +4,7 @@ mod archive; mod builderlab; mod commands; mod deep_link; +mod dictation; mod event_sync; mod events; mod huddle; @@ -36,6 +37,7 @@ use deep_link::{ acknowledge_pending_community_deep_link, handle_deep_link_url, take_pending_community_deep_link, PendingCommunityDeepLinks, }; +use dictation::{push_dictation_audio_pcm, start_dictation, stop_dictation}; use huddle::audio_output::{ get_audio_output_device, list_audio_output_devices, set_audio_output_device, }; @@ -848,6 +850,9 @@ pub fn run() { get_note, get_note_reactions, get_liked_notes, + start_dictation, + stop_dictation, + push_dictation_audio_pcm, start_huddle, join_huddle, leave_huddle, diff --git a/desktop/src/features/huddle/lib/audioWorklet.ts b/desktop/src/features/huddle/lib/audioWorklet.ts index 677f4ac0a0..9ac856adea 100644 --- a/desktop/src/features/huddle/lib/audioWorklet.ts +++ b/desktop/src/features/huddle/lib/audioWorklet.ts @@ -1,20 +1,5 @@ import { listen, type UnlistenFn } from "@tauri-apps/api/event"; - -/** - * Raw binary invoke — uses Tauri's internal IPC for zero-copy ArrayBuffer transfer. - * - * The typed @tauri-apps/api doesn't support raw binary payloads (InvokeBody::Raw). - * This wrapper isolates the internal API dependency to a single call site. - * Tested against Tauri v2. If this breaks on upgrade, only this function needs updating. - */ -function invokeRawBinary(cmd: string, payload: Uint8Array): Promise { - // biome-ignore lint/suspicious/noExplicitAny: Tauri internals have no public type definition - const internals = (window as any).__TAURI_INTERNALS__; - if (!internals?.invoke) { - return Promise.reject(new Error("Tauri internals not available")); - } - return internals.invoke(cmd, payload); -} +import { setupPcmCapture } from "@/shared/lib/pcmCapture"; /** Return type for setupAudioWorklet — stop + mode control. */ export type AudioWorkletHandle = { @@ -52,50 +37,12 @@ export async function setupAudioWorklet( audioTrack: MediaStreamTrack, initialTransmitting = true, ): Promise { - const audioContext = new AudioContext({ sampleRate: 48000 }); - - // Resume after user gesture (required by autoplay policy) - if (audioContext.state === "suspended") { - await audioContext.resume(); - } - - // Load the worklet processor (must live in public/ for Vite to serve it) - await audioContext.audioWorklet.addModule("/worklet.js"); - - const source = audioContext.createMediaStreamSource( - new MediaStream([audioTrack]), + const capture = await setupPcmCapture( + audioTrack, + "push_audio_pcm", + initialTransmitting, ); - const gainNode = audioContext.createGain(); - - const workletNode = new AudioWorkletNode(audioContext, "stt-tap-processor"); - - // Connect: mic → gain → worklet (tap only — no playback) - source.connect(gainNode); - gainNode.connect(workletNode); - - // Set initial PTT state (worklet defaults to transmitting=true). - // In PTT mode, immediately gate audio until the user presses the key. - if (!initialTransmitting) { - workletNode.port.postMessage({ type: "ptt", active: false }); - } - - // Forward PCM batches to Rust via raw binary invoke. - // Direction: worklet→main (receives PCM data from worklet processor). - workletNode.port.onmessage = (event: MessageEvent) => { - const float32 = event.data; - // Fire-and-forget — Rust side uses try_send which drops on backpressure. - // No await: prevents main-thread backpressure from slow Rust processing. - // Create a zero-copy Uint8Array view over the same underlying buffer. - // Rust reinterprets the bytes as f32 on the other side. - invokeRawBinary( - "push_audio_pcm", - new Uint8Array(float32.buffer, float32.byteOffset, float32.byteLength), - ).catch(() => { - /* silently drop — Rust handles backpressure */ - }); - }; - // Track the current mode so PTT events are only forwarded in PTT mode. // In VAD mode, the worklet stays in transmitting=true regardless of // Ctrl+Space presses — prevents accidental muting. (Crossfire fix I1.) @@ -111,7 +58,7 @@ export async function setupAudioWorklet( // Only forward PTT events to the worklet when in PTT mode. // In VAD mode, Ctrl+Space is ignored — the worklet stays open. if (currentMode === "push_to_talk") { - workletNode.port.postMessage({ type: "ptt", active: event.payload }); + capture.setTransmitting(event.payload); } }); } catch { @@ -122,27 +69,20 @@ export async function setupAudioWorklet( return { stop: () => { - workletNode.port.onmessage = null; pttUnlisten?.(); - source.disconnect(); - gainNode.disconnect(); - workletNode.disconnect(); - void audioContext.close(); + capture.stop(); }, setTransmitting: (active: boolean) => { - workletNode.port.postMessage({ type: "ptt", active }); + capture.setTransmitting(active); }, setMode: (mode: "push_to_talk" | "voice_activity") => { currentMode = mode; // When switching to VAD, immediately open the mic. // When switching to PTT, immediately gate until key press. - workletNode.port.postMessage({ - type: "ptt", - active: mode === "voice_activity", - }); + capture.setTransmitting(mode === "voice_activity"); }, setGain: (value: number) => { - gainNode.gain.value = value; + capture.setGain(value); }, }; } diff --git a/desktop/src/features/messages/lib/dictationInsertion.test.mjs b/desktop/src/features/messages/lib/dictationInsertion.test.mjs new file mode 100644 index 0000000000..1d5dd8d326 --- /dev/null +++ b/desktop/src/features/messages/lib/dictationInsertion.test.mjs @@ -0,0 +1,29 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { buildDictationInsertion } from "./dictationInsertion.ts"; + +test("inserts the first transcript without a leading space", () => { + assert.equal( + buildDictationInsertion("", " Hello from voice. "), + "Hello from voice. ", + ); +}); + +test("separates dictation from existing text", () => { + assert.equal( + buildDictationInsertion("d", "Continue this thought."), + " Continue this thought. ", + ); +}); + +test("does not double-space at an existing boundary", () => { + assert.equal( + buildDictationInsertion(" ", "Continue this thought."), + "Continue this thought. ", + ); +}); + +test("ignores an empty recognition result", () => { + assert.equal(buildDictationInsertion("x", " \n "), ""); +}); diff --git a/desktop/src/features/messages/lib/dictationInsertion.ts b/desktop/src/features/messages/lib/dictationInsertion.ts new file mode 100644 index 0000000000..96d5e4e757 --- /dev/null +++ b/desktop/src/features/messages/lib/dictationInsertion.ts @@ -0,0 +1,17 @@ +/** + * Normalize one finalized STT segment for insertion at the editor selection. + * + * Parakeet emits complete utterances without leading/trailing whitespace. Add + * a boundary when dictating after existing text and retain one trailing space + * so the next finalized segment joins naturally. + */ +export function buildDictationInsertion( + previousCharacter: string, + transcript: string, +): string { + const normalized = transcript.trim(); + if (!normalized) return ""; + const prefix = + previousCharacter.length > 0 && !/\s/.test(previousCharacter) ? " " : ""; + return `${prefix}${normalized} `; +} diff --git a/desktop/src/features/messages/ui/MessageComposer.tsx b/desktop/src/features/messages/ui/MessageComposer.tsx index 8c0c574049..4c7bca9a6b 100644 --- a/desktop/src/features/messages/ui/MessageComposer.tsx +++ b/desktop/src/features/messages/ui/MessageComposer.tsx @@ -60,6 +60,7 @@ import { useMentionSendFlow } from "./useMentionSendFlow"; import { usePersistentAgentMentionHydration } from "./usePersistentAgentMentionHydration"; import { useComposerContentState } from "./useComposerContentState"; import { useDraftPersistLifecycle } from "./useDraftPersistSnapshot"; +import { useComposerDictation } from "./useComposerDictation"; type MessageComposerAudienceContext = { type: "thread"; @@ -358,8 +359,12 @@ function MessageComposerImpl({ } }, }); - const linkEditor = useLinkEditor(richText); + const dictation = useComposerDictation({ + disabled, + editor: richText.editor, + sessionKey: effectiveDraftKey, + }); syncContentRefFromEditorRef.current = () => { const markdown = richText.getMarkdown(); contentRef.current = markdown; @@ -598,6 +603,7 @@ function MessageComposerImpl({ // ── Submit message ────────────────────────────────────────────────── const submitMessage = React.useCallback(async () => { + if (dictation.isActive) return; const trimmed = syncComposerContentFromEditor().trim(); // Edit mode @@ -712,6 +718,7 @@ function MessageComposerImpl({ channelId, channelLinks.clearChannels, customEmoji, + dictation.isActive, drafts.loadDraft, emojiAutocomplete.clearEmojis, media.pendingImetaRef, @@ -913,21 +920,12 @@ function MessageComposerImpl({ }); }, [media.setPendingImeta, richText.editor, scrollComposerToBottom]); - // ── Send button state ─────────────────────────────────────────────── - const sendDisabled = React.useMemo( - () => - disabled || - media.isUploading || - mentionSendFlow.isPreparingMentionSend || - (isContentEmpty && media.pendingImeta.length === 0), - [ - disabled, - media.isUploading, - mentionSendFlow.isPreparingMentionSend, - isContentEmpty, - media.pendingImeta.length, - ], - ); + const sendDisabled = + disabled || + dictation.isActive || + media.isUploading || + mentionSendFlow.isPreparingMentionSend || + (isContentEmpty && media.pendingImeta.length === 0); const handleCaptureSelection = React.useCallback(() => { // No-op for Tiptap — selection is managed by ProseMirror. @@ -1077,6 +1075,7 @@ function MessageComposerImpl({ editor={richText.editor} extraActions={toolbarExtraActions} formattingDisabled={disabled} + dictationStatus={dictation.status} isEmojiPickerOpen={isEmojiPickerOpen} isFormattingOpen={isFormattingOpen} isSending={isSending} @@ -1085,6 +1084,7 @@ function MessageComposerImpl({ onEmojiPickerOpenChange={setIsEmojiPickerOpen} onEmojiSelect={insertEmoji} onFormattingToggle={handleFormattingToggle} + onDictationToggle={dictation.toggle} onLinkButton={linkEditor.openFromToolbar} onOpenMentionPicker={openMentionPicker} onPaperclip={handlePaperclipClick} diff --git a/desktop/src/features/messages/ui/MessageComposerToolbar.tsx b/desktop/src/features/messages/ui/MessageComposerToolbar.tsx index 1c8087e992..1ff1f00ce2 100644 --- a/desktop/src/features/messages/ui/MessageComposerToolbar.tsx +++ b/desktop/src/features/messages/ui/MessageComposerToolbar.tsx @@ -1,13 +1,14 @@ import * as React from "react"; import type { Editor } from "@tiptap/react"; import { AnimatePresence, motion } from "motion/react"; -import { ALargeSmall, ArrowUp, AtSign, Paperclip, X } from "lucide-react"; +import { ALargeSmall, ArrowUp, AtSign, Mic, Paperclip, X } from "lucide-react"; import { Button } from "@/shared/ui/button"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/shared/ui/tooltip"; import { ComposerEmojiPicker } from "./ComposerEmojiPicker"; import { FormattingToolbar } from "./FormattingToolbar"; import { SelectionFormattingTray } from "./SelectionFormattingTray"; +import type { ComposerDictationStatus } from "./useComposerDictation"; /** Spring for enter/exit of button groups — all fire simultaneously. */ const presenceSpring = { @@ -22,6 +23,7 @@ export const MessageComposerToolbar = React.memo( editor, extraActions, formattingDisabled, + dictationStatus = "idle", isEmojiPickerOpen, isFormattingOpen, isSending, @@ -30,6 +32,7 @@ export const MessageComposerToolbar = React.memo( onEmojiPickerOpenChange, onEmojiSelect, onFormattingToggle, + onDictationToggle, onLinkButton, onOpenMentionPicker, onPaperclip, @@ -39,6 +42,7 @@ export const MessageComposerToolbar = React.memo( editor: Editor | null; extraActions?: React.ReactNode; formattingDisabled: boolean; + dictationStatus?: ComposerDictationStatus; isEmojiPickerOpen: boolean; isFormattingOpen: boolean; isSending: boolean; @@ -47,6 +51,7 @@ export const MessageComposerToolbar = React.memo( onEmojiPickerOpenChange: (open: boolean) => void; onEmojiSelect: (emoji: string) => void; onFormattingToggle: (pressed: boolean) => void; + onDictationToggle?: () => void; onLinkButton: () => void; onOpenMentionPicker: () => void; onPaperclip: () => void; @@ -231,6 +236,61 @@ export const MessageComposerToolbar = React.memo(
{extraActions} + {onDictationToggle ? ( + + + + + + {dictationStatus === "recording" + ? "Stop dictation" + : dictationStatus === "stopping" + ? "Finishing dictation" + : dictationStatus === "starting" + ? "Starting dictation" + : "Dictate message"} + + + ) : null}