feat(desktop): add local voice dictation to message composer

desktop/playwright.config.ts

@@ -53,6 +53,7 @@ export default defineConfig({
         "**/file-attachment.spec.ts",
         "**/image-attachment-gallery.spec.ts",
         "**/composer-image-draw.spec.ts",
+        "**/composer-dictation.spec.ts",
         "**/video-attachment.spec.ts",
         "**/spoiler.spec.ts",
         "**/composer-link-shortcut.spec.ts",

desktop/src-tauri/Info.plist

@@ -7,7 +7,7 @@
     <key>CFBundleName</key>
     <string>Buzz</string>
     <key>NSMicrophoneUsageDescription</key>
-    <string>Buzz needs microphone access for voice huddles.</string>
+    <string>Buzz needs microphone access for voice dictation and huddles.</string>
     <key>NSCameraUsageDescription</key>
     <string>Buzz needs camera access to record animated avatars.</string>
 </dict>

desktop/src-tauri/src/dictation.rs

@@ -0,0 +1,199 @@
+//! Standalone composer dictation backed by the local huddle STT engine.
+//!
+//! Dictation deliberately owns a separate pipeline and PCM command from
+//! huddles. Microphone audio captured for a draft must never be fanned out to
+//! the huddle relay, even when a huddle is active in the same process.
+
+use std::sync::{atomic::AtomicBool, Arc, LazyLock, Mutex, MutexGuard};
+
+use serde::Serialize;
+use tauri::{AppHandle, Emitter, State};
+
+use crate::{
+    app_state::AppState,
+    huddle::{
+        models::{self, ModelStatus},
+        stt::SttPipeline,
+    },
+};
+
+const MAX_AUDIO_BATCH_BYTES: usize = 100 * 1024;
+
+/// Runtime state for the one process-wide composer dictation session.
+#[derive(Default)]
+struct DictationState {
+    pipeline: Option<Arc<SttPipeline>>,
+    generation: u64,
+}
+
+static DICTATION_STATE: LazyLock<Mutex<DictationState>> =
+    LazyLock::new(|| Mutex::new(DictationState::default()));
+
+fn dictation() -> Result<MutexGuard<'static, DictationState>, String> {
+    DICTATION_STATE.lock().map_err(|error| error.to_string())
+}
+
+#[derive(Clone, Serialize)]
+#[serde(rename_all = "camelCase")]
+struct DictationTranscript {
+    session_id: u64,
+    text: String,
+}
+
+fn model_not_ready_message(status: ModelStatus) -> String {
+    match status {
+        ModelStatus::NotDownloaded => {
+            "Voice input is getting ready. The speech model download has started; try again shortly."
+                .to_string()
+        }
+        ModelStatus::Downloading { progress_percent } => format!(
+            "Voice input is getting ready ({progress_percent}% downloaded). Try again shortly."
+        ),
+        ModelStatus::Ready => "Speech model is not available yet.".to_string(),
+        ModelStatus::Error(error) => format!("Speech model download failed: {error}"),
+    }
+}
+
+/// Start an isolated, continuous-VAD dictation session.
+///
+/// Returns a monotonically increasing session id. Transcript events include
+/// the same id so composers can ignore results from replaced sessions.
+#[tauri::command]
+pub async fn start_dictation(app: AppHandle, state: State<'_, AppState>) -> Result<u64, String> {
+    let manager = models::global_model_manager()
+        .ok_or("model manager unavailable (home directory could not be resolved)")?;
+    manager.start_stt_download(state.http_client.clone());
+    let model_dir = manager
+        .stt_model_dir()
+        .ok_or_else(|| model_not_ready_message(manager.stt_status()))?;
+
+    let (session_id, old_pipeline) = {
+        let mut dictation = dictation()?;
+        dictation.generation = dictation.generation.wrapping_add(1);
+        let session_id = dictation.generation;
+        let old_pipeline = dictation.pipeline.take();
+        (session_id, old_pipeline)
+    };
+
+    if let Some(ref pipeline) = old_pipeline {
+        pipeline.shutdown();
+    }
+    // SttPipeline::drop joins its worker, so never drop it while holding state.
+    drop(old_pipeline);
+
+    let constructed = tokio::task::spawn_blocking(move || {
+        SttPipeline::new(model_dir, Arc::new(AtomicBool::new(false)), None, None)
+    })
+    .await;
+    let (pipeline, mut text_rx) = match constructed {
+        Ok(Ok(result)) => result,
+        Ok(Err(error)) => return Err(error),
+        Err(error) => return Err(format!("failed to start dictation worker: {error}")),
+    };
+    let pipeline = Arc::new(pipeline);
+
+    let installed = {
+        let mut dictation = dictation()?;
+        if dictation.generation != session_id {
+            false
+        } else {
+            dictation.pipeline = Some(Arc::clone(&pipeline));
+            true
+        }
+    };
+    if !installed {
+        pipeline.shutdown();
+        drop(pipeline);
+        return Err("dictation start was superseded".to_string());
+    }
+
+    tauri::async_runtime::spawn(async move {
+        while let Some(text) = text_rx.recv().await {
+            if text.trim().is_empty() {
+                continue;
+            }
+            let is_active = dictation()
+                .map(|dictation| dictation.generation == session_id && dictation.pipeline.is_some())
+                .unwrap_or(false);
+            if !is_active {
+                break;
+            }
+            let _ = app.emit(
+                "dictation-transcript",
+                DictationTranscript { session_id, text },
+            );
+        }
+    });
+
+    Ok(session_id)
+}
+
+/// Stop a dictation session if it is still current.
+///
+/// Stale callers are ignored so cleanup from an old composer cannot stop a
+/// newer session started in another channel or thread.
+#[tauri::command]
+pub fn stop_dictation(session_id: u64) -> Result<(), String> {
+    let old_pipeline = {
+        let mut dictation = dictation()?;
+        if dictation.generation != session_id {
+            return Ok(());
+        }
+        dictation.generation = dictation.generation.wrapping_add(1);
+        dictation.pipeline.take()
+    };
+
+    if let Some(ref pipeline) = old_pipeline {
+        pipeline.shutdown();
+    }
+    drop(old_pipeline);
+    Ok(())
+}
+
+/// Feed raw microphone PCM to composer dictation only.
+///
+/// Expects f32 little-endian samples at 48 kHz mono. This command intentionally
+/// does not touch huddle state or the huddle relay encoder.
+#[tauri::command]
+pub fn push_dictation_audio_pcm(request: tauri::ipc::Request<'_>) -> Result<(), String> {
+    match request.body() {
+        tauri::ipc::InvokeBody::Raw(bytes) => {
+            if bytes.len() > MAX_AUDIO_BATCH_BYTES {
+                return Err(format!(
+                    "audio batch too large: {} bytes (max {})",
+                    bytes.len(),
+                    MAX_AUDIO_BATCH_BYTES
+                ));
+            }
+            if let Some(ref pipeline) = dictation()?.pipeline {
+                pipeline.push_audio(bytes.to_vec())?;
+            }
+            Ok(())
+        }
+        _ => Err("expected raw binary body".to_string()),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::model_not_ready_message;
+    use crate::huddle::models::ModelStatus;
+
+    #[test]
+    fn download_progress_is_actionable() {
+        assert_eq!(
+            model_not_ready_message(ModelStatus::Downloading {
+                progress_percent: 42
+            }),
+            "Voice input is getting ready (42% downloaded). Try again shortly."
+        );
+    }
+
+    #[test]
+    fn download_errors_are_preserved() {
+        assert_eq!(
+            model_not_ready_message(ModelStatus::Error("disk full".to_string())),
+            "Speech model download failed: disk full"
+        );
+    }
+}

desktop/src-tauri/src/lib.rs

@@ -4,6 +4,7 @@ mod archive;
 mod builderlab;
 mod commands;
 mod deep_link;
+mod dictation;
 mod event_sync;
 mod events;
 mod huddle;
@@ -36,6 +37,7 @@ use deep_link::{
     acknowledge_pending_community_deep_link, handle_deep_link_url,
     take_pending_community_deep_link, PendingCommunityDeepLinks,
 };
+use dictation::{push_dictation_audio_pcm, start_dictation, stop_dictation};
 use huddle::audio_output::{
     get_audio_output_device, list_audio_output_devices, set_audio_output_device,
 };
@@ -848,6 +850,9 @@ pub fn run() {
             get_note,
             get_note_reactions,
             get_liked_notes,
+            start_dictation,
+            stop_dictation,
+            push_dictation_audio_pcm,
             start_huddle,
             join_huddle,
             leave_huddle,

desktop/src/features/huddle/lib/audioWorklet.ts

@@ -1,20 +1,5 @@
 import { listen, type UnlistenFn } from "@tauri-apps/api/event";
-
-/**
- * Raw binary invoke — uses Tauri's internal IPC for zero-copy ArrayBuffer transfer.
- *
- * The typed @tauri-apps/api doesn't support raw binary payloads (InvokeBody::Raw).
- * This wrapper isolates the internal API dependency to a single call site.
- * Tested against Tauri v2. If this breaks on upgrade, only this function needs updating.
- */
-function invokeRawBinary(cmd: string, payload: Uint8Array): Promise<unknown> {
-  // biome-ignore lint/suspicious/noExplicitAny: Tauri internals have no public type definition
-  const internals = (window as any).__TAURI_INTERNALS__;
-  if (!internals?.invoke) {
-    return Promise.reject(new Error("Tauri internals not available"));
-  }
-  return internals.invoke(cmd, payload);
-}
+import { setupPcmCapture } from "@/shared/lib/pcmCapture";
 
 /** Return type for setupAudioWorklet — stop + mode control. */
 export type AudioWorkletHandle = {
@@ -52,50 +37,12 @@ export async function setupAudioWorklet(
   audioTrack: MediaStreamTrack,
   initialTransmitting = true,
 ): Promise<AudioWorkletHandle> {
-  const audioContext = new AudioContext({ sampleRate: 48000 });
-
-  // Resume after user gesture (required by autoplay policy)
-  if (audioContext.state === "suspended") {
-    await audioContext.resume();
-  }
-
-  // Load the worklet processor (must live in public/ for Vite to serve it)
-  await audioContext.audioWorklet.addModule("/worklet.js");
-
-  const source = audioContext.createMediaStreamSource(
-    new MediaStream([audioTrack]),
+  const capture = await setupPcmCapture(
+    audioTrack,
+    "push_audio_pcm",
+    initialTransmitting,
   );
 
-  const gainNode = audioContext.createGain();
-
-  const workletNode = new AudioWorkletNode(audioContext, "stt-tap-processor");
-
-  // Connect: mic → gain → worklet (tap only — no playback)
-  source.connect(gainNode);
-  gainNode.connect(workletNode);
-
-  // Set initial PTT state (worklet defaults to transmitting=true).
-  // In PTT mode, immediately gate audio until the user presses the key.
-  if (!initialTransmitting) {
-    workletNode.port.postMessage({ type: "ptt", active: false });
-  }
-
-  // Forward PCM batches to Rust via raw binary invoke.
-  // Direction: worklet→main (receives PCM data from worklet processor).
-  workletNode.port.onmessage = (event: MessageEvent<Float32Array>) => {
-    const float32 = event.data;
-    // Fire-and-forget — Rust side uses try_send which drops on backpressure.
-    // No await: prevents main-thread backpressure from slow Rust processing.
-    // Create a zero-copy Uint8Array view over the same underlying buffer.
-    // Rust reinterprets the bytes as f32 on the other side.
-    invokeRawBinary(
-      "push_audio_pcm",
-      new Uint8Array(float32.buffer, float32.byteOffset, float32.byteLength),
-    ).catch(() => {
-      /* silently drop — Rust handles backpressure */
-    });
-  };
-
   // Track the current mode so PTT events are only forwarded in PTT mode.
   // In VAD mode, the worklet stays in transmitting=true regardless of
   // Ctrl+Space presses — prevents accidental muting. (Crossfire fix I1.)
@@ -111,7 +58,7 @@ export async function setupAudioWorklet(
       // Only forward PTT events to the worklet when in PTT mode.
       // In VAD mode, Ctrl+Space is ignored — the worklet stays open.
       if (currentMode === "push_to_talk") {
-        workletNode.port.postMessage({ type: "ptt", active: event.payload });
+        capture.setTransmitting(event.payload);
       }
     });
   } catch {
@@ -122,27 +69,20 @@ export async function setupAudioWorklet(
 
   return {
     stop: () => {
-      workletNode.port.onmessage = null;
       pttUnlisten?.();
-      source.disconnect();
-      gainNode.disconnect();
-      workletNode.disconnect();
-      void audioContext.close();
+      capture.stop();
     },
     setTransmitting: (active: boolean) => {
-      workletNode.port.postMessage({ type: "ptt", active });
+      capture.setTransmitting(active);
     },
     setMode: (mode: "push_to_talk" | "voice_activity") => {
       currentMode = mode;
       // When switching to VAD, immediately open the mic.
       // When switching to PTT, immediately gate until key press.
-      workletNode.port.postMessage({
-        type: "ptt",
-        active: mode === "voice_activity",
-      });
+      capture.setTransmitting(mode === "voice_activity");
     },
     setGain: (value: number) => {
-      gainNode.gain.value = value;
+      capture.setGain(value);
     },
   };
 }

desktop/src/features/messages/lib/dictationInsertion.test.mjs

@@ -0,0 +1,29 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { buildDictationInsertion } from "./dictationInsertion.ts";
+
+test("inserts the first transcript without a leading space", () => {
+  assert.equal(
+    buildDictationInsertion("", "  Hello from voice.  "),
+    "Hello from voice. ",
+  );
+});
+
+test("separates dictation from existing text", () => {
+  assert.equal(
+    buildDictationInsertion("d", "Continue this thought."),
+    " Continue this thought. ",
+  );
+});
+
+test("does not double-space at an existing boundary", () => {
+  assert.equal(
+    buildDictationInsertion(" ", "Continue this thought."),
+    "Continue this thought. ",
+  );
+});
+
+test("ignores an empty recognition result", () => {
+  assert.equal(buildDictationInsertion("x", " \n "), "");
+});