feat(stt): add local speech dictation with Parakeet TDT

package.json

@@ -236,7 +236,13 @@
   },
   "optionalDependencies": {
     "@rollup/rollup-win32-x64-msvc": "^4.46.2",
-    "electron-winstaller": "^5.4.0"
+    "electron-winstaller": "^5.4.0",
+    "sherpa-onnx": "1.13.8",
+    "sherpa-onnx-darwin-arm64": "1.13.8",
+    "sherpa-onnx-darwin-x64": "1.13.8",
+    "sherpa-onnx-linux-arm64": "1.13.8",
+    "sherpa-onnx-linux-x64": "1.13.8",
+    "sherpa-onnx-win-x64": "1.13.8"
   },
   "resolutions": {
     "@codemirror/language": "6.12.3",

packages/desktop/src/common/adapter/ipcBridge.ts

@@ -49,6 +49,7 @@ import type {
   ProviderHealthCheckResponse,
   UpdateProviderRequest,
 } from '../types/provider/providerApi';
+import type { SpeechModelDownloadStatus, SpeechToTextResult } from '../types/provider/speech';
 import type {
   ITeamAgentRemovedEvent,
   ITeamAgentRenamedEvent,
@@ -2504,3 +2505,24 @@ export const sidebar = {
     (p) => `/api/sidebar/archived/project/${encodeURIComponent(p.project_id)}`
   ),
 };
+
+// ---------------------------------------------------------------------------
+// Local Speech-to-Text (Parakeet TDT)
+// ---------------------------------------------------------------------------
+
+export const speech = {
+  checkModel: bridge.buildProvider<{ isReady: boolean; status: SpeechModelDownloadStatus }, { modelId: string }>(
+    'speech:checkModel'
+  ),
+  downloadModel: bridge.buildProvider<SpeechModelDownloadStatus, { modelId: string }>('speech:downloadModel'),
+  cancelDownload: bridge.buildProvider<boolean, { modelId: string }>('speech:cancelDownload'),
+  transcribe: bridge.buildProvider<SpeechToTextResult, { audioBuffer: number[] | Uint8Array; modelId?: string }>(
+    'speech:transcribe'
+  ),
+  onDownloadProgress: bridge.buildEmitter<{
+    downloadedBytes: number;
+    modelId: string;
+    percent: number;
+    totalBytes: number;
+  }>('speech:downloadProgress'),
+};

packages/desktop/src/common/types/provider/speech.ts

@@ -4,7 +4,24 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-export type SpeechToTextProvider = 'openai' | 'deepgram';
+export type SpeechToTextProvider = 'openai' | 'deepgram' | 'local';
+
+export type LocalSpeechModelId = 'parakeet-tdt-0.6b-v3-int8' | 'parakeet-tdt-0.6b-v2-int8';
+
+export type LocalSpeechToTextConfig = {
+  hotwords?: string;
+  language?: string;
+  model: LocalSpeechModelId | string;
+};
+
+export type SpeechModelDownloadStatus = {
+  downloadedBytes: number;
+  error?: string;
+  modelId: string;
+  progress: number;
+  status: 'idle' | 'downloading' | 'ready' | 'error';
+  totalBytes: number;
+};
 
 export type OpenAISpeechToTextConfig = {
   api_key: string;
@@ -27,10 +44,11 @@ export type DeepgramSpeechToTextConfig = {
 
 export type SpeechToTextConfig = {
   autoSend?: boolean;
-  enabled: boolean;
-  provider: SpeechToTextProvider;
   deepgram?: DeepgramSpeechToTextConfig;
+  enabled: boolean;
+  local?: LocalSpeechToTextConfig;
   openai?: OpenAISpeechToTextConfig;
+  provider: SpeechToTextProvider;
 };
 
 export type SpeechToTextAudioBuffer = Uint8Array | number[] | Record<string, number>;

packages/desktop/src/process/bridge/index.ts

@@ -12,6 +12,7 @@ import { initWindowControlsBridge } from './windowControlsBridge';
 import { initNotificationBridge } from './notificationBridge';
 import { initWebuiBridge } from './webuiBridge';
 import { initThemeBridge } from './themeBridge';
+import { initSpeechBridge } from './speechBridge';
 
 export type BridgeDependencies = Record<string, never>;
 
@@ -24,12 +25,14 @@ export function initAllBridges(_deps: BridgeDependencies = {}): void {
   initNotificationBridge();
   initWebuiBridge();
   initThemeBridge();
+  initSpeechBridge();
 }
 
 export {
   initApplicationBridge,
   initDialogBridge,
   initNotificationBridge,
+  initSpeechBridge,
   initSystemSettingsBridge,
   initThemeBridge,
   initUpdateBridge,

packages/desktop/src/process/bridge/speechBridge.ts

@@ -0,0 +1,37 @@
+/**
+ * @license
+ * Copyright 2026 AionUi (aionui.com)
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { ipcBridge } from '@/common';
+import {
+  cancelModelDownload,
+  downloadModel,
+  getModelStatus,
+  isModelReady,
+  transcribeLocalAudio,
+} from '../services/speech';
+
+export function initSpeechBridge(): void {
+  ipcBridge.speech.checkModel.provider(async ({ modelId }) => {
+    const isReady = await isModelReady(modelId);
+    const status = getModelStatus(modelId);
+    return { isReady, status };
+  });
+
+  ipcBridge.speech.downloadModel.provider(async ({ modelId }) => {
+    return downloadModel(modelId, (progress) => {
+      ipcBridge.speech.onDownloadProgress.emit(progress);
+    });
+  });
+
+  ipcBridge.speech.cancelDownload.provider(({ modelId }) => {
+    return cancelModelDownload(modelId);
+  });
+
+  ipcBridge.speech.transcribe.provider(async ({ audioBuffer, modelId }) => {
+    const bytes = audioBuffer instanceof Uint8Array ? audioBuffer : new Uint8Array(audioBuffer);
+    return transcribeLocalAudio(bytes, modelId);
+  });
+}

packages/desktop/src/process/services/speech/index.ts

@@ -0,0 +1,11 @@
+/**
+ * @license
+ * Copyright 2026 AionUi (aionui.com)
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+export * from './types';
+export * from './modelCatalog';
+export * from './sherpaLoader';
+export * from './modelManager';
+export * from './localSpeechService';

packages/desktop/src/process/services/speech/localSpeechService.ts

@@ -0,0 +1,167 @@
+/**
+ * @license
+ * Copyright 2026 AionUi (aionui.com)
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import path from 'node:path';
+import type { SpeechToTextResult } from '@/common/types/provider/speech';
+import { getLocalSpeechModelManifest } from './modelCatalog';
+import { getModelStorageDir, isModelReady } from './modelManager';
+import { loadSherpaAddon } from './sherpaLoader';
+
+type CachedRecognizer = {
+  modelId: string;
+  recognizer: any;
+};
+
+let activeRecognizerCache: CachedRecognizer | null = null;
+
+function getOrInitRecognizer(modelId: string, sherpa: any): any {
+  if (activeRecognizerCache && activeRecognizerCache.modelId === modelId) {
+    return activeRecognizerCache.recognizer;
+  }
+
+  const manifest = getLocalSpeechModelManifest(modelId);
+  if (!manifest) {
+    throw new Error(`STT_UNKNOWN_MODEL: ${modelId}`);
+  }
+
+  const modelDir = getModelStorageDir(modelId);
+  const encoderPath = path.join(modelDir, 'encoder.int8.onnx');
+  const decoderPath = path.join(modelDir, 'decoder.int8.onnx');
+  const joinerPath = path.join(modelDir, 'joiner.int8.onnx');
+  const tokensPath = path.join(modelDir, 'tokens.txt');
+
+  const config = {
+    decodingMethod: 'greedy_search',
+    featConfig: {
+      featureDim: 80,
+      sampleRate: manifest.sampleRate,
+    },
+    modelConfig: {
+      debug: 0,
+      numThreads: 4,
+      provider: 'cpu',
+      tokens: tokensPath,
+      transducer: {
+        decoder: decoderPath,
+        encoder: encoderPath,
+        joiner: joinerPath,
+      },
+    },
+  };
+
+  const recognizer = sherpa.createOfflineRecognizer(config);
+  activeRecognizerCache = {
+    modelId,
+    recognizer,
+  };
+
+  return recognizer;
+}
+
+function toBuffer(audioBytes: Uint8Array | Buffer | number[] | Record<string, number>): Buffer {
+  if (Buffer.isBuffer(audioBytes)) {
+    return audioBytes;
+  }
+  if (audioBytes instanceof Uint8Array) {
+    return Buffer.from(audioBytes.buffer, audioBytes.byteOffset, audioBytes.byteLength);
+  }
+  if (Array.isArray(audioBytes)) {
+    return Buffer.from(audioBytes);
+  }
+  if (typeof audioBytes === 'object' && audioBytes !== null) {
+    const keys = Object.keys(audioBytes);
+    const buf = Buffer.alloc(keys.length);
+    for (let i = 0; i < keys.length; i++) {
+      buf[i] = (audioBytes as Record<string, number>)[i];
+    }
+    return buf;
+  }
+  return Buffer.alloc(0);
+}
+
+export async function transcribeLocalAudio(
+  audioBytes: Uint8Array | Buffer | number[] | Record<string, number>,
+  modelId = 'parakeet-tdt-0.6b-v3-int8'
+): Promise<SpeechToTextResult> {
+  const ready = await isModelReady(modelId);
+  if (!ready) {
+    throw new Error('STT_LOCAL_MODEL_NOT_DOWNLOADED');
+  }
+
+  const sherpa = loadSherpaAddon();
+  const recognizer = getOrInitRecognizer(modelId, sherpa);
+  const manifest = getLocalSpeechModelManifest(modelId);
+  const targetSampleRate = manifest?.sampleRate ?? 16000;
+
+  const nodeBuffer = toBuffer(audioBytes);
+
+  let samples: Float32Array;
+  let sampleRate: number;
+
+  try {
+    if (typeof sherpa.readWaveFromBinary === 'function') {
+      const wave = sherpa.readWaveFromBinary(nodeBuffer);
+      samples = wave.samples;
+      sampleRate = wave.sampleRate;
+    } else if (typeof sherpa.readWaveFromBinaryData === 'function') {
+      const wave = sherpa.readWaveFromBinaryData(nodeBuffer);
+      samples = wave.samples;
+      sampleRate = wave.sampleRate;
+    } else {
+      throw new Error('No wave parser available');
+    }
+  } catch {
+    // If not a valid WAV header or parsing failed, treat as raw 16-bit PCM (little endian)
+    const int16 = new Int16Array(nodeBuffer.buffer, nodeBuffer.byteOffset, nodeBuffer.byteLength / 2);
+    samples = new Float32Array(int16.length);
+    for (let i = 0; i < int16.length; i++) {
+      samples[i] = int16[i] / 32768.0;
+    }
+    sampleRate = targetSampleRate;
+  }
+
+  let text = '';
+  if (typeof recognizer.createStream === 'function') {
+    // OOP JS / WASM interface
+    const stream = recognizer.createStream();
+    try {
+      stream.acceptWaveform(sampleRate, samples);
+      recognizer.decode(stream);
+      const result = recognizer.getResult(stream);
+      const parsed = typeof result === 'string' ? JSON.parse(result) : result;
+      text = (parsed?.text ?? '').trim();
+    } finally {
+      try {
+        stream.free?.();
+      } catch {
+        // ignore
+      }
+    }
+  } else {
+    // Native C++ NAPI addon interface
+    const stream = sherpa.createOfflineStream(recognizer);
+    try {
+      sherpa.acceptWaveformOffline(stream, { sampleRate, samples });
+      sherpa.decodeOfflineStream(recognizer, stream);
+      const resultJson = sherpa.getOfflineStreamResultAsJson(stream);
+      const parsed = typeof resultJson === 'string' ? JSON.parse(resultJson) : resultJson;
+      text = (parsed?.text ?? '').trim();
+    } finally {
+      // Native addon manages memory or GC
+    }
+  }
+
+  return {
+    language: manifest?.language,
+    model: modelId,
+    provider: 'local',
+    text,
+  };
+}
+
+export function clearLocalRecognizerCache(): void {
+  activeRecognizerCache = null;
+}