From 43b7f3a8c3aba1d781cba803bf4cf1b4281ef8df Mon Sep 17 00:00:00 2001 From: Nirmalkumar Rodrigues <47662869+Nk-rodrigues@users.noreply.github.com> Date: Fri, 16 Oct 2026 22:09:19 +0530 Subject: [PATCH] =?UTF-8?q?=F0=9F=97=A3=EF=B8=8F=20feat:=20First-Class=20g?= =?UTF-8?q?pt-transcribe=20and=20Live=20gpt-live-transcribe=20Speech-to-Te?= =?UTF-8?q?xt?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Recorded transcription: - Send `prompt`, `keywords[]`, and `languages[]` to gpt-transcribe models and never the singular `language`; older models keep `language` and `prompt`. - The user's speech language narrows the admin `languages` list. - Responses carrying detected-language metadata still resolve to `text`. Live transcription (opt-in via `speech.stt.openai.realtime`): - New "Live" engine under Settings > Speech > Engine, offered only when the admin configures it; `speechTab.speechToText.engineSTT: realtime` defaults it. - POST /api/files/speech/stt/realtime exchanges the browser's WebRTC offer for the provider's answer using a 60s client secret, so no credential reaches the browser. Metered by the existing STT limiters and SSRF-guarded. - Deltas stream into the composer; Stop commits the buffer and the final transcript stays editable. Auto Transcribe, Auto Send, cancel, connecting and reconnecting states, and permission/connection/provider errors are handled. Shared silence detection moves to `hooks/Input/silence.ts`. --- api/server/routes/files/speech/index.spec.js | 88 +++ api/server/routes/files/speech/stt.js | 2 + api/server/services/Files/Audio/STTService.js | 11 +- .../services/Files/Audio/STTService.spec.js | 82 ++- .../Files/Audio/getCustomConfigSpeech.js | 2 + .../Files/Audio/getCustomConfigSpeech.spec.js | 47 ++ client/src/common/types.ts | 2 + .../components/Chat/Input/Composer/Bar.tsx | 14 +- .../Nav/Settings/SpeechControls.tsx | 7 +- .../Speech/STT/EngineSTTDropdown.tsx | 17 +- .../STT/__tests__/EngineSTTDropdown.spec.tsx | 84 +++ client/src/data-provider/mutations.ts | 16 + .../__tests__/useSpeechSettingsInit.spec.tsx | 64 ++ .../src/hooks/Config/useSpeechSettingsInit.ts | 8 +- .../src/hooks/Input/__tests__/silence.spec.ts | 109 ++++ .../Input/__tests__/useDictation.spec.tsx | 76 ++- .../useSpeechToTextRealtime.spec.tsx | 585 ++++++++++++++++++ client/src/hooks/Input/silence.ts | 59 ++ client/src/hooks/Input/useDictation.ts | 23 +- .../src/hooks/Input/useSpeechToText.spec.ts | 43 ++ client/src/hooks/Input/useSpeechToText.ts | 24 + .../hooks/Input/useSpeechToTextExternal.ts | 86 +-- .../hooks/Input/useSpeechToTextRealtime.ts | 475 ++++++++++++++ client/src/locales/en/translation.json | 8 + client/src/store/settings.ts | 12 +- librechat.example.yaml | 22 +- packages/api/src/files/index.ts | 2 + packages/api/src/files/realtime.spec.ts | 371 +++++++++++ packages/api/src/files/realtime.ts | 228 +++++++ packages/api/src/files/transcription.spec.ts | 184 ++++++ packages/api/src/files/transcription.ts | 100 +++ packages/data-provider/src/api-endpoints.ts | 2 + packages/data-provider/src/config.spec.ts | 43 +- packages/data-provider/src/config.ts | 35 +- packages/data-provider/src/data-service.ts | 6 + packages/data-provider/src/keys.ts | 1 + packages/data-provider/src/types/files.ts | 34 + 37 files changed, 2865 insertions(+), 107 deletions(-) create mode 100644 api/server/routes/files/speech/index.spec.js create mode 100644 client/src/components/Nav/SettingsTabs/Speech/STT/__tests__/EngineSTTDropdown.spec.tsx create mode 100644 client/src/hooks/Input/__tests__/silence.spec.ts create mode 100644 client/src/hooks/Input/__tests__/useSpeechToTextRealtime.spec.tsx create mode 100644 client/src/hooks/Input/silence.ts create mode 100644 client/src/hooks/Input/useSpeechToTextRealtime.ts create mode 100644 packages/api/src/files/realtime.spec.ts create mode 100644 packages/api/src/files/realtime.ts create mode 100644 packages/api/src/files/transcription.spec.ts create mode 100644 packages/api/src/files/transcription.ts diff --git a/api/server/routes/files/speech/index.spec.js b/api/server/routes/files/speech/index.spec.js new file mode 100644 index 00000000000..d7e9c0cba65 --- /dev/null +++ b/api/server/routes/files/speech/index.spec.js @@ -0,0 +1,88 @@ +const express = require('express'); +const request = require('supertest'); +const { RealtimeTranscriptionErrorCode } = require('librechat-data-provider'); + +const hits = { sttIp: 0, sttUser: 0 }; + +jest.mock('~/server/middleware', () => ({ + createSTTLimiters: jest.fn(() => ({ + sttIpLimiter: (req, res, next) => { + hits.sttIp++; + next(); + }, + sttUserLimiter: (req, res, next) => { + hits.sttUser++; + next(); + }, + })), + createTTSLimiters: jest.fn(() => ({ + ttsIpLimiter: (req, res, next) => next(), + ttsUserLimiter: (req, res, next) => next(), + })), +})); + +jest.mock('~/server/services/Files/Audio', () => ({ + speechToText: jest.fn((req, res) => res.status(200).json({ text: 'recorded' })), +})); +jest.mock('./tts', () => require('express').Router()); +jest.mock('./customConfigSpeech', () => require('express').Router()); + +const liveConfig = { + speech: { + stt: { + openai: { apiKey: 'sk', model: 'gpt-transcribe', realtime: { model: 'gpt-live-transcribe' } }, + }, + }, +}; + +describe('speech route wiring', () => { + let config; + let app; + + beforeAll(() => { + const speech = require('./index'); + app = express(); + app.use(express.json()); + app.use((req, res, next) => { + req.config = config; + next(); + }); + app.use('/api/files/speech', speech); + }); + + beforeEach(() => { + hits.sttIp = 0; + hits.sttUser = 0; + config = liveConfig; + }); + + it('meters the live transcription route with both STT limiters', async () => { + const response = await request(app) + .post('/api/files/speech/stt/realtime') + .send({ sdp: 'not an offer' }); + + expect(response.status).toBe(400); + expect(response.body).toEqual({ error: RealtimeTranscriptionErrorCode.INVALID_OFFER }); + expect(hits.sttIp).toBe(1); + expect(hits.sttUser).toBe(1); + }); + + it('reports live transcription as not configured without a realtime block', async () => { + config = { speech: { stt: { openai: { apiKey: 'sk', model: 'gpt-transcribe' } } } }; + + const response = await request(app) + .post('/api/files/speech/stt/realtime') + .send({ sdp: 'v=0\r\n' }); + + expect(response.status).toBe(404); + expect(response.body).toEqual({ error: RealtimeTranscriptionErrorCode.NOT_CONFIGURED }); + }); + + it('keeps the recorded transcription route in place', async () => { + const response = await request(app).post('/api/files/speech/stt').send({}); + + expect(response.status).toBe(200); + expect(response.body).toEqual({ text: 'recorded' }); + expect(hits.sttIp).toBe(1); + }); +}); diff --git a/api/server/routes/files/speech/stt.js b/api/server/routes/files/speech/stt.js index 663d2e46387..7fd36306b53 100644 --- a/api/server/routes/files/speech/stt.js +++ b/api/server/routes/files/speech/stt.js @@ -1,8 +1,10 @@ const express = require('express'); +const { createRealtimeTranscriptionHandler } = require('@librechat/api'); const { speechToText } = require('~/server/services/Files/Audio'); const router = express.Router(); router.post('/', speechToText); +router.post('/realtime', createRealtimeTranscriptionHandler()); module.exports = router; diff --git a/api/server/services/Files/Audio/STTService.js b/api/server/services/Files/Audio/STTService.js index 1b0f6f59420..2227babc374 100644 --- a/api/server/services/Files/Audio/STTService.js +++ b/api/server/services/Files/Audio/STTService.js @@ -11,6 +11,7 @@ const { getSafeErrorMetadata, applyAxiosProxyConfig, resolveConfigSecret, + getTranscriptionHints, applySSRFSafeAgentIfDirect, contentFilterBlockResponse, contentFilterUninspectableResponse, @@ -200,7 +201,9 @@ class STTService { } /** - * Prepares the request for the OpenAI STT provider. + * Prepares the request for the OpenAI STT provider. `gpt-transcribe` models get the + * `languages`/`keywords`/`prompt` hints; older models keep the singular `language`. + * Axios serializes array hints as repeated `languages[]` / `keywords[]` form fields. * @param {Object} sttSchema - The STT schema for OpenAI. * @param {Stream} audioReadStream - The audio data to be transcribed. * @param {Object} audioFile - The audio file object (unused in OpenAI provider). @@ -214,13 +217,9 @@ class STTService { const data = { file: audioReadStream, model: sttSchema.model, + ...getTranscriptionHints(sttSchema, sttSchema.model, language), }; - const validLanguage = getValidatedLanguageCode(language); - if (validLanguage) { - data.language = validLanguage; - } - const headers = { 'Content-Type': 'multipart/form-data', ...(apiKey && { Authorization: `Bearer ${apiKey}` }), diff --git a/api/server/services/Files/Audio/STTService.spec.js b/api/server/services/Files/Audio/STTService.spec.js index d1a37c4327c..3473f1e76f6 100644 --- a/api/server/services/Files/Audio/STTService.spec.js +++ b/api/server/services/Files/Audio/STTService.spec.js @@ -36,6 +36,7 @@ jest.mock('@librechat/api', () => ({ })), applyAxiosProxyConfig: jest.fn(), resolveConfigSecret: jest.fn((value) => value), + getTranscriptionHints: jest.fn(() => ({})), applySSRFSafeAgentIfDirect: jest.fn(), contentFilterBlockResponse: jest.fn((finding) => ({ error: 'content_filter_block', @@ -61,11 +62,13 @@ jest.mock('~/server/services/Config', () => ({ getAppConfig: jest.fn() })); const fs = require('fs').promises; const axios = require('axios'); +const FormData = require('form-data'); const { logger } = require('@librechat/data-schemas'); const { inspectContent, extractFileContent, getSafeErrorMetadata, + getTranscriptionHints, contentFilterBlockResponse, contentFilterUninspectableResponse, getBlockedUninspectableFileField, @@ -146,6 +149,77 @@ describe('STTService.getProviderSchema provider detection', () => { }); }); +describe('STTService.openAIProvider request fields', () => { + const service = new STTService(); + const audio = Buffer.from('audio'); + + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('builds the hint fields from the schema, its model, and the user language', () => { + const schema = { + apiKey: 'sk', + model: 'gpt-transcribe', + prompt: 'Retail and software terms.', + keywords: ['LibreChat'], + languages: ['de', 'en'], + }; + getTranscriptionHints.mockReturnValueOnce({ + prompt: 'Retail and software terms.', + keywords: ['LibreChat'], + languages: ['de'], + }); + + const [url, data, headers] = service.openAIProvider(schema, audio, {}, 'de-DE'); + + expect(getTranscriptionHints).toHaveBeenCalledWith(schema, 'gpt-transcribe', 'de-DE'); + expect(url).toBe('https://api.openai.com/v1/audio/transcriptions'); + expect(data).toEqual({ + file: audio, + model: 'gpt-transcribe', + prompt: 'Retail and software terms.', + keywords: ['LibreChat'], + languages: ['de'], + }); + expect(headers).toEqual({ 'Content-Type': 'multipart/form-data', Authorization: 'Bearer sk' }); + }); + + it('sends the singular language the hints resolve for older models', () => { + getTranscriptionHints.mockReturnValueOnce({ language: 'en' }); + + const [, data] = service.openAIProvider({ apiKey: 'sk', model: 'whisper-1' }, audio, {}, 'en'); + + expect(data).toEqual({ file: audio, model: 'whisper-1', language: 'en' }); + }); +}); + +describe('STTService.sttRequest response parsing', () => { + const service = new STTService(); + service.providerStrategies.openai = service.openAIProvider; + const request = { + audioBuffer: Buffer.from('audio'), + audioFile: { mimetype: 'audio/webm' }, + language: '', + }; + + beforeEach(() => { + jest.clearAllMocks(); + }); + + it.each([ + ['detected-language metadata', { text: ' Bonjour. ', languages: [{ code: 'fr' }] }], + ['an empty detection result', { text: ' Bonjour. ', languages: [] }], + ['a plain response', { text: ' Bonjour. ' }], + ])('returns the trimmed text from a response with %s', async (_label, responseData) => { + axios.post.mockResolvedValueOnce({ status: 200, data: responseData }); + + await expect( + service.sttRequest('openai', { apiKey: 'sk', model: 'gpt-transcribe' }, request), + ).resolves.toBe('Bonjour.'); + }); +}); + describe('STT audio format validation with MIME normalization', () => { const acceptedFormats = ['flac', 'mp3', 'mp4', 'mpeg', 'mpga', 'm4a', 'ogg', 'wav', 'webm']; @@ -215,8 +289,14 @@ describe('STT error disclosure protection', () => { it('does not log a submitted invalid language value', () => { const submittedLanguage = 'PRIVATE-LANGUAGE@example.com'; const service = new STTService(); + FormData.mockImplementationOnce(() => ({ append: jest.fn(), getHeaders: () => ({}) })); - service.openAIProvider({ model: 'whisper-1' }, Buffer.from('audio'), {}, submittedLanguage); + service.azureOpenAIProvider( + { apiKey: 'key' }, + Buffer.from('audio'), + { originalname: 'audio.webm', mimetype: 'audio/webm' }, + submittedLanguage, + ); expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('Invalid language format')); expect(JSON.stringify(logger.warn.mock.calls)).not.toContain(submittedLanguage); diff --git a/api/server/services/Files/Audio/getCustomConfigSpeech.js b/api/server/services/Files/Audio/getCustomConfigSpeech.js index be3b1810600..5d8191b6561 100644 --- a/api/server/services/Files/Audio/getCustomConfigSpeech.js +++ b/api/server/services/Files/Audio/getCustomConfigSpeech.js @@ -1,4 +1,5 @@ const { logger } = require('@librechat/data-schemas'); +const { isRealtimeTranscriptionAvailable } = require('@librechat/api'); const { getAppConfig } = require('~/server/services/Config'); const LEGACY_EXTERNAL_STT_ENGINES = new Set(['openai', 'azureOpenAI']); @@ -48,6 +49,7 @@ async function getCustomConfigSpeech(req, res) { const ttsExternal = !!appConfig.speech?.tts; let settings = { sttExternal, + sttRealtime: isRealtimeTranscriptionAvailable(appConfig.speech?.stt), ttsExternal, }; diff --git a/api/server/services/Files/Audio/getCustomConfigSpeech.spec.js b/api/server/services/Files/Audio/getCustomConfigSpeech.spec.js index c255839f660..956130ccdcc 100644 --- a/api/server/services/Files/Audio/getCustomConfigSpeech.spec.js +++ b/api/server/services/Files/Audio/getCustomConfigSpeech.spec.js @@ -31,12 +31,58 @@ describe('getCustomConfigSpeech', () => { expect(res.status).toHaveBeenCalledWith(200); expect(res.send).toHaveBeenCalledWith({ sttExternal: true, + sttRealtime: false, ttsExternal: false, engineSTT: 'external', }); }, ); + it('advertises live transcription when the OpenAI provider configures a realtime model', async () => { + const req = { + config: { + speech: { + stt: { + openai: { + apiKey: 'sk', + model: 'gpt-transcribe', + realtime: { model: 'gpt-live-transcribe' }, + }, + }, + speechTab: { speechToText: { engineSTT: 'realtime' } }, + }, + }, + }; + const res = createResponse(); + + await getCustomConfigSpeech(req, res); + + expect(res.send).toHaveBeenCalledWith({ + sttExternal: true, + sttRealtime: true, + ttsExternal: false, + engineSTT: 'realtime', + }); + }); + + it.each([ + ['without a realtime block', { openai: { apiKey: 'sk', model: 'gpt-transcribe' } }], + [ + 'for Azure OpenAI', + { azureOpenAI: { apiKey: 'sk', instanceName: 'i', deploymentName: 'd', apiVersion: 'v' } }, + ], + ])('does not advertise live transcription %s', async (_label, stt) => { + const res = createResponse(); + + await getCustomConfigSpeech({ config: { speech: { stt } } }, res); + + expect(res.send).toHaveBeenCalledWith({ + sttExternal: true, + sttRealtime: false, + ttsExternal: false, + }); + }); + it.each(['browser', 'external'])('preserves the runtime STT engine "%s"', async (engineSTT) => { const req = { config: { @@ -71,6 +117,7 @@ describe('getCustomConfigSpeech', () => { expect(res.status).toHaveBeenCalledWith(200); expect(res.send).toHaveBeenCalledWith({ sttExternal: false, + sttRealtime: false, ttsExternal: true, engineTTS: 'external', }); diff --git a/client/src/common/types.ts b/client/src/common/types.ts index 3e0d75715a3..7a9f990f809 100644 --- a/client/src/common/types.ts +++ b/client/src/common/types.ts @@ -35,6 +35,8 @@ export enum PromptsEditorMode { export enum STTEndpoints { browser = 'browser', external = 'external', + /** Streams the microphone to the provider over WebRTC; offered only when the admin configures it. */ + realtime = 'realtime', } export enum TTSEndpoints { diff --git a/client/src/components/Chat/Input/Composer/Bar.tsx b/client/src/components/Chat/Input/Composer/Bar.tsx index d57ab0e61d0..be451b9ab4d 100644 --- a/client/src/components/Chat/Input/Composer/Bar.tsx +++ b/client/src/components/Chat/Input/Composer/Bar.tsx @@ -299,9 +299,19 @@ function Bar({ let dictationStatus = ''; if (dictation.transcribing) { dictationStatus = localize('com_ui_transcribing'); + } else if (dictation.connection === 'connecting') { + dictationStatus = localize('com_ui_connecting'); + } else if (dictation.connection === 'reconnecting') { + dictationStatus = localize('com_ui_reconnecting'); } else if (dictating) { dictationStatus = localize('com_ui_listening'); } + /* Live dictation has states worth showing in place of the clock: until it + connects nothing is being heard, and a reconnect may lose a few words. */ + const dictationLabel = + dictation.transcribing || dictation.connection != null + ? dictationStatus + : formatElapsed(dictation.elapsed); /* The arrangement is frozen for the length of a recording. The controls shrink to just the elapsed time while one runs, which would otherwise let @@ -489,9 +499,7 @@ function Bar({ )} > diff --git a/client/src/components/Nav/Settings/SpeechControls.tsx b/client/src/components/Nav/Settings/SpeechControls.tsx index 40818131530..8855af39a83 100644 --- a/client/src/components/Nav/Settings/SpeechControls.tsx +++ b/client/src/components/Nav/Settings/SpeechControls.tsx @@ -4,7 +4,12 @@ import { EngineTTSDropdown } from '../SettingsTabs/Speech/TTS'; export function EngineSTTSetting() { const { data } = useGetCustomConfigSpeechQuery(); - return ; + return ( + + ); } export function EngineTTSSetting() { diff --git a/client/src/components/Nav/SettingsTabs/Speech/STT/EngineSTTDropdown.tsx b/client/src/components/Nav/SettingsTabs/Speech/STT/EngineSTTDropdown.tsx index 1cb1d4d7b3f..6e9ef10ab8b 100644 --- a/client/src/components/Nav/SettingsTabs/Speech/STT/EngineSTTDropdown.tsx +++ b/client/src/components/Nav/SettingsTabs/Speech/STT/EngineSTTDropdown.tsx @@ -6,19 +6,22 @@ import store from '~/store'; interface EngineSTTDropdownProps { external: boolean; + /** Whether the admin configured live transcription (`speech.stt.openai.realtime`). */ + realtime?: boolean; } -const EngineSTTDropdown: React.FC = ({ external }) => { +const EngineSTTDropdown: React.FC = ({ external, realtime = false }) => { const localize = useLocalize(); const [engineSTT, setEngineSTT] = useRecoilState(store.engineSTT); const speechToText = useRecoilValue(store.speechToText); - const endpointOptions = external - ? [ - { value: 'browser', label: localize('com_nav_browser') }, - { value: 'external', label: localize('com_nav_external') }, - ] - : [{ value: 'browser', label: localize('com_nav_browser') }]; + const endpointOptions = [{ value: 'browser', label: localize('com_nav_browser') }]; + if (external) { + endpointOptions.push({ value: 'external', label: localize('com_nav_external') }); + } + if (external && realtime) { + endpointOptions.push({ value: 'realtime', label: localize('com_nav_live') }); + } const handleSelect = (value: string) => { setEngineSTT(value); diff --git a/client/src/components/Nav/SettingsTabs/Speech/STT/__tests__/EngineSTTDropdown.spec.tsx b/client/src/components/Nav/SettingsTabs/Speech/STT/__tests__/EngineSTTDropdown.spec.tsx new file mode 100644 index 00000000000..f8e6917faaa --- /dev/null +++ b/client/src/components/Nav/SettingsTabs/Speech/STT/__tests__/EngineSTTDropdown.spec.tsx @@ -0,0 +1,84 @@ +import React from 'react'; +import { RecoilRoot } from 'recoil'; +import { render, screen, fireEvent } from '@testing-library/react'; +import EngineSTTDropdown from '../EngineSTTDropdown'; + +jest.mock('~/hooks', () => ({ + useLocalize: () => (key: string) => key, +})); + +jest.mock('@librechat/client', () => ({ + Dropdown: ({ + value, + options, + onChange, + }: { + value: string; + options: { value: string; label: string }[]; + onChange: (value: string) => void; + }) => ( + + ), +})); + +const optionsOf = () => + Array.from(screen.getByTestId('EngineSTTDropdown').querySelectorAll('option')).map((option) => [ + option.value, + option.textContent, + ]); + +describe('EngineSTTDropdown', () => { + beforeEach(() => { + localStorage.clear(); + }); + + it('offers the live engine only when the server reports it', () => { + render( + + + , + ); + + expect(optionsOf()).toEqual([ + ['browser', 'com_nav_browser'], + ['external', 'com_nav_external'], + ['realtime', 'com_nav_live'], + ]); + + fireEvent.change(screen.getByTestId('EngineSTTDropdown'), { target: { value: 'realtime' } }); + expect(localStorage.getItem('engineSTT')).toBe(JSON.stringify('realtime')); + }); + + it('keeps the existing choices when live transcription is not configured', () => { + render( + + + , + ); + + expect(optionsOf()).toEqual([ + ['browser', 'com_nav_browser'], + ['external', 'com_nav_external'], + ]); + }); + + it('offers only the browser without an external provider', () => { + render( + + + , + ); + + expect(optionsOf()).toEqual([['browser', 'com_nav_browser']]); + }); +}); diff --git a/client/src/data-provider/mutations.ts b/client/src/data-provider/mutations.ts index 5909305a055..701d23ac50b 100644 --- a/client/src/data-provider/mutations.ts +++ b/client/src/data-provider/mutations.ts @@ -1757,6 +1757,22 @@ export const useSpeechToTextMutation = ( }); }; +/* Live speech to text: trades the browser's WebRTC offer for the provider's answer */ +export const useRealtimeTranscriptionMutation = ( + options?: t.RealtimeTranscriptionOptions, +): UseMutationResult< + t.RealtimeTranscriptionResponse, + unknown, + t.RealtimeTranscriptionRequest, + unknown +> => { + return useMutation([MutationKeys.realtimeTranscription], { + mutationFn: (variables: t.RealtimeTranscriptionRequest) => + dataService.createRealtimeTranscription(variables), + ...(options || {}), + }); +}; + /* Text to speech */ export const useTextToSpeechMutation = ( options?: t.TextToSpeechOptions, diff --git a/client/src/hooks/Config/__tests__/useSpeechSettingsInit.spec.tsx b/client/src/hooks/Config/__tests__/useSpeechSettingsInit.spec.tsx index f228a816e87..476809b256e 100644 --- a/client/src/hooks/Config/__tests__/useSpeechSettingsInit.spec.tsx +++ b/client/src/hooks/Config/__tests__/useSpeechSettingsInit.spec.tsx @@ -142,6 +142,70 @@ describe('useSpeechSettingsInit', () => { }, ); + describe('live engine', () => { + it('keeps a saved live engine while the server offers it', async () => { + localStorage.setItem('engineSTT', JSON.stringify('realtime')); + mockUseGetCustomConfigSpeechQuery.mockReturnValue({ + data: { sttExternal: true, sttRealtime: true, ttsExternal: true }, + isFetched: true, + }); + + const { result } = renderHook(() => useSpeechSettingsHarness(), { wrapper }); + + await waitFor(() => expect(result.current.speechSettingsInitialized).toBe(true)); + expect(result.current.engineSTT).toBe('realtime'); + expect(localStorage.getItem('sttRealtime')).toBeNull(); + }); + + it('falls back to recorded transcription when the live block is removed', async () => { + localStorage.setItem('engineSTT', JSON.stringify('realtime')); + mockUseGetCustomConfigSpeechQuery.mockReturnValue({ + data: { sttExternal: true, sttRealtime: false, ttsExternal: true }, + isFetched: true, + }); + + const { result } = renderHook(() => useSpeechSettingsHarness(), { wrapper }); + + await waitFor(() => expect(result.current.engineSTT).toBe('external')); + expect(localStorage.getItem('engineSTT')).toBe(JSON.stringify('external')); + }); + + it('falls back to the browser when no speech provider is left', async () => { + localStorage.setItem('engineSTT', JSON.stringify('realtime')); + mockUseGetCustomConfigSpeechQuery.mockReturnValue({ + data: { sttExternal: false, sttRealtime: false, ttsExternal: false }, + isFetched: true, + }); + + const { result } = renderHook(() => useSpeechSettingsHarness(), { wrapper }); + + await waitFor(() => expect(result.current.engineSTT).toBe('browser')); + }); + + it('seeds the live engine as the server default', async () => { + mockUseGetCustomConfigSpeechQuery.mockReturnValue({ + data: { sttExternal: true, sttRealtime: true, ttsExternal: true, engineSTT: 'realtime' }, + isFetched: true, + }); + + const { result } = renderHook(() => useSpeechSettingsHarness(), { wrapper }); + + await waitFor(() => expect(result.current.engineSTT).toBe('realtime')); + }); + + it('does not seed a live default the server cannot serve', async () => { + mockUseGetCustomConfigSpeechQuery.mockReturnValue({ + data: { sttExternal: true, sttRealtime: false, ttsExternal: true, engineSTT: 'realtime' }, + isFetched: true, + }); + + const { result } = renderHook(() => useSpeechSettingsHarness(), { wrapper }); + + await waitFor(() => expect(result.current.speechSettingsInitialized).toBe(true)); + await waitFor(() => expect(result.current.engineSTT).toBe('external')); + }); + }); + it('normalizes unknown saved engines to browser', async () => { localStorage.setItem('engineSTT', JSON.stringify('unknown-stt')); localStorage.setItem('engineTTS', JSON.stringify('unknown-tts')); diff --git a/client/src/hooks/Config/useSpeechSettingsInit.ts b/client/src/hooks/Config/useSpeechSettingsInit.ts index 5019f4d8136..3bd2033dbe3 100644 --- a/client/src/hooks/Config/useSpeechSettingsInit.ts +++ b/client/src/hooks/Config/useSpeechSettingsInit.ts @@ -59,7 +59,7 @@ export default function useSpeechSettingsInit(isAuthenticated: boolean) { logger.log('Initializing speech settings from config:', data); Object.entries(data).forEach(([key, value]) => { - if (key === 'sttExternal' || key === 'ttsExternal') return; + if (key === 'sttExternal' || key === 'sttRealtime' || key === 'ttsExternal') return; if (localStorage.getItem(key) !== null) return; @@ -74,11 +74,17 @@ export default function useSpeechSettingsInit(isAuthenticated: boolean) { const configuredEngineSTT = hasSavedEngineSTT ? engineSTT : data?.engineSTT; const configuredEngineTTS = hasSavedEngineTTS ? engineTTS : data?.engineTTS; const sttExternalUnavailable = data?.sttExternal != null && !data.sttExternal; + const sttRealtimeUnavailable = data?.sttRealtime != null && !data.sttRealtime; const ttsExternalUnavailable = data?.ttsExternal != null && !data.ttsExternal; if (sttExternalUnavailable && configuredEngineSTT === STTEndpoints.external) { setEngineSTT(STTEndpoints.browser); } + /* Live rides on the external provider, so recorded transcription is the closest + fallback when an admin removes the realtime block but keeps the provider. */ + if (sttRealtimeUnavailable && configuredEngineSTT === STTEndpoints.realtime) { + setEngineSTT(sttExternalUnavailable ? STTEndpoints.browser : STTEndpoints.external); + } if (ttsExternalUnavailable && configuredEngineTTS === TTSEndpoints.external) { setEngineTTS(TTSEndpoints.browser); } diff --git a/client/src/hooks/Input/__tests__/silence.spec.ts b/client/src/hooks/Input/__tests__/silence.spec.ts new file mode 100644 index 00000000000..908e27c848e --- /dev/null +++ b/client/src/hooks/Input/__tests__/silence.spec.ts @@ -0,0 +1,109 @@ +import { monitorSilence, SILENCE_THRESHOLD_MS } from '../silence'; + +class FakeAudioContext { + static instances: FakeAudioContext[] = []; + + state: AudioContextState = 'running'; + level = 0; + close = jest.fn(() => { + this.state = 'closed'; + return Promise.resolve(); + }); + + analyser = { + minDecibels: 0, + frequencyBinCount: 4, + getByteFrequencyData: jest.fn((data: Uint8Array) => data.fill(this.level)), + }; + + constructor() { + FakeAudioContext.instances.push(this); + } + + createMediaStreamSource() { + return { connect: jest.fn() }; + } + + createAnalyser() { + return this.analyser; + } +} + +describe('monitorSilence', () => { + const stream = {} as MediaStream; + let frames: Map; + let nextFrame: number; + let now: number; + const originalAudioContext = global.AudioContext; + + const runFrame = (advanceMs: number) => { + now += advanceMs; + const [id, callback] = frames.entries().next().value as [number, FrameRequestCallback]; + frames.delete(id); + callback(now); + }; + + beforeEach(() => { + FakeAudioContext.instances = []; + frames = new Map(); + nextFrame = 0; + now = 1_000; + jest.spyOn(Date, 'now').mockImplementation(() => now); + jest.spyOn(window, 'requestAnimationFrame').mockImplementation((callback) => { + nextFrame += 1; + frames.set(nextFrame, callback); + return nextFrame; + }); + jest.spyOn(window, 'cancelAnimationFrame').mockImplementation((id) => frames.delete(id)); + (global as { AudioContext?: unknown }).AudioContext = FakeAudioContext; + }); + + afterEach(() => { + jest.restoreAllMocks(); + (global as { AudioContext?: unknown }).AudioContext = originalAudioContext; + }); + + it('calls back once the stream stays quiet past the threshold, after releasing the audio graph', () => { + const onSilence = jest.fn(); + monitorSilence(stream, { minDecibels: -50, onSilence }); + const context = FakeAudioContext.instances[0]; + + expect(context.analyser.minDecibels).toBe(-50); + runFrame(SILENCE_THRESHOLD_MS); + expect(onSilence).not.toHaveBeenCalled(); + + onSilence.mockImplementation(() => expect(context.close).toHaveBeenCalled()); + runFrame(1); + + expect(onSilence).toHaveBeenCalledTimes(1); + expect(frames.size).toBe(0); + }); + + it('keeps listening while sound keeps arriving', () => { + const onSilence = jest.fn(); + monitorSilence(stream, { minDecibels: -45, onSilence, thresholdMs: 100 }); + const context = FakeAudioContext.instances[0]; + context.level = 10; + + for (let i = 0; i < 10; i++) { + runFrame(60); + } + + expect(onSilence).not.toHaveBeenCalled(); + expect(frames.size).toBe(1); + }); + + it('stops the frame loop and closes the context once, however often it is stopped', () => { + const onSilence = jest.fn(); + const stop = monitorSilence(stream, { minDecibels: -45, onSilence }); + const context = FakeAudioContext.instances[0]; + + stop(); + stop(); + + expect(window.cancelAnimationFrame).toHaveBeenCalledTimes(1); + expect(context.close).toHaveBeenCalledTimes(1); + expect(frames.size).toBe(0); + expect(onSilence).not.toHaveBeenCalled(); + }); +}); diff --git a/client/src/hooks/Input/__tests__/useDictation.spec.tsx b/client/src/hooks/Input/__tests__/useDictation.spec.tsx index 330a13ec9f0..803d6e9e6ca 100644 --- a/client/src/hooks/Input/__tests__/useDictation.spec.tsx +++ b/client/src/hooks/Input/__tests__/useDictation.spec.tsx @@ -9,7 +9,7 @@ import useDictation from '../useDictation'; * request, sent by the setting, or thrown away, against both engines. */ -let mockSpeechEndpoint: 'browser' | 'external'; +let mockSpeechEndpoint: 'browser' | 'external' | 'realtime'; let mockSetTextCallback: (text: string) => void; let mockOnTranscriptionComplete: (text: string) => void; let mockOnTranscriptionSettled: () => void; @@ -18,6 +18,7 @@ const mockStop = jest.fn(); const mockAbort = jest.fn(); let mockIsListening: boolean; let mockIsLoading: boolean; +let mockConnection: 'connecting' | 'reconnecting' | null; jest.mock('../useSpeechToText', () => ({ __esModule: true, @@ -35,6 +36,7 @@ jest.mock('../useSpeechToText', () => ({ startRecording: mockStart, stopRecording: mockStop, abortRecording: mockAbort, + connection: mockConnection, }; }, })); @@ -135,6 +137,7 @@ describe('useDictation', () => { mockSpeechEndpoint = 'browser'; mockIsListening = false; mockIsLoading = false; + mockConnection = null; }); afterEach(() => { @@ -453,6 +456,77 @@ describe('useDictation', () => { expect(currentText()).toBe('my unsent draft new words'); }); + describe('live dictation', () => { + beforeEach(() => { + mockSpeechEndpoint = 'realtime'; + }); + + it('writes live text after the draft that was there before recording', () => { + const { result, rerender, currentText } = setup({ draft: 'Dear team,' }); + act(() => result.current.start()); + mockIsListening = true; + act(() => rerender()); + + act(() => mockSetTextCallback('the build')); + expect(currentText()).toBe('Dear team, the build'); + act(() => mockSetTextCallback('the build is green')); + expect(currentText()).toBe('Dear team, the build is green'); + }); + + it('auto-sends the live transcript with the draft in front of it', async () => { + const { result, rerender } = setup({ draft: 'Dear team,', autoSendText: 0 }); + act(() => result.current.start()); + mockIsListening = true; + act(() => rerender()); + + act(() => result.current.stopToComposer()); + await settle(rerender); + act(() => mockOnTranscriptionComplete('the build is green')); + + expect(ask).toHaveBeenCalledWith({ text: 'Dear team, the build is green' }); + }); + + it('reports the connection and starts the clock only once audio can flow', () => { + const { result, rerender } = setup(); + act(() => result.current.start()); + mockIsListening = true; + mockConnection = 'connecting'; + act(() => rerender()); + + expect(result.current.active).toBe(true); + expect(result.current.connection).toBe('connecting'); + act(() => { + jest.advanceTimersByTime(3000); + }); + expect(result.current.elapsed).toBe(0); + + mockConnection = null; + act(() => rerender()); + act(() => { + jest.advanceTimersByTime(2000); + }); + expect(result.current.connection).toBeNull(); + expect(result.current.elapsed).toBe(2); + + mockConnection = 'reconnecting'; + act(() => rerender()); + expect(result.current.connection).toBe('reconnecting'); + }); + + it('lets a take be stopped or cancelled while it is still connecting', () => { + const { result, rerender } = setup(); + act(() => result.current.start()); + mockIsListening = true; + mockConnection = 'connecting'; + act(() => rerender()); + + act(() => result.current.stopToComposer()); + expect(mockStop).toHaveBeenCalledTimes(1); + act(() => result.current.cancel()); + expect(mockAbort).toHaveBeenCalledTimes(1); + }); + }); + /* A run in flight has nothing to send into, so the take is refused outright. `ChatForm` reports a paused `ask_user_question` as NOT submitting for exactly this reason: the run is still open, but the composer has become the diff --git a/client/src/hooks/Input/__tests__/useSpeechToTextRealtime.spec.tsx b/client/src/hooks/Input/__tests__/useSpeechToTextRealtime.spec.tsx new file mode 100644 index 00000000000..6180612d9c2 --- /dev/null +++ b/client/src/hooks/Input/__tests__/useSpeechToTextRealtime.spec.tsx @@ -0,0 +1,585 @@ +import React from 'react'; +import { RecoilRoot } from 'recoil'; +import { act, renderHook } from '@testing-library/react'; +import { RealtimeTranscriptionErrorCode } from 'librechat-data-provider'; +import useSpeechToTextRealtime, { + COMMIT_GRACE_MS, + COMPLETION_TIMEOUT_MS, +} from '../useSpeechToTextRealtime'; +import { monitorSilence } from '../silence'; +import store from '~/store'; + +const mockOpenSession = jest.fn(); +const mockShowToast = jest.fn(); +const mockStopSilence = jest.fn(); + +jest.mock('~/data-provider', () => ({ + useRealtimeTranscriptionMutation: () => ({ mutateAsync: mockOpenSession }), +})); + +jest.mock('@librechat/client', () => ({ + useToastContext: () => ({ showToast: mockShowToast }), +})); + +jest.mock('~/hooks/useLocalize', () => ({ + __esModule: true, + default: () => (key: string) => key, +})); + +jest.mock('../silence', () => ({ + monitorSilence: jest.fn(() => mockStopSilence), +})); + +class FakeDataChannel { + readyState: RTCDataChannelState = 'connecting'; + onopen: (() => void) | null = null; + onclose: (() => void) | null = null; + onmessage: ((event: MessageEvent) => void) | null = null; + send = jest.fn(); + close = jest.fn(() => { + this.readyState = 'closed'; + }); + + constructor(readonly label: string) {} + + open() { + this.readyState = 'open'; + this.onopen?.(); + } + + emit(event: Record) { + this.onmessage?.({ data: JSON.stringify(event) } as MessageEvent); + } + + drop() { + this.readyState = 'closed'; + this.onclose?.(); + } +} + +class FakePeerConnection { + static instances: FakePeerConnection[] = []; + + connectionState: RTCPeerConnectionState = 'new'; + onconnectionstatechange: (() => void) | null = null; + channel: FakeDataChannel | null = null; + addTrack = jest.fn(); + createDataChannel = jest.fn((label: string) => { + this.channel = new FakeDataChannel(label); + return this.channel; + }); + + createOffer = jest.fn(async () => ({ type: 'offer', sdp: 'v=0\r\noffer' })); + setLocalDescription = jest.fn(async () => undefined); + setRemoteDescription = jest.fn(async () => undefined); + close = jest.fn(() => { + this.connectionState = 'closed'; + }); + + constructor() { + FakePeerConnection.instances.push(this); + } + + setState(state: RTCPeerConnectionState) { + this.connectionState = state; + this.onconnectionstatechange?.(); + } +} + +const providerError = (code: string) => + Object.assign(new Error('Request failed'), { + isAxiosError: true, + response: { status: 404, data: { error: code } }, + }); + +let mockGetUserMedia: jest.Mock; +let track: { stop: jest.Mock; kind: string }; + +function setup({ + autoSendText = -1, + autoTranscribeAudio = false, + languageSTT = '', +}: { autoSendText?: number; autoTranscribeAudio?: boolean; languageSTT?: string } = {}) { + const setText = jest.fn(); + const onTranscriptionComplete = jest.fn(); + const onTranscriptionSettled = jest.fn(); + const wrapper = ({ children }: { children: React.ReactNode }) => ( + { + set(store.speechToText, true); + set(store.autoTranscribeAudio, autoTranscribeAudio); + set(store.languageSTT, languageSTT); + }} + > + {children} + + ); + const rendered = renderHook( + () => + useSpeechToTextRealtime( + setText, + onTranscriptionComplete, + onTranscriptionSettled, + autoSendText, + ), + { wrapper }, + ); + return { ...rendered, setText, onTranscriptionComplete, onTranscriptionSettled }; +} + +type Rendered = ReturnType; + +const TAKE = 7; + +/** Starts a take and lets the offer/answer exchange finish. */ +const begin = async (rendered: Rendered) => { + await act(async () => { + await rendered.result.current.startRecording(TAKE); + }); + const pc = FakePeerConnection.instances[FakePeerConnection.instances.length - 1]; + return { pc, channel: pc?.channel as FakeDataChannel }; +}; + +const connect = (pc: FakePeerConnection, channel: FakeDataChannel) => + act(() => { + pc.setState('connected'); + channel.open(); + }); + +const emit = (channel: FakeDataChannel, event: Record) => + act(() => channel.emit(event)); + +const delta = (item_id: string, text: string) => ({ + type: 'conversation.item.input_audio_transcription.delta', + item_id, + delta: text, +}); + +const completed = (item_id: string, transcript: string) => ({ + type: 'conversation.item.input_audio_transcription.completed', + item_id, + transcript, +}); + +const committed = (item_id: string) => ({ type: 'input_audio_buffer.committed', item_id }); + +/** Stops the take and lets the commit go out. */ +const stopAndCommit = (rendered: Rendered) => { + act(() => rendered.result.current.stopRecording()); + act(() => { + jest.advanceTimersByTime(COMMIT_GRACE_MS); + }); +}; + +describe('useSpeechToTextRealtime', () => { + const originalPeerConnection = global.RTCPeerConnection; + const originalMediaDevices = navigator.mediaDevices; + + beforeEach(() => { + jest.useFakeTimers(); + jest.clearAllMocks(); + FakePeerConnection.instances = []; + track = { stop: jest.fn(), kind: 'audio' }; + const stream = { getTracks: () => [track], getAudioTracks: () => [track] }; + mockGetUserMedia = jest.fn().mockResolvedValue(stream); + Object.defineProperty(navigator, 'mediaDevices', { + configurable: true, + value: { getUserMedia: mockGetUserMedia }, + }); + (global as { RTCPeerConnection?: unknown }).RTCPeerConnection = FakePeerConnection; + mockOpenSession.mockResolvedValue({ sdp: 'v=0\r\nanswer' }); + }); + + afterEach(() => { + jest.useRealTimers(); + (global as { RTCPeerConnection?: unknown }).RTCPeerConnection = originalPeerConnection; + Object.defineProperty(navigator, 'mediaDevices', { + configurable: true, + value: originalMediaDevices, + }); + }); + + it('opens the session through the server with the offer and the speech language', async () => { + const rendered = setup({ languageSTT: 'de-DE' }); + const { pc, channel } = await begin(rendered); + + expect(mockGetUserMedia).toHaveBeenCalledWith({ audio: true, video: false }); + expect(pc.addTrack).toHaveBeenCalledWith(track, expect.anything()); + expect(channel.label).toBe('oai-events'); + expect(mockOpenSession).toHaveBeenCalledWith({ sdp: 'v=0\r\noffer', language: 'de-DE' }); + expect(pc.setRemoteDescription).toHaveBeenCalledWith({ type: 'answer', sdp: 'v=0\r\nanswer' }); + expect(rendered.result.current.isListening).toBe(true); + expect(rendered.result.current.connection).toBe('connecting'); + + connect(pc, channel); + + expect(rendered.result.current.connection).toBeNull(); + }); + + it('leaves the language out when the user has not chosen one', async () => { + const rendered = setup(); + await begin(rendered); + + expect(mockOpenSession).toHaveBeenCalledWith({ sdp: 'v=0\r\noffer' }); + }); + + it('writes transcript deltas into the composer while the user speaks', async () => { + const rendered = setup(); + const { pc, channel } = await begin(rendered); + connect(pc, channel); + + emit(channel, delta('item_1', 'Hello')); + emit(channel, delta('item_1', ' world')); + emit(channel, delta('item_2', 'again')); + + expect(rendered.setText.mock.calls).toEqual([ + ['Hello', TAKE], + ['Hello world', TAKE], + ['Hello world again', TAKE], + ]); + }); + + it('commits on stop and settles once the committed item is transcribed', async () => { + const rendered = setup(); + const { pc, channel } = await begin(rendered); + connect(pc, channel); + emit(channel, delta('item_1', 'Hello wor')); + + act(() => rendered.result.current.stopRecording()); + + expect(rendered.result.current.isListening).toBe(false); + expect(rendered.result.current.isLoading).toBe(true); + expect(channel.send).not.toHaveBeenCalled(); + expect(track.stop).not.toHaveBeenCalled(); + + act(() => { + jest.advanceTimersByTime(COMMIT_GRACE_MS); + }); + + expect(channel.send).toHaveBeenCalledWith( + JSON.stringify({ type: 'input_audio_buffer.commit' }), + ); + expect(track.stop).toHaveBeenCalled(); + expect(rendered.onTranscriptionSettled).not.toHaveBeenCalled(); + + emit(channel, committed('item_1')); + emit(channel, completed('item_1', 'Hello world.')); + + expect(rendered.setText).toHaveBeenLastCalledWith('Hello world.', TAKE); + expect(rendered.onTranscriptionSettled).toHaveBeenCalledWith(TAKE); + expect(rendered.result.current.isLoading).toBe(false); + expect(pc.close).toHaveBeenCalled(); + expect(channel.close).toHaveBeenCalled(); + expect(rendered.onTranscriptionComplete).not.toHaveBeenCalled(); + }); + + it('settles when the transcript arrives before the commit is acknowledged', async () => { + const rendered = setup(); + const { pc, channel } = await begin(rendered); + connect(pc, channel); + stopAndCommit(rendered); + + emit(channel, completed('item_1', 'Out of order.')); + expect(rendered.onTranscriptionSettled).not.toHaveBeenCalled(); + + emit(channel, committed('item_1')); + + expect(rendered.setText).toHaveBeenLastCalledWith('Out of order.', TAKE); + expect(rendered.onTranscriptionSettled).toHaveBeenCalledWith(TAKE); + }); + + it('auto-sends the final transcript after the configured delay', async () => { + const rendered = setup({ autoSendText: 2 }); + const { pc, channel } = await begin(rendered); + connect(pc, channel); + stopAndCommit(rendered); + emit(channel, committed('item_1')); + emit(channel, completed('item_1', 'Send this.')); + + act(() => { + jest.advanceTimersByTime(1999); + }); + expect(rendered.onTranscriptionComplete).not.toHaveBeenCalled(); + + act(() => { + jest.advanceTimersByTime(1); + }); + expect(rendered.onTranscriptionComplete).toHaveBeenCalledWith('Send this.', TAKE); + }); + + it('settles quietly when stop arrives before any audio reached the provider', async () => { + const rendered = setup({ autoSendText: 0 }); + const { pc, channel } = await begin(rendered); + connect(pc, channel); + stopAndCommit(rendered); + + emit(channel, { type: 'error', error: { code: 'input_audio_buffer_commit_empty' } }); + + expect(mockShowToast).not.toHaveBeenCalled(); + expect(rendered.onTranscriptionSettled).toHaveBeenCalledWith(TAKE); + expect(rendered.setText).not.toHaveBeenCalled(); + act(() => { + jest.runOnlyPendingTimers(); + }); + expect(rendered.onTranscriptionComplete).not.toHaveBeenCalled(); + }); + + it('drops a cancelled take without writing, settling, or sending anything', async () => { + const rendered = setup({ autoSendText: 0 }); + const { pc, channel } = await begin(rendered); + connect(pc, channel); + emit(channel, delta('item_1', 'Never mind')); + rendered.setText.mockClear(); + + act(() => rendered.result.current.abortRecording()); + + expect(track.stop).toHaveBeenCalled(); + expect(pc.close).toHaveBeenCalled(); + expect(channel.close).toHaveBeenCalled(); + expect(channel.send).not.toHaveBeenCalled(); + expect(rendered.result.current.isListening).toBe(false); + + channel.emit(completed('item_1', 'Never mind.')); + act(() => { + jest.runOnlyPendingTimers(); + }); + + expect(rendered.setText).not.toHaveBeenCalled(); + expect(rendered.onTranscriptionSettled).not.toHaveBeenCalled(); + expect(rendered.onTranscriptionComplete).not.toHaveBeenCalled(); + }); + + it('abandons a take stopped while still connecting, without committing', async () => { + const rendered = setup(); + const { pc, channel } = await begin(rendered); + + act(() => rendered.result.current.stopRecording()); + + expect(channel.send).not.toHaveBeenCalled(); + expect(track.stop).toHaveBeenCalled(); + expect(pc.close).toHaveBeenCalled(); + expect(rendered.result.current.isListening).toBe(false); + expect(rendered.result.current.connection).toBeNull(); + expect(rendered.onTranscriptionSettled).toHaveBeenCalledWith(TAKE); + expect(mockShowToast).not.toHaveBeenCalled(); + }); + + it.each([ + ['NotAllowedError', 'com_ui_microphone_permission_denied'], + ['SecurityError', 'com_ui_microphone_permission_denied'], + ['NotFoundError', 'com_ui_microphone_unavailable'], + ])('reports a %s from the microphone as %s', async (name, message) => { + mockGetUserMedia.mockRejectedValueOnce(new DOMException('denied', name)); + const rendered = setup(); + + await begin(rendered); + + expect(mockShowToast).toHaveBeenCalledWith({ message, status: 'error' }); + expect(mockOpenSession).not.toHaveBeenCalled(); + expect(rendered.result.current.isListening).toBe(false); + expect(rendered.result.current.connection).toBeNull(); + expect(rendered.onTranscriptionSettled).toHaveBeenCalledWith(TAKE); + }); + + it.each([ + [RealtimeTranscriptionErrorCode.NOT_CONFIGURED, 'com_ui_live_transcription_unavailable'], + [RealtimeTranscriptionErrorCode.PROVIDER_ERROR, 'com_ui_live_transcription_start_error'], + ])('reports a %s session refusal and releases the microphone', async (code, message) => { + mockOpenSession.mockRejectedValueOnce(providerError(code)); + const rendered = setup(); + + const { pc } = await begin(rendered); + + expect(mockShowToast).toHaveBeenCalledWith({ message, status: 'error' }); + expect(track.stop).toHaveBeenCalled(); + expect(pc.close).toHaveBeenCalled(); + expect(pc.setRemoteDescription).not.toHaveBeenCalled(); + expect(rendered.result.current.isListening).toBe(false); + expect(rendered.onTranscriptionSettled).toHaveBeenCalledWith(TAKE); + }); + + it('does not touch the microphone where the browser has no WebRTC', async () => { + (global as { RTCPeerConnection?: unknown }).RTCPeerConnection = undefined; + const rendered = setup(); + + await act(async () => { + await rendered.result.current.startRecording(TAKE); + }); + + expect(mockShowToast).toHaveBeenCalledWith({ + message: 'com_ui_live_transcription_unsupported', + status: 'error', + }); + expect(mockGetUserMedia).not.toHaveBeenCalled(); + expect(rendered.result.current.isListening).toBe(false); + }); + + it('shows a reconnect and recovers from it', async () => { + const rendered = setup(); + const { pc, channel } = await begin(rendered); + connect(pc, channel); + + act(() => pc.setState('disconnected')); + expect(rendered.result.current.connection).toBe('reconnecting'); + expect(rendered.result.current.isListening).toBe(true); + + act(() => pc.setState('connected')); + expect(rendered.result.current.connection).toBeNull(); + }); + + it('keeps the text heard so far, without sending it, when the connection is lost', async () => { + const rendered = setup({ autoSendText: 0 }); + const { pc, channel } = await begin(rendered); + connect(pc, channel); + emit(channel, delta('item_1', 'Half a')); + + act(() => pc.setState('failed')); + + expect(mockShowToast).toHaveBeenCalledWith({ + message: 'com_ui_live_transcription_connection_lost', + status: 'error', + }); + expect(rendered.setText).toHaveBeenLastCalledWith('Half a', TAKE); + expect(rendered.onTranscriptionSettled).toHaveBeenCalledWith(TAKE); + expect(rendered.result.current.isListening).toBe(false); + act(() => { + jest.runOnlyPendingTimers(); + }); + expect(rendered.onTranscriptionComplete).not.toHaveBeenCalled(); + }); + + it('treats a session the provider closes as a lost connection', async () => { + const rendered = setup(); + const { pc, channel } = await begin(rendered); + connect(pc, channel); + + act(() => channel.drop()); + + expect(mockShowToast).toHaveBeenCalledWith({ + message: 'com_ui_live_transcription_connection_lost', + status: 'error', + }); + expect(track.stop).toHaveBeenCalled(); + expect(rendered.onTranscriptionSettled).toHaveBeenCalledWith(TAKE); + }); + + it('keeps the text heard so far, without sending it, when the provider reports an error', async () => { + const rendered = setup({ autoSendText: 0 }); + const { pc, channel } = await begin(rendered); + connect(pc, channel); + emit(channel, delta('item_1', 'Partial')); + + emit(channel, { type: 'error', error: { code: 'server_error', message: 'boom' } }); + + expect(mockShowToast).toHaveBeenCalledWith({ + message: 'com_ui_live_transcription_error', + status: 'error', + }); + expect(rendered.setText).toHaveBeenLastCalledWith('Partial', TAKE); + expect(rendered.onTranscriptionSettled).toHaveBeenCalledWith(TAKE); + act(() => { + jest.runOnlyPendingTimers(); + }); + expect(rendered.onTranscriptionComplete).not.toHaveBeenCalled(); + }); + + it('settles a commit that never finalizes, keeping its text for review', async () => { + const rendered = setup({ autoSendText: 0 }); + const { pc, channel } = await begin(rendered); + connect(pc, channel); + emit(channel, delta('item_1', 'Still waiting')); + stopAndCommit(rendered); + + act(() => { + jest.advanceTimersByTime(COMPLETION_TIMEOUT_MS - 1); + }); + expect(rendered.onTranscriptionSettled).not.toHaveBeenCalled(); + + act(() => { + jest.advanceTimersByTime(1); + }); + + expect(mockShowToast).toHaveBeenCalledWith({ + message: 'com_ui_live_transcription_error', + status: 'error', + }); + expect(rendered.setText).toHaveBeenLastCalledWith('Still waiting', TAKE); + expect(rendered.onTranscriptionSettled).toHaveBeenCalledWith(TAKE); + expect(rendered.result.current.isLoading).toBe(false); + act(() => { + jest.runOnlyPendingTimers(); + }); + expect(rendered.onTranscriptionComplete).not.toHaveBeenCalled(); + }); + + it('stops itself after silence when Auto Transcribe is on', async () => { + const rendered = setup({ autoTranscribeAudio: true }); + const { pc, channel } = await begin(rendered); + + expect(monitorSilence).not.toHaveBeenCalled(); + connect(pc, channel); + expect(monitorSilence).toHaveBeenCalledWith(expect.anything(), { + minDecibels: -45, + onSilence: expect.any(Function), + }); + + const { onSilence } = jest.mocked(monitorSilence).mock.calls[0][1]; + act(() => onSilence()); + act(() => { + jest.advanceTimersByTime(COMMIT_GRACE_MS); + }); + + expect(channel.send).toHaveBeenCalledWith( + JSON.stringify({ type: 'input_audio_buffer.commit' }), + ); + expect(rendered.result.current.isLoading).toBe(true); + }); + + it('releases the microphone, session, and silence monitor on unmount', async () => { + const rendered = setup({ autoTranscribeAudio: true }); + const { pc, channel } = await begin(rendered); + connect(pc, channel); + + rendered.unmount(); + + expect(track.stop).toHaveBeenCalled(); + expect(pc.close).toHaveBeenCalled(); + expect(channel.close).toHaveBeenCalled(); + expect(mockStopSilence).toHaveBeenCalled(); + }); + + it('stops a microphone stream that arrives after the composer is gone', async () => { + let grant: (stream: unknown) => void = () => undefined; + mockGetUserMedia.mockReturnValueOnce(new Promise((resolve) => (grant = resolve))); + const rendered = setup(); + + let pending: Promise | undefined; + act(() => { + pending = rendered.result.current.startRecording(TAKE); + }); + rendered.unmount(); + await act(async () => { + grant({ getTracks: () => [track], getAudioTracks: () => [track] }); + await pending; + }); + + expect(track.stop).toHaveBeenCalled(); + expect(FakePeerConnection.instances).toHaveLength(0); + }); + + it('drops a queued auto-send when the composer unmounts', async () => { + const rendered = setup({ autoSendText: 2 }); + const { pc, channel } = await begin(rendered); + connect(pc, channel); + stopAndCommit(rendered); + emit(channel, committed('item_1')); + emit(channel, completed('item_1', 'Queued.')); + + rendered.unmount(); + act(() => { + jest.runOnlyPendingTimers(); + }); + + expect(rendered.onTranscriptionComplete).not.toHaveBeenCalled(); + }); +}); diff --git a/client/src/hooks/Input/silence.ts b/client/src/hooks/Input/silence.ts new file mode 100644 index 00000000000..b3b868f3577 --- /dev/null +++ b/client/src/hooks/Input/silence.ts @@ -0,0 +1,59 @@ +/** How long the microphone has to stay below the noise floor before a take ends itself. */ +export const SILENCE_THRESHOLD_MS = 3000; + +/** + * Calls `onSilence` once the stream has stayed quiet for `thresholdMs`, for the Auto + * Transcribe setting. Returns a stop function that cancels the frame loop and closes + * the audio context; it is safe to call more than once and is called for you before + * `onSilence` runs. Without the close, every take leaves another audio context open + * until the browser refuses to grant one. + */ +export function monitorSilence( + stream: MediaStream, + { + minDecibels, + onSilence, + thresholdMs = SILENCE_THRESHOLD_MS, + }: { minDecibels: number; onSilence: () => void; thresholdMs?: number }, +): () => void { + const audioContext = new AudioContext(); + const analyser = audioContext.createAnalyser(); + analyser.minDecibels = minDecibels; + audioContext.createMediaStreamSource(stream).connect(analyser); + + const domainData = new Uint8Array(analyser.frequencyBinCount); + let lastSoundTime = Date.now(); + let frameId: number | null = null; + let stopped = false; + + const stop = () => { + if (stopped) { + return; + } + stopped = true; + if (frameId !== null) { + window.cancelAnimationFrame(frameId); + frameId = null; + } + if (audioContext.state !== 'closed') { + void audioContext.close().catch(() => undefined); + } + }; + + const detectSound = () => { + frameId = null; + analyser.getByteFrequencyData(domainData); + if (domainData.some((value) => value > 0)) { + lastSoundTime = Date.now(); + } + if (Date.now() - lastSoundTime > thresholdMs) { + stop(); + onSilence(); + return; + } + frameId = window.requestAnimationFrame(detectSound); + }; + + frameId = window.requestAnimationFrame(detectSound); + return stop; +} diff --git a/client/src/hooks/Input/useDictation.ts b/client/src/hooks/Input/useDictation.ts index 0be89d0326a..7e2531d7fd7 100644 --- a/client/src/hooks/Input/useDictation.ts +++ b/client/src/hooks/Input/useDictation.ts @@ -1,5 +1,6 @@ import { useRef, useMemo, useState, useEffect, useCallback } from 'react'; import { useToastContext } from '@librechat/client'; +import type { SpeechConnection } from './useSpeechToTextRealtime'; import type { TAskFunction } from '~/common'; import useGetAudioSettings from './useGetAudioSettings'; import { useChatFormContext } from '~/Providers'; @@ -8,7 +9,9 @@ import useSpeechToText from './useSpeechToText'; import { globalAudioId } from '~/common'; import useLocalize from '../useLocalize'; -const isExternalSTT = (speechToTextEndpoint: string) => speechToTextEndpoint === 'external'; +/** Engines that report only the dictated words, so the pre-recording draft is prefixed here. */ +const isExternalSTT = (speechToTextEndpoint: string) => + speechToTextEndpoint === 'external' || speechToTextEndpoint === 'realtime'; /** How the in-flight transcription should be spent once recording ends. */ type StopMode = 'compose' | 'send' | 'cancel'; @@ -26,6 +29,8 @@ export interface Dictation { stopToComposer: () => void; /** Transcribe and send in one action. */ stopAndSend: () => void; + /** Set while a live session is still connecting or is recovering a dropped connection. */ + connection: SpeechConnection; } /** @@ -204,14 +209,12 @@ export default function useDictation({ setSettling(false); } }, []); - const { isListening, isLoading, startRecording, stopRecording, abortRecording } = useSpeechToText( - setText, - onTranscriptionComplete, - onTranscriptionSettled, - autoSendText, - ); + const { isListening, isLoading, startRecording, stopRecording, abortRecording, connection } = + useSpeechToText(setText, onTranscriptionComplete, onTranscriptionSettled, autoSendText); const active = isListening === true; + /** A live take counts from when audio can reach the provider, not from the click. */ + const recording = active && connection !== 'connecting'; /* Bridges the gap between the recorder being told to stop and the upload starting: `isListening` clears synchronously, while the recorder's own @@ -220,13 +223,13 @@ export default function useDictation({ control it had put away flashes back in and out again. */ const [elapsed, setElapsed] = useState(0); useEffect(() => { - if (!active) { + if (!recording) { setElapsed(0); return; } const timer = setInterval(() => setElapsed((value) => value + 1), 1000); return () => clearInterval(timer); - }, [active]); + }, [recording]); /* The engines only report a completed transcription when Auto Send Text is configured, so an explicit stop-and-send cannot wait for that callback: @@ -339,9 +342,11 @@ export default function useDictation({ cancel, stopToComposer, stopAndSend, + connection: active ? connection : null, }), [ active, + connection, discarded, disabled, isLoading, diff --git a/client/src/hooks/Input/useSpeechToText.spec.ts b/client/src/hooks/Input/useSpeechToText.spec.ts index 4e099d231ca..90aa281be0a 100644 --- a/client/src/hooks/Input/useSpeechToText.spec.ts +++ b/client/src/hooks/Input/useSpeechToText.spec.ts @@ -9,6 +9,10 @@ const mockStartSpeechRecordingBrowser = jest.fn(); const mockStopSpeechRecordingBrowser = jest.fn(); const mockStartSpeechRecordingExternal = jest.fn(); const mockStopSpeechRecordingExternal = jest.fn(); +const mockStartSpeechRecordingRealtime = jest.fn(); +const mockStopSpeechRecordingRealtime = jest.fn(); +const mockAbortSpeechRecordingRealtime = jest.fn(); +let mockRealtimeConnection: 'connecting' | 'reconnecting' | null = null; jest.mock('./useGetAudioSettings', () => ({ __esModule: true, @@ -35,12 +39,25 @@ jest.mock('./useSpeechToTextExternal', () => ({ }), })); +jest.mock('./useSpeechToTextRealtime', () => ({ + __esModule: true, + default: () => ({ + isListening: true, + isLoading: false, + connection: mockRealtimeConnection, + startRecording: mockStartSpeechRecordingRealtime, + stopRecording: mockStopSpeechRecordingRealtime, + abortRecording: mockAbortSpeechRecordingRealtime, + }), +})); + describe('useSpeechToText', () => { beforeEach(() => { jest.clearAllMocks(); mockSpeechToTextEndpoint = 'external'; mockBrowserIsListening = false; mockExternalIsListening = false; + mockRealtimeConnection = null; }); it('selects the externally seeded engine after settings change', () => { @@ -78,4 +95,30 @@ describe('useSpeechToText', () => { expect(mockStartSpeechRecordingExternal).not.toHaveBeenCalled(); expect(mockStopSpeechRecordingBrowser).not.toHaveBeenCalled(); }); + + it('selects the live engine and reports its connection', () => { + mockSpeechToTextEndpoint = 'realtime'; + mockRealtimeConnection = 'connecting'; + const { result } = renderHook(() => useSpeechToText(jest.fn(), jest.fn(), jest.fn(), -1)); + + expect(result.current.isListening).toBe(true); + expect(result.current.connection).toBe('connecting'); + + act(() => result.current.startRecording()); + act(() => result.current.stopRecording()); + act(() => result.current.abortRecording()); + + expect(mockStartSpeechRecordingRealtime).toHaveBeenCalledTimes(1); + expect(mockStopSpeechRecordingRealtime).toHaveBeenCalledTimes(1); + expect(mockAbortSpeechRecordingRealtime).toHaveBeenCalledTimes(1); + expect(mockStartSpeechRecordingExternal).not.toHaveBeenCalled(); + expect(mockStartSpeechRecordingBrowser).not.toHaveBeenCalled(); + }); + + it('reports no connection state for the recorded engines', () => { + mockRealtimeConnection = 'reconnecting'; + const { result } = renderHook(() => useSpeechToText(jest.fn(), jest.fn(), jest.fn(), -1)); + + expect(result.current.connection).toBeNull(); + }); }); diff --git a/client/src/hooks/Input/useSpeechToText.ts b/client/src/hooks/Input/useSpeechToText.ts index 47a10f11ff1..1b52606b833 100644 --- a/client/src/hooks/Input/useSpeechToText.ts +++ b/client/src/hooks/Input/useSpeechToText.ts @@ -1,3 +1,5 @@ +import type { SpeechConnection } from './useSpeechToTextRealtime'; +import useSpeechToTextRealtime from './useSpeechToTextRealtime'; import useSpeechToTextExternal from './useSpeechToTextExternal'; import useSpeechToTextBrowser from './useSpeechToTextBrowser'; import useGetAudioSettings from './useGetAudioSettings'; @@ -15,9 +17,12 @@ const useSpeechToText = ( startRecording: (takeId?: number) => void | Promise; /** Ends capture without producing a transcript. */ abortRecording: () => void; + /** Set while a live session is still being established or recovering. */ + connection: SpeechConnection; } => { const { speechToTextEndpoint } = useGetAudioSettings(); const externalSpeechToText = speechToTextEndpoint === 'external'; + const realtimeSpeechToText = speechToTextEndpoint === 'realtime'; const { isListening: speechIsListeningBrowser, @@ -45,6 +50,24 @@ const useSpeechToText = ( autoSendText, ); + const realtime = useSpeechToTextRealtime( + setText, + onTranscriptionComplete, + onTranscriptionSettled, + autoSendText, + ); + + if (realtimeSpeechToText) { + return { + isLoading: realtime.isLoading, + isListening: realtime.isListening, + stopRecording: realtime.stopRecording, + startRecording: realtime.startRecording, + abortRecording: realtime.abortRecording, + connection: realtime.connection, + }; + } + const isListening = externalSpeechToText ? speechIsListeningExternal : speechIsListeningBrowser; const isLoading = externalSpeechToText ? speechIsLoadingExternal : speechIsLoadingBrowser; @@ -65,6 +88,7 @@ const useSpeechToText = ( stopRecording, startRecording, abortRecording, + connection: null, }; }; diff --git a/client/src/hooks/Input/useSpeechToTextExternal.ts b/client/src/hooks/Input/useSpeechToTextExternal.ts index 33b38acc1dd..f5ab2bc4a55 100644 --- a/client/src/hooks/Input/useSpeechToTextExternal.ts +++ b/client/src/hooks/Input/useSpeechToTextExternal.ts @@ -2,6 +2,7 @@ import { useState, useEffect, useRef } from 'react'; import { useRecoilState } from 'recoil'; import { useToastContext } from '@librechat/client'; import { useSpeechToTextMutation } from '~/data-provider'; +import { monitorSilence } from './silence'; import store from '~/store'; export const getBestSupportedMimeType = ( @@ -42,8 +43,8 @@ const useSpeechToTextExternal = ( ) => { const { showToast } = useToastContext(); const audioStream = useRef(null); - const animationFrameIdRef = useRef(null); - const audioContextRef = useRef(null); + /** Ends the Auto Transcribe silence monitor; set only while one is running. */ + const stopSilenceRef = useRef<(() => void) | null>(null); const mediaRecorderRef = useRef(null); const audioChunksRef = useRef([]); @@ -161,43 +162,6 @@ const useSpeechToTextExternal = ( } }; - const monitorSilence = (stream: MediaStream, stopRecording: () => void) => { - /* Held so it can be closed again: without this the ref below is never - assigned, its guard is always true, and every take leaves another audio - context open until the browser refuses to grant one. */ - const audioContext = new AudioContext(); - audioContextRef.current = audioContext; - const audioStreamSource = audioContext.createMediaStreamSource(stream); - const analyser = audioContext.createAnalyser(); - analyser.minDecibels = minDecibels; - audioStreamSource.connect(analyser); - - const bufferLength = analyser.frequencyBinCount; - const domainData = new Uint8Array(bufferLength); - let lastSoundTime = Date.now(); - - const detectSound = () => { - analyser.getByteFrequencyData(domainData); - const isSoundDetected = domainData.some((value) => value > 0); - - if (isSoundDetected) { - lastSoundTime = Date.now(); - } - - const timeSinceLastSound = Date.now() - lastSoundTime; - const isOverSilenceThreshold = timeSinceLastSound > 3000; - - if (isOverSilenceThreshold) { - stopRecording(); - return; - } - - animationFrameIdRef.current = window.requestAnimationFrame(detectSound); - }; - - animationFrameIdRef.current = window.requestAnimationFrame(detectSound); - }; - /** Stops and drops the capture this hook owns; safe to call more than once. */ const releaseAudioStream = () => { audioStream.current?.getTracks().forEach((track) => track.stop()); @@ -242,8 +206,11 @@ const useSpeechToTextExternal = ( }); mediaRecorderRef.current.addEventListener('stop', handleStop); mediaRecorderRef.current.start(100); - if (!audioContextRef.current && autoTranscribeAudio && speechToText) { - monitorSilence(audioStream.current, stopRecording); + if (!stopSilenceRef.current && autoTranscribeAudio && speechToText) { + stopSilenceRef.current = monitorSilence(audioStream.current, { + minDecibels, + onSilence: stopRecording, + }); } setIsListening(true); } catch (error) { @@ -260,13 +227,10 @@ const useSpeechToTextExternal = ( } }; - /** Releases the silence monitor's audio graph; safe to call more than once. */ - const closeAudioContext = () => { - const context = audioContextRef.current; - audioContextRef.current = null; - if (context != null && context.state !== 'closed') { - void context.close().catch(() => undefined); - } + /** Releases the silence monitor's frame loop and audio graph; safe to call more than once. */ + const stopSilenceMonitor = () => { + stopSilenceRef.current?.(); + stopSilenceRef.current = null; }; const stopRecording = () => { @@ -279,12 +243,7 @@ const useSpeechToTextExternal = ( audioStream.current?.getTracks().forEach((track) => track.stop()); audioStream.current = null; - - if (animationFrameIdRef.current !== null) { - window.cancelAnimationFrame(animationFrameIdRef.current); - animationFrameIdRef.current = null; - } - closeAudioContext(); + stopSilenceMonitor(); setIsListening(false); } else { @@ -328,12 +287,7 @@ const useSpeechToTextExternal = ( audioStream.current?.getTracks().forEach((track) => track.stop()); audioStream.current = null; - - if (animationFrameIdRef.current !== null) { - window.cancelAnimationFrame(animationFrameIdRef.current); - animationFrameIdRef.current = null; - } - closeAudioContext(); + stopSilenceMonitor(); setIsListening(false); }; @@ -364,16 +318,8 @@ const useSpeechToTextExternal = ( audioStream.current?.getTracks().forEach((track) => track.stop()); audioStream.current = null; - if (animationFrameIdRef.current !== null) { - window.cancelAnimationFrame(animationFrameIdRef.current); - animationFrameIdRef.current = null; - } - - const context = audioContextRef.current; - audioContextRef.current = null; - if (context != null && context.state !== 'closed') { - void context.close().catch(() => undefined); - } + stopSilenceRef.current?.(); + stopSilenceRef.current = null; }; }, []); diff --git a/client/src/hooks/Input/useSpeechToTextRealtime.ts b/client/src/hooks/Input/useSpeechToTextRealtime.ts new file mode 100644 index 00000000000..a42b33d4abe --- /dev/null +++ b/client/src/hooks/Input/useSpeechToTextRealtime.ts @@ -0,0 +1,475 @@ +import { useState, useEffect, useRef } from 'react'; +import axios from 'axios'; +import { useRecoilValue } from 'recoil'; +import { useToastContext } from '@librechat/client'; +import { RealtimeTranscriptionErrorCode } from 'librechat-data-provider'; +import { useRealtimeTranscriptionMutation } from '~/data-provider'; +import { monitorSilence } from './silence'; +import useLocalize from '../useLocalize'; +import store from '~/store'; + +/** Where the live session stands while it is not simply listening. */ +export type SpeechConnection = 'connecting' | 'reconnecting' | null; + +/** The event channel the Realtime API reads and writes. */ +const EVENTS_CHANNEL = 'oai-events'; +/** Microphone audio travels by RTP while the commit travels on the data channel, so + * the last words spoken before Stop can still be in flight when the commit lands. */ +export const COMMIT_GRACE_MS = 300; +/** How long a committed take may wait for its final transcript before it is + * settled with whatever text has already arrived. */ +export const COMPLETION_TIMEOUT_MS = 15000; + +type LiveTake = { + takeId?: number; + stream: MediaStream | null; + pc: RTCPeerConnection | null; + channel: RTCDataChannel | null; + /** Item ids in the order they were first heard, so text never reorders. */ + order: string[]; + text: Map; + completed: Set; + /** Set once Stop has asked the provider to finalize the buffer. */ + committing: boolean; + /** The item the commit produced; its completion ends the take. */ + awaitedItem: string | null; + timers: ReturnType[]; + stopSilence: (() => void) | null; +}; + +const getErrorCode = (error: unknown): string | undefined => { + if (!axios.isAxiosError(error)) { + return undefined; + } + const data = error.response?.data as { error?: unknown } | undefined; + return typeof data?.error === 'string' ? data.error : undefined; +}; + +const isPermissionError = (error: unknown) => + error instanceof DOMException && + (error.name === 'NotAllowedError' || error.name === 'SecurityError'); + +/** The transcript so far, one item after another. */ +const transcriptOf = (take: LiveTake) => + take.order + .map((id) => take.text.get(id)?.trim() ?? '') + .filter(Boolean) + .join(' '); + +/** + * Live dictation over a Realtime transcription session. The browser streams the + * microphone to the provider over WebRTC; the server opens the session, so no + * credential reaches the page. Text is written into the composer as it is heard, + * and Stop commits the buffer so the provider can finalize it. + * + * Live transcripts go straight from the provider to the browser: unlike recorded + * audio, they do not pass through the server on their way into the composer. + */ +const useSpeechToTextRealtime = ( + setText: (text: string, takeId?: number) => void, + onTranscriptionComplete: (text: string, takeId?: number) => void, + onTranscriptionSettled: (takeId?: number) => void, + autoSendText: number, +) => { + const localize = useLocalize(); + const { showToast } = useToastContext(); + const [isListening, setIsListening] = useState(false); + const [isLoading, setIsLoading] = useState(false); + const [connection, setConnection] = useState(null); + + const minDecibels = useRecoilValue(store.decibelValue); + const languageSTT = useRecoilValue(store.languageSTT); + const speechToText = useRecoilValue(store.speechToText); + const autoTranscribeAudio = useRecoilValue(store.autoTranscribeAudio); + + const { mutateAsync: openSession } = useRealtimeTranscriptionMutation(); + + const takeRef = useRef(null); + const isMountedRef = useRef(true); + /** Cleared on unmount so a queued auto-send cannot fire into a gone composer. */ + const autoSendTimerRef = useRef | null>(null); + /** Session events arrive through handlers bound when the take started; they read + * the newest callbacks and settings from here rather than from that render. */ + const latest = useRef({ + setText, + onTranscriptionComplete, + onTranscriptionSettled, + autoSendText, + speechToText, + autoTranscribeAudio, + minDecibels, + showToast, + localize, + }); + latest.current = { + setText, + onTranscriptionComplete, + onTranscriptionSettled, + autoSendText, + speechToText, + autoTranscribeAudio, + minDecibels, + showToast, + localize, + }; + + const toastError = (key: Parameters[0]) => { + const { showToast, localize } = latest.current; + showToast({ message: localize(key), status: 'error' }); + }; + + /** Stops the microphone and closes the session; safe to call more than once. */ + const release = (take: LiveTake) => { + take.timers.forEach(clearTimeout); + take.timers = []; + take.stopSilence?.(); + take.stopSilence = null; + take.stream?.getTracks().forEach((track) => track.stop()); + take.stream = null; + if (take.channel) { + take.channel.onopen = null; + take.channel.onclose = null; + take.channel.onmessage = null; + take.channel.close(); + take.channel = null; + } + if (take.pc) { + take.pc.onconnectionstatechange = null; + take.pc.close(); + take.pc = null; + } + }; + + /** Ends the current take and hands its transcript to the composer. */ + const finish = (take: LiveTake, { autoSend }: { autoSend: boolean }) => { + if (takeRef.current !== take) { + return; + } + takeRef.current = null; + release(take); + if (!isMountedRef.current) { + return; + } + setIsListening(false); + setIsLoading(false); + setConnection(null); + + const { autoSendText, speechToText } = latest.current; + const text = transcriptOf(take); + if (text) { + latest.current.setText(text, take.takeId); + } + latest.current.onTranscriptionSettled(take.takeId); + + if (!autoSend || autoSendText < 0 || !speechToText || !text) { + return; + } + if (autoSendTimerRef.current != null) { + clearTimeout(autoSendTimerRef.current); + } + autoSendTimerRef.current = setTimeout(() => { + autoSendTimerRef.current = null; + latest.current.onTranscriptionComplete(text, take.takeId); + }, autoSendText * 1000); + }; + + const publish = (take: LiveTake) => { + latest.current.setText(transcriptOf(take), take.takeId); + }; + + const remember = (take: LiveTake, itemId: string) => { + if (!take.text.has(itemId)) { + take.order.push(itemId); + take.text.set(itemId, ''); + } + }; + + const handleEvent = (take: LiveTake, raw: unknown) => { + if (takeRef.current !== take || typeof raw !== 'string') { + return; + } + let event: Record; + try { + event = JSON.parse(raw); + } catch { + return; + } + const itemId = typeof event.item_id === 'string' ? event.item_id : null; + + switch (event.type) { + case 'conversation.item.input_audio_transcription.delta': { + if (itemId == null || typeof event.delta !== 'string' || take.completed.has(itemId)) { + return; + } + remember(take, itemId); + take.text.set(itemId, (take.text.get(itemId) ?? '') + event.delta); + publish(take); + return; + } + case 'conversation.item.input_audio_transcription.completed': { + if (itemId == null) { + return; + } + remember(take, itemId); + take.text.set(itemId, typeof event.transcript === 'string' ? event.transcript : ''); + take.completed.add(itemId); + publish(take); + if (take.committing && take.awaitedItem === itemId) { + finish(take, { autoSend: true }); + } + return; + } + case 'input_audio_buffer.committed': { + if (!take.committing || itemId == null) { + return; + } + take.awaitedItem = itemId; + remember(take, itemId); + if (take.completed.has(itemId)) { + finish(take, { autoSend: true }); + } + return; + } + case 'conversation.item.input_audio_transcription.failed': { + if (itemId != null && itemId === take.awaitedItem) { + toastError('com_ui_live_transcription_error'); + finish(take, { autoSend: false }); + } + return; + } + case 'error': { + const error = event.error as { code?: unknown } | undefined; + /* Stop before any audio reached the provider: nothing to finalize. */ + if (take.committing && error?.code === 'input_audio_buffer_commit_empty') { + finish(take, { autoSend: true }); + return; + } + toastError('com_ui_live_transcription_error'); + finish(take, { autoSend: false }); + return; + } + default: + return; + } + }; + + const handleConnectionState = (take: LiveTake) => { + if (takeRef.current !== take || take.pc == null) { + return; + } + const state = take.pc.connectionState; + if (state === 'disconnected') { + setConnection('reconnecting'); + } else if (state === 'connected') { + setConnection(take.channel?.readyState === 'open' ? null : 'connecting'); + } else if (state === 'failed') { + toastError('com_ui_live_transcription_connection_lost'); + finish(take, { autoSend: false }); + } + }; + + const handleOpen = (take: LiveTake) => { + if (takeRef.current !== take) { + return; + } + setConnection(null); + const { autoTranscribeAudio, speechToText, minDecibels } = latest.current; + if (autoTranscribeAudio && speechToText && take.stream) { + take.stopSilence = monitorSilence(take.stream, { + minDecibels, + onSilence: () => stopTake(take), + }); + } + }; + + /** Drops a take that never got going, without writing anything. */ + const abandon = (take: LiveTake) => { + if (takeRef.current !== take) { + return; + } + takeRef.current = null; + release(take); + if (!isMountedRef.current) { + return; + } + setIsListening(false); + setIsLoading(false); + setConnection(null); + latest.current.onTranscriptionSettled(take.takeId); + }; + + const startRecording = async (takeId?: number) => { + if (takeRef.current) { + return; + } + if (typeof RTCPeerConnection === 'undefined' || !navigator.mediaDevices?.getUserMedia) { + toastError('com_ui_live_transcription_unsupported'); + return; + } + + const take: LiveTake = { + takeId, + stream: null, + pc: null, + channel: null, + order: [], + text: new Map(), + completed: new Set(), + committing: false, + awaitedItem: null, + timers: [], + stopSilence: null, + }; + takeRef.current = take; + setIsListening(true); + setConnection('connecting'); + + try { + take.stream = await navigator.mediaDevices.getUserMedia({ audio: true, video: false }); + } catch (error) { + if (takeRef.current === take) { + toastError( + isPermissionError(error) + ? 'com_ui_microphone_permission_denied' + : 'com_ui_microphone_unavailable', + ); + abandon(take); + } + return; + } + if (takeRef.current !== take) { + take.stream.getTracks().forEach((track) => track.stop()); + take.stream = null; + return; + } + + try { + const pc = new RTCPeerConnection(); + take.pc = pc; + const stream = take.stream; + stream.getAudioTracks().forEach((track) => pc.addTrack(track, stream)); + const channel = pc.createDataChannel(EVENTS_CHANNEL); + take.channel = channel; + channel.onmessage = (event: MessageEvent) => handleEvent(take, event.data); + channel.onopen = () => handleOpen(take); + channel.onclose = () => { + if (takeRef.current === take) { + toastError('com_ui_live_transcription_connection_lost'); + finish(take, { autoSend: false }); + } + }; + pc.onconnectionstatechange = () => handleConnectionState(take); + + const offer = await pc.createOffer(); + await pc.setLocalDescription(offer); + if (takeRef.current !== take) { + return; + } + const answer = await openSession({ + sdp: offer.sdp ?? '', + ...(languageSTT ? { language: languageSTT } : {}), + }); + if (takeRef.current !== take) { + return; + } + await pc.setRemoteDescription({ type: 'answer', sdp: answer.sdp }); + } catch (error) { + if (takeRef.current !== take) { + return; + } + toastError( + getErrorCode(error) === RealtimeTranscriptionErrorCode.NOT_CONFIGURED + ? 'com_ui_live_transcription_unavailable' + : 'com_ui_live_transcription_start_error', + ); + abandon(take); + } + }; + + /** Asks the provider to finalize what it heard, then waits for the transcript. */ + const stopTake = (take: LiveTake) => { + if (takeRef.current !== take || take.committing) { + return; + } + if (take.channel?.readyState !== 'open') { + /* Still connecting: no audio has reached the provider yet. */ + abandon(take); + return; + } + take.committing = true; + take.stopSilence?.(); + take.stopSilence = null; + setIsListening(false); + setIsLoading(true); + setConnection(null); + + take.timers.push( + setTimeout(() => { + if (takeRef.current !== take) { + return; + } + try { + take.channel?.send(JSON.stringify({ type: 'input_audio_buffer.commit' })); + } catch { + toastError('com_ui_live_transcription_error'); + finish(take, { autoSend: false }); + return; + } + take.stream?.getTracks().forEach((track) => track.stop()); + }, COMMIT_GRACE_MS), + /* A transcript that never finalized may be cut short, so it is kept for + review rather than sent. */ + setTimeout(() => { + if (takeRef.current === take) { + toastError('com_ui_live_transcription_error'); + finish(take, { autoSend: false }); + } + }, COMMIT_GRACE_MS + COMPLETION_TIMEOUT_MS), + ); + }; + + const stopRecording = () => { + if (takeRef.current) { + stopTake(takeRef.current); + } + }; + + /** Drops the take without writing or sending anything. */ + const abortRecording = () => { + const take = takeRef.current; + takeRef.current = null; + if (take) { + release(take); + } + setIsListening(false); + setIsLoading(false); + setConnection(null); + }; + + useEffect(() => { + isMountedRef.current = true; + return () => { + isMountedRef.current = false; + if (autoSendTimerRef.current != null) { + clearTimeout(autoSendTimerRef.current); + autoSendTimerRef.current = null; + } + const take = takeRef.current; + takeRef.current = null; + if (take) { + release(take); + } + }; + }, []); + + return { + isListening, + isLoading, + connection, + startRecording, + stopRecording, + abortRecording, + }; +}; + +export default useSpeechToTextRealtime; diff --git a/client/src/locales/en/translation.json b/client/src/locales/en/translation.json index 0826cd73315..e40b309a88d 100644 --- a/client/src/locales/en/translation.json +++ b/client/src/locales/en/translation.json @@ -957,6 +957,7 @@ "com_nav_lang_vietnamese": "Tiếng Việt", "com_nav_language": "Language", "com_nav_latex_parsing": "Parsing LaTeX in messages (may affect performance)", + "com_nav_live": "Live", "com_nav_log_out": "Log out", "com_nav_long_audio_warning": "Longer texts will take longer to process.", "com_nav_maximize_chat_space": "Maximize chat space", @@ -2171,6 +2172,11 @@ "com_ui_link_refreshed": "Link updated", "com_ui_listening": "Listening", "com_ui_live": "live", + "com_ui_live_transcription_connection_lost": "Live transcription lost its connection. Any text heard so far was kept.", + "com_ui_live_transcription_error": "Live transcription stopped unexpectedly. Any text heard so far was kept.", + "com_ui_live_transcription_start_error": "Could not start live transcription", + "com_ui_live_transcription_unavailable": "Live transcription is not configured on this server", + "com_ui_live_transcription_unsupported": "Your browser does not support live transcription", "com_ui_load_more": "Load more", "com_ui_loading": "Loading...", "com_ui_loading_interactive_view": "Loading interactive view...", @@ -2318,6 +2324,7 @@ "com_ui_message_nav_previous": "Navigate to previous message", "com_ui_message_part_empty": "Message content cannot be empty.", "com_ui_method": "Method", + "com_ui_microphone_permission_denied": "Microphone access was denied. Allow it in your browser settings to dictate.", "com_ui_microphone_unavailable": "Microphone is not available", "com_ui_min_tags": "Cannot remove more values, a minimum of {{0}} are required.", "com_ui_minimal": "Minimal", @@ -2614,6 +2621,7 @@ "com_ui_read_aloud": "Read aloud", "com_ui_read_file": "Read {{0}}", "com_ui_reading_file": "Reading {{0}}", + "com_ui_reconnecting": "Reconnecting", "com_ui_redirect_uri": "Redirect URI", "com_ui_redirect_uri_instructions": "Copy this redirect URI and configure it in your OAuth provider settings.", "com_ui_redirecting_to_provider": "Redirecting to {{0}}, please wait...", diff --git a/client/src/store/settings.ts b/client/src/store/settings.ts index 6c66181d091..19a18ef4b7b 100644 --- a/client/src/store/settings.ts +++ b/client/src/store/settings.ts @@ -3,10 +3,11 @@ import { SettingsViews, LocalStorageKeys } from 'librechat-data-provider'; import type { TOptionSettings } from '~/common'; import { CHAT_TITLE_IN_TAB_KEY } from '~/utils/documentTitle'; import { isDrawerViewport } from '~/utils/breakpoints'; +import { STTEndpoints, TTSEndpoints } from '~/common'; import { atomWithLocalStorage } from '~/store/utils'; -import { STTEndpoints } from '~/common'; -const VALID_SPEECH_ENGINES = new Set(Object.values(STTEndpoints)); +const VALID_STT_ENGINES = new Set(Object.values(STTEndpoints)); +const VALID_TTS_ENGINES = new Set(Object.values(TTSEndpoints)); const LEGACY_EXTERNAL_STT_ENGINES = new Set(['openai', 'azureOpenAI']); const LEGACY_EXTERNAL_TTS_ENGINES = new Set([ 'openai', @@ -17,9 +18,10 @@ const LEGACY_EXTERNAL_TTS_ENGINES = new Set([ const normalizeSavedSpeechEngine = ( engine: string, + validEngines: ReadonlySet, legacyExternalEngines: ReadonlySet, ): string => { - if (VALID_SPEECH_ENGINES.has(engine)) return engine; + if (validEngines.has(engine)) return engine; return legacyExternalEngines.has(engine) ? STTEndpoints.external : STTEndpoints.browser; }; @@ -111,7 +113,7 @@ const localStorageAtoms = { speechToText: atomWithLocalStorage('speechToText', true), engineSTT: atomWithLocalStorage('engineSTT', 'browser', (engine) => - normalizeSavedSpeechEngine(engine, LEGACY_EXTERNAL_STT_ENGINES), + normalizeSavedSpeechEngine(engine, VALID_STT_ENGINES, LEGACY_EXTERNAL_STT_ENGINES), ), languageSTT: atomWithLocalStorage('languageSTT', ''), autoTranscribeAudio: atomWithLocalStorage('autoTranscribeAudio', false), @@ -120,7 +122,7 @@ const localStorageAtoms = { textToSpeech: atomWithLocalStorage('textToSpeech', true), engineTTS: atomWithLocalStorage('engineTTS', 'browser', (engine) => - normalizeSavedSpeechEngine(engine, LEGACY_EXTERNAL_TTS_ENGINES), + normalizeSavedSpeechEngine(engine, VALID_TTS_ENGINES, LEGACY_EXTERNAL_TTS_ENGINES), ), voice: atomWithLocalStorage('voice', undefined), cloudBrowserVoices: atomWithLocalStorage('cloudBrowserVoices', false), diff --git a/librechat.example.yaml b/librechat.example.yaml index f00bd479fa6..d6185cf0008 100644 --- a/librechat.example.yaml +++ b/librechat.example.yaml @@ -511,10 +511,30 @@ registration: # openai: # url: '' # apiKey: '${STT_API_KEY}' -# model: '' +# model: '' # e.g. 'gpt-transcribe', 'gpt-4o-transcribe', 'whisper-1' +# # Optional transcription hints. `keywords` and `languages` are sent only to +# # gpt-transcribe / gpt-live-transcribe; older models receive `prompt` and the +# # user's language as the singular `language` field instead. +# # prompt: 'Internal retail and software terminology may be used.' +# # keywords: ['LibreChat', 'OpenShift'] +# # languages: ['de', 'en'] # narrowed to the user's speech language when one is set +# # Optional live dictation over a Realtime transcription session (WebRTC). When set, +# # users can pick "Live" under Settings > Speech > Engine. The server opens each +# # session with a short-lived client secret, so the API key never reaches the browser. +# # Live audio and transcripts flow directly between the browser and the provider and +# # are not routed through LibreChat, so `filters` rules for the `transcript` field are +# # not applied to them. Live sessions are billed per minute while the microphone is open. +# # realtime: +# # model: 'gpt-live-transcribe' # older realtime models only return text after Stop +# # url: '' # defaults to https://api.openai.com/v1 +# # delay: 'low' # gpt-live-transcribe only: minimal | low | medium | high | xhigh # allowedAddresses: # - 'localhost:8000' # - '127.0.0.1:8000' +# +# speechTab: +# speechToText: +# engineSTT: 'external' # 'browser' | 'external' | 'realtime' (requires stt.openai.realtime) # OCR (Mistral / Mistral-compatible) outbound requests to `ocr.baseURL` are SSRF-guarded at # connect time with the same default-deny for private targets. To point OCR at a private or diff --git a/packages/api/src/files/index.ts b/packages/api/src/files/index.ts index b08ebd4666d..592bf3a40c0 100644 --- a/packages/api/src/files/index.ts +++ b/packages/api/src/files/index.ts @@ -19,6 +19,7 @@ export * from './upload/routing'; export * from './upload/diagnostics'; export * from './upload/fallback'; export * from './rag'; +export * from './realtime'; export * from './rag/context'; export * from './rag/search'; export * from './regexEngine'; @@ -29,4 +30,5 @@ export * from './sweep'; export * from './usage'; export * from './validation'; export * from './text'; +export * from './transcription'; export * from './list'; diff --git a/packages/api/src/files/realtime.spec.ts b/packages/api/src/files/realtime.spec.ts new file mode 100644 index 00000000000..98ef1c1cdcb --- /dev/null +++ b/packages/api/src/files/realtime.spec.ts @@ -0,0 +1,371 @@ +import http from 'http'; +import { logger } from '@librechat/data-schemas'; +import { RealtimeTranscriptionErrorCode } from 'librechat-data-provider'; +import type { Response } from 'express'; +import type { AddressInfo } from 'net'; +import type { STTConfig } from './transcription'; +import { + openRealtimeTranscription, + getRealtimeTranscriptionTarget, + isRealtimeTranscriptionAvailable, + buildRealtimeTranscriptionSession, + createRealtimeTranscriptionHandler, +} from './realtime'; + +const OFFER = 'v=0\r\no=- 1 2 IN IP4 127.0.0.1\r\ns=-\r\nm=audio 9 UDP/TLS/RTP/SAVPF 111\r\n'; +const ANSWER = 'v=0\r\no=- 3 4 IN IP4 0.0.0.0\r\ns=-\r\nm=audio 9 UDP/TLS/RTP/SAVPF 111\r\n'; +const API_KEY = 'sk-test-configured-key'; +const CLIENT_SECRET = 'ek_test_ephemeral_secret'; +const PROVIDER_DETAIL = 'PROVIDER-ECHO private request detail'; + +interface RecordedRequest { + method?: string; + url?: string; + headers: http.IncomingHttpHeaders; + body: string; +} + +type ProviderReply = { status: number; type: string; body: string }; + +const liveConfig = (overrides: Record = {}): STTConfig => ({ + openai: { + apiKey: API_KEY, + model: 'gpt-transcribe', + prompt: 'Retail and software terms.', + keywords: ['LibreChat'], + languages: ['de', 'en'], + realtime: { model: 'gpt-live-transcribe', delay: 'low' }, + ...overrides, + }, +}); + +describe('getRealtimeTranscriptionTarget', () => { + it('is off without an OpenAI realtime block', () => { + expect(getRealtimeTranscriptionTarget(undefined)).toBeNull(); + expect( + getRealtimeTranscriptionTarget({ openai: { apiKey: 'k', model: 'whisper-1' } }), + ).toBeNull(); + expect( + getRealtimeTranscriptionTarget({ + openai: { apiKey: 'k', model: 'm', realtime: { model: ' ' } }, + }), + ).toBeNull(); + }); + + it('is off for Azure OpenAI and for configs the recorded path rejects', () => { + expect( + getRealtimeTranscriptionTarget({ + azureOpenAI: { apiKey: 'k', instanceName: 'i', deploymentName: 'd', apiVersion: 'v' }, + }), + ).toBeNull(); + expect( + getRealtimeTranscriptionTarget({ + ...liveConfig(), + azureOpenAI: { apiKey: 'k', instanceName: 'i', deploymentName: 'd', apiVersion: 'v' }, + }), + ).toBeNull(); + expect(isRealtimeTranscriptionAvailable({ openai: { apiKey: 'k', model: 'm' } })).toBe(false); + }); + + it('defaults to the OpenAI API and normalizes the configured base URL', () => { + expect(getRealtimeTranscriptionTarget(liveConfig())).toMatchObject({ + url: 'https://api.openai.com/v1', + model: 'gpt-live-transcribe', + delay: 'low', + }); + expect( + getRealtimeTranscriptionTarget({ + allowedAddresses: ['10.0.0.5:8443'], + ...liveConfig({ + realtime: { url: 'https://example.openai.azure.com/openai/v1/', model: 'live' }, + }), + }), + ).toMatchObject({ + url: 'https://example.openai.azure.com/openai/v1', + allowedAddresses: ['10.0.0.5:8443'], + }); + expect(isRealtimeTranscriptionAvailable(liveConfig())).toBe(true); + }); +}); + +describe('buildRealtimeTranscriptionSession', () => { + it('builds a manual-commit gpt-live-transcribe session with hints and delay', () => { + const target = getRealtimeTranscriptionTarget(liveConfig())!; + + expect(buildRealtimeTranscriptionSession(target, 'en-US')).toEqual({ + type: 'transcription', + audio: { + input: { + format: { type: 'audio/pcm', rate: 24000 }, + transcription: { + model: 'gpt-live-transcribe', + prompt: 'Retail and software terms.', + keywords: ['LibreChat'], + languages: ['en'], + delay: 'low', + }, + turn_detection: null, + }, + }, + }); + }); + + it('gives older realtime models the singular language and no hint-only fields', () => { + const target = getRealtimeTranscriptionTarget( + liveConfig({ realtime: { model: 'gpt-4o-mini-transcribe', delay: 'low' } }), + )!; + + const { transcription, turn_detection } = buildRealtimeTranscriptionSession(target, 'de-DE') + .audio.input; + + expect(transcription).toEqual({ + model: 'gpt-4o-mini-transcribe', + prompt: 'Retail and software terms.', + language: 'de', + }); + expect(turn_detection).toBeNull(); + }); +}); + +describe('openRealtimeTranscription', () => { + let server: http.Server; + let baseUrl: string; + let allowedAddresses: string[]; + let requests: RecordedRequest[]; + let replies: Record; + + beforeAll(async () => { + server = http.createServer((req, res) => { + const chunks: Buffer[] = []; + req.on('data', (chunk: Buffer) => chunks.push(chunk)); + req.on('end', () => { + requests.push({ + method: req.method, + url: req.url, + headers: req.headers, + body: Buffer.concat(chunks).toString('utf8'), + }); + const reply = replies[req.url ?? ''] ?? { status: 404, type: 'text/plain', body: '' }; + res.writeHead(reply.status, { 'Content-Type': reply.type }); + res.end(reply.body); + }); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const { port } = server.address() as AddressInfo; + baseUrl = `http://127.0.0.1:${port}/v1`; + allowedAddresses = [`127.0.0.1:${port}`]; + }); + + afterAll(async () => { + await new Promise((resolve) => server.close(resolve)); + }); + + beforeEach(() => { + requests = []; + replies = { + '/v1/realtime/client_secrets': { + status: 200, + type: 'application/json', + body: JSON.stringify({ value: CLIENT_SECRET, expires_at: 1 }), + }, + '/v1/realtime/calls': { status: 201, type: 'application/sdp', body: ANSWER }, + }; + jest.spyOn(logger, 'error').mockImplementation(() => logger); + }); + + afterEach(() => { + jest.restoreAllMocks(); + }); + + const localConfig = (overrides: Record = {}): STTConfig => ({ + allowedAddresses, + ...liveConfig({ + realtime: { url: baseUrl, model: 'gpt-live-transcribe', delay: 'minimal' }, + ...overrides, + }), + }); + + it('mints a client secret with the server key, then exchanges the SDP with that secret', async () => { + const result = await openRealtimeTranscription(localConfig(), { + sdp: OFFER, + language: 'zh-TW', + }); + + expect(result).toEqual({ ok: true, sdp: ANSWER }); + expect(requests).toHaveLength(2); + + const [secretRequest, callRequest] = requests; + expect(secretRequest.method).toBe('POST'); + expect(secretRequest.url).toBe('/v1/realtime/client_secrets'); + expect(secretRequest.headers.authorization).toBe(`Bearer ${API_KEY}`); + expect(JSON.parse(secretRequest.body)).toEqual({ + expires_after: { anchor: 'created_at', seconds: 60 }, + session: { + type: 'transcription', + audio: { + input: { + format: { type: 'audio/pcm', rate: 24000 }, + transcription: { + model: 'gpt-live-transcribe', + prompt: 'Retail and software terms.', + keywords: ['LibreChat'], + languages: ['zh-tw'], + delay: 'minimal', + }, + turn_detection: null, + }, + }, + }, + }); + + expect(callRequest.url).toBe('/v1/realtime/calls'); + expect(callRequest.headers.authorization).toBe(`Bearer ${CLIENT_SECRET}`); + expect(callRequest.headers['content-type']).toBe('application/sdp'); + expect(callRequest.body).toBe(OFFER); + }); + + it('returns no credential to the browser', async () => { + const result = await openRealtimeTranscription(localConfig(), { sdp: OFFER }); + + expect(JSON.stringify(result)).not.toContain(API_KEY); + expect(JSON.stringify(result)).not.toContain(CLIENT_SECRET); + }); + + it('reports a missing realtime block as not configured without calling the provider', async () => { + const result = await openRealtimeTranscription( + { allowedAddresses, openai: { apiKey: API_KEY, model: 'gpt-transcribe' } }, + { sdp: OFFER }, + ); + + expect(result).toEqual({ + ok: false, + status: 404, + code: RealtimeTranscriptionErrorCode.NOT_CONFIGURED, + }); + expect(requests).toHaveLength(0); + }); + + it.each([ + ['a missing offer', undefined], + ['a non-string offer', { type: 'offer' }], + ['a non-SDP offer', 'hello'], + ['an oversized offer', `v=0\r\n${'a'.repeat(64 * 1024)}`], + ])('rejects %s without calling the provider', async (_label, sdp) => { + const result = await openRealtimeTranscription(localConfig(), { sdp } as never); + + expect(result).toEqual({ + ok: false, + status: 400, + code: RealtimeTranscriptionErrorCode.INVALID_OFFER, + }); + expect(requests).toHaveLength(0); + }); + + it('ignores an oversized language instead of forwarding it', async () => { + await openRealtimeTranscription(localConfig(), { sdp: OFFER, language: 'e'.repeat(36) }); + + const { transcription } = JSON.parse(requests[0].body).session.audio.input; + expect(transcription.languages).toEqual(['de', 'en']); + }); + + it('hides provider detail when the client secret is refused', async () => { + replies['/v1/realtime/client_secrets'] = { + status: 401, + type: 'application/json', + body: JSON.stringify({ error: { message: PROVIDER_DETAIL } }), + }; + + const result = await openRealtimeTranscription(localConfig(), { sdp: OFFER }); + + expect(result).toEqual({ + ok: false, + status: 502, + code: RealtimeTranscriptionErrorCode.PROVIDER_ERROR, + }); + expect(requests).toHaveLength(1); + expect(logger.error).toHaveBeenCalledWith( + '[STT] Failed to open a live transcription session:', + { type: 'Error', status: 401 }, + ); + expect(JSON.stringify(result)).not.toContain(PROVIDER_DETAIL); + expect(JSON.stringify(jest.mocked(logger.error).mock.calls)).not.toContain(PROVIDER_DETAIL); + }); + + it('fails when the provider answers without a client secret', async () => { + replies['/v1/realtime/client_secrets'] = { + status: 200, + type: 'application/json', + body: JSON.stringify({ expires_at: 1 }), + }; + + const result = await openRealtimeTranscription(localConfig(), { sdp: OFFER }); + + expect(result).toMatchObject({ + ok: false, + code: RealtimeTranscriptionErrorCode.PROVIDER_ERROR, + }); + expect(requests).toHaveLength(1); + }); + + it.each([ + ['rejects the call', { status: 400, type: 'application/json', body: '{"error":{}}' }], + ['answers with something other than SDP', { status: 201, type: 'text/plain', body: 'nope' }], + ])('fails when the provider %s', async (_label, reply) => { + replies['/v1/realtime/calls'] = reply; + + const result = await openRealtimeTranscription(localConfig(), { sdp: OFFER }); + + expect(result).toMatchObject({ + ok: false, + status: 502, + code: RealtimeTranscriptionErrorCode.PROVIDER_ERROR, + }); + expect(requests).toHaveLength(2); + }); + + it('blocks a private realtime URL unless allowedAddresses exempts it', async () => { + const blocked = await openRealtimeTranscription( + { ...localConfig(), allowedAddresses: undefined }, + { sdp: OFFER }, + ); + + expect(blocked).toMatchObject({ + ok: false, + code: RealtimeTranscriptionErrorCode.PROVIDER_ERROR, + }); + expect(requests).toHaveLength(0); + }); +}); + +describe('createRealtimeTranscriptionHandler', () => { + const createResponse = () => { + const res = { status: jest.fn(), json: jest.fn() }; + res.status.mockReturnValue(res); + res.json.mockReturnValue(res); + return res; + }; + + it('responds with the coded error when live transcription is off', async () => { + const res = createResponse(); + + await createRealtimeTranscriptionHandler()( + { config: { speech: {} }, body: { sdp: OFFER } } as never, + res as unknown as Response, + ); + + expect(res.status).toHaveBeenCalledWith(404); + expect(res.json).toHaveBeenCalledWith({ error: RealtimeTranscriptionErrorCode.NOT_CONFIGURED }); + }); + + it('responds with the coded error for an invalid offer', async () => { + const res = createResponse(); + + await createRealtimeTranscriptionHandler()( + { config: { speech: { stt: liveConfig() } }, body: {} } as never, + res as unknown as Response, + ); + + expect(res.status).toHaveBeenCalledWith(400); + expect(res.json).toHaveBeenCalledWith({ error: RealtimeTranscriptionErrorCode.INVALID_OFFER }); + }); +}); diff --git a/packages/api/src/files/realtime.ts b/packages/api/src/files/realtime.ts new file mode 100644 index 00000000000..b89def3ae8d --- /dev/null +++ b/packages/api/src/files/realtime.ts @@ -0,0 +1,228 @@ +import axios from 'axios'; +import { logger } from '@librechat/data-schemas'; +import { + STTProviders, + sttRealtimeDelays, + listConfiguredSpeechProviders, + RealtimeTranscriptionErrorCode, +} from 'librechat-data-provider'; +import type { RealtimeTranscriptionRequest } from 'librechat-data-provider'; +import type { AppConfig } from '@librechat/data-schemas'; +import type { Request, Response } from 'express'; +import type { AxiosRequestConfig } from 'axios'; +import type { OpenAISTTConfig, TranscriptionHints, STTConfig } from './transcription'; +import { getTranscriptionHints, usesTranscriptionHints } from './transcription'; +import { applySSRFSafeAgentIfDirect } from '~/auth/agent'; +import { applyAxiosProxyConfig } from '~/utils/proxy'; +import { resolveConfigSecret } from '~/admin/secrets'; +import { getSafeErrorMetadata } from '~/utils'; + +const DEFAULT_REALTIME_URL = 'https://api.openai.com/v1'; +/** Browser offers are a few KB; the cap only bounds what is forwarded upstream. */ +const MAX_OFFER_LENGTH = 64 * 1024; +const MAX_LANGUAGE_LENGTH = 35; +/** The secret is spent on the SDP exchange that immediately follows it. */ +const CLIENT_SECRET_TTL_SECONDS = 60; +const REQUEST_TIMEOUT_MS = 15_000; + +type RealtimeDelay = (typeof sttRealtimeDelays)[number]; + +export interface RealtimeTranscriptionTarget { + openai: OpenAISTTConfig; + /** Realtime API base URL without a trailing slash. */ + url: string; + model: string; + delay?: RealtimeDelay; + allowedAddresses?: string[]; +} + +export interface RealtimeTranscriptionSession { + type: 'transcription'; + audio: { + input: { + format: { type: 'audio/pcm'; rate: 24000 }; + transcription: TranscriptionHints & { model: string; delay?: RealtimeDelay }; + turn_detection: null; + }; + }; +} + +export type RealtimeTranscriptionResult = + | { ok: true; sdp: string } + | { ok: false; status: number; code: RealtimeTranscriptionErrorCode }; + +/** + * Resolves the live transcription target, or `null` when live dictation is off. It + * requires the same single-provider STT config the recorded path accepts, so live + * dictation is never offered on a config the rest of speech-to-text would reject. + */ +export function getRealtimeTranscriptionTarget( + stt?: STTConfig | null, +): RealtimeTranscriptionTarget | null { + const providers = listConfiguredSpeechProviders(stt as Record | undefined); + if (providers.length !== 1 || providers[0][0] !== STTProviders.OPENAI) { + return null; + } + const openai = stt?.openai; + const realtime = openai?.realtime; + const model = realtime?.model?.trim(); + if (openai == null || !model) { + return null; + } + const delay = sttRealtimeDelays.find((value) => value === realtime?.delay); + return { + openai, + model, + url: (realtime?.url?.trim() || DEFAULT_REALTIME_URL).replace(/\/+$/, ''), + ...(delay && { delay }), + ...(stt?.allowedAddresses && { allowedAddresses: stt.allowedAddresses }), + }; +} + +export function isRealtimeTranscriptionAvailable(stt?: STTConfig | null): boolean { + return getRealtimeTranscriptionTarget(stt) != null; +} + +/** + * A transcription-only session. Turn detection is off so a turn ends only when the + * browser commits the audio buffer: `gpt-live-transcribe` has no server VAD, and for + * older models it keeps a pause from splitting one dictation into several items. + */ +export function buildRealtimeTranscriptionSession( + target: Pick, + language?: string | null, +): RealtimeTranscriptionSession { + const transcription: RealtimeTranscriptionSession['audio']['input']['transcription'] = { + model: target.model, + ...getTranscriptionHints(target.openai, target.model, language), + }; + if (target.delay && usesTranscriptionHints(target.model)) { + transcription.delay = target.delay; + } + return { + type: 'transcription', + audio: { + input: { + format: { type: 'audio/pcm', rate: 24000 }, + transcription, + turn_detection: null, + }, + }, + }; +} + +function guarded( + url: string, + config: AxiosRequestConfig, + allowedAddresses?: string[], +): AxiosRequestConfig { + applyAxiosProxyConfig(config, url); + applySSRFSafeAgentIfDirect(config, url, allowedAddresses); + return config; +} + +/** + * Opens a live transcription call for the browser's WebRTC offer and returns the + * provider's answer. The configured key only mints a short-lived client secret, and + * that secret is spent here on the SDP exchange, so neither credential reaches the + * browser. Azure OpenAI's `/realtime/calls` accepts only these ephemeral secrets. + */ +export async function createRealtimeTranscriptionCall( + target: RealtimeTranscriptionTarget, + offer: string, + language?: string | null, +): Promise { + const apiKey = resolveConfigSecret(target.openai.apiKey) || ''; + const secretUrl = `${target.url}/realtime/client_secrets`; + const secretResponse = await axios.post( + secretUrl, + { + expires_after: { anchor: 'created_at', seconds: CLIENT_SECRET_TTL_SECONDS }, + session: buildRealtimeTranscriptionSession(target, language), + }, + guarded( + secretUrl, + { + headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${apiKey}` }, + timeout: REQUEST_TIMEOUT_MS, + }, + target.allowedAddresses, + ), + ); + const secret: unknown = secretResponse.data?.value; + if (typeof secret !== 'string' || secret.length === 0) { + throw new Error('The provider did not return a client secret'); + } + + const callsUrl = `${target.url}/realtime/calls`; + const answer = await axios.post( + callsUrl, + offer, + guarded( + callsUrl, + { + headers: { 'Content-Type': 'application/sdp', Authorization: `Bearer ${secret}` }, + responseType: 'text', + timeout: REQUEST_TIMEOUT_MS, + }, + target.allowedAddresses, + ), + ); + if (typeof answer.data !== 'string' || !answer.data.startsWith('v=0')) { + throw new Error('The provider did not return an SDP answer'); + } + return answer.data; +} + +function isOffer(sdp: unknown): sdp is string { + return typeof sdp === 'string' && sdp.length <= MAX_OFFER_LENGTH && sdp.startsWith('v=0'); +} + +/** Validates the browser's offer and opens the call. Provider failures are logged without their content. */ +export async function openRealtimeTranscription( + stt: STTConfig | null | undefined, + body: Partial | undefined, +): Promise { + const target = getRealtimeTranscriptionTarget(stt); + if (target == null) { + return { ok: false, status: 404, code: RealtimeTranscriptionErrorCode.NOT_CONFIGURED }; + } + const sdp = body?.sdp; + if (!isOffer(sdp)) { + return { ok: false, status: 400, code: RealtimeTranscriptionErrorCode.INVALID_OFFER }; + } + const language = + typeof body?.language === 'string' && body.language.length <= MAX_LANGUAGE_LENGTH + ? body.language + : undefined; + try { + return { ok: true, sdp: await createRealtimeTranscriptionCall(target, sdp, language) }; + } catch (error) { + logger.error('[STT] Failed to open a live transcription session:', getSafeErrorMetadata(error)); + return { ok: false, status: 502, code: RealtimeTranscriptionErrorCode.PROVIDER_ERROR }; + } +} + +type RealtimeTranscriptionHttpRequest = Request< + unknown, + unknown, + Partial | undefined +> & { config?: AppConfig }; + +/** + * `POST /api/files/speech/stt/realtime`. Transcripts stream from the provider straight + * to the browser, so unlike the recorded path they are not inspected by transcript + * content filters. + */ +export function createRealtimeTranscriptionHandler() { + return async function createRealtimeTranscription( + req: RealtimeTranscriptionHttpRequest, + res: Response, + ): Promise { + const result = await openRealtimeTranscription(req.config?.speech?.stt, req.body); + if (!result.ok) { + return res.status(result.status).json({ error: result.code }); + } + return res.status(200).json({ sdp: result.sdp }); + }; +} diff --git a/packages/api/src/files/transcription.spec.ts b/packages/api/src/files/transcription.spec.ts new file mode 100644 index 00000000000..215400535dc --- /dev/null +++ b/packages/api/src/files/transcription.spec.ts @@ -0,0 +1,184 @@ +import http from 'http'; +import axios from 'axios'; +import { Readable } from 'stream'; +import { logger } from '@librechat/data-schemas'; +import type { AddressInfo } from 'net'; +import { + getTranscriptionHints, + usesTranscriptionHints, + normalizeTranscriptionLanguage, +} from './transcription'; + +jest.mock('@librechat/data-schemas', () => ({ + logger: { warn: jest.fn(), error: jest.fn(), debug: jest.fn() }, +})); + +const hintConfig = { + prompt: ' A support call about LibreChat. ', + keywords: ['LibreChat', 'OpenShift'], + languages: ['de', 'EN'], +}; + +describe('usesTranscriptionHints', () => { + it.each([ + 'gpt-transcribe', + 'gpt-live-transcribe', + 'gpt-transcribe-2026-03-01', + 'GPT-Transcribe', + 'openai/gpt-transcribe', + ])('treats %s as a hint model', (model) => { + expect(usesTranscriptionHints(model)).toBe(true); + }); + + it.each([ + 'whisper-1', + 'gpt-4o-transcribe', + 'gpt-4o-mini-transcribe', + 'gpt-4o-transcribe-diarize', + 'gpt-transcriber', + '', + undefined, + null, + ])('treats %s as a legacy model', (model) => { + expect(usesTranscriptionHints(model)).toBe(false); + }); +}); + +describe('normalizeTranscriptionLanguage', () => { + it('reduces locales to ISO 639-1 for legacy models', () => { + expect(normalizeTranscriptionLanguage('en-US', false)).toBe('en'); + expect(normalizeTranscriptionLanguage('zh-TW', false)).toBe('zh'); + expect(normalizeTranscriptionLanguage('yue', false)).toBeUndefined(); + }); + + it('keeps regional Chinese and ISO 639-3 codes for hint models', () => { + expect(normalizeTranscriptionLanguage('zh-TW', true)).toBe('zh-tw'); + expect(normalizeTranscriptionLanguage('zh-HK', true)).toBe('zh-hk'); + expect(normalizeTranscriptionLanguage('pt-br', true)).toBe('pt'); + expect(normalizeTranscriptionLanguage('yue', true)).toBe('yue'); + }); + + it.each(['xx-piglatin', 'en_US', 'english', 'PRIVATE@example.com'])( + 'drops %s instead of sending it', + (language) => { + expect(normalizeTranscriptionLanguage(language, true)).toBeUndefined(); + expect(normalizeTranscriptionLanguage(language, false)).toBeUndefined(); + }, + ); + + it('does not log the submitted value it drops', () => { + const submitted = 'PRIVATE-LANGUAGE@example.com'; + + normalizeTranscriptionLanguage(submitted, true); + + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('Invalid language format')); + expect(JSON.stringify(jest.mocked(logger.warn).mock.calls)).not.toContain(submitted); + }); + + it('ignores an empty language', () => { + expect(normalizeTranscriptionLanguage('', true)).toBeUndefined(); + expect(normalizeTranscriptionLanguage(undefined, false)).toBeUndefined(); + }); +}); + +describe('getTranscriptionHints', () => { + it('sends languages, keywords, and prompt to gpt-transcribe and never the singular language', () => { + const hints = getTranscriptionHints(hintConfig, 'gpt-transcribe'); + + expect(hints).toEqual({ + prompt: 'A support call about LibreChat.', + keywords: ['LibreChat', 'OpenShift'], + languages: ['de', 'en'], + }); + expect(hints).not.toHaveProperty('language'); + }); + + it("narrows the admin languages to the user's language", () => { + expect(getTranscriptionHints(hintConfig, 'gpt-transcribe', 'en-GB').languages).toEqual(['en']); + expect(getTranscriptionHints(hintConfig, 'gpt-transcribe', 'zh-TW').languages).toEqual([ + 'zh-tw', + ]); + }); + + it('falls back to the admin languages when the user language is not a language code', () => { + expect(getTranscriptionHints(hintConfig, 'gpt-transcribe', 'xx-piglatin').languages).toEqual([ + 'de', + 'en', + ]); + }); + + it('sends the user language as languages[] even without admin hints', () => { + expect(getTranscriptionHints(undefined, 'gpt-transcribe', 'fr-FR')).toEqual({ + languages: ['fr'], + }); + }); + + it('omits empty hint fields', () => { + expect( + getTranscriptionHints({ prompt: ' ', keywords: [], languages: [] }, 'gpt-transcribe'), + ).toEqual({}); + }); + + it.each(['whisper-1', 'gpt-4o-transcribe', 'gpt-4o-mini-transcribe'])( + 'keeps the singular language for %s and drops hints it would reject', + (model) => { + const hints = getTranscriptionHints(hintConfig, model, 'en-US'); + + expect(hints).toEqual({ prompt: 'A support call about LibreChat.', language: 'en' }); + expect(hints).not.toHaveProperty('languages'); + expect(hints).not.toHaveProperty('keywords'); + }, + ); + + it('leaves a legacy request unchanged when no hints are configured', () => { + expect(getTranscriptionHints(undefined, 'whisper-1')).toEqual({}); + expect(getTranscriptionHints({}, 'whisper-1', 'de-DE')).toEqual({ language: 'de' }); + }); +}); + +describe('multipart serialization', () => { + let server: http.Server; + let url: string; + let received: string; + + beforeAll(async () => { + server = http.createServer((req, res) => { + const chunks: Buffer[] = []; + req.on('data', (chunk: Buffer) => chunks.push(chunk)); + req.on('end', () => { + received = Buffer.concat(chunks).toString('utf8'); + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({ text: 'ok' })); + }); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + url = `http://127.0.0.1:${(server.address() as AddressInfo).port}/audio/transcriptions`; + }); + + afterAll(async () => { + await new Promise((resolve) => server.close(resolve)); + }); + + const fieldValues = (name: string) => + [ + ...received.matchAll( + new RegExp(`name="${name.replace(/[[\]]/g, '\\$&')}"\\r\\n\\r\\n([^\\r]*)`, 'g'), + ), + ].map((match) => match[1]); + + it('posts array hints as the repeated languages[] and keywords[] fields OpenAI documents', async () => { + const file = Readable.from(Buffer.from('audio')); + Object.assign(file, { path: 'audio.webm' }); + + await axios.post( + url, + { file, model: 'gpt-transcribe', ...getTranscriptionHints(hintConfig, 'gpt-transcribe') }, + { headers: { 'Content-Type': 'multipart/form-data' }, proxy: false }, + ); + + expect(fieldValues('languages[]')).toEqual(['de', 'en']); + expect(fieldValues('keywords[]')).toEqual(['LibreChat', 'OpenShift']); + expect(fieldValues('prompt')).toEqual(['A support call about LibreChat.']); + expect(fieldValues('language')).toEqual([]); + }); +}); diff --git a/packages/api/src/files/transcription.ts b/packages/api/src/files/transcription.ts new file mode 100644 index 00000000000..ea06bc22059 --- /dev/null +++ b/packages/api/src/files/transcription.ts @@ -0,0 +1,100 @@ +import { logger } from '@librechat/data-schemas'; +import type { TCustomConfig } from 'librechat-data-provider'; + +export type STTConfig = NonNullable['stt']>; +export type OpenAISTTConfig = NonNullable; + +/** Request hints for a transcription, shared by the recorded and live paths. */ +export interface TranscriptionHints { + prompt?: string; + keywords?: string[]; + /** Sent only to `gpt-transcribe` models, which reject the singular `language`. */ + languages?: string[]; + /** Sent only to older models (Whisper, `gpt-4o-transcribe`), which reject `languages`. */ + language?: string; +} + +/** `gpt-transcribe`, `gpt-live-transcribe`, and their dated snapshots. */ +const HINT_MODEL_PATTERN = /(^|\/)gpt-(live-)?transcribe(-|$)/i; +/** OpenAI accepts these regional codes as-is; every other locale is reduced to its language. */ +const REGIONAL_LANGUAGES = new Set(['zh-cn', 'zh-tw', 'zh-hk']); + +/** + * Whether the model takes `languages`, `keywords`, and `prompt` hints. These models + * replace the singular `language` field with `languages`, and the API rejects a + * request that carries both, so callers must pick one shape per model. + */ +export function usesTranscriptionHints(model?: string | null): boolean { + return typeof model === 'string' && HINT_MODEL_PATTERN.test(model); +} + +/** + * Reduces a locale like `en-US` to the code the model accepts, or `undefined` when it + * is not a language code at all (e.g. the `xx-piglatin` browser-only option). Older + * models take ISO 639-1 only; hint models also take ISO 639-3 and regional Chinese. + */ +export function normalizeTranscriptionLanguage( + language: string | null | undefined, + hints: boolean, +): string | undefined { + if (typeof language !== 'string' || language.length === 0) { + return undefined; + } + const normalized = language.trim().toLowerCase(); + if (hints && REGIONAL_LANGUAGES.has(normalized)) { + return normalized; + } + const pattern = hints ? /^([a-z]{2,3})(-[a-z]{2})?$/ : /^([a-z]{2})(-[a-z]{2})?$/; + const match = pattern.exec(normalized); + if (match == null) { + logger.warn( + '[STT] Invalid language format. Expected a language code like "en" or "en-US". Skipping language parameter.', + ); + return undefined; + } + return match[1]; +} + +/** + * Builds the hints for one transcription. The user's language setting narrows the + * admin's `languages` list to that one language; without it the admin list applies. + * `keywords` and `languages` exist only on hint models, so older models receive the + * singular `language` and an optional `prompt`, exactly as before this config existed. + */ +export function getTranscriptionHints( + config: Pick | undefined, + model: string | undefined, + userLanguage?: string | null, +): TranscriptionHints { + const hints: TranscriptionHints = {}; + const prompt = config?.prompt?.trim(); + if (prompt) { + hints.prompt = prompt; + } + + if (!usesTranscriptionHints(model)) { + const language = normalizeTranscriptionLanguage(userLanguage, false); + if (language) { + hints.language = language; + } + return hints; + } + + const keywords = config?.keywords?.filter( + (keyword): keyword is string => typeof keyword === 'string' && keyword.length > 0, + ); + if (keywords?.length) { + hints.keywords = keywords; + } + + const userCode = normalizeTranscriptionLanguage(userLanguage, true); + const languages = userCode + ? [userCode] + : (config?.languages ?? []) + .map((code) => (typeof code === 'string' ? code.toLowerCase() : undefined)) + .filter((code): code is string => !!code); + if (languages.length > 0) { + hints.languages = [...new Set(languages)]; + } + return hints; +} diff --git a/packages/data-provider/src/api-endpoints.ts b/packages/data-provider/src/api-endpoints.ts index 84c45ca04ae..48c11598371 100644 --- a/packages/data-provider/src/api-endpoints.ts +++ b/packages/data-provider/src/api-endpoints.ts @@ -418,6 +418,8 @@ export const speech = () => `${files()}/speech`; export const speechToText = () => `${speech()}/stt`; +export const speechToTextRealtime = () => `${speechToText()}/realtime`; + export const textToSpeech = () => `${speech()}/tts`; export const textToSpeechManual = () => `${textToSpeech()}/manual`; diff --git a/packages/data-provider/src/config.spec.ts b/packages/data-provider/src/config.spec.ts index 3340ca64712..bd3d1c7f3fa 100644 --- a/packages/data-provider/src/config.spec.ts +++ b/packages/data-provider/src/config.spec.ts @@ -1573,8 +1573,49 @@ describe('agent background task config', () => { }); }); +describe('speech-to-text OpenAI schema', () => { + const parseStt = (openai: Record) => + configSchema.safeParse({ + version: '1.0', + speech: { stt: { openai: { apiKey: 'sk', model: 'gpt-transcribe', ...openai } } }, + }); + + it('keeps a configuration without hints or a live block valid', () => { + expect(parseStt({}).success).toBe(true); + }); + + it('accepts transcription hints and a live block', () => { + const result = parseStt({ + prompt: 'Retail and software terminology.', + keywords: ['LibreChat', 'OpenShift'], + languages: ['de', 'EN', 'zh-tw', 'yue'], + realtime: { model: 'gpt-live-transcribe', delay: 'low', url: 'https://api.openai.com/v1' }, + }); + + expect(result.success).toBe(true); + }); + + it.each(['', 'two\nlines', ''])('rejects the keyword %j', (keyword) => { + expect(parseStt({ keywords: [keyword] }).success).toBe(false); + }); + + it.each(['english', 'en_US', 'en-USA'])('rejects the language %j', (language) => { + expect(parseStt({ languages: [language] }).success).toBe(false); + }); + + it('requires a model for live transcription', () => { + expect(parseStt({ realtime: {} }).success).toBe(false); + }); + + it('rejects an unknown live delay', () => { + expect(parseStt({ realtime: { model: 'gpt-live-transcribe', delay: 'instant' } }).success).toBe( + false, + ); + }); +}); + describe('speechTab schema', () => { - it.each(['browser', 'external', 'openai', 'azureOpenAI'])( + it.each(['browser', 'external', 'realtime', 'openai', 'azureOpenAI'])( 'accepts the speech-to-text engine "%s"', (engineSTT) => { const result = configSchema.safeParse({ diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index 403b94976c8..4754c4196ca 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -2389,11 +2389,42 @@ const ttsSchema = z.object({ localai: ttsLocalaiSchema.optional(), }); +/** Latency presets for live transcription; lower values return text sooner but less accurately. */ +export const sttRealtimeDelays = ['minimal', 'low', 'medium', 'high', 'xhigh'] as const; + +/** OpenAI rejects the whole request when a keyword spans lines or contains angle brackets. */ +const sttKeywordSchema = z + .string() + .min(1) + .refine((keyword) => !/[<>\r\n]/.test(keyword), { + message: 'Keywords cannot contain "<", ">", or line breaks', + }); + +/** ISO 639-1, ISO 639-3, or a regional code such as `zh-tw`. */ +const sttLanguageSchema = z.string().regex(/^[a-z]{2,3}(-[a-z]{2})?$/i, { + message: 'Languages must be ISO 639-1 or 639-3 codes, optionally with a region (e.g. "zh-tw")', +}); + +const sttRealtimeSchema = z.object({ + /** Realtime API base URL; defaults to `https://api.openai.com/v1`. */ + url: z.string().optional(), + model: z.string(), + delay: z.enum(sttRealtimeDelays).optional(), +}); + const sttOpenaiSchema = z.object({ url: z.string().optional(), apiKey: z.string(), apiKeyPreview: apiKeyPreviewSchema, model: z.string(), + /** Free-form context about the audio, sent with every transcription. */ + prompt: z.string().optional(), + /** Literal terms (names, acronyms) for `gpt-transcribe` models to spell exactly. */ + keywords: z.array(sttKeywordSchema).optional(), + /** Expected languages for `gpt-transcribe` models when the user has not picked one. */ + languages: z.array(sttLanguageSchema).optional(), + /** Enables live dictation over the Realtime API; the recorded path keeps using `model`. */ + realtime: sttRealtimeSchema.optional(), }); const sttAzureOpenAISchema = z.object({ @@ -2447,7 +2478,9 @@ const speechTab = z .or( z.object({ /** Provider names remain valid for backward compatibility and are normalized for clients. */ - engineSTT: z.enum(['browser', 'external', 'openai', 'azureOpenAI']).optional(), + engineSTT: z + .enum(['browser', 'external', 'realtime', 'openai', 'azureOpenAI']) + .optional(), languageSTT: z.string().optional(), autoTranscribeAudio: z.boolean().optional(), decibelValue: z.number().optional(), diff --git a/packages/data-provider/src/data-service.ts b/packages/data-provider/src/data-service.ts index 0e86c35f04b..572898cbfac 100644 --- a/packages/data-provider/src/data-service.ts +++ b/packages/data-provider/src/data-service.ts @@ -944,6 +944,12 @@ export const speechToText = (data: FormData): Promise => return request.postMultiPart(endpoints.speechToText(), data); }; +export const createRealtimeTranscription = ( + data: f.RealtimeTranscriptionRequest, +): Promise => { + return request.post(endpoints.speechToTextRealtime(), data); +}; + export const textToSpeech = (data: FormData): Promise => { return request.postTTS(endpoints.textToSpeechManual(), data); }; diff --git a/packages/data-provider/src/keys.ts b/packages/data-provider/src/keys.ts index c439d5b8b30..94f5331ddb0 100644 --- a/packages/data-provider/src/keys.ts +++ b/packages/data-provider/src/keys.ts @@ -150,6 +150,7 @@ export enum MutationKeys { refreshToken = 'refreshToken', avatarUpload = 'avatarUpload', speechToText = 'speechToText', + realtimeTranscription = 'realtimeTranscription', textToSpeech = 'textToSpeech', assistantAvatarUpload = 'assistantAvatarUpload', agentAvatarUpload = 'agentAvatarUpload', diff --git a/packages/data-provider/src/types/files.ts b/packages/data-provider/src/types/files.ts index 39650f3fa5a..cf068c0b6e4 100644 --- a/packages/data-provider/src/types/files.ts +++ b/packages/data-provider/src/types/files.ts @@ -266,6 +266,31 @@ export type SpeechToTextResponse = { text: string; }; +/** Stable, machine-readable error codes for live transcription session failures. */ +export const RealtimeTranscriptionErrorCode = { + /** Live transcription is not configured (`speech.stt.openai.realtime`). */ + NOT_CONFIGURED: 'stt_realtime_not_configured', + /** The browser's SDP offer is missing, malformed, or too large. */ + INVALID_OFFER: 'stt_realtime_invalid_offer', + /** The provider refused or failed to open the session. No provider detail is disclosed. */ + PROVIDER_ERROR: 'stt_realtime_provider_error', +} as const; + +export type RealtimeTranscriptionErrorCode = + (typeof RealtimeTranscriptionErrorCode)[keyof typeof RealtimeTranscriptionErrorCode]; + +/** The browser's WebRTC offer for a live transcription session. */ +export type RealtimeTranscriptionRequest = { + sdp: string; + /** The user's speech language setting (e.g. `en-US`); the server normalizes it. */ + language?: string; +}; + +/** The provider's WebRTC answer; no credential is returned to the browser. */ +export type RealtimeTranscriptionResponse = { + sdp: string; +}; + export type VoiceResponse = string[]; export type UploadMutationOptions = { @@ -286,6 +311,15 @@ export type SpeechToTextOptions = { onError?: (error: unknown, variables: FormData, context?: unknown) => void; }; +export type RealtimeTranscriptionOptions = { + onSuccess?: ( + data: RealtimeTranscriptionResponse, + variables: RealtimeTranscriptionRequest, + context?: unknown, + ) => void; + onError?: (error: unknown, variables: RealtimeTranscriptionRequest, context?: unknown) => void; +}; + export type TextToSpeechOptions = { onSuccess?: (data: ArrayBuffer, variables: FormData, context?: unknown) => void; onMutate?: (variables: FormData) => void | Promise;