🗣️ feat: First-Class gpt-transcribe and Live gpt-live-transcribe Speech-to-Text

api/server/routes/files/speech/index.spec.js

@@ -0,0 +1,88 @@
+const express = require('express');
+const request = require('supertest');
+const { RealtimeTranscriptionErrorCode } = require('librechat-data-provider');
+
+const hits = { sttIp: 0, sttUser: 0 };
+
+jest.mock('~/server/middleware', () => ({
+  createSTTLimiters: jest.fn(() => ({
+    sttIpLimiter: (req, res, next) => {
+      hits.sttIp++;
+      next();
+    },
+    sttUserLimiter: (req, res, next) => {
+      hits.sttUser++;
+      next();
+    },
+  })),
+  createTTSLimiters: jest.fn(() => ({
+    ttsIpLimiter: (req, res, next) => next(),
+    ttsUserLimiter: (req, res, next) => next(),
+  })),
+}));
+
+jest.mock('~/server/services/Files/Audio', () => ({
+  speechToText: jest.fn((req, res) => res.status(200).json({ text: 'recorded' })),
+}));
+jest.mock('./tts', () => require('express').Router());
+jest.mock('./customConfigSpeech', () => require('express').Router());
+
+const liveConfig = {
+  speech: {
+    stt: {
+      openai: { apiKey: 'sk', model: 'gpt-transcribe', realtime: { model: 'gpt-live-transcribe' } },
+    },
+  },
+};
+
+describe('speech route wiring', () => {
+  let config;
+  let app;
+
+  beforeAll(() => {
+    const speech = require('./index');
+    app = express();
+    app.use(express.json());
+    app.use((req, res, next) => {
+      req.config = config;
+      next();
+    });
+    app.use('/api/files/speech', speech);
+  });
+
+  beforeEach(() => {
+    hits.sttIp = 0;
+    hits.sttUser = 0;
+    config = liveConfig;
+  });
+
+  it('meters the live transcription route with both STT limiters', async () => {
+    const response = await request(app)
+      .post('/api/files/speech/stt/realtime')
+      .send({ sdp: 'not an offer' });
+
+    expect(response.status).toBe(400);
+    expect(response.body).toEqual({ error: RealtimeTranscriptionErrorCode.INVALID_OFFER });
+    expect(hits.sttIp).toBe(1);
+    expect(hits.sttUser).toBe(1);
+  });
+
+  it('reports live transcription as not configured without a realtime block', async () => {
+    config = { speech: { stt: { openai: { apiKey: 'sk', model: 'gpt-transcribe' } } } };
+
+    const response = await request(app)
+      .post('/api/files/speech/stt/realtime')
+      .send({ sdp: 'v=0\r\n' });
+
+    expect(response.status).toBe(404);
+    expect(response.body).toEqual({ error: RealtimeTranscriptionErrorCode.NOT_CONFIGURED });
+  });
+
+  it('keeps the recorded transcription route in place', async () => {
+    const response = await request(app).post('/api/files/speech/stt').send({});
+
+    expect(response.status).toBe(200);
+    expect(response.body).toEqual({ text: 'recorded' });
+    expect(hits.sttIp).toBe(1);
+  });
+});

api/server/routes/files/speech/stt.js

@@ -1,8 +1,10 @@
 const express = require('express');
+const { createRealtimeTranscriptionHandler } = require('@librechat/api');
 const { speechToText } = require('~/server/services/Files/Audio');
 
 const router = express.Router();
 
 router.post('/', speechToText);
+router.post('/realtime', createRealtimeTranscriptionHandler());
 
 module.exports = router;

api/server/services/Files/Audio/STTService.js

@@ -11,6 +11,7 @@ const {
   getSafeErrorMetadata,
   applyAxiosProxyConfig,
   resolveConfigSecret,
+  getTranscriptionHints,
   applySSRFSafeAgentIfDirect,
   contentFilterBlockResponse,
   contentFilterUninspectableResponse,
@@ -200,7 +201,9 @@ class STTService {
   }
 
   /**
-   * Prepares the request for the OpenAI STT provider.
+   * Prepares the request for the OpenAI STT provider. `gpt-transcribe` models get the
+   * `languages`/`keywords`/`prompt` hints; older models keep the singular `language`.
+   * Axios serializes array hints as repeated `languages[]` / `keywords[]` form fields.
    * @param {Object} sttSchema - The STT schema for OpenAI.
    * @param {Stream} audioReadStream - The audio data to be transcribed.
    * @param {Object} audioFile - The audio file object (unused in OpenAI provider).
@@ -214,13 +217,9 @@ class STTService {
     const data = {
       file: audioReadStream,
       model: sttSchema.model,
+      ...getTranscriptionHints(sttSchema, sttSchema.model, language),
     };
 
-    const validLanguage = getValidatedLanguageCode(language);
-    if (validLanguage) {
-      data.language = validLanguage;
-    }
-
     const headers = {
       'Content-Type': 'multipart/form-data',
       ...(apiKey && { Authorization: `Bearer ${apiKey}` }),

api/server/services/Files/Audio/STTService.spec.js

@@ -36,6 +36,7 @@ jest.mock('@librechat/api', () => ({
   })),
   applyAxiosProxyConfig: jest.fn(),
   resolveConfigSecret: jest.fn((value) => value),
+  getTranscriptionHints: jest.fn(() => ({})),
   applySSRFSafeAgentIfDirect: jest.fn(),
   contentFilterBlockResponse: jest.fn((finding) => ({
     error: 'content_filter_block',
@@ -61,11 +62,13 @@ jest.mock('~/server/services/Config', () => ({ getAppConfig: jest.fn() }));
 
 const fs = require('fs').promises;
 const axios = require('axios');
+const FormData = require('form-data');
 const { logger } = require('@librechat/data-schemas');
 const {
   inspectContent,
   extractFileContent,
   getSafeErrorMetadata,
+  getTranscriptionHints,
   contentFilterBlockResponse,
   contentFilterUninspectableResponse,
   getBlockedUninspectableFileField,
@@ -146,6 +149,77 @@ describe('STTService.getProviderSchema provider detection', () => {
   });
 });
 
+describe('STTService.openAIProvider request fields', () => {
+  const service = new STTService();
+  const audio = Buffer.from('audio');
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  it('builds the hint fields from the schema, its model, and the user language', () => {
+    const schema = {
+      apiKey: 'sk',
+      model: 'gpt-transcribe',
+      prompt: 'Retail and software terms.',
+      keywords: ['LibreChat'],
+      languages: ['de', 'en'],
+    };
+    getTranscriptionHints.mockReturnValueOnce({
+      prompt: 'Retail and software terms.',
+      keywords: ['LibreChat'],
+      languages: ['de'],
+    });
+
+    const [url, data, headers] = service.openAIProvider(schema, audio, {}, 'de-DE');
+
+    expect(getTranscriptionHints).toHaveBeenCalledWith(schema, 'gpt-transcribe', 'de-DE');
+    expect(url).toBe('https://api.openai.com/v1/audio/transcriptions');
+    expect(data).toEqual({
+      file: audio,
+      model: 'gpt-transcribe',
+      prompt: 'Retail and software terms.',
+      keywords: ['LibreChat'],
+      languages: ['de'],
+    });
+    expect(headers).toEqual({ 'Content-Type': 'multipart/form-data', Authorization: 'Bearer sk' });
+  });
+
+  it('sends the singular language the hints resolve for older models', () => {
+    getTranscriptionHints.mockReturnValueOnce({ language: 'en' });
+
+    const [, data] = service.openAIProvider({ apiKey: 'sk', model: 'whisper-1' }, audio, {}, 'en');
+
+    expect(data).toEqual({ file: audio, model: 'whisper-1', language: 'en' });
+  });
+});
+
+describe('STTService.sttRequest response parsing', () => {
+  const service = new STTService();
+  service.providerStrategies.openai = service.openAIProvider;
+  const request = {
+    audioBuffer: Buffer.from('audio'),
+    audioFile: { mimetype: 'audio/webm' },
+    language: '',
+  };
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  it.each([
+    ['detected-language metadata', { text: ' Bonjour. ', languages: [{ code: 'fr' }] }],
+    ['an empty detection result', { text: ' Bonjour. ', languages: [] }],
+    ['a plain response', { text: ' Bonjour. ' }],
+  ])('returns the trimmed text from a response with %s', async (_label, responseData) => {
+    axios.post.mockResolvedValueOnce({ status: 200, data: responseData });
+
+    await expect(
+      service.sttRequest('openai', { apiKey: 'sk', model: 'gpt-transcribe' }, request),
+    ).resolves.toBe('Bonjour.');
+  });
+});
+
 describe('STT audio format validation with MIME normalization', () => {
   const acceptedFormats = ['flac', 'mp3', 'mp4', 'mpeg', 'mpga', 'm4a', 'ogg', 'wav', 'webm'];
 
@@ -215,8 +289,14 @@ describe('STT error disclosure protection', () => {
   it('does not log a submitted invalid language value', () => {
     const submittedLanguage = 'PRIVATE-LANGUAGE@example.com';
     const service = new STTService();
+    FormData.mockImplementationOnce(() => ({ append: jest.fn(), getHeaders: () => ({}) }));
 
-    service.openAIProvider({ model: 'whisper-1' }, Buffer.from('audio'), {}, submittedLanguage);
+    service.azureOpenAIProvider(
+      { apiKey: 'key' },
+      Buffer.from('audio'),
+      { originalname: 'audio.webm', mimetype: 'audio/webm' },
+      submittedLanguage,
+    );
 
     expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('Invalid language format'));
     expect(JSON.stringify(logger.warn.mock.calls)).not.toContain(submittedLanguage);

api/server/services/Files/Audio/getCustomConfigSpeech.js

@@ -1,4 +1,5 @@
 const { logger } = require('@librechat/data-schemas');
+const { isRealtimeTranscriptionAvailable } = require('@librechat/api');
 const { getAppConfig } = require('~/server/services/Config');
 
 const LEGACY_EXTERNAL_STT_ENGINES = new Set(['openai', 'azureOpenAI']);
@@ -48,6 +49,7 @@ async function getCustomConfigSpeech(req, res) {
     const ttsExternal = !!appConfig.speech?.tts;
     let settings = {
       sttExternal,
+      sttRealtime: isRealtimeTranscriptionAvailable(appConfig.speech?.stt),
       ttsExternal,
     };
 

api/server/services/Files/Audio/getCustomConfigSpeech.spec.js

@@ -31,12 +31,58 @@ describe('getCustomConfigSpeech', () => {
       expect(res.status).toHaveBeenCalledWith(200);
       expect(res.send).toHaveBeenCalledWith({
         sttExternal: true,
+        sttRealtime: false,
         ttsExternal: false,
         engineSTT: 'external',
       });
     },
   );
 
+  it('advertises live transcription when the OpenAI provider configures a realtime model', async () => {
+    const req = {
+      config: {
+        speech: {
+          stt: {
+            openai: {
+              apiKey: 'sk',
+              model: 'gpt-transcribe',
+              realtime: { model: 'gpt-live-transcribe' },
+            },
+          },
+          speechTab: { speechToText: { engineSTT: 'realtime' } },
+        },
+      },
+    };
+    const res = createResponse();
+
+    await getCustomConfigSpeech(req, res);
+
+    expect(res.send).toHaveBeenCalledWith({
+      sttExternal: true,
+      sttRealtime: true,
+      ttsExternal: false,
+      engineSTT: 'realtime',
+    });
+  });
+
+  it.each([
+    ['without a realtime block', { openai: { apiKey: 'sk', model: 'gpt-transcribe' } }],
+    [
+      'for Azure OpenAI',
+      { azureOpenAI: { apiKey: 'sk', instanceName: 'i', deploymentName: 'd', apiVersion: 'v' } },
+    ],
+  ])('does not advertise live transcription %s', async (_label, stt) => {
+    const res = createResponse();
+
+    await getCustomConfigSpeech({ config: { speech: { stt } } }, res);
+
+    expect(res.send).toHaveBeenCalledWith({
+      sttExternal: true,
+      sttRealtime: false,
+      ttsExternal: false,
+    });
+  });
+
   it.each(['browser', 'external'])('preserves the runtime STT engine "%s"', async (engineSTT) => {
     const req = {
       config: {
@@ -71,6 +117,7 @@ describe('getCustomConfigSpeech', () => {
       expect(res.status).toHaveBeenCalledWith(200);
       expect(res.send).toHaveBeenCalledWith({
         sttExternal: false,
+        sttRealtime: false,
         ttsExternal: true,
         engineTTS: 'external',
       });

client/src/common/types.ts

@@ -35,6 +35,8 @@ export enum PromptsEditorMode {
 export enum STTEndpoints {
   browser = 'browser',
   external = 'external',
+  /** Streams the microphone to the provider over WebRTC; offered only when the admin configures it. */
+  realtime = 'realtime',
 }
 
 export enum TTSEndpoints {

client/src/components/Chat/Input/Composer/Bar.tsx

@@ -299,9 +299,19 @@ function Bar({
   let dictationStatus = '';
   if (dictation.transcribing) {
     dictationStatus = localize('com_ui_transcribing');
+  } else if (dictation.connection === 'connecting') {
+    dictationStatus = localize('com_ui_connecting');
+  } else if (dictation.connection === 'reconnecting') {
+    dictationStatus = localize('com_ui_reconnecting');
   } else if (dictating) {
     dictationStatus = localize('com_ui_listening');
   }
+  /* Live dictation has states worth showing in place of the clock: until it
+     connects nothing is being heard, and a reconnect may lose a few words. */
+  const dictationLabel =
+    dictation.transcribing || dictation.connection != null
+      ? dictationStatus
+      : formatElapsed(dictation.elapsed);
 
   /* The arrangement is frozen for the length of a recording. The controls
      shrink to just the elapsed time while one runs, which would otherwise let
@@ -489,9 +499,7 @@ function Bar({
               )}
             >
               <span className="text-text-secondary text-xs tabular-nums" aria-hidden="true">
-                {dictation.transcribing
-                  ? localize('com_ui_transcribing')
-                  : formatElapsed(dictation.elapsed)}
+                {dictationLabel}
               </span>
             </div>
           </div>

client/src/components/Nav/Settings/SpeechControls.tsx

@@ -4,7 +4,12 @@ import { EngineTTSDropdown } from '../SettingsTabs/Speech/TTS';
 
 export function EngineSTTSetting() {
   const { data } = useGetCustomConfigSpeechQuery();
-  return <EngineSTTDropdown external={Boolean(data?.sttExternal)} />;
+  return (
+    <EngineSTTDropdown
+      external={Boolean(data?.sttExternal)}
+      realtime={Boolean(data?.sttRealtime)}
+    />
+  );
 }
 
 export function EngineTTSSetting() {