From 132ed3580de83385d194541694dc66584e64896b Mon Sep 17 00:00:00 2001 From: Prasenjit Chakraborty <280958636+iprasen@users.noreply.github.com> Date: Mon, 28 Sep 2026 23:56:27 +0530 Subject: [PATCH] feat(tools): add Azure OpenAI Sora video generation tool (#7702) --- api/app/clients/tools/index.js | 2 + api/app/clients/tools/manifest.json | 13 + api/app/clients/tools/structured/Sora.js | 293 ++++++++++++++++++ .../tools/structured/specs/Sora.spec.js | 222 +++++++++++++ api/app/clients/tools/util/handleTools.js | 3 + packages/data-provider/src/config.ts | 4 + 6 files changed, 537 insertions(+) create mode 100644 api/app/clients/tools/structured/Sora.js create mode 100644 api/app/clients/tools/structured/specs/Sora.spec.js diff --git a/api/app/clients/tools/index.js b/api/app/clients/tools/index.js index bb58e81221c..adfc5f9efcd 100644 --- a/api/app/clients/tools/index.js +++ b/api/app/clients/tools/index.js @@ -12,6 +12,7 @@ const TraversaalSearch = require('./structured/TraversaalSearch'); const createOpenAIImageTools = require('./structured/OpenAIImageTools'); const TavilySearchResults = require('./structured/TavilySearchResults'); const createGeminiImageTool = require('./structured/GeminiImageGen'); +const Sora = require('./structured/Sora'); module.exports = { ...manifest, @@ -27,4 +28,5 @@ module.exports = { TavilySearchResults, createOpenAIImageTools, createGeminiImageTool, + Sora, }; diff --git a/api/app/clients/tools/manifest.json b/api/app/clients/tools/manifest.json index e9775511a26..d6289a0166d 100644 --- a/api/app/clients/tools/manifest.json +++ b/api/app/clients/tools/manifest.json @@ -173,5 +173,18 @@ "optional": true } ] + }, + { + "name": "Sora Video Generation", + "pluginKey": "sora", + "description": "[Sora] Create high-quality video scenes from natural language descriptions via Azure OpenAI or OpenAI.", + "icon": "assets/openai.svg", + "authConfig": [ + { + "authField": "SORA_API_KEY||AZURE_OPENAI_API_KEY", + "label": "API Key", + "description": "API Key for Azure OpenAI or OpenAI Sora video generation." + } + ] } ] diff --git a/api/app/clients/tools/structured/Sora.js b/api/app/clients/tools/structured/Sora.js new file mode 100644 index 00000000000..6d4c3102fd9 --- /dev/null +++ b/api/app/clients/tools/structured/Sora.js @@ -0,0 +1,293 @@ +const path = require('path'); +const { v4: uuidv4 } = require('uuid'); +const fetch = globalThis.fetch ?? require('undici').fetch; +const { logger } = require('@librechat/data-schemas'); +const { Tool } = require('@librechat/agents/langchain/tools'); +const { + extractBaseURL, + getProxyDispatcher, + getEnvProxyDispatcher, + createMinimalRetentionRequest, +} = require('@librechat/api'); +const { FileContext, ContentTypes } = require('librechat-data-provider'); + +const soraJsonSchema = { + type: 'object', + properties: { + prompt: { + type: 'string', + maxLength: 4000, + description: + 'A detailed natural language description of the video scene to generate.', + }, + size: { + type: 'string', + enum: ['1280x720', '720x1280', '1920x1080', '1080x1920'], + description: + 'Resolution of the video. Default is 1280x720 (landscape). Use 720x1280 for portrait.', + }, + duration: { + type: 'number', + enum: [5, 10], + description: 'Duration of the generated video in seconds (5 or 10). Default is 5.', + }, + }, + required: ['prompt'], +}; + +const displayMessage = + "Sora generated a video. The video is displayed directly in the chat preview. Do not repeat the prompt description in detail."; + +class Sora extends Tool { + constructor(fields = {}) { + super(); + this.override = fields.override ?? false; + this.returnMetadata = fields.returnMetadata ?? false; + this.userId = fields.userId; + this.tenantId = fields.req?.user?.tenantId; + this.retentionRequest = createMinimalRetentionRequest(fields.req); + this.fileStrategy = fields.fileStrategy; + this.isAgent = fields.isAgent; + + if (this.isAgent) { + this.responseFormat = 'content_and_artifact'; + } + if (fields.processFileURL) { + this.processFileURL = fields.processFileURL.bind(this); + } + + this.apiKey = + fields.SORA_API_KEY ?? + fields.AZURE_OPENAI_API_KEY ?? + process.env.SORA_API_KEY ?? + process.env.AZURE_OPENAI_API_KEY ?? + process.env.OPENAI_API_KEY ?? + ''; + + if (!this.apiKey && !this.override) { + throw new Error('Missing SORA_API_KEY or AZURE_OPENAI_API_KEY environment variable.'); + } + + this.baseURL = + fields.SORA_BASEURL ?? + process.env.SORA_BASEURL ?? + process.env.AZURE_OPENAI_BASEURL ?? + (process.env.SORA_REVERSE_PROXY ? extractBaseURL(process.env.SORA_REVERSE_PROXY) : 'https://api.openai.com/v1'); + + this.apiVersion = + fields.SORA_AZURE_API_VERSION ?? + process.env.SORA_AZURE_API_VERSION ?? + process.env.AZURE_OPENAI_API_VERSION ?? + '2025-05-01-preview'; + + this.model = fields.model ?? process.env.SORA_MODEL ?? 'sora'; + this.pollInterval = fields.pollInterval ?? 2500; + this.pollTimeout = fields.pollTimeout ?? 180000; + + this.name = 'sora'; + this.description = `Use Sora to generate video scenes from detailed natural language descriptions via Azure OpenAI or OpenAI. + - Specify scene details, motion, lighting, and camera movement in the prompt. + - Optional parameters include size ('1280x720', '720x1280', '1920x1080', '1080x1920') and duration in seconds (5 or 10).`; + this.schema = soraJsonSchema; + } + + static get jsonSchema() { + return soraJsonSchema; + } + + replaceUnwantedChars(inputString) { + return inputString + .replace(/\r\n|\r|\n/g, ' ') + .replace(/"/g, '') + .trim(); + } + + wrapInMarkdown(videoUrl) { + return `[Generated Video](${videoUrl})\n\n`; + } + + getHeaders() { + const isAzure = Boolean(this.apiVersion && this.baseURL.includes('openai.azure.com')); + const headers = { + 'Content-Type': 'application/json', + }; + if (isAzure) { + headers['api-key'] = this.apiKey; + } else { + headers['Authorization'] = `Bearer ${this.apiKey}`; + } + return headers; + } + + getJobUrl(jobId) { + const isAzure = Boolean(this.apiVersion && this.baseURL.includes('openai.azure.com')); + const base = this.baseURL.replace(/\/+$/, ''); + if (jobId) { + return isAzure + ? `${base}/openai/v1/video/generations/jobs/${jobId}?api-version=${this.apiVersion}` + : `${base}/video/generations/jobs/${jobId}`; + } + return isAzure + ? `${base}/openai/v1/video/generations/jobs?api-version=${this.apiVersion}` + : `${base}/video/generations/jobs`; + } + + async pollJob(jobId, fetchOptions = {}) { + const startTime = Date.now(); + const pollUrl = this.getJobUrl(jobId); + + while (Date.now() - startTime < this.pollTimeout) { + await new Promise((resolve) => setTimeout(resolve, this.pollInterval)); + + const res = await fetch(pollUrl, { + method: 'GET', + headers: this.getHeaders(), + ...fetchOptions, + }); + + if (!res.ok) { + const errorText = await res.text(); + throw new Error(`Failed to check video generation job status: ${res.status} ${errorText}`); + } + + const data = await res.json(); + const status = data.status?.toLowerCase(); + + if (status === 'succeeded' || status === 'completed') { + const videoUrl = + data.generations?.[0]?.url ?? + data.output?.url ?? + data.result?.url ?? + data.url; + if (!videoUrl) { + throw new Error('Video generation succeeded but no download URL was returned.'); + } + return videoUrl; + } + + if (status === 'failed' || status === 'cancelled') { + const errorMsg = data.error?.message ?? data.failure_reason ?? 'Video generation failed.'; + throw new Error(errorMsg); + } + } + + throw new Error(`Video generation timed out after ${this.pollTimeout / 1000} seconds.`); + } + + async _call(data) { + const { prompt, size = '1280x720', duration = 5 } = data; + if (!prompt) { + throw new Error('Missing required field: prompt'); + } + + const fetchOptions = {}; + const proxyDispatcher = getProxyDispatcher() || getEnvProxyDispatcher(); + if (proxyDispatcher) { + fetchOptions.dispatcher = proxyDispatcher; + } + + const payload = { + prompt: this.replaceUnwantedChars(prompt), + model: this.model, + size, + duration: Number(duration), + }; + + let startRes; + try { + startRes = await fetch(this.getJobUrl(), { + method: 'POST', + headers: this.getHeaders(), + body: JSON.stringify(payload), + ...fetchOptions, + }); + } catch (error) { + logger.error('[Sora] Error initiating video generation job:', error); + return this.returnValue( + `Failed to connect to video generation service: ${error.message}`, + ); + } + + if (!startRes.ok) { + const errText = await startRes.text(); + logger.error('[Sora] Video job creation failed:', { status: startRes.status, errText }); + return this.returnValue( + `Video generation request failed (${startRes.status}): ${errText}`, + ); + } + + const jobData = await startRes.json(); + const jobId = jobData.id; + + if (!jobId) { + // In case synchronous URL was returned directly + const directUrl = jobData.generations?.[0]?.url || jobData.url; + if (directUrl) { + return this.handleCompletedVideo(directUrl, fetchOptions); + } + return this.returnValue('No job ID returned from video generation service.'); + } + + let theVideoUrl; + try { + theVideoUrl = await this.pollJob(jobId, fetchOptions); + } catch (pollError) { + logger.error('[Sora] Error during video generation polling:', pollError); + return this.returnValue(`Error generating video: ${pollError.message}`); + } + + return this.handleCompletedVideo(theVideoUrl, fetchOptions); + } + + async handleCompletedVideo(theVideoUrl, fetchOptions = {}) { + if (this.isAgent) { + const content = [ + { + type: ContentTypes.VIDEO_URL, + video_url: { + url: theVideoUrl, + }, + }, + ]; + const response = [ + { + type: ContentTypes.TEXT, + text: displayMessage, + }, + ]; + return [response, { content }]; + } + + const videoName = `video-${uuidv4()}.mp4`; + + if (this.processFileURL) { + try { + const result = await this.processFileURL({ + URL: theVideoUrl, + basePath: 'videos', + userId: this.userId, + fileName: videoName, + fileStrategy: this.fileStrategy, + context: FileContext.video_generation ?? 'video_generation', + tenantId: this.tenantId, + req: this.retentionRequest, + }); + + if (this.returnMetadata) { + this.result = result; + } else { + this.result = this.wrapInMarkdown(result.filepath ?? theVideoUrl); + } + } catch (saveError) { + logger.error('[Sora] Error saving video file locally:', saveError); + this.result = this.wrapInMarkdown(theVideoUrl); + } + } else { + this.result = this.wrapInMarkdown(theVideoUrl); + } + + return this.returnValue(this.result); + } +} + +module.exports = Sora; diff --git a/api/app/clients/tools/structured/specs/Sora.spec.js b/api/app/clients/tools/structured/specs/Sora.spec.js new file mode 100644 index 00000000000..1be3d7d3f81 --- /dev/null +++ b/api/app/clients/tools/structured/specs/Sora.spec.js @@ -0,0 +1,222 @@ +const { logger } = require('@librechat/data-schemas'); +const { ContentTypes } = require('librechat-data-provider'); +const Sora = require('../Sora'); + +jest.mock('@librechat/data-schemas', () => ({ + logger: { + info: jest.fn(), + warn: jest.fn(), + debug: jest.fn(), + error: jest.fn(), + }, +})); + +jest.mock('@librechat/api', () => ({ + extractBaseURL: jest.fn((url) => url), + getProxyDispatcher: jest.fn(() => null), + getEnvProxyDispatcher: jest.fn(() => null), + createMinimalRetentionRequest: jest.fn((req) => req), +})); + +jest.mock('undici', () => ({ + fetch: jest.fn(), +})); + +const { fetch } = require('undici'); + +describe('Sora Structured Tool', () => { + let originalEnv; + const mockApiKey = 'test-sora-api-key'; + + beforeAll(() => { + originalEnv = { ...process.env }; + }); + + beforeEach(() => { + jest.clearAllMocks(); + process.env = { ...originalEnv }; + process.env.SORA_API_KEY = mockApiKey; + process.env.SORA_BASEURL = 'https://my-azure-resource.openai.azure.com'; + process.env.SORA_AZURE_API_VERSION = '2025-05-01-preview'; + }); + + afterAll(() => { + process.env = originalEnv; + }); + + describe('Initialization & Configuration', () => { + it('initializes successfully with environment variables', () => { + const tool = new Sora(); + expect(tool.name).toBe('sora'); + expect(tool.apiKey).toBe(mockApiKey); + expect(tool.baseURL).toBe('https://my-azure-resource.openai.azure.com'); + expect(tool.apiVersion).toBe('2025-05-01-preview'); + }); + + it('initializes with Azure API Key fallback', () => { + delete process.env.SORA_API_KEY; + process.env.AZURE_OPENAI_API_KEY = 'azure-key-123'; + const tool = new Sora(); + expect(tool.apiKey).toBe('azure-key-123'); + }); + + it('throws error when no API key is provided and override is false', () => { + delete process.env.SORA_API_KEY; + delete process.env.AZURE_OPENAI_API_KEY; + delete process.env.OPENAI_API_KEY; + + expect(() => new Sora()).toThrow('Missing SORA_API_KEY or AZURE_OPENAI_API_KEY environment variable.'); + }); + + it('does not throw when override is true and API key is missing', () => { + delete process.env.SORA_API_KEY; + delete process.env.AZURE_OPENAI_API_KEY; + delete process.env.OPENAI_API_KEY; + + expect(() => new Sora({ override: true })).not.toThrow(); + }); + + it('sets responseFormat for Agent mode', () => { + const tool = new Sora({ isAgent: true }); + expect(tool.responseFormat).toBe('content_and_artifact'); + }); + }); + + describe('Headers & URL Routing', () => { + it('generates Azure headers with api-key', () => { + const tool = new Sora(); + const headers = tool.getHeaders(); + expect(headers['api-key']).toBe(mockApiKey); + expect(headers['Content-Type']).toBe('application/json'); + expect(headers['Authorization']).toBeUndefined(); + }); + + it('generates standard OpenAI Bearer header when not Azure', () => { + const tool = new Sora({ + SORA_BASEURL: 'https://api.openai.com/v1', + SORA_AZURE_API_VERSION: '', + }); + const headers = tool.getHeaders(); + expect(headers['Authorization']).toBe(`Bearer ${mockApiKey}`); + expect(headers['api-key']).toBeUndefined(); + }); + + it('constructs correct Azure job URLs with api-version query param', () => { + const tool = new Sora(); + const submitUrl = tool.getJobUrl(); + expect(submitUrl).toBe( + 'https://my-azure-resource.openai.azure.com/openai/v1/video/generations/jobs?api-version=2025-05-01-preview', + ); + + const pollUrl = tool.getJobUrl('job-1234'); + expect(pollUrl).toBe( + 'https://my-azure-resource.openai.azure.com/openai/v1/video/generations/jobs/job-1234?api-version=2025-05-01-preview', + ); + }); + }); + + describe('Execution & Polling Loop', () => { + it('successfully submits job and polls until completion', async () => { + const tool = new Sora({ pollInterval: 10, pollTimeout: 5000 }); + + // Mock 1: Initial POST job submission returns job ID + fetch.mockResolvedValueOnce({ + ok: true, + status: 201, + json: async () => ({ id: 'job-abc', status: 'queued' }), + }); + + // Mock 2: First poll returns running + fetch.mockResolvedValueOnce({ + ok: true, + status: 200, + json: async () => ({ id: 'job-abc', status: 'running' }), + }); + + // Mock 3: Second poll returns succeeded with video URL + fetch.mockResolvedValueOnce({ + ok: true, + status: 200, + json: async () => ({ + id: 'job-abc', + status: 'succeeded', + generations: [{ url: 'https://storage.azure.com/videos/output.mp4' }], + }), + }); + + const result = await tool._call({ + prompt: 'A cyberpunk city in the rain with neon lights', + size: '1280x720', + duration: 5, + }); + + expect(result).toContain('https://storage.azure.com/videos/output.mp4'); + expect(result).toContain('