From 815219fd92f178fe01eac9f58de7699550604379 Mon Sep 17 00:00:00 2001 From: Mihidum Hettiyahandi <55163074+mihidumh@users.noreply.github.com> Date: Tue, 29 Sep 2026 09:19:06 +1000 Subject: [PATCH 1/6] =?UTF-8?q?=F0=9F=93=8E=20fix:=20Keep=20Provider=20Upl?= =?UTF-8?q?oads=20the=20Provider=20Cannot=20Read=20Inline=20Out=20of=20Fil?= =?UTF-8?q?e=20Parts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The inherited supportedMimeTypes list routes files to "Upload to Provider" (#13550), but some types on it can never go inline: - Archives (zip, tar, gzip, epub, octet-stream) go to the provider only when the endpoint lists them itself, in BaseClient and child runs. - Gemini, native or behind an OpenAI-compatible gateway (detected by model name, as #16055 does for Claude), gets PDF and textual types plus types the endpoint lists; Office documents are skipped like the Anthropic filter from #14535. Textual application/* types (JSON, YAML, XML, SQL) go to Gemini as a text part; text/* stays inline. - application/sql, x-sh and xml go to other OpenAI-like providers as a text part unless the endpoint lists them: Azure OpenAI rejects them as a file part with a 400 that recurs on every later turn. Co-Authored-By: Claude Opus 5.5 --- api/app/clients/BaseClient.js | 12 +- api/app/clients/specs/BaseClient.test.js | 57 ++++ packages/api/src/agents/files/encode.spec.ts | 38 +++ packages/api/src/agents/files/encode.ts | 4 +- .../api/src/files/encode/document.spec.ts | 250 +++++++++++++++++- packages/api/src/files/encode/document.ts | 99 +++++-- .../files/encode/processAttachments.spec.ts | 63 ++++- packages/api/src/files/encode/utils.ts | 33 +++ 8 files changed, 528 insertions(+), 28 deletions(-) diff --git a/api/app/clients/BaseClient.js b/api/app/clients/BaseClient.js index 4a4f6975bda..3136a1bdaa6 100644 --- a/api/app/clients/BaseClient.js +++ b/api/app/clients/BaseClient.js @@ -14,6 +14,7 @@ const { encodeAndFormatVideos, getTransactionsConfig, encodeAndFormatDocuments, + isProviderDocumentCandidate, getLangfuseTraceMessageFields, isContentFilterError, assertModelBoundProviderContent, @@ -1842,15 +1843,18 @@ class BaseClient { categorizedAttachments.audios.push(file); allFiles.push(file); } else if ( - file.type && - deliveryRouting?.endpointConfig.supportedMimeTypes && - deliveryRouting.fileConfig.checkType( + isProviderDocumentCandidate( file.type, - deliveryRouting.endpointConfig.supportedMimeTypes, + deliveryRouting?.fileConfig, + deliveryRouting?.endpointConfig.supportedMimeTypes, ) ) { categorizedAttachments.documents.push(file); allFiles.push(file); + } else if (deliveryPath === 'provider') { + logger.warn( + `[BaseClient] Not sending "${file.filename}" (${file.type}) to the provider: list this type in the endpoint's own supportedMimeTypes to send it`, + ); } } diff --git a/api/app/clients/specs/BaseClient.test.js b/api/app/clients/specs/BaseClient.test.js index 2c675aa5dcd..7421237a5f4 100644 --- a/api/app/clients/specs/BaseClient.test.js +++ b/api/app/clients/specs/BaseClient.test.js @@ -3560,6 +3560,63 @@ describe('BaseClient', () => { }); }; + describe('provider documents under the inherited supportedMimeTypes list', () => { + /** A file the user sent through "Upload to Provider"; the endpoint sets no MIME list. */ + const providerFile = (file_id, type) => ({ + user: 'user1', + file_id, + filename: file_id, + filepath: `/uploads/${file_id}`, + type, + bytes: 100, + source: 'local', + llmDeliveryPath: 'provider', + metadata: { destinationChosen: true }, + }); + + const withSupportedMimeTypes = (supportedMimeTypes) => { + TestClient.options.req = { + config: { + fileConfig: { + endpoints: { [EModelEndpoint.openAI]: { fileLimit: 10, supportedMimeTypes } }, + }, + }, + }; + TestClient.options.agent = routedAgent({ + provider: EModelEndpoint.openAI, + endpoint: EModelEndpoint.openAI, + }); + }; + + test('skips archives but keeps other listed types without an endpoint opt-in', async () => { + withSupportedMimeTypes(undefined); + const { logger } = require('@librechat/data-schemas'); + const warn = jest.spyOn(logger, 'warn').mockImplementation(() => {}); + const sql = providerFile('query.sql', 'application/sql'); + const zip = providerFile('archive.zip', 'application/zip'); + const pdf = providerFile('report.pdf', 'application/pdf'); + + try { + const result = await TestClient.processAttachments({}, [sql, zip, pdf]); + + expect(TestClient.addDocuments).toHaveBeenCalledWith(expect.anything(), [sql, pdf]); + expect(result).toEqual([sql, pdf]); + expect(warn).toHaveBeenCalledWith(expect.stringContaining('"archive.zip"')); + } finally { + warn.mockRestore(); + } + }); + + test('sends a binary type the endpoint lists explicitly', async () => { + withSupportedMimeTypes(['^application/zip$']); + const zip = providerFile('archive.zip', 'application/zip'); + + await TestClient.processAttachments({}, [zip]); + + expect(TestClient.addDocuments).toHaveBeenCalledWith(expect.anything(), [zip]); + }); + }); + test('keeps a none image in returned files without adding image URLs', async () => { routeTo('none', 'image/*'); const message = {}; diff --git a/packages/api/src/agents/files/encode.spec.ts b/packages/api/src/agents/files/encode.spec.ts index 10c29025be9..8f2cdf67d3b 100644 --- a/packages/api/src/agents/files/encode.spec.ts +++ b/packages/api/src/agents/files/encode.spec.ts @@ -349,6 +349,44 @@ describe('createRunFileMessageEncoder', () => { expect(harness.encodeDocuments).not.toHaveBeenCalled(); }); + /** A file the user sent through "Upload to Provider": the chosen path survives child routing. */ + const providerChosen = (file_id: string, type: string): TFile => ({ + ...pdf, + file_id, + filename: file_id, + type, + text: undefined, + metadata: { destinationChosen: true }, + }); + + it('keeps archives out of the document encoder under the inherited MIME list', async () => { + const harness = setup(); + await harness.encode( + [providerChosen('zip', 'application/zip'), providerChosen('sql', 'application/sql')], + 'child', + ); + expect(harness.encodeDocuments).toHaveBeenCalledTimes(1); + expect(harness.encodeDocuments).toHaveBeenCalledWith( + harness.req, + [expect.objectContaining({ file_id: 'sql' })], + expect.anything(), + harness.getStrategyFunctions, + ); + }); + + it('sends a binary type to the document encoder when the endpoint lists it', async () => { + const harness = setup({ + fileConfig: { endpoints: { openAI: { supportedMimeTypes: ['^application/zip$'] } } }, + }); + await harness.encode([providerChosen('zip', 'application/zip')], 'child'); + expect(harness.encodeDocuments).toHaveBeenCalledWith( + harness.req, + [expect.objectContaining({ file_id: 'zip' })], + expect.anything(), + harness.getStrategyFunctions, + ); + }); + it.each([{ disabled: true }, { supportedMimeTypes: ['image/png'] }, { fileSizeLimit: 0.000001 }])( 'rejects incompatible child policy before reading bytes: %j', async (policy) => { diff --git a/packages/api/src/agents/files/encode.ts b/packages/api/src/agents/files/encode.ts index e04d6cbed86..b64d347fdd7 100644 --- a/packages/api/src/agents/files/encode.ts +++ b/packages/api/src/agents/files/encode.ts @@ -23,6 +23,7 @@ import { } from '../attachments'; import { assertModelBoundContent } from '~/middleware/modelBoundContent'; import { filterFilesByEndpointRuntimeConfig } from '~/files/filter'; +import { isProviderDocumentCandidate } from '~/files/encode/utils'; import { countTokens } from '~/utils/tokenizer'; type ContentBlock = Exclude[number]; @@ -176,8 +177,7 @@ export function createRunFileMessageEncoder( } else if (file.type.startsWith('video/')) { videos.push(file); } else if ( - endpointConfig.supportedMimeTypes && - fileConfig.checkType?.(file.type, endpointConfig.supportedMimeTypes) + isProviderDocumentCandidate(file.type, fileConfig, endpointConfig.supportedMimeTypes) ) { documents.push(file); } diff --git a/packages/api/src/files/encode/document.spec.ts b/packages/api/src/files/encode/document.spec.ts index f58d591ed67..448c4710b9b 100644 --- a/packages/api/src/files/encode/document.spec.ts +++ b/packages/api/src/files/encode/document.spec.ts @@ -15,6 +15,7 @@ jest.mock('./utils', () => ({ getFileStream: jest.fn(), getConfiguredFileSizeLimit: jest.fn(), isAttachmentObjectNotFoundError: jest.fn(() => false), + isConfiguredProviderMediaType: jest.requireActual('./utils').isConfiguredProviderMediaType, })); import { validatePdf, validateBedrockDocument } from '~/files/validation'; @@ -75,6 +76,23 @@ describe('encodeAndFormatDocuments - fileConfig integration', () => { : undefined, }); + /** Helper to create a mock request whose endpoint lists its own `supportedMimeTypes` */ + const createOptInRequest = ( + supportedMimeTypes: string[], + provider: string = Providers.OPENAI, + ): Partial => ({ + config: { + fileConfig: { + endpoints: { + [provider]: { + fileSizeLimit: 30, + supportedMimeTypes, + }, + }, + }, + }, + }); + /** Helper to create a mock PDF file */ const createMockFile = (sizeInMB: number): IMongoFile => ({ @@ -944,7 +962,7 @@ describe('encodeAndFormatDocuments - fileConfig integration', () => { const result = await encodeAndFormatDocuments( req, [file], - { provider: Providers.OPENAI, model: 'gemini-auto-latest' }, + { provider: Providers.OPENAI, model: 'gpt-5.4' }, mockStrategyFunctions, ); @@ -1045,9 +1063,9 @@ describe('encodeAndFormatDocuments - fileConfig integration', () => { expect(result.files).toHaveLength(1); }); - it('should format XLSX for Google/VertexAI as media block', async () => { - const req = createMockRequest(25) as ServerRequest; + it('should format XLSX for Google/VertexAI as media block when the endpoint lists it', async () => { const mimeType = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'; + const req = createOptInRequest([mimeType], Providers.GOOGLE) as ServerRequest; const file = createMockDocFile(2, mimeType, 'report.xlsx'); const mockContent = Buffer.from('xlsx-binary').toString('base64'); @@ -1102,6 +1120,232 @@ describe('encodeAndFormatDocuments - fileConfig integration', () => { expect(result.files).toHaveLength(1); }); + describe('textual documents without an endpoint opt-in', () => { + const sql = 'SELECT 1;'; + + const encodeSql = async ( + req: ServerRequest, + params: Parameters[2], + ) => { + const file = createMockDocFile(1, 'application/sql', 'query.sql'); + mockedGetFileStream.mockResolvedValue({ + file, + content: Buffer.from(sql).toString('base64'), + metadata: file, + }); + return encodeAndFormatDocuments(req, [file], params, mockStrategyFunctions); + }; + + it.each([ + ['OpenAI chat completions', Providers.OPENAI, false], + ['OpenAI responses API', Providers.OPENAI, true], + ['OpenRouter', Providers.OPENROUTER, false], + ['Google', Providers.GOOGLE, false], + ['Vertex AI', Providers.VERTEXAI, false], + ])( + 'sends application/sql to %s as a text part under the inherited list', + async (_label, provider, useResponsesApi) => { + const req = createMockRequest(30, provider) as ServerRequest; + + const result = await encodeSql(req, { provider, useResponsesApi }); + + expect(result.documents).toEqual([{ type: 'text', text: `File: "query.sql"\n\n${sql}` }]); + expect(result.files).toHaveLength(1); + }, + ); + + it('sends a textual type as a text part when no fileConfig is set', async () => { + const req = createMockRequest() as ServerRequest; + + const result = await encodeSql(req, { provider: Providers.OPENAI }); + + expect(result.documents).toEqual([{ type: 'text', text: `File: "query.sql"\n\n${sql}` }]); + }); + + it('sends textual documents as text parts to Claude through a gateway on the responses API', async () => { + const req = createMockRequest(30) as ServerRequest; + + const result = await encodeSql(req, { + provider: Providers.OPENAI, + model: 'claude-auto-latest', + useResponsesApi: true, + }); + + expect(result.documents).toEqual([{ type: 'text', text: `File: "query.sql"\n\n${sql}` }]); + }); + + it('keeps application/sql as a file part when the endpoint lists it', async () => { + const req = createOptInRequest(['^application/sql$']) as ServerRequest; + + const result = await encodeSql(req, { provider: Providers.OPENAI }); + + expect(result.documents).toEqual([ + { + type: 'file', + file: { + filename: 'query.sql', + file_data: `data:application/sql;base64,${Buffer.from(sql).toString('base64')}`, + }, + }, + ]); + }); + + it('keeps application/sql as an input_file on the responses API when the endpoint lists it', async () => { + const req = createOptInRequest(['^application/sql$']) as ServerRequest; + + const result = await encodeSql(req, { provider: Providers.OPENAI, useResponsesApi: true }); + + expect(result.documents).toMatchObject([{ type: 'input_file', filename: 'query.sql' }]); + }); + + it('keeps a PDF as a file part under the inherited list', async () => { + const req = createMockRequest(30) as ServerRequest; + const file = createMockFile(1); + const content = Buffer.from('%PDF-1.4').toString('base64'); + mockedGetFileStream.mockResolvedValue({ file, content, metadata: file }); + mockedValidatePdf.mockResolvedValue({ isValid: true }); + + const result = await encodeAndFormatDocuments( + req, + [file], + { provider: Providers.OPENAI }, + mockStrategyFunctions, + ); + + expect(result.documents).toMatchObject([ + { type: 'file', file: { file_data: `data:application/pdf;base64,${content}` } }, + ]); + }); + + it('keeps application/json as a file part for OpenAI under the inherited list', async () => { + const req = createMockRequest(30) as ServerRequest; + const file = createMockDocFile(1, 'application/json', 'data.json'); + const content = Buffer.from('{"a":1}').toString('base64'); + mockedGetFileStream.mockResolvedValue({ file, content, metadata: file }); + + const result = await encodeAndFormatDocuments( + req, + [file], + { provider: Providers.OPENAI }, + mockStrategyFunctions, + ); + + expect(result.documents).toMatchObject([ + { type: 'file', file: { file_data: `data:application/json;base64,${content}` } }, + ]); + }); + + it.each([Providers.GOOGLE, Providers.VERTEXAI])( + 'keeps text/csv as inline media for %s under the inherited list', + async (provider) => { + const req = createMockRequest(30, provider) as ServerRequest; + const file = createMockDocFile(1, 'text/csv', 'data.csv'); + const content = Buffer.from('a,b\n1,2').toString('base64'); + mockedGetFileStream.mockResolvedValue({ file, content, metadata: file }); + + const result = await encodeAndFormatDocuments( + req, + [file], + { provider }, + mockStrategyFunctions, + ); + + expect(result.documents).toEqual([ + { type: 'media', mimeType: 'text/csv', data: content }, + ]); + }, + ); + + it('sends application/xml to OpenAI as a text part under the inherited list', async () => { + const req = createMockRequest(30) as ServerRequest; + const file = createMockDocFile(1, 'application/xml', 'data.xml'); + const content = Buffer.from('1').toString('base64'); + mockedGetFileStream.mockResolvedValue({ file, content, metadata: file }); + + const result = await encodeAndFormatDocuments( + req, + [file], + { provider: Providers.OPENAI }, + mockStrategyFunctions, + ); + + expect(result.documents).toEqual([{ type: 'text', text: 'File: "data.xml"\n\n1' }]); + }); + + it.each(['application/yaml', 'application/typescript'])( + 'keeps %s as a file part for OpenAI under the inherited list', + async (mimeType) => { + const req = createMockRequest(30) as ServerRequest; + const file = createMockDocFile(1, mimeType, 'file'); + const content = Buffer.from('a: 1').toString('base64'); + mockedGetFileStream.mockResolvedValue({ file, content, metadata: file }); + + const result = await encodeAndFormatDocuments( + req, + [file], + { provider: Providers.OPENAI }, + mockStrategyFunctions, + ); + + expect(result.documents).toMatchObject([{ type: 'file' }]); + }, + ); + + it.each([ + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + 'application/vnd.openxmlformats-officedocument.presentationml.presentation', + ])('skips %s for Google under the inherited list', async (mimeType) => { + const req = createMockRequest(30, Providers.GOOGLE) as ServerRequest; + const file = createMockDocFile(1, mimeType, 'office-file'); + + const result = await encodeAndFormatDocuments( + req, + [file], + { provider: Providers.GOOGLE }, + mockStrategyFunctions, + ); + + expect(result.documents).toHaveLength(0); + expect(mockedGetFileStream).not.toHaveBeenCalled(); + }); + + it('skips XLSX for Gemini through an OpenAI-compatible gateway', async () => { + const req = createMockRequest(30) as ServerRequest; + const file = createMockDocFile( + 1, + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + 'report.xlsx', + ); + + const result = await encodeAndFormatDocuments( + req, + [file], + { provider: Providers.OPENAI, model: 'gemini-auto-latest' }, + mockStrategyFunctions, + ); + + expect(result.documents).toHaveLength(0); + expect(mockedGetFileStream).not.toHaveBeenCalled(); + }); + + it('sends application/json to Gemini through a gateway as a text part', async () => { + const req = createMockRequest(30) as ServerRequest; + const file = createMockDocFile(1, 'application/json', 'data.json'); + const content = Buffer.from('{"a":1}').toString('base64'); + mockedGetFileStream.mockResolvedValue({ file, content, metadata: file }); + + const result = await encodeAndFormatDocuments( + req, + [file], + { provider: Providers.OPENAI, model: 'gemini-auto-latest', useResponsesApi: true }, + mockStrategyFunctions, + ); + + expect(result.documents).toEqual([{ type: 'text', text: 'File: "data.json"\n\n{"a":1}' }]); + }); + }); + it('should skip non-Bedrock-document types for Bedrock provider', async () => { const req = createMockRequest() as ServerRequest; const file = createMockDocFile(1, 'application/zip', 'archive.zip'); diff --git a/packages/api/src/files/encode/document.ts b/packages/api/src/files/encode/document.ts index 57ef72db9a3..048258d2405 100644 --- a/packages/api/src/files/encode/document.ts +++ b/packages/api/src/files/encode/document.ts @@ -18,6 +18,7 @@ import type { import { getFileStream, getConfiguredFileSizeLimit, + isConfiguredProviderMediaType, isAttachmentObjectNotFoundError, } from './utils'; import { validatePdf, validateBedrockDocument } from '~/files/validation'; @@ -58,9 +59,58 @@ function usesAnthropicDocumentCapabilities(provider: Providers, model?: string): ); } +const isGoogleProvider = (provider: Providers): boolean => + provider === Providers.GOOGLE || provider === Providers.VERTEXAI; + +/** + * Whether the model behind this provider is Gemini, which rejects inline Office documents + * and textual `application/*` types (JSON, SQL) with a 400. OpenAI-compatible gateways + * report an OpenAI-like provider for Gemini models. + */ +function usesGeminiDocumentCapabilities(provider: Providers, model?: string): boolean { + return ( + isGoogleProvider(provider) || + (isOpenAILikeProvider(provider) && (model?.toLowerCase().includes('gemini') ?? false)) + ); +} + +/** + * Textual types that OpenAI-compatible gateways can reject as a `file` part (Azure OpenAI + * answers 400 "Invalid file data" for these), so they go as text unless the endpoint lists + * them. `text/*`, JSON, YAML and TypeScript stay file parts. + */ +const textPartApplicationTypes = new Set([ + 'application/sql', + 'application/x-sh', + 'application/xml', +]); + +/** + * Whether a document goes as a text part because the endpoint's own `supportedMimeTypes` + * does not list it. Gemini accepts `text/*` inline but rejects every textual + * `application/*` type (JSON, YAML, XML, SQL). + */ +function sendsAsTextWithoutOptIn(provider: Providers, mimeType: string, model?: string): boolean { + if (usesGeminiDocumentCapabilities(provider, model)) { + return !mimeType.startsWith('text/') && isAnthropicTextDocumentType(mimeType); + } + return textPartApplicationTypes.has(mimeType); +} + +/** A textual file as a plain text part, which every provider and API shape accepts. */ +function formatTextDocumentBlock(filename: string, content: string): DocumentBlock { + return { + type: 'text', + text: `File: "${filename}"\n\n${Buffer.from(content, 'base64').toString('utf8')}`, + }; +} + /** * Formats a base64-encoded document into the appropriate provider-specific block. * Returns `null` when the provider has no matching handler. + * + * `optedIn` is true when the endpoint's own `supportedMimeTypes` lists the type, rather + * than the built-in list it inherits. */ function formatDocumentBlock( provider: Providers, @@ -69,6 +119,7 @@ function formatDocumentBlock( filename: string | undefined, useResponsesApi: boolean | undefined, model?: string, + optedIn = false, ): DocumentBlock | null { if (provider === Providers.ANTHROPIC) { const source = getAnthropicDocumentSource(mimeType, content); @@ -89,7 +140,13 @@ function formatDocumentBlock( return document; } - if (provider === Providers.GOOGLE || provider === Providers.VERTEXAI) { + const resolvedFilename = filename ?? 'document'; + + if (!optedIn && sendsAsTextWithoutOptIn(provider, mimeType, model)) { + return formatTextDocumentBlock(resolvedFilename, content); + } + + if (isGoogleProvider(provider)) { return { type: 'media', mimeType, @@ -97,8 +154,6 @@ function formatDocumentBlock( }; } - const resolvedFilename = filename ?? 'document'; - /* A gateway translates an OpenAI `file` part into a base64 document with the file's own * media type, which Claude rejects for anything but PDF. Send textual files as text. */ if ( @@ -106,10 +161,7 @@ function formatDocumentBlock( isAnthropicTextDocumentType(mimeType) && usesAnthropicDocumentCapabilities(provider, model) ) { - return { - type: 'text', - text: `File: "${resolvedFilename}"\n\n${Buffer.from(content, 'base64').toString('utf8')}`, - }; + return formatTextDocumentBlock(resolvedFilename, content); } if (useResponsesApi) { @@ -136,26 +188,37 @@ function formatDocumentBlock( /** * Filters out files the provider's document path cannot send to the model. * Claude rejects non-PDF binary documents with a 400 that recurs on every retry, - * including when it is reached through an OpenAI-compatible gateway. Unsupported - * types are skipped instead of bricking the conversation. + * including when it is reached through an OpenAI-compatible gateway. Gemini rejects + * inline Office documents the same way, so for Gemini a type other than PDF or text + * goes only when the endpoint lists it. Unsupported types are skipped instead of + * bricking the conversation. */ function filterProviderDocumentFiles( provider: Providers, files: IMongoFile[], - model?: string, + model: string | undefined, + isOptedIn: (mimeType: string) => boolean, ): IMongoFile[] { if (provider === Providers.BEDROCK) { return files.filter((file) => isBedrockDocumentType(file.type)); } - if (!usesAnthropicDocumentCapabilities(provider, model)) { + let label: string; + let isSupported: (file: IMongoFile) => boolean; + if (usesAnthropicDocumentCapabilities(provider, model)) { + label = 'Claude'; + isSupported = (file) => isAnthropicDocumentType(file.type); + } else if (usesGeminiDocumentCapabilities(provider, model)) { + label = 'Gemini'; + isSupported = (file) => isAnthropicDocumentType(file.type) || isOptedIn(file.type ?? ''); + } else { return files; } const processable: IMongoFile[] = []; const skipped: string[] = []; for (const file of files) { - if (isAnthropicDocumentType(file.type)) { + if (isSupported(file)) { processable.push(file); } else { skipped.push(`"${file.filename}" (${file.type})`); @@ -164,7 +227,7 @@ function filterProviderDocumentFiles( if (skipped.length) { console.warn( - `Skipping attachment(s) unsupported by Claude document input: ${skipped.join(', ')}`, + `Skipping attachment(s) unsupported by ${label} document input: ${skipped.join(', ')}`, ); } @@ -192,8 +255,11 @@ function getBase64DecodedByteCount(content: string): number { * - **Bedrock**: Only encodes types in `bedrockDocumentFormats`; all others are skipped. * - **Anthropic**: Only encodes PDFs (base64 source) and textual types (plain-text source); * all others are skipped. + * - **Google/Vertex**: Encodes PDFs and textual types, plus types the endpoint's own + * `supportedMimeTypes` lists; all others are skipped. * - **PDF**: Validated via `validatePdf` before encoding. - * - **Generic types**: Encoded with a provider-specific size check. + * - **Generic types**: Encoded with a provider-specific size check. Textual types a + * provider can reject as a file part go as a text part unless the endpoint lists them. */ export async function encodeAndFormatDocuments( req: ServerRequest, @@ -216,7 +282,9 @@ export async function encodeAndFormatDocuments( return result; } - const processableFiles = filterProviderDocumentFiles(provider, files, model); + const isOptedIn = (mimeType: string) => + isConfiguredProviderMediaType(req, { provider, endpoint }, mimeType); + const processableFiles = filterProviderDocumentFiles(provider, files, model, isOptedIn); if (!processableFiles.length) { return result; @@ -325,6 +393,7 @@ export async function encodeAndFormatDocuments( file.filename, useResponsesApi, model, + isOptedIn(mimeType), ); if (block) { result.documents.push(block); diff --git a/packages/api/src/files/encode/processAttachments.spec.ts b/packages/api/src/files/encode/processAttachments.spec.ts index 01855dbf67f..ce0c00e78e0 100644 --- a/packages/api/src/files/encode/processAttachments.spec.ts +++ b/packages/api/src/files/encode/processAttachments.spec.ts @@ -6,6 +6,7 @@ import { isBedrockDocumentType, } from 'librechat-data-provider'; import type { FileConfig, EndpointFileConfig } from 'librechat-data-provider'; +import { isProviderDocumentCandidate } from './utils'; /** * Mirrors the categorization logic from BaseClient.processAttachments. @@ -49,10 +50,7 @@ function categorizeFile( } else if (file.type?.startsWith('audio/')) { return 'audios'; } else if ( - file.type && - mergedFileConfig && - endpointFileConfig?.supportedMimeTypes && - mergedFileConfig.checkType?.(file.type, endpointFileConfig.supportedMimeTypes) + isProviderDocumentCandidate(file.type, mergedFileConfig, endpointFileConfig?.supportedMimeTypes) ) { return 'documents'; } @@ -185,4 +183,61 @@ describe('processAttachments — supportedMimeTypes routing logic', () => { const result = categorizeFile({ type: 'text/csv', embedded: true }, false, merged, epConfig); expect(result).toBe('skipped'); }); + + describe('inherited default supportedMimeTypes', () => { + /** An endpoint with no `supportedMimeTypes` of its own inherits the built-in list. */ + const inherited = () => { + const merged = mergeFileConfig({ endpoints: { [endpoint]: { fileLimit: 5 } } }); + return { merged, epConfig: getEndpointFileConfig({ fileConfig: merged, endpoint }) }; + }; + + it.each([ + 'application/zip', + 'application/x-zip-compressed', + 'application/x-tar', + 'application/epub+zip', + ])('does not opt archive %s into the provider document path', (type) => { + const { merged, epConfig } = inherited(); + expect(merged.checkType?.(type, epConfig?.supportedMimeTypes ?? [])).toBe(true); + expect(categorizeFile({ type }, false, merged, epConfig)).toBe('skipped'); + }); + + it.each([ + 'application/sql', + 'application/x-sh', + 'application/json', + 'text/plain', + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + ])('keeps routing %s to documents', (type) => { + const { merged, epConfig } = inherited(); + expect(categorizeFile({ type }, false, merged, epConfig)).toBe('documents'); + }); + + it('keeps images, PDF, audio and video on their own branches', () => { + const { merged, epConfig } = inherited(); + expect(categorizeFile({ type: 'image/png' }, false, merged, epConfig)).toBe('images'); + expect(categorizeFile({ type: 'application/pdf' }, false, merged, epConfig)).toBe( + 'documents', + ); + expect(categorizeFile({ type: 'audio/mp3' }, false, merged, epConfig)).toBe('audios'); + expect(categorizeFile({ type: 'video/mp4' }, false, merged, epConfig)).toBe('videos'); + }); + + it('keeps Bedrock document types for Bedrock', () => { + const { merged, epConfig } = inherited(); + const docx = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'; + expect(categorizeFile({ type: docx }, true, merged, epConfig)).toBe('documents'); + }); + }); + + it('should skip textual types an explicit allowlist leaves out', () => { + const { merged, epConfig } = resolveConfig(['^image/.*', '^application/pdf$']); + expect(categorizeFile({ type: 'application/sql' }, false, merged, epConfig)).toBe('skipped'); + }); + + it('should route zip to documents when an explicit allowlist lists it', () => { + const { merged, epConfig } = resolveConfig(['^application/zip$']); + expect(categorizeFile({ type: 'application/zip' }, false, merged, epConfig)).toBe('documents'); + }); }); diff --git a/packages/api/src/files/encode/utils.ts b/packages/api/src/files/encode/utils.ts index 9c39ce2c13c..e69a4b1b694 100644 --- a/packages/api/src/files/encode/utils.ts +++ b/packages/api/src/files/encode/utils.ts @@ -6,6 +6,7 @@ import { isExplicitMimeConfig, getEndpointFileConfig, } from 'librechat-data-provider'; +import type { FileConfig, RegexLike } from 'librechat-data-provider'; import type { ServerRequest, StrategyFunctions, ProcessedFile } from '~/types'; import type { StoredFileRef } from '~/storage/path'; import { resolveDownloadPath } from '~/storage/path'; @@ -111,6 +112,38 @@ export const isConfiguredProviderMediaType = ( return fileConfig.checkType?.(mimeType, types) ?? false; }; +/** Archive and container types that no provider accepts as an inline document. */ +const providerArchiveMimeTypes = new Set([ + 'application/zip', + 'application/x-zip-compressed', + 'application/x-tar', + 'application/gzip', + 'application/x-gzip', + 'application/epub+zip', + 'application/octet-stream', +]); + +/** + * Whether a provider-bound attachment that has no dedicated branch (image, PDF, Bedrock + * document, audio, video) goes to the document encoder. The endpoint's `supportedMimeTypes` + * decides, including the built-in list an unconfigured endpoint inherits. That inherited + * list also allows archives for other upload paths, and no provider accepts one inline, so + * an archive goes to the provider only when the admin lists it for the endpoint. + */ +export const isProviderDocumentCandidate = ( + mimeType: string | null | undefined, + fileConfig: Pick | undefined, + supportedMimeTypes: RegexLike[] | undefined, +): boolean => { + if (!mimeType || !supportedMimeTypes) { + return false; + } + if (!isExplicitMimeConfig(supportedMimeTypes) && providerArchiveMimeTypes.has(mimeType)) { + return false; + } + return fileConfig?.checkType?.(mimeType, supportedMimeTypes) ?? false; +}; + /** * Maps accepted audio MIME types to the format token OpenAI-compatible providers expect. * Mirrors the canonicalization in `api/server/services/Files/Audio/STTService.js`: the From 5ec8bb31185fc299e4bd1efc4bbe4a40a5828690 Mon Sep 17 00:00:00 2001 From: Mihidum Hettiyahandi <55163074+mihidumh@users.noreply.github.com> Date: Tue, 29 Sep 2026 10:54:11 +1000 Subject: [PATCH 2/6] =?UTF-8?q?=F0=9F=A7=B9=20fix:=20Classify=20Gemini=20T?= =?UTF-8?q?ext=20Documents=20Like=20the=20Router=20Does?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Gemini document filter and its text-part decision used the Anthropic text list, which lacks application/vnd.coffeescript and application/x-yaml. The router sends those to the provider as natively readable text, so the filter dropped an inherited CoffeeScript file. Both now use isNativelyReadableText, the classification that routes the file. The BaseClient skip warning no longer tells admins to list the type: for Claude and Bedrock an explicitly listed archive is still filtered out. Co-Authored-By: Claude Opus 5.5 --- api/app/clients/BaseClient.js | 2 +- .../api/src/files/encode/document.spec.ts | 22 +++++++++++++++++++ packages/api/src/files/encode/document.ts | 11 +++++++--- 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/api/app/clients/BaseClient.js b/api/app/clients/BaseClient.js index 3136a1bdaa6..69615e0101e 100644 --- a/api/app/clients/BaseClient.js +++ b/api/app/clients/BaseClient.js @@ -1853,7 +1853,7 @@ class BaseClient { allFiles.push(file); } else if (deliveryPath === 'provider') { logger.warn( - `[BaseClient] Not sending "${file.filename}" (${file.type}) to the provider: list this type in the endpoint's own supportedMimeTypes to send it`, + `[BaseClient] Not sending "${file.filename}" (${file.type}) to the provider: this type is not an inline document for this endpoint`, ); } } diff --git a/packages/api/src/files/encode/document.spec.ts b/packages/api/src/files/encode/document.spec.ts index 448c4710b9b..68d5c3be5ef 100644 --- a/packages/api/src/files/encode/document.spec.ts +++ b/packages/api/src/files/encode/document.spec.ts @@ -1344,6 +1344,28 @@ describe('encodeAndFormatDocuments - fileConfig integration', () => { expect(result.documents).toEqual([{ type: 'text', text: 'File: "data.json"\n\n{"a":1}' }]); }); + + it.each([ + [Providers.GOOGLE, undefined], + [Providers.OPENAI, 'gemini-auto-latest'], + ])( + 'sends inherited application/vnd.coffeescript to Gemini (%s) as a text part', + async (provider, model) => { + const req = createMockRequest(30, provider) as ServerRequest; + const file = createMockDocFile(1, 'application/vnd.coffeescript', 'app.coffee'); + const content = Buffer.from('x = 1').toString('base64'); + mockedGetFileStream.mockResolvedValue({ file, content, metadata: file }); + + const result = await encodeAndFormatDocuments( + req, + [file], + { provider, model }, + mockStrategyFunctions, + ); + + expect(result.documents).toEqual([{ type: 'text', text: 'File: "app.coffee"\n\nx = 1' }]); + }, + ); }); it('should skip non-Bedrock-document types for Bedrock provider', async () => { diff --git a/packages/api/src/files/encode/document.ts b/packages/api/src/files/encode/document.ts index 048258d2405..326aeeb6d21 100644 --- a/packages/api/src/files/encode/document.ts +++ b/packages/api/src/files/encode/document.ts @@ -3,6 +3,7 @@ import { isOpenAILikeProvider, isBedrockDocumentType, bedrockDocumentFormats, + isNativelyReadableText, isAnthropicDocumentType, isDocumentSupportedProvider, isAnthropicTextDocumentType, @@ -88,11 +89,12 @@ const textPartApplicationTypes = new Set([ /** * Whether a document goes as a text part because the endpoint's own `supportedMimeTypes` * does not list it. Gemini accepts `text/*` inline but rejects every textual - * `application/*` type (JSON, YAML, XML, SQL). + * `application/*` type (JSON, YAML, XML, SQL, CoffeeScript). "Textual" is the same + * classification that routes a file to the provider (`isNativelyReadableText`). */ function sendsAsTextWithoutOptIn(provider: Providers, mimeType: string, model?: string): boolean { if (usesGeminiDocumentCapabilities(provider, model)) { - return !mimeType.startsWith('text/') && isAnthropicTextDocumentType(mimeType); + return !mimeType.startsWith('text/') && isNativelyReadableText(mimeType); } return textPartApplicationTypes.has(mimeType); } @@ -210,7 +212,10 @@ function filterProviderDocumentFiles( isSupported = (file) => isAnthropicDocumentType(file.type); } else if (usesGeminiDocumentCapabilities(provider, model)) { label = 'Gemini'; - isSupported = (file) => isAnthropicDocumentType(file.type) || isOptedIn(file.type ?? ''); + isSupported = (file) => + file.type === 'application/pdf' || + isNativelyReadableText(file.type ?? '') || + isOptedIn(file.type ?? ''); } else { return files; } From a3155b430d89fc097d1e885749bcf1ec58b796fc Mon Sep 17 00:00:00 2001 From: Mihidum Hettiyahandi <55163074+mihidumh@users.noreply.github.com> Date: Wed, 30 Sep 2026 09:04:32 +1000 Subject: [PATCH 3/6] =?UTF-8?q?=F0=9F=A7=AA=20test:=20Recover=20a=20Conver?= =?UTF-8?q?sation=20Stuck=20on=20an=20Earlier=20Archive=20Upload?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A conversation breaks when an archive from an earlier message goes out as a file part on every later turn. The replay path (addPreviousAttachments → processAttachments) now has a test: the archive is skipped with a warning, the PDF from the same message is still sent, and message_file_map holds only the PDF. Co-Authored-By: Claude Opus 5.5 --- api/app/clients/specs/BaseClient.test.js | 34 ++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/api/app/clients/specs/BaseClient.test.js b/api/app/clients/specs/BaseClient.test.js index 7421237a5f4..be5ed4fe265 100644 --- a/api/app/clients/specs/BaseClient.test.js +++ b/api/app/clients/specs/BaseClient.test.js @@ -3607,6 +3607,40 @@ describe('BaseClient', () => { } }); + test('recovers a conversation whose earlier message holds an archive', async () => { + /* The archive that 400s on every later turn arrives through history, not this turn's + * upload, so the replay path has to skip it too. */ + withSupportedMimeTypes(undefined); + TestClient.options.req.user = { id: 'user1' }; + TestClient.options.resendFiles = true; + TestClient.addFileContextToMessage = jest.fn(); + TestClient.assertHistoricalAttachmentLimits = undefined; + TestClient.checkVisionRequest = jest.fn(); + TestClient.message_file_map = undefined; + const { logger } = require('@librechat/data-schemas'); + const warn = jest.spyOn(logger, 'warn').mockImplementation(() => {}); + const zip = providerFile('archive.zip', 'application/zip'); + const pdf = providerFile('report.pdf', 'application/pdf'); + getFiles.mockResolvedValueOnce([zip, pdf]); + + try { + const [message] = await TestClient.addPreviousAttachments([ + { + messageId: 'msg-earlier', + text: 'Here are the files', + files: [{ file_id: 'archive.zip' }, { file_id: 'report.pdf' }], + }, + ]); + + expect(TestClient.addDocuments).toHaveBeenCalledTimes(1); + expect(TestClient.addDocuments).toHaveBeenCalledWith(message, [pdf]); + expect(TestClient.message_file_map['msg-earlier']).toEqual([pdf]); + expect(warn).toHaveBeenCalledWith(expect.stringContaining('"archive.zip"')); + } finally { + warn.mockRestore(); + } + }); + test('sends a binary type the endpoint lists explicitly', async () => { withSupportedMimeTypes(['^application/zip$']); const zip = providerFile('archive.zip', 'application/zip'); From 8c83da126548e9d8eb4efde5c8156fa0f4ae7942 Mon Sep 17 00:00:00 2001 From: Mihidum Hettiyahandi <55163074+mihidumh@users.noreply.github.com> Date: Wed, 30 Sep 2026 13:33:24 +1000 Subject: [PATCH 4/6] =?UTF-8?q?=F0=9F=9A=AB=20fix:=20Reject=20Unreadable?= =?UTF-8?q?=20Provider=20Attachments=20and=20Bound=20Their=20Text?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses the three findings of the review on a3155b430. Unsupported attachments are now visible. The document encoder returns the files it skipped in `omitted` (reason `unsupported_type` or `text_limit`), and BaseClient collects them together with the provider-bound files that no encoder takes. On the current turn, processAttachments throws AgentAttachmentUnsupportedError (code AGENT_ATTACHMENT_UNSUPPORTED, 415). The error message names each file, it is an attachment error to isAgentAttachmentLimitError, and the turn fails before the user message is saved. On history replay the turn continues: the file is left out of message_file_map, the message carries a text note that names it, and the server logs the omission with the message id. The child-run encoder rejects a share with a provider-bound file that it cannot encode, in validate() and before it reads any bytes, so an archive-only share is no longer answered with no file message. Decoded file text is bounded. The text-part fallback and the plain-text source of a native Anthropic document now go through processTextWithTokenLimit with fileTokenLimit. fileContextCharLimit is tracked per request, so it covers the current turn, history replay and child runs together. The text is cut to the remaining characters before tokens are counted, and a file that gets no budget is reported as `text_limit`. A native Google/Vertex upload allowlist no longer turns off safe encoding. Listing a type for the native API does not change what that API accepts. So on native Google/Vertex, textual application types always go as text, and Office files are omitted whatever supportedMimeTypes says. The opt-in still works for Gemini behind an OpenAI-compatible gateway. Co-Authored-By: Claude Opus 5.5 --- api/app/clients/BaseClient.js | 63 +++++- api/app/clients/specs/BaseClient.test.js | 121 ++++++++++-- api/server/controllers/agents/client.js | 13 +- api/server/controllers/agents/client.test.js | 23 ++- api/typedefs.js | 6 + packages/api/src/agents/attachments.ts | 44 ++++- packages/api/src/agents/errors.ts | 2 + packages/api/src/agents/files/encode.spec.ts | 52 +++-- packages/api/src/agents/files/encode.ts | 55 ++++-- .../api/src/files/encode/document.spec.ts | 181 ++++++++++++++++-- packages/api/src/files/encode/document.ts | 126 ++++++++++-- packages/api/src/types/files.ts | 13 ++ 12 files changed, 600 insertions(+), 99 deletions(-) diff --git a/api/app/clients/BaseClient.js b/api/app/clients/BaseClient.js index 69615e0101e..7fe98811ccf 100644 --- a/api/app/clients/BaseClient.js +++ b/api/app/clients/BaseClient.js @@ -15,6 +15,7 @@ const { getTransactionsConfig, encodeAndFormatDocuments, isProviderDocumentCandidate, + AgentAttachmentUnsupportedError, getLangfuseTraceMessageFields, isContentFilterError, assertModelBoundProviderContent, @@ -1699,7 +1700,12 @@ class BaseClient { ); } - async addDocuments(message, attachments) { + /** + * @param {TMessage} message + * @param {MongoFile[]} attachments + * @param {OmittedAttachment[]} [omissions] - Receives the files the encoder did not send. + */ + async addDocuments(message, attachments, omissions) { const documentResult = await encodeAndFormatDocuments( this.options.req, attachments, @@ -1715,6 +1721,9 @@ class BaseClient { documentResult.documents && documentResult.documents.length ? documentResult.documents : undefined; + if (omissions && documentResult.omitted?.length) { + omissions.push(...documentResult.omitted); + } return documentResult.files; } @@ -1790,7 +1799,18 @@ class BaseClient { return resolveTurnLLMDeliveryPath(this.options.agent?.deliveryRouting, file, fileConsumers); } - async processAttachments(message, attachments, fileConsumers) { + /** + * A provider-bound file the model would not receive is rejected on the current turn with + * {@link AgentAttachmentUnsupportedError}, so the model never answers as if it had read + * it. On history replay the same file is left out with a note in the message instead: + * throwing there would fail every later turn of the conversation. + * + * @param {TMessage} message + * @param {MongoFile[]} attachments + * @param {TurnFileConsumers} [fileConsumers] + * @param {{ historical?: boolean }} [options] + */ + async processAttachments(message, attachments, fileConsumers, options = {}) { const categorizedAttachments = { images: [], videos: [], @@ -1798,6 +1818,8 @@ class BaseClient { documents: [], }; + /** @type {OmittedAttachment[]} */ + const omissions = []; const allFiles = []; const provider = this.options.agent?.provider ?? this.options.endpoint; const isBedrock = provider === EModelEndpoint.bedrock; @@ -1852,9 +1874,12 @@ class BaseClient { categorizedAttachments.documents.push(file); allFiles.push(file); } else if (deliveryPath === 'provider') { - logger.warn( - `[BaseClient] Not sending "${file.filename}" (${file.type}) to the provider: this type is not an inline document for this endpoint`, - ); + omissions.push({ + ...(file.file_id && { file_id: file.file_id }), + filename: file.filename, + type: file.type, + reason: 'unsupported_type', + }); } } @@ -1863,7 +1888,7 @@ class BaseClient { ? this.addImageURLs(message, categorizedAttachments.images) : Promise.resolve([]), categorizedAttachments.documents.length > 0 - ? this.addDocuments(message, categorizedAttachments.documents) + ? this.addDocuments(message, categorizedAttachments.documents, omissions) : Promise.resolve([]), categorizedAttachments.videos.length > 0 ? this.addVideos(message, categorizedAttachments.videos) @@ -1875,7 +1900,29 @@ class BaseClient { allFiles.push(...imageFiles); - const seenFileIds = new Set(); + if (omissions.length > 0) { + if (!options.historical) { + throw new AgentAttachmentUnsupportedError(omissions); + } + logger.warn('[BaseClient] Attachments from history not sent to the model', { + messageId: message.messageId, + omitted: omissions, + }); + message.documents = [ + ...(message.documents ?? []), + ...omissions.map((file) => ({ + type: 'text', + text: `File "${file.filename}" was attached to this message but was not sent to the model: ${ + file.reason === 'text_limit' + ? 'the file text limit for this request was reached.' + : `this model cannot read ${file.type || 'this file type'}.` + }`, + })), + ]; + } + + const omittedFileIds = new Set(omissions.map((file) => file.file_id).filter(Boolean)); + const seenFileIds = new Set(omittedFileIds); const uniqueFiles = []; for (const file of allFiles) { @@ -2032,7 +2079,7 @@ class BaseClient { const [, processedFiles] = await Promise.all([ this.addFileContextToMessage(message, contextFiles), - this.processAttachments(message, contextFiles), + this.processAttachments(message, contextFiles, undefined, { historical: true }), ]); const processedFileIds = new Set( diff --git a/api/app/clients/specs/BaseClient.test.js b/api/app/clients/specs/BaseClient.test.js index be5ed4fe265..dc5208a72ee 100644 --- a/api/app/clients/specs/BaseClient.test.js +++ b/api/app/clients/specs/BaseClient.test.js @@ -5,6 +5,8 @@ const { resolveTurnDeliveryRouting, buildSteerMedia, Tokenizer, + isAgentAttachmentLimitError, + AgentAttachmentUnsupportedError, } = require('@librechat/api'); const { FakeClient, initializeFakeClient } = require('./FakeClient'); @@ -2945,7 +2947,9 @@ describe('BaseClient', () => { {}, ); expect(TestClient.addFileContextToMessage).toHaveBeenCalledWith(message, [ownerFile]); - expect(TestClient.processAttachments).toHaveBeenCalledWith(message, [ownerFile]); + expect(TestClient.processAttachments).toHaveBeenCalledWith(message, [ownerFile], undefined, { + historical: true, + }); expect(message.fileContext).toBe('authorized owner text'); expect(message.files).toEqual([ expect.objectContaining({ @@ -3588,23 +3592,54 @@ describe('BaseClient', () => { }); }; - test('skips archives but keeps other listed types without an endpoint opt-in', async () => { + test('rejects a current-turn archive the endpoint does not list, naming the file', async () => { withSupportedMimeTypes(undefined); - const { logger } = require('@librechat/data-schemas'); - const warn = jest.spyOn(logger, 'warn').mockImplementation(() => {}); const sql = providerFile('query.sql', 'application/sql'); const zip = providerFile('archive.zip', 'application/zip'); const pdf = providerFile('report.pdf', 'application/pdf'); - try { - const result = await TestClient.processAttachments({}, [sql, zip, pdf]); + const rejection = TestClient.processAttachments({}, [sql, zip, pdf]); - expect(TestClient.addDocuments).toHaveBeenCalledWith(expect.anything(), [sql, pdf]); - expect(result).toEqual([sql, pdf]); - expect(warn).toHaveBeenCalledWith(expect.stringContaining('"archive.zip"')); - } finally { - warn.mockRestore(); - } + await expect(rejection).rejects.toBeInstanceOf(AgentAttachmentUnsupportedError); + await expect(rejection).rejects.toMatchObject({ + code: 'AGENT_ATTACHMENT_UNSUPPORTED', + message: expect.stringContaining('"archive.zip" (application/zip)'), + attachments: [ + { + file_id: 'archive.zip', + filename: 'archive.zip', + type: 'application/zip', + reason: 'unsupported_type', + }, + ], + }); + expect(isAgentAttachmentLimitError(await rejection.catch((error) => error))).toBe(true); + expect(TestClient.addDocuments).toHaveBeenCalledWith( + expect.anything(), + [sql, pdf], + expect.any(Array), + ); + }); + + test('rejects a current-turn file the document encoder leaves out', async () => { + /* A DOCX sent to Gemini passes routing but not the encoder, which used to leave the + * file on the message while the model received none of it. */ + withSupportedMimeTypes(undefined); + const csv = providerFile('table.csv', 'text/csv'); + TestClient.addDocuments = jest.fn(async (_message, _files, omissions) => { + omissions.push({ + file_id: 'table.csv', + filename: 'table.csv', + type: 'text/csv', + reason: 'text_limit', + }); + return []; + }); + + await expect(TestClient.processAttachments({}, [csv])).rejects.toMatchObject({ + code: 'AGENT_ATTACHMENT_UNSUPPORTED', + message: expect.stringContaining('exceeds the file text limit'), + }); }); test('recovers a conversation whose earlier message holds an archive', async () => { @@ -3633,21 +3668,75 @@ describe('BaseClient', () => { ]); expect(TestClient.addDocuments).toHaveBeenCalledTimes(1); - expect(TestClient.addDocuments).toHaveBeenCalledWith(message, [pdf]); + expect(TestClient.addDocuments).toHaveBeenCalledWith(message, [pdf], expect.any(Array)); expect(TestClient.message_file_map['msg-earlier']).toEqual([pdf]); - expect(warn).toHaveBeenCalledWith(expect.stringContaining('"archive.zip"')); + /* The omission is explicit to the model and in the log, not silent. */ + expect(message.documents).toEqual([ + { type: 'file' }, + { + type: 'text', + text: 'File "archive.zip" was attached to this message but was not sent to the model: this model cannot read application/zip.', + }, + ]); + expect(warn).toHaveBeenCalledWith( + '[BaseClient] Attachments from history not sent to the model', + expect.objectContaining({ + messageId: 'msg-earlier', + omitted: [expect.objectContaining({ file_id: 'archive.zip' })], + }), + ); } finally { warn.mockRestore(); } }); + test('leaves a file the encoder omitted from history out of the message file map', async () => { + withSupportedMimeTypes(undefined); + TestClient.options.req.user = { id: 'user1' }; + TestClient.options.resendFiles = true; + TestClient.addFileContextToMessage = jest.fn(); + TestClient.assertHistoricalAttachmentLimits = undefined; + TestClient.checkVisionRequest = jest.fn(); + TestClient.message_file_map = undefined; + const docx = providerFile( + 'report.docx', + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + ); + TestClient.addDocuments = jest.fn(async (_message, _files, omissions) => { + omissions.push({ + file_id: docx.file_id, + filename: docx.filename, + type: docx.type, + reason: 'unsupported_type', + }); + return []; + }); + getFiles.mockResolvedValueOnce([docx]); + + const [message] = await TestClient.addPreviousAttachments([ + { messageId: 'msg-docx', text: 'Read this', files: [{ file_id: 'report.docx' }] }, + ]); + + expect(TestClient.message_file_map['msg-docx']).toEqual([]); + expect(message.documents).toEqual([ + expect.objectContaining({ + type: 'text', + text: expect.stringContaining('"report.docx" was attached to this message'), + }), + ]); + }); + test('sends a binary type the endpoint lists explicitly', async () => { withSupportedMimeTypes(['^application/zip$']); const zip = providerFile('archive.zip', 'application/zip'); await TestClient.processAttachments({}, [zip]); - expect(TestClient.addDocuments).toHaveBeenCalledWith(expect.anything(), [zip]); + expect(TestClient.addDocuments).toHaveBeenCalledWith( + expect.anything(), + [zip], + expect.any(Array), + ); }); }); @@ -4224,7 +4313,7 @@ describe('BaseClient', () => { expect(result).toEqual([file]); expect(message.documents).toEqual([{ type: 'file' }]); - expect(TestClient.addDocuments).toHaveBeenCalledWith(message, [file]); + expect(TestClient.addDocuments).toHaveBeenCalledWith(message, [file], expect.any(Array)); }); }); }); diff --git a/api/server/controllers/agents/client.js b/api/server/controllers/agents/client.js index ae72f09200a..bfbbccf25a7 100644 --- a/api/server/controllers/agents/client.js +++ b/api/server/controllers/agents/client.js @@ -381,7 +381,7 @@ class AgentClient extends BaseClient { ); } - async addDocuments(message, attachments) { + async addDocuments(message, attachments, omissions) { const memoryContext = { req: this.options.req, conversationId: this.conversationId, @@ -390,13 +390,13 @@ class AgentClient extends BaseClient { }; logAgentMemorySnapshot('before_encode_documents', memoryContext); try { - return await super.addDocuments(message, attachments); + return await super.addDocuments(message, attachments, omissions); } finally { logAgentMemorySnapshot('after_encode_documents', memoryContext); } } - async processAttachments(message, attachments, fileConsumers) { + async processAttachments(message, attachments, fileConsumers, options) { const modelBoundAttachments = this.getModelBoundAttachmentsForEndpoint(attachments); const processableAttachments = this.getProcessableAttachmentsForEndpoint( attachments, @@ -416,7 +416,12 @@ class AgentClient extends BaseClient { }; logAgentMemorySnapshot('before_process_attachments', memoryContext); try { - return await super.processAttachments(message, processableAttachments, fileConsumers); + return await super.processAttachments( + message, + processableAttachments, + fileConsumers, + options, + ); } finally { logAgentMemorySnapshot('after_process_attachments', memoryContext); } diff --git a/api/server/controllers/agents/client.test.js b/api/server/controllers/agents/client.test.js index a6a3991c421..e1b9451caec 100644 --- a/api/server/controllers/agents/client.test.js +++ b/api/server/controllers/agents/client.test.js @@ -6815,7 +6815,7 @@ describe('AgentClient - titleConvo', () => { const files = await client.processAttachments(message, [currentFile]); - expect(client.addDocuments).toHaveBeenCalledWith(message, [currentFile]); + expect(client.addDocuments).toHaveBeenCalledWith(message, [currentFile], []); expect(message.documents).toEqual([ expect.objectContaining({ type: 'input_file', @@ -6826,6 +6826,27 @@ describe('AgentClient - titleConvo', () => { }, ); + it('rejects a current-turn archive the user sent to the provider, naming the file', async () => { + const currentFile = { + ...makeUploadedFile('zip-file', 'bundle.zip', 'application/zip'), + llmDeliveryPath: 'provider', + metadata: { destinationChosen: true }, + }; + client.addDocuments = jest.fn(async () => []); + + await expect( + client.processAttachments( + { messageId: 'msg-1', text: 'Read this bundle.', isCreatedByUser: true }, + [currentFile], + ), + ).rejects.toMatchObject({ + code: 'AGENT_ATTACHMENT_UNSUPPORTED', + status: 415, + message: expect.stringContaining('"bundle.zip" (application/zip)'), + }); + expect(client.addDocuments).not.toHaveBeenCalled(); + }); + it.each(['current', 'history', 'history-disabled'])( 'resolves %s tool-routed text only for an authorized handoff without a reader', async (location) => { diff --git a/api/typedefs.js b/api/typedefs.js index 81bc2b2ea90..d8eefc9e47a 100644 --- a/api/typedefs.js +++ b/api/typedefs.js @@ -868,6 +868,12 @@ * @memberof typedefs */ +/** + * @exports OmittedAttachment + * @typedef {import('@librechat/api').OmittedAttachment} OmittedAttachment + * @memberof typedefs + */ + /** * @exports ISession * @typedef {import('@librechat/data-schemas').ISession} ISession diff --git a/packages/api/src/agents/attachments.ts b/packages/api/src/agents/attachments.ts index 447af3c9218..5bf4a543331 100644 --- a/packages/api/src/agents/attachments.ts +++ b/packages/api/src/agents/attachments.ts @@ -8,9 +8,9 @@ import { } from 'librechat-data-provider'; import type { IMongoFile } from '@librechat/data-schemas'; import type { TokenCountFn } from '~/utils/text'; -import type { ServerRequest } from '~/types'; +import type { OmittedAttachment, ServerRequest } from '~/types'; import { filterFilesByEndpointRuntimeConfig } from '~/files/filter'; -import { AGENT_ATTACHMENT_LIMIT_EXCEEDED } from './errors'; +import { AGENT_ATTACHMENT_LIMIT_EXCEEDED, AGENT_ATTACHMENT_UNSUPPORTED } from './errors'; import { countTokens } from '~/utils/tokenizer'; import { extractFileContext } from '~/files'; @@ -217,6 +217,35 @@ export class AgentAttachmentLimitError extends Error { } } +/** + * A current-turn attachment the model would not receive: its type is not an inline + * document for this model, or the turn's file text budget is spent. Rejecting the turn + * keeps the model from answering as if it had read the file. + */ +export class AgentAttachmentUnsupportedError extends Error { + readonly code: typeof AGENT_ATTACHMENT_UNSUPPORTED = AGENT_ATTACHMENT_UNSUPPORTED; + readonly status = 415; + readonly statusCode = 415; + + constructor(readonly attachments: OmittedAttachment[]) { + const describe = (reason: OmittedAttachment['reason']) => + attachments + .filter((file) => file.reason === reason) + .map((file) => (file.type ? `"${file.filename}" (${file.type})` : `"${file.filename}"`)) + .join(', '); + const unsupported = describe('unsupported_type'); + const overLimit = describe('text_limit'); + const sentences = [ + unsupported && + `This model cannot read ${unsupported}. Remove it, or upload it as text or to the code environment, and try again.`, + overLimit && + `${overLimit} exceeds the file text limit for this turn. Remove some attachments or use smaller files and try again.`, + ].filter(Boolean); + super(sentences.join(' ')); + this.name = 'AgentAttachmentUnsupportedError'; + } +} + export class AgentAttachmentPolicyError extends Error { readonly code: typeof AGENT_ATTACHMENT_LIMIT_EXCEEDED = AGENT_ATTACHMENT_LIMIT_EXCEEDED; readonly status = 413; @@ -232,8 +261,15 @@ export class AgentAttachmentPolicyError extends Error { export function isAgentAttachmentLimitError( error: unknown, -): error is AgentAttachmentLimitError | AgentAttachmentPolicyError { - return error instanceof AgentAttachmentLimitError || error instanceof AgentAttachmentPolicyError; +): error is + | AgentAttachmentLimitError + | AgentAttachmentPolicyError + | AgentAttachmentUnsupportedError { + return ( + error instanceof AgentAttachmentLimitError || + error instanceof AgentAttachmentPolicyError || + error instanceof AgentAttachmentUnsupportedError + ); } export function collectAgentAttachmentStats( diff --git a/packages/api/src/agents/errors.ts b/packages/api/src/agents/errors.ts index 37f6d1b2071..97ddb44057c 100644 --- a/packages/api/src/agents/errors.ts +++ b/packages/api/src/agents/errors.ts @@ -9,10 +9,12 @@ import { OboTokenResolutionError } from '~/mcp/oauth/obo'; export const AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE = 'AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE'; export const AGENT_ATTACHMENT_LIMIT_EXCEEDED = 'AGENT_ATTACHMENT_LIMIT_EXCEEDED'; +export const AGENT_ATTACHMENT_UNSUPPORTED = 'AGENT_ATTACHMENT_UNSUPPORTED'; const FATAL_AGENT_INITIALIZATION_CODES = new Set( [ AGENT_ATTACHMENT_LIMIT_EXCEEDED, + AGENT_ATTACHMENT_UNSUPPORTED, ErrorTypes.RESOURCE_RECOVERY_REQUIRED, ErrorTypes.STATEFUL_CODE_ENVIRONMENT_NOT_ALLOWED, ErrorTypes.CODE_WORKSPACE_UNAVAILABLE, diff --git a/packages/api/src/agents/files/encode.spec.ts b/packages/api/src/agents/files/encode.spec.ts index 8f2cdf67d3b..070d1bf731b 100644 --- a/packages/api/src/agents/files/encode.spec.ts +++ b/packages/api/src/agents/files/encode.spec.ts @@ -2,7 +2,11 @@ import { FileContext, FileSources, ImageDetail } from 'librechat-data-provider'; import type { TFile } from 'librechat-data-provider'; import type { RunFileEncodingAgent, RunFileMessageEncoderDeps } from './encode'; import type { ServerRequest } from '~/types'; -import { AgentAttachmentLimitError, AgentAttachmentPolicyError } from '../attachments'; +import { + AgentAttachmentLimitError, + AgentAttachmentPolicyError, + AgentAttachmentUnsupportedError, +} from '../attachments'; import { resolveTurnDeliveryRouting } from './delivery'; import { createRunFileMessageEncoder } from './encode'; @@ -359,19 +363,41 @@ describe('createRunFileMessageEncoder', () => { metadata: { destinationChosen: true }, }); - it('keeps archives out of the document encoder under the inherited MIME list', async () => { - const harness = setup(); - await harness.encode( + it.each([ + ['an archive-only share', [providerChosen('zip', 'application/zip')]], + [ + 'a share that mixes an archive with a document', [providerChosen('zip', 'application/zip'), providerChosen('sql', 'application/sql')], - 'child', - ); - expect(harness.encodeDocuments).toHaveBeenCalledTimes(1); - expect(harness.encodeDocuments).toHaveBeenCalledWith( - harness.req, - [expect.objectContaining({ file_id: 'sql' })], - expect.anything(), - harness.getStrategyFunctions, - ); + ], + ])('rejects %s under the inherited MIME list before reading bytes', async (_label, files) => { + const harness = setup(); + expect(() => harness.validate(files, 'child')).toThrow(AgentAttachmentUnsupportedError); + const rejection = harness.encode(files, 'child'); + await expect(rejection).rejects.toBeInstanceOf(AgentAttachmentUnsupportedError); + await expect(rejection).rejects.toMatchObject({ + code: 'AGENT_ATTACHMENT_UNSUPPORTED', + attachments: [expect.objectContaining({ file_id: 'zip', reason: 'unsupported_type' })], + }); + expect(harness.encodeDocuments).not.toHaveBeenCalled(); + }); + + it('rejects the share when the document encoder omits a file', async () => { + const harness = setup(); + const omitted = { + file_id: 'docx', + filename: 'report.docx', + type: 'application/msword', + reason: 'unsupported_type' as const, + }; + jest.mocked(harness.encodeDocuments).mockResolvedValueOnce({ + documents: [], + omitted: [omitted], + }); + + await expect(harness.encode([pdf], 'child')).rejects.toMatchObject({ + code: 'AGENT_ATTACHMENT_UNSUPPORTED', + attachments: [omitted], + }); }); it('sends a binary type to the document encoder when the endpoint lists it', async () => { diff --git a/packages/api/src/agents/files/encode.ts b/packages/api/src/agents/files/encode.ts index b64d347fdd7..efce4da4f95 100644 --- a/packages/api/src/agents/files/encode.ts +++ b/packages/api/src/agents/files/encode.ts @@ -13,13 +13,14 @@ import type { TurnFileConsumers, } from 'librechat-data-provider'; import type { BaseMessage } from '@librechat/agents/langchain'; -import type { ServerRequest, StrategyFunctions } from '~/types'; +import type { OmittedAttachment, ServerRequest, StrategyFunctions } from '~/types'; import type { TokenCountFn } from '~/utils/text'; import { isToolOwnedAttachment, isModelBoundAttachmentFile, assertAgentAttachmentLimits, AgentAttachmentPolicyError, + AgentAttachmentUnsupportedError, } from '../attachments'; import { assertModelBoundContent } from '~/middleware/modelBoundContent'; import { filterFilesByEndpointRuntimeConfig } from '~/files/filter'; @@ -60,7 +61,7 @@ export interface RunFileMessageEncoderDeps { req: ServerRequest; getAgent: (agentId: string) => RunFileEncodingAgent | undefined; encodeImages: MediaEncoder<{ image_urls: ContentBlock[] }>; - encodeDocuments: MediaEncoder<{ documents: ContentBlock[] }>; + encodeDocuments: MediaEncoder<{ documents: ContentBlock[]; omitted?: OmittedAttachment[] }>; encodeAudios: MediaEncoder<{ audios: ContentBlock[] }>; encodeVideos: MediaEncoder<{ videos: ContentBlock[] }>; getStrategyFunctions: (source: string) => StrategyFunctions; @@ -129,31 +130,18 @@ export function createRunFileMessageEncoder( endpoint, }); assertModelBoundContent({ filters: deps.req.config?.filters, files: sharedFiles }); - return { agent, params, sharedFiles, fileConfig, endpointConfig }; - } - function validate(files: TFile[], agentId: string): void { - if (files.length > 0) prepare(files, agentId); - } - - async function encode(files: TFile[], agentId: string): Promise { - if (files.length === 0) return []; - const { agent, params, sharedFiles, fileConfig, endpointConfig } = prepare(files, agentId); const images: TFile[] = []; const documents: TFile[] = []; const audios: TFile[] = []; const videos: TFile[] = []; const textFiles: TFile[] = []; + const unsupported: OmittedAttachment[] = []; for (const file of sharedFiles) { const deliveryPath = file.llmDeliveryPath; if (deliveryPath === 'none') { continue; } - if (deliveryPath === 'text' && !file.text) { - throw new Error( - `Shared file "${file.filename}" requires extracted text for this agent. Attach a text version or use an agent that supports the original file.`, - ); - } if (deliveryPath == null || deliveryPath === 'text') { textFiles.push(file); } @@ -180,6 +168,38 @@ export function createRunFileMessageEncoder( isProviderDocumentCandidate(file.type, fileConfig, endpointConfig.supportedMimeTypes) ) { documents.push(file); + } else if (deliveryPath === 'provider') { + unsupported.push({ + ...(file.file_id && { file_id: file.file_id }), + filename: file.filename, + type: file.type, + reason: 'unsupported_type', + }); + } + } + /* A provider-bound file with no encoder would reach the child as nothing at all, so the + * share is rejected before the child runs rather than answered without the file. */ + if (unsupported.length > 0) { + throw new AgentAttachmentUnsupportedError(unsupported); + } + return { params, sharedFiles, images, documents, audios, videos, textFiles }; + } + + function validate(files: TFile[], agentId: string): void { + if (files.length > 0) prepare(files, agentId); + } + + async function encode(files: TFile[], agentId: string): Promise { + if (files.length === 0) return []; + const { params, sharedFiles, images, documents, audios, videos, textFiles } = prepare( + files, + agentId, + ); + for (const file of sharedFiles) { + if (file.llmDeliveryPath === 'text' && !file.text) { + throw new Error( + `Shared file "${file.filename}" requires extracted text for this agent. Attach a text version or use an agent that supports the original file.`, + ); } } @@ -196,6 +216,9 @@ export function createRunFileMessageEncoder( ? deps.extractText({ attachments: textFiles, req: deps.req, tokenCountFn: countTokens }) : Promise.resolve(undefined), ]); + if (documentResult.omitted?.length) { + throw new AgentAttachmentUnsupportedError(documentResult.omitted); + } if ( !text && imageResult.image_urls.length === 0 && diff --git a/packages/api/src/files/encode/document.spec.ts b/packages/api/src/files/encode/document.spec.ts index 68d5c3be5ef..890b499e9ea 100644 --- a/packages/api/src/files/encode/document.spec.ts +++ b/packages/api/src/files/encode/document.spec.ts @@ -1063,32 +1063,88 @@ describe('encodeAndFormatDocuments - fileConfig integration', () => { expect(result.files).toHaveLength(1); }); - it('should format XLSX for Google/VertexAI as media block when the endpoint lists it', async () => { - const mimeType = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'; - const req = createOptInRequest([mimeType], Providers.GOOGLE) as ServerRequest; - const file = createMockDocFile(2, mimeType, 'report.xlsx'); + describe('native Google/Vertex ignores the upload allowlist', () => { + const officeTypes = [ + ['XLSX', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', 'r.xlsx'], + [ + 'DOCX', + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + 'r.docx', + ], + ] as const; + + it.each( + [Providers.GOOGLE, Providers.VERTEXAI].flatMap((provider) => + officeTypes.map( + ([label, mimeType, filename]) => + [provider, label, mimeType, filename] as [Providers, string, string, string], + ), + ), + )( + 'omits %s %s even when the endpoint lists it', + async (provider, _label, mimeType, filename) => { + const req = createOptInRequest([mimeType], provider) as ServerRequest; + const file = createMockDocFile(1, mimeType, filename); - const mockContent = Buffer.from('xlsx-binary').toString('base64'); - mockedGetFileStream.mockResolvedValue({ - file, - content: mockContent, - metadata: file, - }); + const result = await encodeAndFormatDocuments( + req, + [file], + { provider }, + mockStrategyFunctions, + ); - const result = await encodeAndFormatDocuments( - req, - [file], - { provider: Providers.GOOGLE }, - mockStrategyFunctions, + expect(result.documents).toHaveLength(0); + expect(result.files).toHaveLength(0); + expect(result.omitted).toEqual([ + { file_id: file.file_id, filename, type: mimeType, reason: 'unsupported_type' }, + ]); + expect(mockedGetFileStream).not.toHaveBeenCalled(); + }, ); - expect(result.documents).toHaveLength(1); - expect(result.documents[0]).toMatchObject({ - type: 'media', - mimeType, - data: mockContent, + it.each([Providers.GOOGLE, Providers.VERTEXAI])( + 'sends a listed application/sql to %s as a text part, not inline media', + async (provider) => { + const req = createOptInRequest(['^application/sql$'], provider) as ServerRequest; + const file = createMockDocFile(1, 'application/sql', 'query.sql'); + const content = Buffer.from('SELECT 1;').toString('base64'); + mockedGetFileStream.mockResolvedValue({ file, content, metadata: file }); + + const result = await encodeAndFormatDocuments( + req, + [file], + { provider }, + mockStrategyFunctions, + ); + + expect(result.documents).toEqual([ + { type: 'text', text: 'File: "query.sql"\n\nSELECT 1;' }, + ]); + }, + ); + + it('keeps a listed XLSX as a file part for Gemini behind a gateway', async () => { + const mimeType = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'; + const req = createOptInRequest([mimeType]) as ServerRequest; + const file = createMockDocFile(1, mimeType, 'report.xlsx'); + const content = Buffer.from('xlsx-binary').toString('base64'); + mockedGetFileStream.mockResolvedValue({ file, content, metadata: file }); + + const result = await encodeAndFormatDocuments( + req, + [file], + { provider: Providers.OPENAI, model: 'gemini-auto-latest' }, + mockStrategyFunctions, + ); + + expect(result.documents).toEqual([ + { + type: 'file', + file: { filename: 'report.xlsx', file_data: `data:${mimeType};base64,${content}` }, + }, + ]); + expect(result.omitted).toBeUndefined(); }); - expect(result.files).toHaveLength(1); }); it('should format text/plain for standard OpenAI-like provider as file block', async () => { @@ -1368,6 +1424,89 @@ describe('encodeAndFormatDocuments - fileConfig integration', () => { ); }); + describe('decoded text budgets', () => { + const createBudgetRequest = (fileTokenLimit?: number, fileContextCharLimit?: number) => + ({ + body: fileTokenLimit != null ? { fileTokenLimit } : {}, + config: { fileConfig: { fileContextCharLimit } }, + }) as unknown as ServerRequest; + + const mockSqlFile = (filename: string, bytes: number) => { + const file = createMockDocFile(bytes / (1024 * 1024), 'application/sql', filename); + const content = Buffer.from('SELECT 1;\n'.repeat(bytes / 10)).toString('base64'); + mockedGetFileStream.mockResolvedValueOnce({ file, content, metadata: file }); + return file; + }; + + it('bounds a 2 MB SQL text part by the token and character limits', async () => { + const req = createBudgetRequest(8, 100); + const file = mockSqlFile('big.sql', 2_000_000); + + const result = await encodeAndFormatDocuments( + req, + [file], + { provider: Providers.OPENAI }, + mockStrategyFunctions, + ); + + expect(result.documents).toHaveLength(1); + const block = result.documents[0] as { type: string; text: string }; + expect(block.type).toBe('text'); + expect(block.text.startsWith('File: "big.sql"')).toBe(true); + expect(block.text.length).toBeLessThanOrEqual(100); + expect(result.files).toHaveLength(1); + }); + + it('shares the character limit across calls of one request and omits past it', async () => { + const req = createBudgetRequest(undefined, 100); + + const first = await encodeAndFormatDocuments( + req, + [mockSqlFile('a.sql', 1000)], + { provider: Providers.OPENAI }, + mockStrategyFunctions, + ); + const secondFile = mockSqlFile('b.sql', 1000); + const second = await encodeAndFormatDocuments( + req, + [secondFile], + { provider: Providers.OPENAI }, + mockStrategyFunctions, + ); + + expect((first.documents[0] as { text: string }).text).toHaveLength(100); + expect(second.documents).toHaveLength(0); + expect(second.files).toHaveLength(0); + expect(second.omitted).toEqual([ + { + file_id: secondFile.file_id, + filename: 'b.sql', + type: 'application/sql', + reason: 'text_limit', + }, + ]); + }); + + it('bounds the plain-text source of a native Anthropic document', async () => { + const req = createBudgetRequest(undefined, 50); + const file = createMockDocFile(1, 'text/plain', 'notes.txt'); + const content = Buffer.from('x'.repeat(500)).toString('base64'); + mockedGetFileStream.mockResolvedValue({ file, content, metadata: file }); + + const result = await encodeAndFormatDocuments( + req, + [file], + { provider: Providers.ANTHROPIC }, + mockStrategyFunctions, + ); + + expect(result.documents[0]).toMatchObject({ + type: 'document', + source: { type: 'text', data: 'x'.repeat(50) }, + }); + }); + }); + it('should skip non-Bedrock-document types for Bedrock provider', async () => { const req = createMockRequest() as ServerRequest; const file = createMockDocFile(1, 'application/zip', 'archive.zip'); diff --git a/packages/api/src/files/encode/document.ts b/packages/api/src/files/encode/document.ts index 326aeeb6d21..6290b04b0ad 100644 --- a/packages/api/src/files/encode/document.ts +++ b/packages/api/src/files/encode/document.ts @@ -7,15 +7,20 @@ import { isAnthropicDocumentType, isDocumentSupportedProvider, isAnthropicTextDocumentType, + mergeFileConfig, } from 'librechat-data-provider'; import type { IMongoFile } from '@librechat/data-schemas'; import type { DocumentBlock, AnthropicDocumentBlock, + AttachmentOmissionReason, + OmittedAttachment, StrategyFunctions, DocumentResult, ServerRequest, } from '~/types'; +import { processTextWithTokenLimit } from '~/utils/text'; +import { countTokens } from '~/utils/tokenizer'; import { getFileStream, getConfiguredFileSizeLimit, @@ -193,21 +198,20 @@ function formatDocumentBlock( * including when it is reached through an OpenAI-compatible gateway. Gemini rejects * inline Office documents the same way, so for Gemini a type other than PDF or text * goes only when the endpoint lists it. Unsupported types are skipped instead of - * bricking the conversation. + * bricking the conversation, and returned so the caller can report them. */ function filterProviderDocumentFiles( provider: Providers, files: IMongoFile[], model: string | undefined, isOptedIn: (mimeType: string) => boolean, -): IMongoFile[] { - if (provider === Providers.BEDROCK) { - return files.filter((file) => isBedrockDocumentType(file.type)); - } - +): { processable: IMongoFile[]; skipped: IMongoFile[] } { let label: string; let isSupported: (file: IMongoFile) => boolean; - if (usesAnthropicDocumentCapabilities(provider, model)) { + if (provider === Providers.BEDROCK) { + label = 'Bedrock'; + isSupported = (file) => isBedrockDocumentType(file.type); + } else if (usesAnthropicDocumentCapabilities(provider, model)) { label = 'Claude'; isSupported = (file) => isAnthropicDocumentType(file.type); } else if (usesGeminiDocumentCapabilities(provider, model)) { @@ -217,26 +221,94 @@ function filterProviderDocumentFiles( isNativelyReadableText(file.type ?? '') || isOptedIn(file.type ?? ''); } else { - return files; + return { processable: files, skipped: [] }; } const processable: IMongoFile[] = []; - const skipped: string[] = []; + const skipped: IMongoFile[] = []; for (const file of files) { if (isSupported(file)) { processable.push(file); } else { - skipped.push(`"${file.filename}" (${file.type})`); + skipped.push(file); } } if (skipped.length) { console.warn( - `Skipping attachment(s) unsupported by ${label} document input: ${skipped.join(', ')}`, + `Skipping attachment(s) unsupported by ${label} document input: ${skipped + .map((file) => `"${file.filename}" (${file.type})`) + .join(', ')}`, ); } - return processable; + return { processable, skipped }; +} + +const toOmittedAttachment = ( + file: Pick, + reason: AttachmentOmissionReason, +): OmittedAttachment => ({ + ...(file.file_id && { file_id: file.file_id }), + filename: file.filename ?? 'document', + type: file.type ?? '', + reason, +}); + +/** + * Decoded file text already sent per request. `fileContextCharLimit` then spans every + * encoder call of one request: the current turn, history replay and child runs. + */ +const decodedTextCharsByRequest = new WeakMap(); + +/** + * Bounds decoded file text with the limits that govern extracted file text: + * `fileTokenLimit` per file and `fileContextCharLimit` across the request. + * Returns `null` when the request's character budget is already spent. + */ +async function limitDecodedText(req: ServerRequest, text: string): Promise { + const fileConfig = mergeFileConfig(req.config?.fileConfig); + const charLimit = fileConfig.fileContextCharLimit; + const used = decodedTextCharsByRequest.get(req) ?? 0; + let limited = text; + if (charLimit) { + const remaining = charLimit - used; + if (remaining <= 0) { + return null; + } + /* Cut to the character budget first, so the token count never runs on the whole file. */ + limited = limited.slice(0, remaining); + } + const tokenLimit = req.body?.fileTokenLimit ?? fileConfig.fileTokenLimit; + if (tokenLimit) { + ({ text: limited } = await processTextWithTokenLimit({ + text: limited, + tokenLimit, + tokenCountFn: countTokens, + })); + } + decodedTextCharsByRequest.set(req, used + limited.length); + return limited; +} + +/** + * Applies {@link limitDecodedText} to a block that carries decoded file text: the text + * part fallback, and the plain-text source of a native Anthropic document. Other blocks + * pass through. Returns `null` when no budget is left for the text. + */ +async function limitTextBlock( + req: ServerRequest, + block: DocumentBlock, +): Promise { + if (block.type === 'text') { + const text = await limitDecodedText(req, block.text); + return text == null ? null : { ...block, text }; + } + if (block.type === 'document' && 'source' in block && block.source.type === 'text') { + const data = await limitDecodedText(req, block.source.data); + return data == null ? null : { ...block, source: { ...block.source, data } }; + } + return block; } function getBase64DecodedByteCount(content: string): number { @@ -260,11 +332,16 @@ function getBase64DecodedByteCount(content: string): number { * - **Bedrock**: Only encodes types in `bedrockDocumentFormats`; all others are skipped. * - **Anthropic**: Only encodes PDFs (base64 source) and textual types (plain-text source); * all others are skipped. - * - **Google/Vertex**: Encodes PDFs and textual types, plus types the endpoint's own - * `supportedMimeTypes` lists; all others are skipped. + * - **Google/Vertex**: Encodes PDFs and textual types; all others are skipped. The native + * API's limits do not change with the upload allowlist, so `supportedMimeTypes` widens + * only Gemini behind a gateway. * - **PDF**: Validated via `validatePdf` before encoding. * - **Generic types**: Encoded with a provider-specific size check. Textual types a * provider can reject as a file part go as a text part unless the endpoint lists them. + * - **Decoded text**: Bounded by `fileTokenLimit` per file and `fileContextCharLimit` + * across the request. + * + * Skipped files are returned in `omitted`, so the caller can reject or report them. */ export async function encodeAndFormatDocuments( req: ServerRequest, @@ -287,9 +364,20 @@ export async function encodeAndFormatDocuments( return result; } + /* An admin's allowlist can widen what a gateway forwards, but not what the native + * Google/Vertex API accepts. */ const isOptedIn = (mimeType: string) => + !isGoogleProvider(provider) && isConfiguredProviderMediaType(req, { provider, endpoint }, mimeType); - const processableFiles = filterProviderDocumentFiles(provider, files, model, isOptedIn); + const { processable: processableFiles, skipped } = filterProviderDocumentFiles( + provider, + files, + model, + isOptedIn, + ); + if (skipped.length) { + result.omitted = skipped.map((file) => toOmittedAttachment(file, 'unsupported_type')); + } if (!processableFiles.length) { return result; @@ -391,7 +479,7 @@ export async function encodeAndFormatDocuments( ); } - const block = formatDocumentBlock( + const formatted = formatDocumentBlock( provider, mimeType, content, @@ -400,9 +488,15 @@ export async function encodeAndFormatDocuments( model, isOptedIn(mimeType), ); + if (!formatted) { + continue; + } + const block = await limitTextBlock(req, formatted); if (block) { result.documents.push(block); result.files.push(metadata); + } else { + (result.omitted ??= []).push(toOmittedAttachment(file, 'text_limit')); } } } diff --git a/packages/api/src/types/files.ts b/packages/api/src/types/files.ts index 83c3bfc1230..885d8f7ecaf 100644 --- a/packages/api/src/types/files.ts +++ b/packages/api/src/types/files.ts @@ -124,6 +124,17 @@ export type DocumentBlock = | OpenAIInputFileBlock | BedrockDocumentBlock; +/** Why a provider-bound attachment was not sent to the model. */ +export type AttachmentOmissionReason = 'unsupported_type' | 'text_limit'; + +/** A provider-bound attachment the model did not receive. */ +export interface OmittedAttachment { + file_id?: string; + filename: string; + type: string; + reason: AttachmentOmissionReason; +} + export interface DocumentResult { documents: DocumentBlock[]; files: Array<{ @@ -134,6 +145,8 @@ export interface DocumentResult { filename: string; type: string; }>; + /** Attachments this call did not send, so the caller can reject or report them. */ + omitted?: OmittedAttachment[]; } /** Google audio block format */ From e5fc787629b402f15021d8ac1ee59bdf89c6ad73 Mon Sep 17 00:00:00 2001 From: Mihidum Hettiyahandi <55163074+mihidumh@users.noreply.github.com> Date: Fri, 2 Oct 2026 10:34:30 +1000 Subject: [PATCH 5/6] =?UTF-8?q?=F0=9F=A7=B9=20chore:=20Sort=20Imports=20in?= =?UTF-8?q?=20the=20Attachment=20Encoders?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The "Static checks" job failed on import order in attachments.ts and document.ts. This is the output of `npm run sort-imports` for the two files. No code change. Co-Authored-By: Claude Opus 5.5 --- packages/api/src/agents/attachments.ts | 4 ++-- packages/api/src/files/encode/document.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/api/src/agents/attachments.ts b/packages/api/src/agents/attachments.ts index 5bf4a543331..692a3e53fda 100644 --- a/packages/api/src/agents/attachments.ts +++ b/packages/api/src/agents/attachments.ts @@ -7,10 +7,10 @@ import { getEndpointFileConfig, } from 'librechat-data-provider'; import type { IMongoFile } from '@librechat/data-schemas'; -import type { TokenCountFn } from '~/utils/text'; import type { OmittedAttachment, ServerRequest } from '~/types'; -import { filterFilesByEndpointRuntimeConfig } from '~/files/filter'; +import type { TokenCountFn } from '~/utils/text'; import { AGENT_ATTACHMENT_LIMIT_EXCEEDED, AGENT_ATTACHMENT_UNSUPPORTED } from './errors'; +import { filterFilesByEndpointRuntimeConfig } from '~/files/filter'; import { countTokens } from '~/utils/tokenizer'; import { extractFileContext } from '~/files'; diff --git a/packages/api/src/files/encode/document.ts b/packages/api/src/files/encode/document.ts index 6290b04b0ad..7baa81145b3 100644 --- a/packages/api/src/files/encode/document.ts +++ b/packages/api/src/files/encode/document.ts @@ -19,8 +19,6 @@ import type { DocumentResult, ServerRequest, } from '~/types'; -import { processTextWithTokenLimit } from '~/utils/text'; -import { countTokens } from '~/utils/tokenizer'; import { getFileStream, getConfiguredFileSizeLimit, @@ -28,7 +26,9 @@ import { isAttachmentObjectNotFoundError, } from './utils'; import { validatePdf, validateBedrockDocument } from '~/files/validation'; +import { processTextWithTokenLimit } from '~/utils/text'; import { runGuardedEncode } from './memoryGuard'; +import { countTokens } from '~/utils/tokenizer'; /** Anthropic only accepts PDFs as base64 documents; textual types must use a text source */ function getAnthropicDocumentSource( From 99bc9973fdda1f9b25c82e4e594c3c83af6892bc Mon Sep 17 00:00:00 2001 From: Mihidum Hettiyahandi <55163074+mihidumh@users.noreply.github.com> Date: Tue, 6 Oct 2026 10:16:27 +1100 Subject: [PATCH 6/6] =?UTF-8?q?=F0=9F=93=9D=20fix:=20Send=20Textual=20Uplo?= =?UTF-8?q?ads=20to=20OpenAI=20Chat=20Completions=20as=20Text=20Parts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OpenAI chat completions accepts only PDF as `file.file_data` and answers 400 "Expected a base64-encoded data URL with an application/pdf MIME type" for `text/plain`, CSV, HTML and JSON, which recurs on every later turn because the file is re-sent from history. Azure OpenAI accepts those types, and the code cannot tell the two apart behind a gateway, so a textual type now goes as a text part under the inherited list, which both read. An endpoint that lists the type in its own `supportedMimeTypes` keeps the file part. The responses API keeps `input_file`, which OpenAI documents for textual types. Measured by @tommctech on v0.8.8 with `legacyFileUploadUX: true` on api.openai.com (#16473). --- .../api/src/files/encode/document.spec.ts | 91 +++++++++++++++++-- packages/api/src/files/encode/document.ts | 25 +++-- 2 files changed, 100 insertions(+), 16 deletions(-) diff --git a/packages/api/src/files/encode/document.spec.ts b/packages/api/src/files/encode/document.spec.ts index 890b499e9ea..0beb6f36eb9 100644 --- a/packages/api/src/files/encode/document.spec.ts +++ b/packages/api/src/files/encode/document.spec.ts @@ -949,7 +949,7 @@ describe('encodeAndFormatDocuments - fileConfig integration', () => { expect(result.files).toHaveLength(1); }); - it('should keep textual documents as file parts for non-Claude OpenAI models', async () => { + it('should send textual documents as text parts for non-Claude OpenAI models on chat completions', async () => { const req = createMockRequest(30) as ServerRequest; const file = createMockDocFile(1, 'text/html', 'page.html'); const mockContent = Buffer.from('

hi

').toString('base64'); @@ -966,10 +966,31 @@ describe('encodeAndFormatDocuments - fileConfig integration', () => { mockStrategyFunctions, ); + expect(result.documents).toEqual([{ type: 'text', text: 'File: "page.html"\n\n

hi

' }]); + }); + + it('should keep textual documents as input_file parts for non-Claude OpenAI models on the responses API', async () => { + const req = createMockRequest(30) as ServerRequest; + const file = createMockDocFile(1, 'text/html', 'page.html'); + const mockContent = Buffer.from('

hi

').toString('base64'); + mockedGetFileStream.mockResolvedValue({ + file, + content: mockContent, + metadata: file, + }); + + const result = await encodeAndFormatDocuments( + req, + [file], + { provider: Providers.OPENAI, model: 'gpt-5.4', useResponsesApi: true }, + mockStrategyFunctions, + ); + expect(result.documents).toEqual([ { - type: 'file', - file: { filename: 'page.html', file_data: `data:text/html;base64,${mockContent}` }, + type: 'input_file', + filename: 'page.html', + file_data: `data:text/html;base64,${mockContent}`, }, ]); }); @@ -1147,8 +1168,8 @@ describe('encodeAndFormatDocuments - fileConfig integration', () => { }); }); - it('should format text/plain for standard OpenAI-like provider as file block', async () => { - const req = createMockRequest(15) as ServerRequest; + it('should keep text/plain as a file block for a standard OpenAI-like provider when the endpoint lists it', async () => { + const req = createOptInRequest(['^text/plain$']) as ServerRequest; const file = createMockDocFile(1, 'text/plain', 'readme.txt'); const mockContent = Buffer.from('readme content').toString('base64'); @@ -1273,8 +1294,60 @@ describe('encodeAndFormatDocuments - fileConfig integration', () => { ]); }); - it('keeps application/json as a file part for OpenAI under the inherited list', async () => { - const req = createMockRequest(30) as ServerRequest; + it.each([ + ['text/plain', 'fixture.txt', 'plain text'], + ['text/csv', 'fixture.csv', 'a,b\n1,2'], + ['text/html', 'fixture.html', '

hi

'], + ['application/json', 'fixture.json', '{"a":1}'], + ])( + 'sends %s to OpenAI chat completions as a text part under the inherited list', + async (mimeType, filename, text) => { + const req = createMockRequest(30) as ServerRequest; + const file = createMockDocFile(1, mimeType, filename); + const content = Buffer.from(text).toString('base64'); + mockedGetFileStream.mockResolvedValue({ file, content, metadata: file }); + + const result = await encodeAndFormatDocuments( + req, + [file], + { provider: Providers.OPENAI, model: 'gpt-5.1' }, + mockStrategyFunctions, + ); + + expect(result.documents).toEqual([ + { type: 'text', text: `File: "${filename}"\n\n${text}` }, + ]); + expect(result.files).toHaveLength(1); + }, + ); + + it.each([ + ['text/plain', 'fixture.txt'], + ['text/csv', 'fixture.csv'], + ['application/json', 'fixture.json'], + ])( + 'keeps %s as an input_file on the OpenAI responses API under the inherited list', + async (mimeType, filename) => { + const req = createMockRequest(30) as ServerRequest; + const file = createMockDocFile(1, mimeType, filename); + const content = Buffer.from('x').toString('base64'); + mockedGetFileStream.mockResolvedValue({ file, content, metadata: file }); + + const result = await encodeAndFormatDocuments( + req, + [file], + { provider: Providers.OPENAI, useResponsesApi: true }, + mockStrategyFunctions, + ); + + expect(result.documents).toEqual([ + { type: 'input_file', filename, file_data: `data:${mimeType};base64,${content}` }, + ]); + }, + ); + + it('keeps application/json as a file part for OpenAI when the endpoint lists it', async () => { + const req = createOptInRequest(['^application/json$']) as ServerRequest; const file = createMockDocFile(1, 'application/json', 'data.json'); const content = Buffer.from('{"a":1}').toString('base64'); mockedGetFileStream.mockResolvedValue({ file, content, metadata: file }); @@ -1329,7 +1402,7 @@ describe('encodeAndFormatDocuments - fileConfig integration', () => { }); it.each(['application/yaml', 'application/typescript'])( - 'keeps %s as a file part for OpenAI under the inherited list', + 'sends %s to OpenAI chat completions as a text part under the inherited list', async (mimeType) => { const req = createMockRequest(30) as ServerRequest; const file = createMockDocFile(1, mimeType, 'file'); @@ -1343,7 +1416,7 @@ describe('encodeAndFormatDocuments - fileConfig integration', () => { mockStrategyFunctions, ); - expect(result.documents).toMatchObject([{ type: 'file' }]); + expect(result.documents).toEqual([{ type: 'text', text: 'File: "file"\n\na: 1' }]); }, ); diff --git a/packages/api/src/files/encode/document.ts b/packages/api/src/files/encode/document.ts index 7baa81145b3..1a3510cad86 100644 --- a/packages/api/src/files/encode/document.ts +++ b/packages/api/src/files/encode/document.ts @@ -81,9 +81,9 @@ function usesGeminiDocumentCapabilities(provider: Providers, model?: string): bo } /** - * Textual types that OpenAI-compatible gateways can reject as a `file` part (Azure OpenAI - * answers 400 "Invalid file data" for these), so they go as text unless the endpoint lists - * them. `text/*`, JSON, YAML and TypeScript stay file parts. + * Textual types that OpenAI-compatible gateways reject as a `file` part on either API shape + * (Azure OpenAI answers 400 "Invalid file data" for these), so they go as text unless the + * endpoint lists them. */ const textPartApplicationTypes = new Set([ 'application/sql', @@ -94,14 +94,25 @@ const textPartApplicationTypes = new Set([ /** * Whether a document goes as a text part because the endpoint's own `supportedMimeTypes` * does not list it. Gemini accepts `text/*` inline but rejects every textual - * `application/*` type (JSON, YAML, XML, SQL, CoffeeScript). "Textual" is the same + * `application/*` type (JSON, YAML, XML, SQL, CoffeeScript). OpenAI chat completions + * accepts only PDF as `file.file_data` and answers 400 for `text/plain`, CSV, HTML and + * JSON, while Azure OpenAI accepts those; a text part is read by both. The responses API + * documents textual `input_file` types, so it keeps them. "Textual" is the same * classification that routes a file to the provider (`isNativelyReadableText`). */ -function sendsAsTextWithoutOptIn(provider: Providers, mimeType: string, model?: string): boolean { +function sendsAsTextWithoutOptIn( + provider: Providers, + mimeType: string, + useResponsesApi: boolean | undefined, + model?: string, +): boolean { if (usesGeminiDocumentCapabilities(provider, model)) { return !mimeType.startsWith('text/') && isNativelyReadableText(mimeType); } - return textPartApplicationTypes.has(mimeType); + if (textPartApplicationTypes.has(mimeType)) { + return true; + } + return !useResponsesApi && isNativelyReadableText(mimeType); } /** A textual file as a plain text part, which every provider and API shape accepts. */ @@ -149,7 +160,7 @@ function formatDocumentBlock( const resolvedFilename = filename ?? 'document'; - if (!optedIn && sendsAsTextWithoutOptIn(provider, mimeType, model)) { + if (!optedIn && sendsAsTextWithoutOptIn(provider, mimeType, useResponsesApi, model)) { return formatTextDocumentBlock(resolvedFilename, content); }