diff --git a/api/app/clients/BaseClient.js b/api/app/clients/BaseClient.js index 4a4f6975bda..7fe98811ccf 100644 --- a/api/app/clients/BaseClient.js +++ b/api/app/clients/BaseClient.js @@ -14,6 +14,8 @@ const { encodeAndFormatVideos, getTransactionsConfig, encodeAndFormatDocuments, + isProviderDocumentCandidate, + AgentAttachmentUnsupportedError, getLangfuseTraceMessageFields, isContentFilterError, assertModelBoundProviderContent, @@ -1698,7 +1700,12 @@ class BaseClient { ); } - async addDocuments(message, attachments) { + /** + * @param {TMessage} message + * @param {MongoFile[]} attachments + * @param {OmittedAttachment[]} [omissions] - Receives the files the encoder did not send. + */ + async addDocuments(message, attachments, omissions) { const documentResult = await encodeAndFormatDocuments( this.options.req, attachments, @@ -1714,6 +1721,9 @@ class BaseClient { documentResult.documents && documentResult.documents.length ? documentResult.documents : undefined; + if (omissions && documentResult.omitted?.length) { + omissions.push(...documentResult.omitted); + } return documentResult.files; } @@ -1789,7 +1799,18 @@ class BaseClient { return resolveTurnLLMDeliveryPath(this.options.agent?.deliveryRouting, file, fileConsumers); } - async processAttachments(message, attachments, fileConsumers) { + /** + * A provider-bound file the model would not receive is rejected on the current turn with + * {@link AgentAttachmentUnsupportedError}, so the model never answers as if it had read + * it. On history replay the same file is left out with a note in the message instead: + * throwing there would fail every later turn of the conversation. + * + * @param {TMessage} message + * @param {MongoFile[]} attachments + * @param {TurnFileConsumers} [fileConsumers] + * @param {{ historical?: boolean }} [options] + */ + async processAttachments(message, attachments, fileConsumers, options = {}) { const categorizedAttachments = { images: [], videos: [], @@ -1797,6 +1818,8 @@ class BaseClient { documents: [], }; + /** @type {OmittedAttachment[]} */ + const omissions = []; const allFiles = []; const provider = this.options.agent?.provider ?? this.options.endpoint; const isBedrock = provider === EModelEndpoint.bedrock; @@ -1842,15 +1865,21 @@ class BaseClient { categorizedAttachments.audios.push(file); allFiles.push(file); } else if ( - file.type && - deliveryRouting?.endpointConfig.supportedMimeTypes && - deliveryRouting.fileConfig.checkType( + isProviderDocumentCandidate( file.type, - deliveryRouting.endpointConfig.supportedMimeTypes, + deliveryRouting?.fileConfig, + deliveryRouting?.endpointConfig.supportedMimeTypes, ) ) { categorizedAttachments.documents.push(file); allFiles.push(file); + } else if (deliveryPath === 'provider') { + omissions.push({ + ...(file.file_id && { file_id: file.file_id }), + filename: file.filename, + type: file.type, + reason: 'unsupported_type', + }); } } @@ -1859,7 +1888,7 @@ class BaseClient { ? this.addImageURLs(message, categorizedAttachments.images) : Promise.resolve([]), categorizedAttachments.documents.length > 0 - ? this.addDocuments(message, categorizedAttachments.documents) + ? this.addDocuments(message, categorizedAttachments.documents, omissions) : Promise.resolve([]), categorizedAttachments.videos.length > 0 ? this.addVideos(message, categorizedAttachments.videos) @@ -1871,7 +1900,29 @@ class BaseClient { allFiles.push(...imageFiles); - const seenFileIds = new Set(); + if (omissions.length > 0) { + if (!options.historical) { + throw new AgentAttachmentUnsupportedError(omissions); + } + logger.warn('[BaseClient] Attachments from history not sent to the model', { + messageId: message.messageId, + omitted: omissions, + }); + message.documents = [ + ...(message.documents ?? []), + ...omissions.map((file) => ({ + type: 'text', + text: `File "${file.filename}" was attached to this message but was not sent to the model: ${ + file.reason === 'text_limit' + ? 'the file text limit for this request was reached.' + : `this model cannot read ${file.type || 'this file type'}.` + }`, + })), + ]; + } + + const omittedFileIds = new Set(omissions.map((file) => file.file_id).filter(Boolean)); + const seenFileIds = new Set(omittedFileIds); const uniqueFiles = []; for (const file of allFiles) { @@ -2028,7 +2079,7 @@ class BaseClient { const [, processedFiles] = await Promise.all([ this.addFileContextToMessage(message, contextFiles), - this.processAttachments(message, contextFiles), + this.processAttachments(message, contextFiles, undefined, { historical: true }), ]); const processedFileIds = new Set( diff --git a/api/app/clients/specs/BaseClient.test.js b/api/app/clients/specs/BaseClient.test.js index 2c675aa5dcd..dc5208a72ee 100644 --- a/api/app/clients/specs/BaseClient.test.js +++ b/api/app/clients/specs/BaseClient.test.js @@ -5,6 +5,8 @@ const { resolveTurnDeliveryRouting, buildSteerMedia, Tokenizer, + isAgentAttachmentLimitError, + AgentAttachmentUnsupportedError, } = require('@librechat/api'); const { FakeClient, initializeFakeClient } = require('./FakeClient'); @@ -2945,7 +2947,9 @@ describe('BaseClient', () => { {}, ); expect(TestClient.addFileContextToMessage).toHaveBeenCalledWith(message, [ownerFile]); - expect(TestClient.processAttachments).toHaveBeenCalledWith(message, [ownerFile]); + expect(TestClient.processAttachments).toHaveBeenCalledWith(message, [ownerFile], undefined, { + historical: true, + }); expect(message.fileContext).toBe('authorized owner text'); expect(message.files).toEqual([ expect.objectContaining({ @@ -3560,6 +3564,182 @@ describe('BaseClient', () => { }); }; + describe('provider documents under the inherited supportedMimeTypes list', () => { + /** A file the user sent through "Upload to Provider"; the endpoint sets no MIME list. */ + const providerFile = (file_id, type) => ({ + user: 'user1', + file_id, + filename: file_id, + filepath: `/uploads/${file_id}`, + type, + bytes: 100, + source: 'local', + llmDeliveryPath: 'provider', + metadata: { destinationChosen: true }, + }); + + const withSupportedMimeTypes = (supportedMimeTypes) => { + TestClient.options.req = { + config: { + fileConfig: { + endpoints: { [EModelEndpoint.openAI]: { fileLimit: 10, supportedMimeTypes } }, + }, + }, + }; + TestClient.options.agent = routedAgent({ + provider: EModelEndpoint.openAI, + endpoint: EModelEndpoint.openAI, + }); + }; + + test('rejects a current-turn archive the endpoint does not list, naming the file', async () => { + withSupportedMimeTypes(undefined); + const sql = providerFile('query.sql', 'application/sql'); + const zip = providerFile('archive.zip', 'application/zip'); + const pdf = providerFile('report.pdf', 'application/pdf'); + + const rejection = TestClient.processAttachments({}, [sql, zip, pdf]); + + await expect(rejection).rejects.toBeInstanceOf(AgentAttachmentUnsupportedError); + await expect(rejection).rejects.toMatchObject({ + code: 'AGENT_ATTACHMENT_UNSUPPORTED', + message: expect.stringContaining('"archive.zip" (application/zip)'), + attachments: [ + { + file_id: 'archive.zip', + filename: 'archive.zip', + type: 'application/zip', + reason: 'unsupported_type', + }, + ], + }); + expect(isAgentAttachmentLimitError(await rejection.catch((error) => error))).toBe(true); + expect(TestClient.addDocuments).toHaveBeenCalledWith( + expect.anything(), + [sql, pdf], + expect.any(Array), + ); + }); + + test('rejects a current-turn file the document encoder leaves out', async () => { + /* A DOCX sent to Gemini passes routing but not the encoder, which used to leave the + * file on the message while the model received none of it. */ + withSupportedMimeTypes(undefined); + const csv = providerFile('table.csv', 'text/csv'); + TestClient.addDocuments = jest.fn(async (_message, _files, omissions) => { + omissions.push({ + file_id: 'table.csv', + filename: 'table.csv', + type: 'text/csv', + reason: 'text_limit', + }); + return []; + }); + + await expect(TestClient.processAttachments({}, [csv])).rejects.toMatchObject({ + code: 'AGENT_ATTACHMENT_UNSUPPORTED', + message: expect.stringContaining('exceeds the file text limit'), + }); + }); + + test('recovers a conversation whose earlier message holds an archive', async () => { + /* The archive that 400s on every later turn arrives through history, not this turn's + * upload, so the replay path has to skip it too. */ + withSupportedMimeTypes(undefined); + TestClient.options.req.user = { id: 'user1' }; + TestClient.options.resendFiles = true; + TestClient.addFileContextToMessage = jest.fn(); + TestClient.assertHistoricalAttachmentLimits = undefined; + TestClient.checkVisionRequest = jest.fn(); + TestClient.message_file_map = undefined; + const { logger } = require('@librechat/data-schemas'); + const warn = jest.spyOn(logger, 'warn').mockImplementation(() => {}); + const zip = providerFile('archive.zip', 'application/zip'); + const pdf = providerFile('report.pdf', 'application/pdf'); + getFiles.mockResolvedValueOnce([zip, pdf]); + + try { + const [message] = await TestClient.addPreviousAttachments([ + { + messageId: 'msg-earlier', + text: 'Here are the files', + files: [{ file_id: 'archive.zip' }, { file_id: 'report.pdf' }], + }, + ]); + + expect(TestClient.addDocuments).toHaveBeenCalledTimes(1); + expect(TestClient.addDocuments).toHaveBeenCalledWith(message, [pdf], expect.any(Array)); + expect(TestClient.message_file_map['msg-earlier']).toEqual([pdf]); + /* The omission is explicit to the model and in the log, not silent. */ + expect(message.documents).toEqual([ + { type: 'file' }, + { + type: 'text', + text: 'File "archive.zip" was attached to this message but was not sent to the model: this model cannot read application/zip.', + }, + ]); + expect(warn).toHaveBeenCalledWith( + '[BaseClient] Attachments from history not sent to the model', + expect.objectContaining({ + messageId: 'msg-earlier', + omitted: [expect.objectContaining({ file_id: 'archive.zip' })], + }), + ); + } finally { + warn.mockRestore(); + } + }); + + test('leaves a file the encoder omitted from history out of the message file map', async () => { + withSupportedMimeTypes(undefined); + TestClient.options.req.user = { id: 'user1' }; + TestClient.options.resendFiles = true; + TestClient.addFileContextToMessage = jest.fn(); + TestClient.assertHistoricalAttachmentLimits = undefined; + TestClient.checkVisionRequest = jest.fn(); + TestClient.message_file_map = undefined; + const docx = providerFile( + 'report.docx', + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + ); + TestClient.addDocuments = jest.fn(async (_message, _files, omissions) => { + omissions.push({ + file_id: docx.file_id, + filename: docx.filename, + type: docx.type, + reason: 'unsupported_type', + }); + return []; + }); + getFiles.mockResolvedValueOnce([docx]); + + const [message] = await TestClient.addPreviousAttachments([ + { messageId: 'msg-docx', text: 'Read this', files: [{ file_id: 'report.docx' }] }, + ]); + + expect(TestClient.message_file_map['msg-docx']).toEqual([]); + expect(message.documents).toEqual([ + expect.objectContaining({ + type: 'text', + text: expect.stringContaining('"report.docx" was attached to this message'), + }), + ]); + }); + + test('sends a binary type the endpoint lists explicitly', async () => { + withSupportedMimeTypes(['^application/zip$']); + const zip = providerFile('archive.zip', 'application/zip'); + + await TestClient.processAttachments({}, [zip]); + + expect(TestClient.addDocuments).toHaveBeenCalledWith( + expect.anything(), + [zip], + expect.any(Array), + ); + }); + }); + test('keeps a none image in returned files without adding image URLs', async () => { routeTo('none', 'image/*'); const message = {}; @@ -4133,7 +4313,7 @@ describe('BaseClient', () => { expect(result).toEqual([file]); expect(message.documents).toEqual([{ type: 'file' }]); - expect(TestClient.addDocuments).toHaveBeenCalledWith(message, [file]); + expect(TestClient.addDocuments).toHaveBeenCalledWith(message, [file], expect.any(Array)); }); }); }); diff --git a/api/server/controllers/agents/client.js b/api/server/controllers/agents/client.js index ae72f09200a..bfbbccf25a7 100644 --- a/api/server/controllers/agents/client.js +++ b/api/server/controllers/agents/client.js @@ -381,7 +381,7 @@ class AgentClient extends BaseClient { ); } - async addDocuments(message, attachments) { + async addDocuments(message, attachments, omissions) { const memoryContext = { req: this.options.req, conversationId: this.conversationId, @@ -390,13 +390,13 @@ class AgentClient extends BaseClient { }; logAgentMemorySnapshot('before_encode_documents', memoryContext); try { - return await super.addDocuments(message, attachments); + return await super.addDocuments(message, attachments, omissions); } finally { logAgentMemorySnapshot('after_encode_documents', memoryContext); } } - async processAttachments(message, attachments, fileConsumers) { + async processAttachments(message, attachments, fileConsumers, options) { const modelBoundAttachments = this.getModelBoundAttachmentsForEndpoint(attachments); const processableAttachments = this.getProcessableAttachmentsForEndpoint( attachments, @@ -416,7 +416,12 @@ class AgentClient extends BaseClient { }; logAgentMemorySnapshot('before_process_attachments', memoryContext); try { - return await super.processAttachments(message, processableAttachments, fileConsumers); + return await super.processAttachments( + message, + processableAttachments, + fileConsumers, + options, + ); } finally { logAgentMemorySnapshot('after_process_attachments', memoryContext); } diff --git a/api/server/controllers/agents/client.test.js b/api/server/controllers/agents/client.test.js index a6a3991c421..e1b9451caec 100644 --- a/api/server/controllers/agents/client.test.js +++ b/api/server/controllers/agents/client.test.js @@ -6815,7 +6815,7 @@ describe('AgentClient - titleConvo', () => { const files = await client.processAttachments(message, [currentFile]); - expect(client.addDocuments).toHaveBeenCalledWith(message, [currentFile]); + expect(client.addDocuments).toHaveBeenCalledWith(message, [currentFile], []); expect(message.documents).toEqual([ expect.objectContaining({ type: 'input_file', @@ -6826,6 +6826,27 @@ describe('AgentClient - titleConvo', () => { }, ); + it('rejects a current-turn archive the user sent to the provider, naming the file', async () => { + const currentFile = { + ...makeUploadedFile('zip-file', 'bundle.zip', 'application/zip'), + llmDeliveryPath: 'provider', + metadata: { destinationChosen: true }, + }; + client.addDocuments = jest.fn(async () => []); + + await expect( + client.processAttachments( + { messageId: 'msg-1', text: 'Read this bundle.', isCreatedByUser: true }, + [currentFile], + ), + ).rejects.toMatchObject({ + code: 'AGENT_ATTACHMENT_UNSUPPORTED', + status: 415, + message: expect.stringContaining('"bundle.zip" (application/zip)'), + }); + expect(client.addDocuments).not.toHaveBeenCalled(); + }); + it.each(['current', 'history', 'history-disabled'])( 'resolves %s tool-routed text only for an authorized handoff without a reader', async (location) => { diff --git a/api/typedefs.js b/api/typedefs.js index 81bc2b2ea90..d8eefc9e47a 100644 --- a/api/typedefs.js +++ b/api/typedefs.js @@ -868,6 +868,12 @@ * @memberof typedefs */ +/** + * @exports OmittedAttachment + * @typedef {import('@librechat/api').OmittedAttachment} OmittedAttachment + * @memberof typedefs + */ + /** * @exports ISession * @typedef {import('@librechat/data-schemas').ISession} ISession diff --git a/packages/api/src/agents/attachments.ts b/packages/api/src/agents/attachments.ts index 447af3c9218..692a3e53fda 100644 --- a/packages/api/src/agents/attachments.ts +++ b/packages/api/src/agents/attachments.ts @@ -7,10 +7,10 @@ import { getEndpointFileConfig, } from 'librechat-data-provider'; import type { IMongoFile } from '@librechat/data-schemas'; +import type { OmittedAttachment, ServerRequest } from '~/types'; import type { TokenCountFn } from '~/utils/text'; -import type { ServerRequest } from '~/types'; +import { AGENT_ATTACHMENT_LIMIT_EXCEEDED, AGENT_ATTACHMENT_UNSUPPORTED } from './errors'; import { filterFilesByEndpointRuntimeConfig } from '~/files/filter'; -import { AGENT_ATTACHMENT_LIMIT_EXCEEDED } from './errors'; import { countTokens } from '~/utils/tokenizer'; import { extractFileContext } from '~/files'; @@ -217,6 +217,35 @@ export class AgentAttachmentLimitError extends Error { } } +/** + * A current-turn attachment the model would not receive: its type is not an inline + * document for this model, or the turn's file text budget is spent. Rejecting the turn + * keeps the model from answering as if it had read the file. + */ +export class AgentAttachmentUnsupportedError extends Error { + readonly code: typeof AGENT_ATTACHMENT_UNSUPPORTED = AGENT_ATTACHMENT_UNSUPPORTED; + readonly status = 415; + readonly statusCode = 415; + + constructor(readonly attachments: OmittedAttachment[]) { + const describe = (reason: OmittedAttachment['reason']) => + attachments + .filter((file) => file.reason === reason) + .map((file) => (file.type ? `"${file.filename}" (${file.type})` : `"${file.filename}"`)) + .join(', '); + const unsupported = describe('unsupported_type'); + const overLimit = describe('text_limit'); + const sentences = [ + unsupported && + `This model cannot read ${unsupported}. Remove it, or upload it as text or to the code environment, and try again.`, + overLimit && + `${overLimit} exceeds the file text limit for this turn. Remove some attachments or use smaller files and try again.`, + ].filter(Boolean); + super(sentences.join(' ')); + this.name = 'AgentAttachmentUnsupportedError'; + } +} + export class AgentAttachmentPolicyError extends Error { readonly code: typeof AGENT_ATTACHMENT_LIMIT_EXCEEDED = AGENT_ATTACHMENT_LIMIT_EXCEEDED; readonly status = 413; @@ -232,8 +261,15 @@ export class AgentAttachmentPolicyError extends Error { export function isAgentAttachmentLimitError( error: unknown, -): error is AgentAttachmentLimitError | AgentAttachmentPolicyError { - return error instanceof AgentAttachmentLimitError || error instanceof AgentAttachmentPolicyError; +): error is + | AgentAttachmentLimitError + | AgentAttachmentPolicyError + | AgentAttachmentUnsupportedError { + return ( + error instanceof AgentAttachmentLimitError || + error instanceof AgentAttachmentPolicyError || + error instanceof AgentAttachmentUnsupportedError + ); } export function collectAgentAttachmentStats( diff --git a/packages/api/src/agents/errors.ts b/packages/api/src/agents/errors.ts index 37f6d1b2071..97ddb44057c 100644 --- a/packages/api/src/agents/errors.ts +++ b/packages/api/src/agents/errors.ts @@ -9,10 +9,12 @@ import { OboTokenResolutionError } from '~/mcp/oauth/obo'; export const AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE = 'AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE'; export const AGENT_ATTACHMENT_LIMIT_EXCEEDED = 'AGENT_ATTACHMENT_LIMIT_EXCEEDED'; +export const AGENT_ATTACHMENT_UNSUPPORTED = 'AGENT_ATTACHMENT_UNSUPPORTED'; const FATAL_AGENT_INITIALIZATION_CODES = new Set( [ AGENT_ATTACHMENT_LIMIT_EXCEEDED, + AGENT_ATTACHMENT_UNSUPPORTED, ErrorTypes.RESOURCE_RECOVERY_REQUIRED, ErrorTypes.STATEFUL_CODE_ENVIRONMENT_NOT_ALLOWED, ErrorTypes.CODE_WORKSPACE_UNAVAILABLE, diff --git a/packages/api/src/agents/files/encode.spec.ts b/packages/api/src/agents/files/encode.spec.ts index 10c29025be9..070d1bf731b 100644 --- a/packages/api/src/agents/files/encode.spec.ts +++ b/packages/api/src/agents/files/encode.spec.ts @@ -2,7 +2,11 @@ import { FileContext, FileSources, ImageDetail } from 'librechat-data-provider'; import type { TFile } from 'librechat-data-provider'; import type { RunFileEncodingAgent, RunFileMessageEncoderDeps } from './encode'; import type { ServerRequest } from '~/types'; -import { AgentAttachmentLimitError, AgentAttachmentPolicyError } from '../attachments'; +import { + AgentAttachmentLimitError, + AgentAttachmentPolicyError, + AgentAttachmentUnsupportedError, +} from '../attachments'; import { resolveTurnDeliveryRouting } from './delivery'; import { createRunFileMessageEncoder } from './encode'; @@ -349,6 +353,66 @@ describe('createRunFileMessageEncoder', () => { expect(harness.encodeDocuments).not.toHaveBeenCalled(); }); + /** A file the user sent through "Upload to Provider": the chosen path survives child routing. */ + const providerChosen = (file_id: string, type: string): TFile => ({ + ...pdf, + file_id, + filename: file_id, + type, + text: undefined, + metadata: { destinationChosen: true }, + }); + + it.each([ + ['an archive-only share', [providerChosen('zip', 'application/zip')]], + [ + 'a share that mixes an archive with a document', + [providerChosen('zip', 'application/zip'), providerChosen('sql', 'application/sql')], + ], + ])('rejects %s under the inherited MIME list before reading bytes', async (_label, files) => { + const harness = setup(); + expect(() => harness.validate(files, 'child')).toThrow(AgentAttachmentUnsupportedError); + const rejection = harness.encode(files, 'child'); + await expect(rejection).rejects.toBeInstanceOf(AgentAttachmentUnsupportedError); + await expect(rejection).rejects.toMatchObject({ + code: 'AGENT_ATTACHMENT_UNSUPPORTED', + attachments: [expect.objectContaining({ file_id: 'zip', reason: 'unsupported_type' })], + }); + expect(harness.encodeDocuments).not.toHaveBeenCalled(); + }); + + it('rejects the share when the document encoder omits a file', async () => { + const harness = setup(); + const omitted = { + file_id: 'docx', + filename: 'report.docx', + type: 'application/msword', + reason: 'unsupported_type' as const, + }; + jest.mocked(harness.encodeDocuments).mockResolvedValueOnce({ + documents: [], + omitted: [omitted], + }); + + await expect(harness.encode([pdf], 'child')).rejects.toMatchObject({ + code: 'AGENT_ATTACHMENT_UNSUPPORTED', + attachments: [omitted], + }); + }); + + it('sends a binary type to the document encoder when the endpoint lists it', async () => { + const harness = setup({ + fileConfig: { endpoints: { openAI: { supportedMimeTypes: ['^application/zip$'] } } }, + }); + await harness.encode([providerChosen('zip', 'application/zip')], 'child'); + expect(harness.encodeDocuments).toHaveBeenCalledWith( + harness.req, + [expect.objectContaining({ file_id: 'zip' })], + expect.anything(), + harness.getStrategyFunctions, + ); + }); + it.each([{ disabled: true }, { supportedMimeTypes: ['image/png'] }, { fileSizeLimit: 0.000001 }])( 'rejects incompatible child policy before reading bytes: %j', async (policy) => { diff --git a/packages/api/src/agents/files/encode.ts b/packages/api/src/agents/files/encode.ts index e04d6cbed86..efce4da4f95 100644 --- a/packages/api/src/agents/files/encode.ts +++ b/packages/api/src/agents/files/encode.ts @@ -13,16 +13,18 @@ import type { TurnFileConsumers, } from 'librechat-data-provider'; import type { BaseMessage } from '@librechat/agents/langchain'; -import type { ServerRequest, StrategyFunctions } from '~/types'; +import type { OmittedAttachment, ServerRequest, StrategyFunctions } from '~/types'; import type { TokenCountFn } from '~/utils/text'; import { isToolOwnedAttachment, isModelBoundAttachmentFile, assertAgentAttachmentLimits, AgentAttachmentPolicyError, + AgentAttachmentUnsupportedError, } from '../attachments'; import { assertModelBoundContent } from '~/middleware/modelBoundContent'; import { filterFilesByEndpointRuntimeConfig } from '~/files/filter'; +import { isProviderDocumentCandidate } from '~/files/encode/utils'; import { countTokens } from '~/utils/tokenizer'; type ContentBlock = Exclude[number]; @@ -59,7 +61,7 @@ export interface RunFileMessageEncoderDeps { req: ServerRequest; getAgent: (agentId: string) => RunFileEncodingAgent | undefined; encodeImages: MediaEncoder<{ image_urls: ContentBlock[] }>; - encodeDocuments: MediaEncoder<{ documents: ContentBlock[] }>; + encodeDocuments: MediaEncoder<{ documents: ContentBlock[]; omitted?: OmittedAttachment[] }>; encodeAudios: MediaEncoder<{ audios: ContentBlock[] }>; encodeVideos: MediaEncoder<{ videos: ContentBlock[] }>; getStrategyFunctions: (source: string) => StrategyFunctions; @@ -128,31 +130,18 @@ export function createRunFileMessageEncoder( endpoint, }); assertModelBoundContent({ filters: deps.req.config?.filters, files: sharedFiles }); - return { agent, params, sharedFiles, fileConfig, endpointConfig }; - } - function validate(files: TFile[], agentId: string): void { - if (files.length > 0) prepare(files, agentId); - } - - async function encode(files: TFile[], agentId: string): Promise { - if (files.length === 0) return []; - const { agent, params, sharedFiles, fileConfig, endpointConfig } = prepare(files, agentId); const images: TFile[] = []; const documents: TFile[] = []; const audios: TFile[] = []; const videos: TFile[] = []; const textFiles: TFile[] = []; + const unsupported: OmittedAttachment[] = []; for (const file of sharedFiles) { const deliveryPath = file.llmDeliveryPath; if (deliveryPath === 'none') { continue; } - if (deliveryPath === 'text' && !file.text) { - throw new Error( - `Shared file "${file.filename}" requires extracted text for this agent. Attach a text version or use an agent that supports the original file.`, - ); - } if (deliveryPath == null || deliveryPath === 'text') { textFiles.push(file); } @@ -176,10 +165,41 @@ export function createRunFileMessageEncoder( } else if (file.type.startsWith('video/')) { videos.push(file); } else if ( - endpointConfig.supportedMimeTypes && - fileConfig.checkType?.(file.type, endpointConfig.supportedMimeTypes) + isProviderDocumentCandidate(file.type, fileConfig, endpointConfig.supportedMimeTypes) ) { documents.push(file); + } else if (deliveryPath === 'provider') { + unsupported.push({ + ...(file.file_id && { file_id: file.file_id }), + filename: file.filename, + type: file.type, + reason: 'unsupported_type', + }); + } + } + /* A provider-bound file with no encoder would reach the child as nothing at all, so the + * share is rejected before the child runs rather than answered without the file. */ + if (unsupported.length > 0) { + throw new AgentAttachmentUnsupportedError(unsupported); + } + return { params, sharedFiles, images, documents, audios, videos, textFiles }; + } + + function validate(files: TFile[], agentId: string): void { + if (files.length > 0) prepare(files, agentId); + } + + async function encode(files: TFile[], agentId: string): Promise { + if (files.length === 0) return []; + const { params, sharedFiles, images, documents, audios, videos, textFiles } = prepare( + files, + agentId, + ); + for (const file of sharedFiles) { + if (file.llmDeliveryPath === 'text' && !file.text) { + throw new Error( + `Shared file "${file.filename}" requires extracted text for this agent. Attach a text version or use an agent that supports the original file.`, + ); } } @@ -196,6 +216,9 @@ export function createRunFileMessageEncoder( ? deps.extractText({ attachments: textFiles, req: deps.req, tokenCountFn: countTokens }) : Promise.resolve(undefined), ]); + if (documentResult.omitted?.length) { + throw new AgentAttachmentUnsupportedError(documentResult.omitted); + } if ( !text && imageResult.image_urls.length === 0 && diff --git a/packages/api/src/files/encode/document.spec.ts b/packages/api/src/files/encode/document.spec.ts index f58d591ed67..0beb6f36eb9 100644 --- a/packages/api/src/files/encode/document.spec.ts +++ b/packages/api/src/files/encode/document.spec.ts @@ -15,6 +15,7 @@ jest.mock('./utils', () => ({ getFileStream: jest.fn(), getConfiguredFileSizeLimit: jest.fn(), isAttachmentObjectNotFoundError: jest.fn(() => false), + isConfiguredProviderMediaType: jest.requireActual('./utils').isConfiguredProviderMediaType, })); import { validatePdf, validateBedrockDocument } from '~/files/validation'; @@ -75,6 +76,23 @@ describe('encodeAndFormatDocuments - fileConfig integration', () => { : undefined, }); + /** Helper to create a mock request whose endpoint lists its own `supportedMimeTypes` */ + const createOptInRequest = ( + supportedMimeTypes: string[], + provider: string = Providers.OPENAI, + ): Partial => ({ + config: { + fileConfig: { + endpoints: { + [provider]: { + fileSizeLimit: 30, + supportedMimeTypes, + }, + }, + }, + }, + }); + /** Helper to create a mock PDF file */ const createMockFile = (sizeInMB: number): IMongoFile => ({ @@ -931,7 +949,27 @@ describe('encodeAndFormatDocuments - fileConfig integration', () => { expect(result.files).toHaveLength(1); }); - it('should keep textual documents as file parts for non-Claude OpenAI models', async () => { + it('should send textual documents as text parts for non-Claude OpenAI models on chat completions', async () => { + const req = createMockRequest(30) as ServerRequest; + const file = createMockDocFile(1, 'text/html', 'page.html'); + const mockContent = Buffer.from('

hi

').toString('base64'); + mockedGetFileStream.mockResolvedValue({ + file, + content: mockContent, + metadata: file, + }); + + const result = await encodeAndFormatDocuments( + req, + [file], + { provider: Providers.OPENAI, model: 'gpt-5.4' }, + mockStrategyFunctions, + ); + + expect(result.documents).toEqual([{ type: 'text', text: 'File: "page.html"\n\n

hi

' }]); + }); + + it('should keep textual documents as input_file parts for non-Claude OpenAI models on the responses API', async () => { const req = createMockRequest(30) as ServerRequest; const file = createMockDocFile(1, 'text/html', 'page.html'); const mockContent = Buffer.from('

hi

').toString('base64'); @@ -944,14 +982,15 @@ describe('encodeAndFormatDocuments - fileConfig integration', () => { const result = await encodeAndFormatDocuments( req, [file], - { provider: Providers.OPENAI, model: 'gemini-auto-latest' }, + { provider: Providers.OPENAI, model: 'gpt-5.4', useResponsesApi: true }, mockStrategyFunctions, ); expect(result.documents).toEqual([ { - type: 'file', - file: { filename: 'page.html', file_data: `data:text/html;base64,${mockContent}` }, + type: 'input_file', + filename: 'page.html', + file_data: `data:text/html;base64,${mockContent}`, }, ]); }); @@ -1045,36 +1084,92 @@ describe('encodeAndFormatDocuments - fileConfig integration', () => { expect(result.files).toHaveLength(1); }); - it('should format XLSX for Google/VertexAI as media block', async () => { - const req = createMockRequest(25) as ServerRequest; - const mimeType = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'; - const file = createMockDocFile(2, mimeType, 'report.xlsx'); - - const mockContent = Buffer.from('xlsx-binary').toString('base64'); - mockedGetFileStream.mockResolvedValue({ - file, - content: mockContent, - metadata: file, - }); + describe('native Google/Vertex ignores the upload allowlist', () => { + const officeTypes = [ + ['XLSX', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', 'r.xlsx'], + [ + 'DOCX', + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + 'r.docx', + ], + ] as const; + + it.each( + [Providers.GOOGLE, Providers.VERTEXAI].flatMap((provider) => + officeTypes.map( + ([label, mimeType, filename]) => + [provider, label, mimeType, filename] as [Providers, string, string, string], + ), + ), + )( + 'omits %s %s even when the endpoint lists it', + async (provider, _label, mimeType, filename) => { + const req = createOptInRequest([mimeType], provider) as ServerRequest; + const file = createMockDocFile(1, mimeType, filename); + + const result = await encodeAndFormatDocuments( + req, + [file], + { provider }, + mockStrategyFunctions, + ); + + expect(result.documents).toHaveLength(0); + expect(result.files).toHaveLength(0); + expect(result.omitted).toEqual([ + { file_id: file.file_id, filename, type: mimeType, reason: 'unsupported_type' }, + ]); + expect(mockedGetFileStream).not.toHaveBeenCalled(); + }, + ); - const result = await encodeAndFormatDocuments( - req, - [file], - { provider: Providers.GOOGLE }, - mockStrategyFunctions, + it.each([Providers.GOOGLE, Providers.VERTEXAI])( + 'sends a listed application/sql to %s as a text part, not inline media', + async (provider) => { + const req = createOptInRequest(['^application/sql$'], provider) as ServerRequest; + const file = createMockDocFile(1, 'application/sql', 'query.sql'); + const content = Buffer.from('SELECT 1;').toString('base64'); + mockedGetFileStream.mockResolvedValue({ file, content, metadata: file }); + + const result = await encodeAndFormatDocuments( + req, + [file], + { provider }, + mockStrategyFunctions, + ); + + expect(result.documents).toEqual([ + { type: 'text', text: 'File: "query.sql"\n\nSELECT 1;' }, + ]); + }, ); - expect(result.documents).toHaveLength(1); - expect(result.documents[0]).toMatchObject({ - type: 'media', - mimeType, - data: mockContent, + it('keeps a listed XLSX as a file part for Gemini behind a gateway', async () => { + const mimeType = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'; + const req = createOptInRequest([mimeType]) as ServerRequest; + const file = createMockDocFile(1, mimeType, 'report.xlsx'); + const content = Buffer.from('xlsx-binary').toString('base64'); + mockedGetFileStream.mockResolvedValue({ file, content, metadata: file }); + + const result = await encodeAndFormatDocuments( + req, + [file], + { provider: Providers.OPENAI, model: 'gemini-auto-latest' }, + mockStrategyFunctions, + ); + + expect(result.documents).toEqual([ + { + type: 'file', + file: { filename: 'report.xlsx', file_data: `data:${mimeType};base64,${content}` }, + }, + ]); + expect(result.omitted).toBeUndefined(); }); - expect(result.files).toHaveLength(1); }); - it('should format text/plain for standard OpenAI-like provider as file block', async () => { - const req = createMockRequest(15) as ServerRequest; + it('should keep text/plain as a file block for a standard OpenAI-like provider when the endpoint lists it', async () => { + const req = createOptInRequest(['^text/plain$']) as ServerRequest; const file = createMockDocFile(1, 'text/plain', 'readme.txt'); const mockContent = Buffer.from('readme content').toString('base64'); @@ -1102,6 +1197,389 @@ describe('encodeAndFormatDocuments - fileConfig integration', () => { expect(result.files).toHaveLength(1); }); + describe('textual documents without an endpoint opt-in', () => { + const sql = 'SELECT 1;'; + + const encodeSql = async ( + req: ServerRequest, + params: Parameters[2], + ) => { + const file = createMockDocFile(1, 'application/sql', 'query.sql'); + mockedGetFileStream.mockResolvedValue({ + file, + content: Buffer.from(sql).toString('base64'), + metadata: file, + }); + return encodeAndFormatDocuments(req, [file], params, mockStrategyFunctions); + }; + + it.each([ + ['OpenAI chat completions', Providers.OPENAI, false], + ['OpenAI responses API', Providers.OPENAI, true], + ['OpenRouter', Providers.OPENROUTER, false], + ['Google', Providers.GOOGLE, false], + ['Vertex AI', Providers.VERTEXAI, false], + ])( + 'sends application/sql to %s as a text part under the inherited list', + async (_label, provider, useResponsesApi) => { + const req = createMockRequest(30, provider) as ServerRequest; + + const result = await encodeSql(req, { provider, useResponsesApi }); + + expect(result.documents).toEqual([{ type: 'text', text: `File: "query.sql"\n\n${sql}` }]); + expect(result.files).toHaveLength(1); + }, + ); + + it('sends a textual type as a text part when no fileConfig is set', async () => { + const req = createMockRequest() as ServerRequest; + + const result = await encodeSql(req, { provider: Providers.OPENAI }); + + expect(result.documents).toEqual([{ type: 'text', text: `File: "query.sql"\n\n${sql}` }]); + }); + + it('sends textual documents as text parts to Claude through a gateway on the responses API', async () => { + const req = createMockRequest(30) as ServerRequest; + + const result = await encodeSql(req, { + provider: Providers.OPENAI, + model: 'claude-auto-latest', + useResponsesApi: true, + }); + + expect(result.documents).toEqual([{ type: 'text', text: `File: "query.sql"\n\n${sql}` }]); + }); + + it('keeps application/sql as a file part when the endpoint lists it', async () => { + const req = createOptInRequest(['^application/sql$']) as ServerRequest; + + const result = await encodeSql(req, { provider: Providers.OPENAI }); + + expect(result.documents).toEqual([ + { + type: 'file', + file: { + filename: 'query.sql', + file_data: `data:application/sql;base64,${Buffer.from(sql).toString('base64')}`, + }, + }, + ]); + }); + + it('keeps application/sql as an input_file on the responses API when the endpoint lists it', async () => { + const req = createOptInRequest(['^application/sql$']) as ServerRequest; + + const result = await encodeSql(req, { provider: Providers.OPENAI, useResponsesApi: true }); + + expect(result.documents).toMatchObject([{ type: 'input_file', filename: 'query.sql' }]); + }); + + it('keeps a PDF as a file part under the inherited list', async () => { + const req = createMockRequest(30) as ServerRequest; + const file = createMockFile(1); + const content = Buffer.from('%PDF-1.4').toString('base64'); + mockedGetFileStream.mockResolvedValue({ file, content, metadata: file }); + mockedValidatePdf.mockResolvedValue({ isValid: true }); + + const result = await encodeAndFormatDocuments( + req, + [file], + { provider: Providers.OPENAI }, + mockStrategyFunctions, + ); + + expect(result.documents).toMatchObject([ + { type: 'file', file: { file_data: `data:application/pdf;base64,${content}` } }, + ]); + }); + + it.each([ + ['text/plain', 'fixture.txt', 'plain text'], + ['text/csv', 'fixture.csv', 'a,b\n1,2'], + ['text/html', 'fixture.html', '

hi

'], + ['application/json', 'fixture.json', '{"a":1}'], + ])( + 'sends %s to OpenAI chat completions as a text part under the inherited list', + async (mimeType, filename, text) => { + const req = createMockRequest(30) as ServerRequest; + const file = createMockDocFile(1, mimeType, filename); + const content = Buffer.from(text).toString('base64'); + mockedGetFileStream.mockResolvedValue({ file, content, metadata: file }); + + const result = await encodeAndFormatDocuments( + req, + [file], + { provider: Providers.OPENAI, model: 'gpt-5.1' }, + mockStrategyFunctions, + ); + + expect(result.documents).toEqual([ + { type: 'text', text: `File: "${filename}"\n\n${text}` }, + ]); + expect(result.files).toHaveLength(1); + }, + ); + + it.each([ + ['text/plain', 'fixture.txt'], + ['text/csv', 'fixture.csv'], + ['application/json', 'fixture.json'], + ])( + 'keeps %s as an input_file on the OpenAI responses API under the inherited list', + async (mimeType, filename) => { + const req = createMockRequest(30) as ServerRequest; + const file = createMockDocFile(1, mimeType, filename); + const content = Buffer.from('x').toString('base64'); + mockedGetFileStream.mockResolvedValue({ file, content, metadata: file }); + + const result = await encodeAndFormatDocuments( + req, + [file], + { provider: Providers.OPENAI, useResponsesApi: true }, + mockStrategyFunctions, + ); + + expect(result.documents).toEqual([ + { type: 'input_file', filename, file_data: `data:${mimeType};base64,${content}` }, + ]); + }, + ); + + it('keeps application/json as a file part for OpenAI when the endpoint lists it', async () => { + const req = createOptInRequest(['^application/json$']) as ServerRequest; + const file = createMockDocFile(1, 'application/json', 'data.json'); + const content = Buffer.from('{"a":1}').toString('base64'); + mockedGetFileStream.mockResolvedValue({ file, content, metadata: file }); + + const result = await encodeAndFormatDocuments( + req, + [file], + { provider: Providers.OPENAI }, + mockStrategyFunctions, + ); + + expect(result.documents).toMatchObject([ + { type: 'file', file: { file_data: `data:application/json;base64,${content}` } }, + ]); + }); + + it.each([Providers.GOOGLE, Providers.VERTEXAI])( + 'keeps text/csv as inline media for %s under the inherited list', + async (provider) => { + const req = createMockRequest(30, provider) as ServerRequest; + const file = createMockDocFile(1, 'text/csv', 'data.csv'); + const content = Buffer.from('a,b\n1,2').toString('base64'); + mockedGetFileStream.mockResolvedValue({ file, content, metadata: file }); + + const result = await encodeAndFormatDocuments( + req, + [file], + { provider }, + mockStrategyFunctions, + ); + + expect(result.documents).toEqual([ + { type: 'media', mimeType: 'text/csv', data: content }, + ]); + }, + ); + + it('sends application/xml to OpenAI as a text part under the inherited list', async () => { + const req = createMockRequest(30) as ServerRequest; + const file = createMockDocFile(1, 'application/xml', 'data.xml'); + const content = Buffer.from('1').toString('base64'); + mockedGetFileStream.mockResolvedValue({ file, content, metadata: file }); + + const result = await encodeAndFormatDocuments( + req, + [file], + { provider: Providers.OPENAI }, + mockStrategyFunctions, + ); + + expect(result.documents).toEqual([{ type: 'text', text: 'File: "data.xml"\n\n1' }]); + }); + + it.each(['application/yaml', 'application/typescript'])( + 'sends %s to OpenAI chat completions as a text part under the inherited list', + async (mimeType) => { + const req = createMockRequest(30) as ServerRequest; + const file = createMockDocFile(1, mimeType, 'file'); + const content = Buffer.from('a: 1').toString('base64'); + mockedGetFileStream.mockResolvedValue({ file, content, metadata: file }); + + const result = await encodeAndFormatDocuments( + req, + [file], + { provider: Providers.OPENAI }, + mockStrategyFunctions, + ); + + expect(result.documents).toEqual([{ type: 'text', text: 'File: "file"\n\na: 1' }]); + }, + ); + + it.each([ + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + 'application/vnd.openxmlformats-officedocument.presentationml.presentation', + ])('skips %s for Google under the inherited list', async (mimeType) => { + const req = createMockRequest(30, Providers.GOOGLE) as ServerRequest; + const file = createMockDocFile(1, mimeType, 'office-file'); + + const result = await encodeAndFormatDocuments( + req, + [file], + { provider: Providers.GOOGLE }, + mockStrategyFunctions, + ); + + expect(result.documents).toHaveLength(0); + expect(mockedGetFileStream).not.toHaveBeenCalled(); + }); + + it('skips XLSX for Gemini through an OpenAI-compatible gateway', async () => { + const req = createMockRequest(30) as ServerRequest; + const file = createMockDocFile( + 1, + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + 'report.xlsx', + ); + + const result = await encodeAndFormatDocuments( + req, + [file], + { provider: Providers.OPENAI, model: 'gemini-auto-latest' }, + mockStrategyFunctions, + ); + + expect(result.documents).toHaveLength(0); + expect(mockedGetFileStream).not.toHaveBeenCalled(); + }); + + it('sends application/json to Gemini through a gateway as a text part', async () => { + const req = createMockRequest(30) as ServerRequest; + const file = createMockDocFile(1, 'application/json', 'data.json'); + const content = Buffer.from('{"a":1}').toString('base64'); + mockedGetFileStream.mockResolvedValue({ file, content, metadata: file }); + + const result = await encodeAndFormatDocuments( + req, + [file], + { provider: Providers.OPENAI, model: 'gemini-auto-latest', useResponsesApi: true }, + mockStrategyFunctions, + ); + + expect(result.documents).toEqual([{ type: 'text', text: 'File: "data.json"\n\n{"a":1}' }]); + }); + + it.each([ + [Providers.GOOGLE, undefined], + [Providers.OPENAI, 'gemini-auto-latest'], + ])( + 'sends inherited application/vnd.coffeescript to Gemini (%s) as a text part', + async (provider, model) => { + const req = createMockRequest(30, provider) as ServerRequest; + const file = createMockDocFile(1, 'application/vnd.coffeescript', 'app.coffee'); + const content = Buffer.from('x = 1').toString('base64'); + mockedGetFileStream.mockResolvedValue({ file, content, metadata: file }); + + const result = await encodeAndFormatDocuments( + req, + [file], + { provider, model }, + mockStrategyFunctions, + ); + + expect(result.documents).toEqual([{ type: 'text', text: 'File: "app.coffee"\n\nx = 1' }]); + }, + ); + }); + + describe('decoded text budgets', () => { + const createBudgetRequest = (fileTokenLimit?: number, fileContextCharLimit?: number) => + ({ + body: fileTokenLimit != null ? { fileTokenLimit } : {}, + config: { fileConfig: { fileContextCharLimit } }, + }) as unknown as ServerRequest; + + const mockSqlFile = (filename: string, bytes: number) => { + const file = createMockDocFile(bytes / (1024 * 1024), 'application/sql', filename); + const content = Buffer.from('SELECT 1;\n'.repeat(bytes / 10)).toString('base64'); + mockedGetFileStream.mockResolvedValueOnce({ file, content, metadata: file }); + return file; + }; + + it('bounds a 2 MB SQL text part by the token and character limits', async () => { + const req = createBudgetRequest(8, 100); + const file = mockSqlFile('big.sql', 2_000_000); + + const result = await encodeAndFormatDocuments( + req, + [file], + { provider: Providers.OPENAI }, + mockStrategyFunctions, + ); + + expect(result.documents).toHaveLength(1); + const block = result.documents[0] as { type: string; text: string }; + expect(block.type).toBe('text'); + expect(block.text.startsWith('File: "big.sql"')).toBe(true); + expect(block.text.length).toBeLessThanOrEqual(100); + expect(result.files).toHaveLength(1); + }); + + it('shares the character limit across calls of one request and omits past it', async () => { + const req = createBudgetRequest(undefined, 100); + + const first = await encodeAndFormatDocuments( + req, + [mockSqlFile('a.sql', 1000)], + { provider: Providers.OPENAI }, + mockStrategyFunctions, + ); + const secondFile = mockSqlFile('b.sql', 1000); + const second = await encodeAndFormatDocuments( + req, + [secondFile], + { provider: Providers.OPENAI }, + mockStrategyFunctions, + ); + + expect((first.documents[0] as { text: string }).text).toHaveLength(100); + expect(second.documents).toHaveLength(0); + expect(second.files).toHaveLength(0); + expect(second.omitted).toEqual([ + { + file_id: secondFile.file_id, + filename: 'b.sql', + type: 'application/sql', + reason: 'text_limit', + }, + ]); + }); + + it('bounds the plain-text source of a native Anthropic document', async () => { + const req = createBudgetRequest(undefined, 50); + const file = createMockDocFile(1, 'text/plain', 'notes.txt'); + const content = Buffer.from('x'.repeat(500)).toString('base64'); + mockedGetFileStream.mockResolvedValue({ file, content, metadata: file }); + + const result = await encodeAndFormatDocuments( + req, + [file], + { provider: Providers.ANTHROPIC }, + mockStrategyFunctions, + ); + + expect(result.documents[0]).toMatchObject({ + type: 'document', + source: { type: 'text', data: 'x'.repeat(50) }, + }); + }); + }); + it('should skip non-Bedrock-document types for Bedrock provider', async () => { const req = createMockRequest() as ServerRequest; const file = createMockDocFile(1, 'application/zip', 'archive.zip'); diff --git a/packages/api/src/files/encode/document.ts b/packages/api/src/files/encode/document.ts index 57ef72db9a3..1a3510cad86 100644 --- a/packages/api/src/files/encode/document.ts +++ b/packages/api/src/files/encode/document.ts @@ -3,14 +3,18 @@ import { isOpenAILikeProvider, isBedrockDocumentType, bedrockDocumentFormats, + isNativelyReadableText, isAnthropicDocumentType, isDocumentSupportedProvider, isAnthropicTextDocumentType, + mergeFileConfig, } from 'librechat-data-provider'; import type { IMongoFile } from '@librechat/data-schemas'; import type { DocumentBlock, AnthropicDocumentBlock, + AttachmentOmissionReason, + OmittedAttachment, StrategyFunctions, DocumentResult, ServerRequest, @@ -18,10 +22,13 @@ import type { import { getFileStream, getConfiguredFileSizeLimit, + isConfiguredProviderMediaType, isAttachmentObjectNotFoundError, } from './utils'; import { validatePdf, validateBedrockDocument } from '~/files/validation'; +import { processTextWithTokenLimit } from '~/utils/text'; import { runGuardedEncode } from './memoryGuard'; +import { countTokens } from '~/utils/tokenizer'; /** Anthropic only accepts PDFs as base64 documents; textual types must use a text source */ function getAnthropicDocumentSource( @@ -58,9 +65,70 @@ function usesAnthropicDocumentCapabilities(provider: Providers, model?: string): ); } +const isGoogleProvider = (provider: Providers): boolean => + provider === Providers.GOOGLE || provider === Providers.VERTEXAI; + +/** + * Whether the model behind this provider is Gemini, which rejects inline Office documents + * and textual `application/*` types (JSON, SQL) with a 400. OpenAI-compatible gateways + * report an OpenAI-like provider for Gemini models. + */ +function usesGeminiDocumentCapabilities(provider: Providers, model?: string): boolean { + return ( + isGoogleProvider(provider) || + (isOpenAILikeProvider(provider) && (model?.toLowerCase().includes('gemini') ?? false)) + ); +} + +/** + * Textual types that OpenAI-compatible gateways reject as a `file` part on either API shape + * (Azure OpenAI answers 400 "Invalid file data" for these), so they go as text unless the + * endpoint lists them. + */ +const textPartApplicationTypes = new Set([ + 'application/sql', + 'application/x-sh', + 'application/xml', +]); + +/** + * Whether a document goes as a text part because the endpoint's own `supportedMimeTypes` + * does not list it. Gemini accepts `text/*` inline but rejects every textual + * `application/*` type (JSON, YAML, XML, SQL, CoffeeScript). OpenAI chat completions + * accepts only PDF as `file.file_data` and answers 400 for `text/plain`, CSV, HTML and + * JSON, while Azure OpenAI accepts those; a text part is read by both. The responses API + * documents textual `input_file` types, so it keeps them. "Textual" is the same + * classification that routes a file to the provider (`isNativelyReadableText`). + */ +function sendsAsTextWithoutOptIn( + provider: Providers, + mimeType: string, + useResponsesApi: boolean | undefined, + model?: string, +): boolean { + if (usesGeminiDocumentCapabilities(provider, model)) { + return !mimeType.startsWith('text/') && isNativelyReadableText(mimeType); + } + if (textPartApplicationTypes.has(mimeType)) { + return true; + } + return !useResponsesApi && isNativelyReadableText(mimeType); +} + +/** A textual file as a plain text part, which every provider and API shape accepts. */ +function formatTextDocumentBlock(filename: string, content: string): DocumentBlock { + return { + type: 'text', + text: `File: "${filename}"\n\n${Buffer.from(content, 'base64').toString('utf8')}`, + }; +} + /** * Formats a base64-encoded document into the appropriate provider-specific block. * Returns `null` when the provider has no matching handler. + * + * `optedIn` is true when the endpoint's own `supportedMimeTypes` lists the type, rather + * than the built-in list it inherits. */ function formatDocumentBlock( provider: Providers, @@ -69,6 +137,7 @@ function formatDocumentBlock( filename: string | undefined, useResponsesApi: boolean | undefined, model?: string, + optedIn = false, ): DocumentBlock | null { if (provider === Providers.ANTHROPIC) { const source = getAnthropicDocumentSource(mimeType, content); @@ -89,7 +158,13 @@ function formatDocumentBlock( return document; } - if (provider === Providers.GOOGLE || provider === Providers.VERTEXAI) { + const resolvedFilename = filename ?? 'document'; + + if (!optedIn && sendsAsTextWithoutOptIn(provider, mimeType, useResponsesApi, model)) { + return formatTextDocumentBlock(resolvedFilename, content); + } + + if (isGoogleProvider(provider)) { return { type: 'media', mimeType, @@ -97,8 +172,6 @@ function formatDocumentBlock( }; } - const resolvedFilename = filename ?? 'document'; - /* A gateway translates an OpenAI `file` part into a base64 document with the file's own * media type, which Claude rejects for anything but PDF. Send textual files as text. */ if ( @@ -106,10 +179,7 @@ function formatDocumentBlock( isAnthropicTextDocumentType(mimeType) && usesAnthropicDocumentCapabilities(provider, model) ) { - return { - type: 'text', - text: `File: "${resolvedFilename}"\n\n${Buffer.from(content, 'base64').toString('utf8')}`, - }; + return formatTextDocumentBlock(resolvedFilename, content); } if (useResponsesApi) { @@ -136,39 +206,120 @@ function formatDocumentBlock( /** * Filters out files the provider's document path cannot send to the model. * Claude rejects non-PDF binary documents with a 400 that recurs on every retry, - * including when it is reached through an OpenAI-compatible gateway. Unsupported - * types are skipped instead of bricking the conversation. + * including when it is reached through an OpenAI-compatible gateway. Gemini rejects + * inline Office documents the same way, so for Gemini a type other than PDF or text + * goes only when the endpoint lists it. Unsupported types are skipped instead of + * bricking the conversation, and returned so the caller can report them. */ function filterProviderDocumentFiles( provider: Providers, files: IMongoFile[], - model?: string, -): IMongoFile[] { + model: string | undefined, + isOptedIn: (mimeType: string) => boolean, +): { processable: IMongoFile[]; skipped: IMongoFile[] } { + let label: string; + let isSupported: (file: IMongoFile) => boolean; if (provider === Providers.BEDROCK) { - return files.filter((file) => isBedrockDocumentType(file.type)); - } - - if (!usesAnthropicDocumentCapabilities(provider, model)) { - return files; + label = 'Bedrock'; + isSupported = (file) => isBedrockDocumentType(file.type); + } else if (usesAnthropicDocumentCapabilities(provider, model)) { + label = 'Claude'; + isSupported = (file) => isAnthropicDocumentType(file.type); + } else if (usesGeminiDocumentCapabilities(provider, model)) { + label = 'Gemini'; + isSupported = (file) => + file.type === 'application/pdf' || + isNativelyReadableText(file.type ?? '') || + isOptedIn(file.type ?? ''); + } else { + return { processable: files, skipped: [] }; } const processable: IMongoFile[] = []; - const skipped: string[] = []; + const skipped: IMongoFile[] = []; for (const file of files) { - if (isAnthropicDocumentType(file.type)) { + if (isSupported(file)) { processable.push(file); } else { - skipped.push(`"${file.filename}" (${file.type})`); + skipped.push(file); } } if (skipped.length) { console.warn( - `Skipping attachment(s) unsupported by Claude document input: ${skipped.join(', ')}`, + `Skipping attachment(s) unsupported by ${label} document input: ${skipped + .map((file) => `"${file.filename}" (${file.type})`) + .join(', ')}`, ); } - return processable; + return { processable, skipped }; +} + +const toOmittedAttachment = ( + file: Pick, + reason: AttachmentOmissionReason, +): OmittedAttachment => ({ + ...(file.file_id && { file_id: file.file_id }), + filename: file.filename ?? 'document', + type: file.type ?? '', + reason, +}); + +/** + * Decoded file text already sent per request. `fileContextCharLimit` then spans every + * encoder call of one request: the current turn, history replay and child runs. + */ +const decodedTextCharsByRequest = new WeakMap(); + +/** + * Bounds decoded file text with the limits that govern extracted file text: + * `fileTokenLimit` per file and `fileContextCharLimit` across the request. + * Returns `null` when the request's character budget is already spent. + */ +async function limitDecodedText(req: ServerRequest, text: string): Promise { + const fileConfig = mergeFileConfig(req.config?.fileConfig); + const charLimit = fileConfig.fileContextCharLimit; + const used = decodedTextCharsByRequest.get(req) ?? 0; + let limited = text; + if (charLimit) { + const remaining = charLimit - used; + if (remaining <= 0) { + return null; + } + /* Cut to the character budget first, so the token count never runs on the whole file. */ + limited = limited.slice(0, remaining); + } + const tokenLimit = req.body?.fileTokenLimit ?? fileConfig.fileTokenLimit; + if (tokenLimit) { + ({ text: limited } = await processTextWithTokenLimit({ + text: limited, + tokenLimit, + tokenCountFn: countTokens, + })); + } + decodedTextCharsByRequest.set(req, used + limited.length); + return limited; +} + +/** + * Applies {@link limitDecodedText} to a block that carries decoded file text: the text + * part fallback, and the plain-text source of a native Anthropic document. Other blocks + * pass through. Returns `null` when no budget is left for the text. + */ +async function limitTextBlock( + req: ServerRequest, + block: DocumentBlock, +): Promise { + if (block.type === 'text') { + const text = await limitDecodedText(req, block.text); + return text == null ? null : { ...block, text }; + } + if (block.type === 'document' && 'source' in block && block.source.type === 'text') { + const data = await limitDecodedText(req, block.source.data); + return data == null ? null : { ...block, source: { ...block.source, data } }; + } + return block; } function getBase64DecodedByteCount(content: string): number { @@ -192,8 +343,16 @@ function getBase64DecodedByteCount(content: string): number { * - **Bedrock**: Only encodes types in `bedrockDocumentFormats`; all others are skipped. * - **Anthropic**: Only encodes PDFs (base64 source) and textual types (plain-text source); * all others are skipped. + * - **Google/Vertex**: Encodes PDFs and textual types; all others are skipped. The native + * API's limits do not change with the upload allowlist, so `supportedMimeTypes` widens + * only Gemini behind a gateway. * - **PDF**: Validated via `validatePdf` before encoding. - * - **Generic types**: Encoded with a provider-specific size check. + * - **Generic types**: Encoded with a provider-specific size check. Textual types a + * provider can reject as a file part go as a text part unless the endpoint lists them. + * - **Decoded text**: Bounded by `fileTokenLimit` per file and `fileContextCharLimit` + * across the request. + * + * Skipped files are returned in `omitted`, so the caller can reject or report them. */ export async function encodeAndFormatDocuments( req: ServerRequest, @@ -216,7 +375,20 @@ export async function encodeAndFormatDocuments( return result; } - const processableFiles = filterProviderDocumentFiles(provider, files, model); + /* An admin's allowlist can widen what a gateway forwards, but not what the native + * Google/Vertex API accepts. */ + const isOptedIn = (mimeType: string) => + !isGoogleProvider(provider) && + isConfiguredProviderMediaType(req, { provider, endpoint }, mimeType); + const { processable: processableFiles, skipped } = filterProviderDocumentFiles( + provider, + files, + model, + isOptedIn, + ); + if (skipped.length) { + result.omitted = skipped.map((file) => toOmittedAttachment(file, 'unsupported_type')); + } if (!processableFiles.length) { return result; @@ -318,17 +490,24 @@ export async function encodeAndFormatDocuments( ); } - const block = formatDocumentBlock( + const formatted = formatDocumentBlock( provider, mimeType, content, file.filename, useResponsesApi, model, + isOptedIn(mimeType), ); + if (!formatted) { + continue; + } + const block = await limitTextBlock(req, formatted); if (block) { result.documents.push(block); result.files.push(metadata); + } else { + (result.omitted ??= []).push(toOmittedAttachment(file, 'text_limit')); } } } diff --git a/packages/api/src/files/encode/processAttachments.spec.ts b/packages/api/src/files/encode/processAttachments.spec.ts index 01855dbf67f..ce0c00e78e0 100644 --- a/packages/api/src/files/encode/processAttachments.spec.ts +++ b/packages/api/src/files/encode/processAttachments.spec.ts @@ -6,6 +6,7 @@ import { isBedrockDocumentType, } from 'librechat-data-provider'; import type { FileConfig, EndpointFileConfig } from 'librechat-data-provider'; +import { isProviderDocumentCandidate } from './utils'; /** * Mirrors the categorization logic from BaseClient.processAttachments. @@ -49,10 +50,7 @@ function categorizeFile( } else if (file.type?.startsWith('audio/')) { return 'audios'; } else if ( - file.type && - mergedFileConfig && - endpointFileConfig?.supportedMimeTypes && - mergedFileConfig.checkType?.(file.type, endpointFileConfig.supportedMimeTypes) + isProviderDocumentCandidate(file.type, mergedFileConfig, endpointFileConfig?.supportedMimeTypes) ) { return 'documents'; } @@ -185,4 +183,61 @@ describe('processAttachments — supportedMimeTypes routing logic', () => { const result = categorizeFile({ type: 'text/csv', embedded: true }, false, merged, epConfig); expect(result).toBe('skipped'); }); + + describe('inherited default supportedMimeTypes', () => { + /** An endpoint with no `supportedMimeTypes` of its own inherits the built-in list. */ + const inherited = () => { + const merged = mergeFileConfig({ endpoints: { [endpoint]: { fileLimit: 5 } } }); + return { merged, epConfig: getEndpointFileConfig({ fileConfig: merged, endpoint }) }; + }; + + it.each([ + 'application/zip', + 'application/x-zip-compressed', + 'application/x-tar', + 'application/epub+zip', + ])('does not opt archive %s into the provider document path', (type) => { + const { merged, epConfig } = inherited(); + expect(merged.checkType?.(type, epConfig?.supportedMimeTypes ?? [])).toBe(true); + expect(categorizeFile({ type }, false, merged, epConfig)).toBe('skipped'); + }); + + it.each([ + 'application/sql', + 'application/x-sh', + 'application/json', + 'text/plain', + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + ])('keeps routing %s to documents', (type) => { + const { merged, epConfig } = inherited(); + expect(categorizeFile({ type }, false, merged, epConfig)).toBe('documents'); + }); + + it('keeps images, PDF, audio and video on their own branches', () => { + const { merged, epConfig } = inherited(); + expect(categorizeFile({ type: 'image/png' }, false, merged, epConfig)).toBe('images'); + expect(categorizeFile({ type: 'application/pdf' }, false, merged, epConfig)).toBe( + 'documents', + ); + expect(categorizeFile({ type: 'audio/mp3' }, false, merged, epConfig)).toBe('audios'); + expect(categorizeFile({ type: 'video/mp4' }, false, merged, epConfig)).toBe('videos'); + }); + + it('keeps Bedrock document types for Bedrock', () => { + const { merged, epConfig } = inherited(); + const docx = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'; + expect(categorizeFile({ type: docx }, true, merged, epConfig)).toBe('documents'); + }); + }); + + it('should skip textual types an explicit allowlist leaves out', () => { + const { merged, epConfig } = resolveConfig(['^image/.*', '^application/pdf$']); + expect(categorizeFile({ type: 'application/sql' }, false, merged, epConfig)).toBe('skipped'); + }); + + it('should route zip to documents when an explicit allowlist lists it', () => { + const { merged, epConfig } = resolveConfig(['^application/zip$']); + expect(categorizeFile({ type: 'application/zip' }, false, merged, epConfig)).toBe('documents'); + }); }); diff --git a/packages/api/src/files/encode/utils.ts b/packages/api/src/files/encode/utils.ts index 9c39ce2c13c..e69a4b1b694 100644 --- a/packages/api/src/files/encode/utils.ts +++ b/packages/api/src/files/encode/utils.ts @@ -6,6 +6,7 @@ import { isExplicitMimeConfig, getEndpointFileConfig, } from 'librechat-data-provider'; +import type { FileConfig, RegexLike } from 'librechat-data-provider'; import type { ServerRequest, StrategyFunctions, ProcessedFile } from '~/types'; import type { StoredFileRef } from '~/storage/path'; import { resolveDownloadPath } from '~/storage/path'; @@ -111,6 +112,38 @@ export const isConfiguredProviderMediaType = ( return fileConfig.checkType?.(mimeType, types) ?? false; }; +/** Archive and container types that no provider accepts as an inline document. */ +const providerArchiveMimeTypes = new Set([ + 'application/zip', + 'application/x-zip-compressed', + 'application/x-tar', + 'application/gzip', + 'application/x-gzip', + 'application/epub+zip', + 'application/octet-stream', +]); + +/** + * Whether a provider-bound attachment that has no dedicated branch (image, PDF, Bedrock + * document, audio, video) goes to the document encoder. The endpoint's `supportedMimeTypes` + * decides, including the built-in list an unconfigured endpoint inherits. That inherited + * list also allows archives for other upload paths, and no provider accepts one inline, so + * an archive goes to the provider only when the admin lists it for the endpoint. + */ +export const isProviderDocumentCandidate = ( + mimeType: string | null | undefined, + fileConfig: Pick | undefined, + supportedMimeTypes: RegexLike[] | undefined, +): boolean => { + if (!mimeType || !supportedMimeTypes) { + return false; + } + if (!isExplicitMimeConfig(supportedMimeTypes) && providerArchiveMimeTypes.has(mimeType)) { + return false; + } + return fileConfig?.checkType?.(mimeType, supportedMimeTypes) ?? false; +}; + /** * Maps accepted audio MIME types to the format token OpenAI-compatible providers expect. * Mirrors the canonicalization in `api/server/services/Files/Audio/STTService.js`: the diff --git a/packages/api/src/types/files.ts b/packages/api/src/types/files.ts index 83c3bfc1230..885d8f7ecaf 100644 --- a/packages/api/src/types/files.ts +++ b/packages/api/src/types/files.ts @@ -124,6 +124,17 @@ export type DocumentBlock = | OpenAIInputFileBlock | BedrockDocumentBlock; +/** Why a provider-bound attachment was not sent to the model. */ +export type AttachmentOmissionReason = 'unsupported_type' | 'text_limit'; + +/** A provider-bound attachment the model did not receive. */ +export interface OmittedAttachment { + file_id?: string; + filename: string; + type: string; + reason: AttachmentOmissionReason; +} + export interface DocumentResult { documents: DocumentBlock[]; files: Array<{ @@ -134,6 +145,8 @@ export interface DocumentResult { filename: string; type: string; }>; + /** Attachments this call did not send, so the caller can reject or report them. */ + omitted?: OmittedAttachment[]; } /** Google audio block format */