📎 fix: Keep Provider Uploads the Provider Cannot Read Inline Out of File Parts

api/app/clients/BaseClient.js

@@ -14,6 +14,8 @@ const {
   encodeAndFormatVideos,
   getTransactionsConfig,
   encodeAndFormatDocuments,
+  isProviderDocumentCandidate,
+  AgentAttachmentUnsupportedError,
   getLangfuseTraceMessageFields,
   isContentFilterError,
   assertModelBoundProviderContent,
@@ -1698,7 +1700,12 @@ class BaseClient {
     );
   }
 
-  async addDocuments(message, attachments) {
+  /**
+   * @param {TMessage} message
+   * @param {MongoFile[]} attachments
+   * @param {OmittedAttachment[]} [omissions] - Receives the files the encoder did not send.
+   */
+  async addDocuments(message, attachments, omissions) {
     const documentResult = await encodeAndFormatDocuments(
       this.options.req,
       attachments,
@@ -1714,6 +1721,9 @@ class BaseClient {
       documentResult.documents && documentResult.documents.length
         ? documentResult.documents
         : undefined;
+    if (omissions && documentResult.omitted?.length) {
+      omissions.push(...documentResult.omitted);
+    }
     return documentResult.files;
   }
 
@@ -1789,14 +1799,27 @@ class BaseClient {
     return resolveTurnLLMDeliveryPath(this.options.agent?.deliveryRouting, file, fileConsumers);
   }
 
-  async processAttachments(message, attachments, fileConsumers) {
+  /**
+   * A provider-bound file the model would not receive is rejected on the current turn with
+   * {@link AgentAttachmentUnsupportedError}, so the model never answers as if it had read
+   * it. On history replay the same file is left out with a note in the message instead:
+   * throwing there would fail every later turn of the conversation.
+   *
+   * @param {TMessage} message
+   * @param {MongoFile[]} attachments
+   * @param {TurnFileConsumers} [fileConsumers]
+   * @param {{ historical?: boolean }} [options]
+   */
+  async processAttachments(message, attachments, fileConsumers, options = {}) {
     const categorizedAttachments = {
       images: [],
       videos: [],
       audios: [],
       documents: [],
     };
 
+    /** @type {OmittedAttachment[]} */
+    const omissions = [];
     const allFiles = [];
     const provider = this.options.agent?.provider ?? this.options.endpoint;
     const isBedrock = provider === EModelEndpoint.bedrock;
@@ -1842,15 +1865,21 @@ class BaseClient {
         categorizedAttachments.audios.push(file);
         allFiles.push(file);
       } else if (
-        file.type &&
-        deliveryRouting?.endpointConfig.supportedMimeTypes &&
-        deliveryRouting.fileConfig.checkType(
+        isProviderDocumentCandidate(
           file.type,
-          deliveryRouting.endpointConfig.supportedMimeTypes,
+          deliveryRouting?.fileConfig,
+          deliveryRouting?.endpointConfig.supportedMimeTypes,
         )
       ) {
         categorizedAttachments.documents.push(file);
         allFiles.push(file);
+      } else if (deliveryPath === 'provider') {
+        omissions.push({
+          ...(file.file_id && { file_id: file.file_id }),
+          filename: file.filename,
+          type: file.type,
+          reason: 'unsupported_type',
+        });
       }
     }
 
@@ -1859,7 +1888,7 @@ class BaseClient {
         ? this.addImageURLs(message, categorizedAttachments.images)
         : Promise.resolve([]),
       categorizedAttachments.documents.length > 0
-        ? this.addDocuments(message, categorizedAttachments.documents)
+        ? this.addDocuments(message, categorizedAttachments.documents, omissions)
         : Promise.resolve([]),
       categorizedAttachments.videos.length > 0
         ? this.addVideos(message, categorizedAttachments.videos)
@@ -1871,7 +1900,29 @@ class BaseClient {
 
     allFiles.push(...imageFiles);
 
-    const seenFileIds = new Set();
+    if (omissions.length > 0) {
+      if (!options.historical) {
+        throw new AgentAttachmentUnsupportedError(omissions);
+      }
+      logger.warn('[BaseClient] Attachments from history not sent to the model', {
+        messageId: message.messageId,
+        omitted: omissions,
+      });
+      message.documents = [
+        ...(message.documents ?? []),
+        ...omissions.map((file) => ({
+          type: 'text',
+          text: `File "${file.filename}" was attached to this message but was not sent to the model: ${
+            file.reason === 'text_limit'
+              ? 'the file text limit for this request was reached.'
+              : `this model cannot read ${file.type || 'this file type'}.`
+          }`,
+        })),
+      ];
+    }
+
+    const omittedFileIds = new Set(omissions.map((file) => file.file_id).filter(Boolean));
+    const seenFileIds = new Set(omittedFileIds);
     const uniqueFiles = [];
 
     for (const file of allFiles) {
@@ -2028,7 +2079,7 @@ class BaseClient {
 
       const [, processedFiles] = await Promise.all([
         this.addFileContextToMessage(message, contextFiles),
-        this.processAttachments(message, contextFiles),
+        this.processAttachments(message, contextFiles, undefined, { historical: true }),
       ]);
 
       const processedFileIds = new Set(

api/app/clients/specs/BaseClient.test.js

@@ -5,6 +5,8 @@ const {
   resolveTurnDeliveryRouting,
   buildSteerMedia,
   Tokenizer,
+  isAgentAttachmentLimitError,
+  AgentAttachmentUnsupportedError,
 } = require('@librechat/api');
 const { FakeClient, initializeFakeClient } = require('./FakeClient');
 
@@ -2945,7 +2947,9 @@ describe('BaseClient', () => {
         {},
       );
       expect(TestClient.addFileContextToMessage).toHaveBeenCalledWith(message, [ownerFile]);
-      expect(TestClient.processAttachments).toHaveBeenCalledWith(message, [ownerFile]);
+      expect(TestClient.processAttachments).toHaveBeenCalledWith(message, [ownerFile], undefined, {
+        historical: true,
+      });
       expect(message.fileContext).toBe('authorized owner text');
       expect(message.files).toEqual([
         expect.objectContaining({
@@ -3560,6 +3564,182 @@ describe('BaseClient', () => {
       });
     };
 
+    describe('provider documents under the inherited supportedMimeTypes list', () => {
+      /** A file the user sent through "Upload to Provider"; the endpoint sets no MIME list. */
+      const providerFile = (file_id, type) => ({
+        user: 'user1',
+        file_id,
+        filename: file_id,
+        filepath: `/uploads/${file_id}`,
+        type,
+        bytes: 100,
+        source: 'local',
+        llmDeliveryPath: 'provider',
+        metadata: { destinationChosen: true },
+      });
+
+      const withSupportedMimeTypes = (supportedMimeTypes) => {
+        TestClient.options.req = {
+          config: {
+            fileConfig: {
+              endpoints: { [EModelEndpoint.openAI]: { fileLimit: 10, supportedMimeTypes } },
+            },
+          },
+        };
+        TestClient.options.agent = routedAgent({
+          provider: EModelEndpoint.openAI,
+          endpoint: EModelEndpoint.openAI,
+        });
+      };
+
+      test('rejects a current-turn archive the endpoint does not list, naming the file', async () => {
+        withSupportedMimeTypes(undefined);
+        const sql = providerFile('query.sql', 'application/sql');
+        const zip = providerFile('archive.zip', 'application/zip');
+        const pdf = providerFile('report.pdf', 'application/pdf');
+
+        const rejection = TestClient.processAttachments({}, [sql, zip, pdf]);
+
+        await expect(rejection).rejects.toBeInstanceOf(AgentAttachmentUnsupportedError);
+        await expect(rejection).rejects.toMatchObject({
+          code: 'AGENT_ATTACHMENT_UNSUPPORTED',
+          message: expect.stringContaining('"archive.zip" (application/zip)'),
+          attachments: [
+            {
+              file_id: 'archive.zip',
+              filename: 'archive.zip',
+              type: 'application/zip',
+              reason: 'unsupported_type',
+            },
+          ],
+        });
+        expect(isAgentAttachmentLimitError(await rejection.catch((error) => error))).toBe(true);
+        expect(TestClient.addDocuments).toHaveBeenCalledWith(
+          expect.anything(),
+          [sql, pdf],
+          expect.any(Array),
+        );
+      });
+
+      test('rejects a current-turn file the document encoder leaves out', async () => {
+        /* A DOCX sent to Gemini passes routing but not the encoder, which used to leave the
+         * file on the message while the model received none of it. */
+        withSupportedMimeTypes(undefined);
+        const csv = providerFile('table.csv', 'text/csv');
+        TestClient.addDocuments = jest.fn(async (_message, _files, omissions) => {
+          omissions.push({
+            file_id: 'table.csv',
+            filename: 'table.csv',
+            type: 'text/csv',
+            reason: 'text_limit',
+          });
+          return [];
+        });
+
+        await expect(TestClient.processAttachments({}, [csv])).rejects.toMatchObject({
+          code: 'AGENT_ATTACHMENT_UNSUPPORTED',
+          message: expect.stringContaining('exceeds the file text limit'),
+        });
+      });
+
+      test('recovers a conversation whose earlier message holds an archive', async () => {
+        /* The archive that 400s on every later turn arrives through history, not this turn's
+         * upload, so the replay path has to skip it too. */
+        withSupportedMimeTypes(undefined);
+        TestClient.options.req.user = { id: 'user1' };
+        TestClient.options.resendFiles = true;
+        TestClient.addFileContextToMessage = jest.fn();
+        TestClient.assertHistoricalAttachmentLimits = undefined;
+        TestClient.checkVisionRequest = jest.fn();
+        TestClient.message_file_map = undefined;
+        const { logger } = require('@librechat/data-schemas');
+        const warn = jest.spyOn(logger, 'warn').mockImplementation(() => {});
+        const zip = providerFile('archive.zip', 'application/zip');
+        const pdf = providerFile('report.pdf', 'application/pdf');
+        getFiles.mockResolvedValueOnce([zip, pdf]);
+
+        try {
+          const [message] = await TestClient.addPreviousAttachments([
+            {
+              messageId: 'msg-earlier',
+              text: 'Here are the files',
+              files: [{ file_id: 'archive.zip' }, { file_id: 'report.pdf' }],
+            },
+          ]);
+
+          expect(TestClient.addDocuments).toHaveBeenCalledTimes(1);
+          expect(TestClient.addDocuments).toHaveBeenCalledWith(message, [pdf], expect.any(Array));
+          expect(TestClient.message_file_map['msg-earlier']).toEqual([pdf]);
+          /* The omission is explicit to the model and in the log, not silent. */
+          expect(message.documents).toEqual([
+            { type: 'file' },
+            {
+              type: 'text',
+              text: 'File "archive.zip" was attached to this message but was not sent to the model: this model cannot read application/zip.',
+            },
+          ]);
+          expect(warn).toHaveBeenCalledWith(
+            '[BaseClient] Attachments from history not sent to the model',
+            expect.objectContaining({
+              messageId: 'msg-earlier',
+              omitted: [expect.objectContaining({ file_id: 'archive.zip' })],
+            }),
+          );
+        } finally {
+          warn.mockRestore();
+        }
+      });
+
+      test('leaves a file the encoder omitted from history out of the message file map', async () => {
+        withSupportedMimeTypes(undefined);
+        TestClient.options.req.user = { id: 'user1' };
+        TestClient.options.resendFiles = true;
+        TestClient.addFileContextToMessage = jest.fn();
+        TestClient.assertHistoricalAttachmentLimits = undefined;
+        TestClient.checkVisionRequest = jest.fn();
+        TestClient.message_file_map = undefined;
+        const docx = providerFile(
+          'report.docx',
+          'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+        );
+        TestClient.addDocuments = jest.fn(async (_message, _files, omissions) => {
+          omissions.push({
+            file_id: docx.file_id,
+            filename: docx.filename,
+            type: docx.type,
+            reason: 'unsupported_type',
+          });
+          return [];
+        });
+        getFiles.mockResolvedValueOnce([docx]);
+
+        const [message] = await TestClient.addPreviousAttachments([
+          { messageId: 'msg-docx', text: 'Read this', files: [{ file_id: 'report.docx' }] },
+        ]);
+
+        expect(TestClient.message_file_map['msg-docx']).toEqual([]);
+        expect(message.documents).toEqual([
+          expect.objectContaining({
+            type: 'text',
+            text: expect.stringContaining('"report.docx" was attached to this message'),
+          }),
+        ]);
+      });
+
+      test('sends a binary type the endpoint lists explicitly', async () => {
+        withSupportedMimeTypes(['^application/zip$']);
+        const zip = providerFile('archive.zip', 'application/zip');
+
+        await TestClient.processAttachments({}, [zip]);
+
+        expect(TestClient.addDocuments).toHaveBeenCalledWith(
+          expect.anything(),
+          [zip],
+          expect.any(Array),
+        );
+      });
+    });
+
     test('keeps a none image in returned files without adding image URLs', async () => {
       routeTo('none', 'image/*');
       const message = {};
@@ -4133,7 +4313,7 @@ describe('BaseClient', () => {
 
       expect(result).toEqual([file]);
       expect(message.documents).toEqual([{ type: 'file' }]);
-      expect(TestClient.addDocuments).toHaveBeenCalledWith(message, [file]);
+      expect(TestClient.addDocuments).toHaveBeenCalledWith(message, [file], expect.any(Array));
     });
   });
 });

api/server/controllers/agents/client.js

@@ -381,7 +381,7 @@ class AgentClient extends BaseClient {
     );
   }
 
-  async addDocuments(message, attachments) {
+  async addDocuments(message, attachments, omissions) {
     const memoryContext = {
       req: this.options.req,
       conversationId: this.conversationId,
@@ -390,13 +390,13 @@ class AgentClient extends BaseClient {
     };
     logAgentMemorySnapshot('before_encode_documents', memoryContext);
     try {
-      return await super.addDocuments(message, attachments);
+      return await super.addDocuments(message, attachments, omissions);
     } finally {
       logAgentMemorySnapshot('after_encode_documents', memoryContext);
     }
   }
 
-  async processAttachments(message, attachments, fileConsumers) {
+  async processAttachments(message, attachments, fileConsumers, options) {
     const modelBoundAttachments = this.getModelBoundAttachmentsForEndpoint(attachments);
     const processableAttachments = this.getProcessableAttachmentsForEndpoint(
       attachments,
@@ -416,7 +416,12 @@ class AgentClient extends BaseClient {
     };
     logAgentMemorySnapshot('before_process_attachments', memoryContext);
     try {
-      return await super.processAttachments(message, processableAttachments, fileConsumers);
+      return await super.processAttachments(
+        message,
+        processableAttachments,
+        fileConsumers,
+        options,
+      );
     } finally {
       logAgentMemorySnapshot('after_process_attachments', memoryContext);
     }