diff --git a/api/app/clients/BaseClient.js b/api/app/clients/BaseClient.js index f02af652d4b..8a097ba8790 100644 --- a/api/app/clients/BaseClient.js +++ b/api/app/clients/BaseClient.js @@ -10,10 +10,15 @@ const { extractFileContext, getReferencedQuotes, applyTurnDelivery, + prepareTurnFiles, + getTurnTextOptions, encodeAndFormatAudios, encodeAndFormatVideos, getTransactionsConfig, encodeAndFormatDocuments, + encodeNativeDocuments, + recordNativeRejections, + prepareMessageAttachments, getLangfuseTraceMessageFields, isContentFilterError, assertModelBoundProviderContent, @@ -859,7 +864,7 @@ class BaseClient { if (this.options.resendFiles !== false && this.authorizedHistoricalFiles == null) { const historicalFileState = collectModelBoundHistoricalFileIdState(modelBoundStoredMessages); this.modelBoundHistoricalFileIdsOverflowed ||= historicalFileState.overflowed; - const files = this.resolveTurnAttachments( + const files = await this.prepareTurnAttachments( await getOwnerHistoricalFiles(historicalFileState.fileIds, this.options.req?.user), ); this.authorizedHistoricalFiles = new Map( @@ -1739,17 +1744,24 @@ class BaseClient { } async addDocuments(message, attachments) { - const documentResult = await encodeAndFormatDocuments( - this.options.req, + const documentResult = await encodeNativeDocuments( attachments, - { - provider: this.options.agent?.provider ?? this.options.endpoint, - endpoint: this.options.agent?.endpoint ?? this.options.endpoint, - useResponsesApi: this.usesResponsesApi(), - model: this.modelOptions?.model ?? this.model, - }, - getStrategyFunctions, + this.options.agent, + (files, onValidationFailure) => + encodeAndFormatDocuments( + this.options.req, + files, + { + provider: this.options.agent?.provider ?? this.options.endpoint, + endpoint: this.options.agent?.endpoint ?? this.options.endpoint, + useResponsesApi: this.usesResponsesApi(), + model: this.modelOptions?.model ?? this.model, + onValidationFailure, + }, + getStrategyFunctions, + ), ); + recordNativeRejections(this.getConversationAgents(), documentResult.rejected); message.documents = documentResult.documents && documentResult.documents.length ? documentResult.documents @@ -1757,6 +1769,11 @@ class BaseClient { return documentResult.files; } + /** The agents whose messages carry this conversation's encoded attachments. */ + getConversationAgents() { + return [this.options.agent]; + } + async addVideos(message, attachments) { const videoResult = await encodeAndFormatVideos( this.options.req, @@ -1800,6 +1817,7 @@ class BaseClient { attachments: textAttachments, req: this.options?.req, tokenCountFn: (text) => countTokens(text), + ...getTurnTextOptions(this.options.agent?.deliveryRouting), }); if (fileContext) { @@ -1825,6 +1843,26 @@ class BaseClient { }); } + /** {@link resolveTurnAttachments}, deriving text the turn's reading needs first. */ + prepareTurnAttachments(files, fileConsumers = this.options.agent?.fileConsumers) { + return prepareTurnFiles({ + routing: this.options.agent?.deliveryRouting, + files, + consumers: fileConsumers, + signal: this.options.abortController?.signal, + }); + } + + processMessageAttachments(message, files, fileConsumers, contextFiles) { + return prepareMessageAttachments({ + client: this, + message, + files, + consumers: fileConsumers, + contextFiles, + }); + } + getAttachmentDeliveryPath(file, fileConsumers = this.options.agent?.fileConsumers) { return resolveTurnLLMDeliveryPath(this.options.agent?.deliveryRouting, file, fileConsumers); } @@ -1949,7 +1987,7 @@ class BaseClient { const historicalFileState = collectModelBoundHistoricalFileIdState(_messages); this.modelBoundHistoricalFileIdsOverflowed ||= historicalFileState.overflowed; const authorizedFilesById = new Map(); - const files = this.resolveTurnAttachments( + const files = await this.prepareTurnAttachments( await getOwnerHistoricalFiles(historicalFileState.fileIds, this.options.req?.user), ); const nonSteerReplayFileIds = collectModelBoundHistoricalFileIdState( @@ -2066,16 +2104,10 @@ class BaseClient { return message; } - const [, processedFiles] = await Promise.all([ - this.addFileContextToMessage(message, contextFiles), - this.processAttachments(message, contextFiles), - ]); + const processedFiles = await this.processMessageAttachments(message, contextFiles); - const processedFileIds = new Set( - (processedFiles ?? []).map((file) => file?.file_id).filter(Boolean), - ); - this.message_file_map[message.messageId] = contextFiles.filter( - (file) => processedFileIds.has(file?.file_id) && isModelBoundAttachmentFile(file), + this.message_file_map[message.messageId] = (processedFiles ?? []).filter( + isModelBoundAttachmentFile, ); return message; }; diff --git a/api/app/clients/specs/BaseClient.test.js b/api/app/clients/specs/BaseClient.test.js index cd6ac2c34f1..a8ef61b1be9 100644 --- a/api/app/clients/specs/BaseClient.test.js +++ b/api/app/clients/specs/BaseClient.test.js @@ -1,3 +1,6 @@ +const fs = require('fs'); +const os = require('os'); +const path = require('path'); const { Constants, ContentTypes, EModelEndpoint } = require('librechat-data-provider'); const BaseClientClass = require('../BaseClient'); const { @@ -8,6 +11,8 @@ const { getPrivateTextInspectionTokens, assertModelBoundContent, resolveTurnDeliveryRouting, + buildTurnReadingContext, + buildMessageFiles, buildSteerMedia, Tokenizer, } = require('@librechat/api'); @@ -3059,6 +3064,34 @@ describe('BaseClient', () => { expect(userSave[0].files).toHaveLength(1); expect(userSave[0].files[0].file_id).toBe('file-abc'); }); + + test('saves the user row with the files buildMessageFiles derives from the request and attachments', async () => { + const brief = { + user: 'user-1', + file_id: 'brief', + filename: 'brief.pdf', + filepath: '/uploads/brief.pdf', + type: 'application/pdf', + bytes: 2048, + source: 'local', + text: 'extracted text', + _id: 'mongo-brief', + }; + const requestFiles = [{ file_id: 'brief' }, { file_id: 'scan' }]; + TestClient.options.req = { body: { files: requestFiles } }; + TestClient.options.attachments = [brief]; + TestClient.saveMessageToDatabase = jest.fn().mockResolvedValue({ message: {} }); + + await TestClient.sendMessage('Hello'); + + const userSave = TestClient.saveMessageToDatabase.mock.calls.find( + ([msg]) => msg.isCreatedByUser, + ); + const expected = buildMessageFiles(requestFiles, [brief]); + expect(expected).toHaveLength(1); + expect(userSave[0].files).toEqual(expected); + expect(TestClient.options.attachments).toEqual([brief]); + }); }); describe('addPreviousAttachments authorization', () => { @@ -3481,6 +3514,70 @@ describe('BaseClient', () => { expect(JSON.stringify(secondMessage)).not.toContain('second-forged'); }); + describe('text a deferred file needs on a later turn', () => { + const deferredWorkbook = { + file_id: 'deferred-xlsx', + filename: 'quarterly.xlsx', + filepath: '/uploads/quarterly.xlsx', + source: 'local', + type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + bytes: 2048, + user: 'user-1', + context: 'message_attachment', + llmDeliveryPath: 'none', + metadata: { destinationChosen: false, textDerivation: { outcome: 'deferred', at: 1 } }, + }; + const derivedMarker = { outcome: 'complete', extractor: 'document_parser', at: 2 }; + + test('prepares the replayed file once and hands the prepared copy to the limit check and file context', async () => { + const deriveText = jest.fn(async () => ({ + status: 'derived', + text: 'Q1,1200', + textDerivation: derivedMarker, + })); + getFiles.mockResolvedValueOnce([deferredWorkbook]); + TestClient.options.req.config = { + fileConfig: { + endpoints: { [EModelEndpoint.openAI]: { llmDeliveryPolicy: 'automatic' } }, + }, + }; + TestClient.options.agent = { + provider: EModelEndpoint.openAI, + fileConsumers: { executeCode: false, fileSearch: false }, + }; + const routing = resolveTurnDeliveryRouting({ + agent: TestClient.options.agent, + config: TestClient.options.req.config, + }); + routing.reading = buildTurnReadingContext({ + routing, + provider: EModelEndpoint.openAI, + fileTokenLimit: 100000, + configuredFileSizeLimit: undefined, + countTokens: (text) => text.length, + deriveText, + }); + TestClient.options.agent.deliveryRouting = routing; + TestClient.assertHistoricalAttachmentLimits = jest.fn(async (files) => files); + const prepareTurnAttachments = jest.spyOn(TestClient, 'prepareTurnAttachments'); + + const [message] = await TestClient.addPreviousAttachments([ + { messageId: 'msg-xlsx', text: 'Totals?', files: [{ file_id: 'deferred-xlsx' }] }, + ]); + + const prepared = { + ...deferredWorkbook, + text: 'Q1,1200', + llmDeliveryPath: 'text', + metadata: { ...deferredWorkbook.metadata, textDerivation: derivedMarker }, + }; + expect(prepareTurnAttachments).toHaveBeenCalledTimes(1); + expect(TestClient.assertHistoricalAttachmentLimits).toHaveBeenCalledWith([prepared]); + expect(message.fileContext).toBe('Q1,1200'); + expect(deferredWorkbook.text).toBeUndefined(); + }); + }); + test('extracts historical file context while encoding provider attachments', async () => { getFiles.mockResolvedValueOnce([ownerFile]); const fileContext = deferred(); @@ -3503,8 +3600,7 @@ describe('BaseClient', () => { return messages; }); - await Promise.resolve(); - await Promise.resolve(); + await new Promise((resolve) => setImmediate(resolve)); expect(TestClient.addFileContextToMessage).toHaveBeenCalledTimes(1); expect(TestClient.processAttachments).toHaveBeenCalledTimes(1); @@ -4535,6 +4631,133 @@ describe('BaseClient', () => { }); }); +describe('BaseClient attachment text under a reading context', () => { + const longNote = { + file_id: 'long-note', + filename: 'notes.txt', + source: 'text', + type: 'text/plain', + text: 'quarterly revenue grew '.repeat(200), + }; + + beforeEach(() => { + jest.spyOn(Tokenizer, 'initEncoding').mockResolvedValue(undefined); + jest.spyOn(Tokenizer, 'getTokenCount').mockImplementation((text) => text.length); + }); + + afterEach(() => { + jest.restoreAllMocks(); + }); + + /** The truncation notice is emitted only when the turn reading context's text options + * reach `extractFileContext`; the notice wording itself belongs to packages/api. */ + test('extracts file context with the text options of the turn reading context', async () => { + const client = initializeFakeClient(apiKey, { modelOptions: { model: 'gpt-4o-mini' } }, []); + const config = { + fileConfig: { endpoints: { [EModelEndpoint.openAI]: { llmDeliveryPolicy: 'automatic' } } }, + }; + client.options.req = { body: { fileTokenLimit: 20 }, config }; + client.options.agent = { + provider: EModelEndpoint.openAI, + fileConsumers: { executeCode: false, fileSearch: false }, + }; + const routing = resolveTurnDeliveryRouting({ agent: client.options.agent, config }); + routing.reading = buildTurnReadingContext({ + routing, + provider: EModelEndpoint.openAI, + fileTokenLimit: 20, + configuredFileSizeLimit: undefined, + countTokens: (text) => text.length, + }); + client.options.agent.deliveryRouting = routing; + const message = {}; + + await client.addFileContextToMessage(message, [longNote], client.options.agent.fileConsumers); + + expect(message.fileContext).toContain('# "notes.txt"'); + expect(message.fileContext).toContain( + '[Truncated: only the beginning of "notes.txt" fits; the rest is omitted.]', + ); + }); +}); + +describe('BaseClient native documents under a reading context', () => { + const brokenPdf = { + user: 'user1', + file_id: 'broken-pdf', + filename: 'brief.pdf', + filepath: '/uploads/brief.pdf', + type: 'application/pdf', + bytes: 18, + source: 'local', + context: 'message_attachment', + llmDeliveryPath: 'provider', + metadata: { destinationChosen: false }, + }; + const rejection = [{ file_id: 'broken-pdf', reason: 'integrity' }]; + let uploads; + + beforeAll(() => { + uploads = fs.mkdtempSync(path.join(os.tmpdir(), 'baseclient-documents-')); + fs.writeFileSync(path.join(uploads, 'brief.pdf'), 'not a pdf document'); + }); + + afterAll(() => { + fs.rmSync(uploads, { recursive: true, force: true }); + }); + + const routeBrokenPdf = () => { + const client = initializeFakeClient(apiKey, { modelOptions: { model: 'claude-sonnet-4' } }, []); + const config = { + paths: { uploads }, + fileConfig: { endpoints: { [EModelEndpoint.anthropic]: { llmDeliveryPolicy: 'automatic' } } }, + }; + client.options.req = { config, user: { id: 'user1' } }; + client.options.agent = { + provider: EModelEndpoint.anthropic, + endpoint: EModelEndpoint.anthropic, + fileConsumers: { executeCode: false, fileSearch: true }, + currentRequestAttachments: [brokenPdf], + }; + const routing = resolveTurnDeliveryRouting({ agent: client.options.agent, config }); + routing.reading = buildTurnReadingContext({ + routing, + provider: EModelEndpoint.anthropic, + fileTokenLimit: 100000, + configuredFileSizeLimit: undefined, + countTokens: (text) => text.length, + }); + client.options.agent.deliveryRouting = routing; + return { client, routing }; + }; + + test('leaves a rejected document out of the forwarded files and records it on the reading context', async () => { + const { client, routing } = routeBrokenPdf(); + const recordRejections = jest.spyOn(routing.reading, 'recordRejections'); + const message = {}; + + const files = await client.addDocuments(message, [brokenPdf]); + + expect(files).toEqual([]); + expect(message.documents).toBeUndefined(); + expect(recordRejections).toHaveBeenCalledWith(rejection); + }); + + test('records the rejection on every agent the conversation agents list', async () => { + const { client } = routeBrokenPdf(); + const { routing: handoffRouting } = routeBrokenPdf(); + const recordRejections = jest.spyOn(handoffRouting.reading, 'recordRejections'); + client.getConversationAgents = () => [ + client.options.agent, + { deliveryRouting: handoffRouting }, + ]; + + await client.addDocuments({}, [brokenPdf]); + + expect(recordRejections).toHaveBeenCalledWith(rejection); + }); +}); + describe('BaseClient compaction turns', () => { const compactionOptions = { modelOptions: { model: 'gpt-4o-mini', temperature: 0 } }; const compactionHistory = [ diff --git a/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js b/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js index 1d0790a95aa..4476a453497 100644 --- a/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js +++ b/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js @@ -299,7 +299,8 @@ jest.mock('@librechat/api', () => ({ return { text, fileIds: fileIds.sort() }; }), getViolationInfo: (...args) => mockGetViolationInfo(...args), - buildMessageFiles: jest.fn(() => []), + /** Real, because the files the final user-message save carries are under test. */ + buildMessageFiles: jest.requireActual('@librechat/api').buildMessageFiles, resolveTitleTiming: jest.fn(() => 'immediate'), createConvoPersistenceSignal: jest.requireActual('@librechat/api').createConvoPersistenceSignal, recoverTurnMessageReference: jest.requireActual('@librechat/api').recoverTurnMessageReference, @@ -3771,6 +3772,91 @@ describe('ResumableAgentController resume metadata', () => { } } + it('persists the user message without file text and fails the turn when search preparation fails', async () => { + const api = jest.requireActual('@librechat/api'); + const file = { + user: 'user-123', + file_id: 'search-pdf', + filename: 'search.pdf', + filepath: '/uploads/search.pdf', + type: 'application/pdf', + bytes: 3 * 1024 * 1024, + object: 'file', + embedded: false, + usage: 0, + source: 'local', + context: 'message_attachment', + llmDeliveryPath: 'none', + metadata: { destinationChosen: false }, + text: 'complete cached text was not sent', + }; + const preparation = new Map(); + const agent = { + id: 'agent-1', + fileConsumers: { executeCode: false, fileSearch: true }, + currentRequestAttachments: [file], + provisionState: { + codeEnvFiles: [], + vectorDBFiles: [file], + searchPreparation: preparation, + aliveFileIds: new Set(), + agentScopedFileIds: new Set(), + }, + }; + const refs = [{ file_id: file.file_id }]; + const message = { + messageId: 'user-message', + parentMessageId: 'prior-response', + conversationId, + text: 'Search the attached PDF.', + files: api.buildMessageFiles(refs, [file]), + }; + const provisionFiles = api.createProvisionFilesCallback({ + req: { user: { id: 'user-123' } }, + agentToolContexts: new Map([[agent.id, { provisionState: agent.provisionState }]]), + provisionToCodeEnv: jest.fn(), + provisionToVectorDB: jest.fn().mockResolvedValue({ embedded: false, fileUpdate: null }), + updateFile: jest.fn(), + updateCodeEnvRef: jest.fn(), + addEmbeddedEntity: jest.fn(), + }); + const client = { + options: { attachments: [file], agent }, + sendMessage: jest.fn(async (_text, options) => { + options.onStart(message, 'user-message_'); + await provisionFiles(['file_search'], agent.id); + }), + }; + mockGenerationJobManager.completeJob.mockImplementation( + async (_streamId, error, _createdAt, options) => { + await options.beforeErrorPublication(); + const saved = mockSaveMessage.mock.calls.find( + ([, row]) => row.messageId === message.messageId, + )?.[1]; + expect(saved.files[0]).toMatchObject({ file_id: file.file_id }); + expect(saved.files[0]).not.toHaveProperty('text'); + expect(saved.files[0]).not.toHaveProperty('reading'); + expect(error).toBe(new api.FileSearchPreparationError().message); + return true; + }, + ); + + await AgentController( + createFailedRequest({ text: message.text, files: refs }), + createResumableResponse(), + jest.fn(), + jest.fn().mockResolvedValue({ client }), + null, + ); + await flushBackgroundGeneration(); + expect(mockGenerationJobManager.completeJob).toHaveBeenCalled(); + const saved = mockSaveMessage.mock.calls.find( + ([, row]) => row.messageId === message.messageId, + )?.[1]; + expect(saved.files[0]).not.toHaveProperty('reading'); + expect(agent.provisionState.vectorDBFiles).toEqual([file]); + }); + it.each(['history', 'exact-model'])( 'does not persist a protected turn rejected by %s policy during terminal recovery', async (boundary) => { @@ -3895,6 +3981,28 @@ describe('ResumableAgentController resume metadata', () => { ); }); + it('saves an edited resubmission that fails early with the file refs the request carried', async () => { + const initializeClient = jest.fn().mockRejectedValue(new Error('model unavailable')); + const replayedFile = { + file_id: 'book', + filename: 'book.xlsx', + type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + }; + + await AgentController( + createFailedRequest({ files: [replayedFile] }), + createResumableResponse(), + jest.fn(), + initializeClient, + null, + ); + + const [, savedUser] = mockSaveMessage.mock.calls.find( + ([, message]) => message?.messageId === 'user-message', + ); + expect(savedUser.files).toEqual([replayedFile]); + }); + it('allows a follow-up to chain from the persisted failed response', async () => { const initializeClient = jest.fn().mockRejectedValue(new Error('model unavailable')); await AgentController( diff --git a/api/server/controllers/agents/__tests__/resume.spec.js b/api/server/controllers/agents/__tests__/resume.spec.js index b5baf0c4a71..f3d36626db7 100644 --- a/api/server/controllers/agents/__tests__/resume.spec.js +++ b/api/server/controllers/agents/__tests__/resume.spec.js @@ -3655,6 +3655,116 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => { }); }); + /** A resumed turn whose only attachment File Search prepares with `outcome`. */ + const prepareResumedSearchTurn = (outcome) => { + const { + resolveTurnDeliveryRouting, + buildTurnReadingContext, + buildMessageFiles, + createProvisionFilesCallback, + FileSearchPreparationError, + } = jest.requireActual('@librechat/api'); + const file = { + user: USER_ID, + file_id: 'search-brief', + filename: 'search-brief.pdf', + filepath: '/uploads/search-brief.pdf', + type: 'application/pdf', + bytes: 15 * 1024 * 1024, + source: 'local', + context: 'message_attachment', + llmDeliveryPath: 'none', + metadata: { destinationChosen: false }, + text: 'cached text must not reach status', + }; + const preparation = new Map(); + const provisionState = { + codeEnvFiles: [], + vectorDBFiles: [file], + aliveFileIds: new Set(), + agentScopedFileIds: new Set(), + searchPreparation: preparation, + }; + const agent = { + id: AGENT_ID, + provider: 'openAI', + endpoint: 'openAI', + fileConsumers: { executeCode: false, fileSearch: true }, + currentRequestAttachments: [file], + provisionState, + }; + agent.deliveryRouting = resolveTurnDeliveryRouting({ + agent, + config: { fileConfig: { endpoints: { openAI: { llmDeliveryPolicy: 'automatic' } } } }, + }); + agent.deliveryRouting.reading = buildTurnReadingContext({ + routing: agent.deliveryRouting, + provider: 'openAI', + fileTokenLimit: 100000, + configuredFileSizeLimit: undefined, + countTokens: (text) => text.length, + }); + agent.deliveryRouting.reading.setSearchEvidence({ + queued: [file.file_id], + registered: [], + preparation, + }); + const refs = buildMessageFiles([{ file_id: file.file_id }], [file]); + mockGetMessages.mockResolvedValue([{ files: refs }]); + mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob()); + const context = { agentId: AGENT_ID, provisionState }; + const provisionFiles = createProvisionFilesCallback({ + req: { user: { id: USER_ID } }, + agentToolContexts: new Map([[AGENT_ID, context]]), + provisionToVectorDB: async () => ({ embedded: outcome !== 'failure' }), + provisionToCodeEnv: jest.fn(), + updateFile: jest.fn(), + updateCodeEnvRef: jest.fn(), + addEmbeddedEntity: jest.fn(), + }); + const client = makeClient({ options: { attachments: [file], agent } }); + client.resumeCompletion.mockImplementation(async () => { + await provisionFiles(['file_search'], AGENT_ID); + }); + mockInitializeClient.mockResolvedValue({ client, userMCPAuthMap: {} }); + mockSaveMessage.mockImplementation(async (_ctx, message) => message); + return { file, refs, FileSearchPreparationError }; + }; + + it('fails the resumed turn with FileSearchPreparationError when search preparation fails', async () => { + const { file, FileSearchPreparationError } = prepareResumedSearchTurn('failure'); + + const response = await post(approveBody()); + expect(response.status).toBe(200); + await settled; + await flush(); + + expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith( + CONVO_ID, + new FileSearchPreparationError().message, + 1000, + ); + expect(JSON.stringify(mockGenerationJobManager.completeJob.mock.calls)).not.toContain( + file.text, + ); + expect(mockGenerationJobManager.publishTerminalClaim).not.toHaveBeenCalled(); + }); + + it('finalizes the resumed turn with the restored refs as they are, without file text, once search preparation succeeds', async () => { + const { file, refs } = prepareResumedSearchTurn('success'); + + const response = await post(approveBody()); + expect(response.status).toBe(200); + await settled; + await flush(); + + expect(mockGenerationJobManager.publishTerminalClaim).toHaveBeenCalledTimes(1); + const [, finalEvent] = mockGenerationJobManager.publishTerminalClaim.mock.calls[0]; + expect(finalEvent.requestMessage.files).toEqual(refs); + expect(finalEvent.requestMessage.files[0]).not.toHaveProperty('reading'); + expect(JSON.stringify(finalEvent)).not.toContain(file.text); + }); + it('persists the response, claims terminal ownership, emits done, finishes, and prunes', async () => { mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob()); await post(approveBody()); diff --git a/api/server/controllers/agents/client.js b/api/server/controllers/agents/client.js index 746cb5b7e4e..33c64fee324 100644 --- a/api/server/controllers/agents/client.js +++ b/api/server/controllers/agents/client.js @@ -150,6 +150,10 @@ const { createAgentMemoryCallback, assertAgentAttachmentLimits, assertAgentAttachmentTopology, + allocateTurnAttachmentsWithHistory, + allocateTurnAttachmentsWithRetainedContext, + admitNativeFallbackAttachments, + retainNativeAttachmentTopology, collectHistoricalAttachmentIds, admitSteerAttachmentHistory, rollbackSteerAttachmentHistory, @@ -159,6 +163,7 @@ const { buildAgentScopedContext, buildAgentScopedAttachmentMap, resolveScopedTurnAttachments, + prepareScopedTurnCandidates, buildAgentContextAttachmentsByAgentId, buildSkillPrimeContentParts, buildInitialToolSessions, @@ -407,6 +412,11 @@ class AgentClient extends BaseClient { ); } + /** The primary agent and the handoff agents that share its conversation payload. */ + getConversationAgents() { + return [this.options.agent, ...(this.agentConfigs?.values() ?? [])]; + } + async addDocuments(message, attachments) { const memoryContext = { req: this.options.req, @@ -422,6 +432,10 @@ class AgentClient extends BaseClient { } } + admitPreparedAttachments(files, fileConsumers) { + return admitNativeFallbackAttachments(this, files, fileConsumers); + } + async processAttachments(message, attachments, fileConsumers) { const modelBoundAttachments = this.getModelBoundAttachmentsForEndpoint(attachments); const processableAttachments = this.getProcessableAttachmentsForEndpoint( @@ -469,6 +483,7 @@ class AgentClient extends BaseClient { attachmentsByAgentId, sharedRunAttachmentIds: sharedAttachmentIds, req: this.options.req, + consumers: this.options.agent?.fileConsumers, endpoint: this.options.agent?.endpoint ?? this.options.endpoint ?? EModelEndpoint.agents, endpointType: this.options.endpointType, endpointsByAgentId, @@ -505,6 +520,7 @@ class AgentClient extends BaseClient { historicalFileIds, scopedAttachmentsByAgentId, req: this.options.req, + consumers: this.options.agent?.fileConsumers, endpoint: this.options.agent?.endpoint ?? this.options.endpoint, endpointType: this.options.endpointType, endpointsByAgentId: this.turnAttachmentEndpointsByAgentId, @@ -527,7 +543,34 @@ class AgentClient extends BaseClient { } async assertHistoricalAttachmentLimits(historicalAttachments) { - const currentAttachments = this.options.attachments ? await this.options.attachments : []; + const agents = collectReachableAgents([ + this.options.agent, + ...(this.agentConfigs?.values() ?? []), + ]); + const endpointsByAgentId = new Map( + agents + .filter((agent) => agent?.id) + .map((agent) => [ + agent.id, + { + endpoint: agent.endpoint, + endpointType: agent === this.options.agent ? this.options.endpointType : undefined, + }, + ]), + ); + const currentAttachments = await allocateTurnAttachmentsWithHistory({ + agent: this.options.agent, + historical: historicalAttachments, + current: this.options.attachments ? await this.options.attachments : [], + req: this.options.req, + endpoint: this.options.agent?.endpoint ?? this.options.endpoint, + endpointType: this.options.endpointType, + endpointsByAgentId, + onAllocated: (files) => { + this.options.attachments = files; + }, + signal: this.options.abortController?.signal, + }); const compatibleHistoricalAttachments = this.getModelBoundAttachmentsForEndpoint(historicalAttachments); const compatibleCurrentAttachments = @@ -539,36 +582,28 @@ class AgentClient extends BaseClient { ); const sharedAttachments = [...compatibleHistoricalAttachments, ...compatibleCurrentAttachments]; const sharedAttachmentIds = collectFileIds(sharedAttachments); - const agents = collectReachableAgents([ - this.options.agent, - ...(this.agentConfigs?.values() ?? []), - ]); const scopedAttachmentMap = this.getFilteredScopedAttachmentMap( sharedAttachmentIds, this.options.agentContextAttachmentsByAgentId, agents, ); - const endpointsByAgentId = new Map( - agents - .filter((agent) => agent?.id) - .map((agent) => [ - agent.id, - { - endpoint: agent.endpoint, - endpointType: agent === this.options.agent ? this.options.endpointType : undefined, - }, - ]), - ); assertAgentAttachmentTopology({ sharedAttachments, historicalFileIds: this.turnHistoricalAttachmentIds, scopedAttachmentsByAgentId: scopedAttachmentMap, req: this.options.req, + consumers: this.options.agent?.fileConsumers, endpoint: this.options.agent?.endpoint ?? this.options.endpoint, endpointType: this.options.endpointType, endpointsByAgentId, }); this.assertTurnAttachmentLimits(sharedAttachments, [...scopedAttachmentMap.values()].flat()); + retainNativeAttachmentTopology( + this, + sharedAttachments, + scopedAttachmentMap, + endpointsByAgentId, + ); return compatibleHistoricalAttachments; } @@ -2474,9 +2509,6 @@ class AgentClient extends BaseClient { files: [...modelBoundFileContexts], }); const requestAttachmentsSource = this.options.attachments; - const requestAttachments = requestAttachmentsSource ? await requestAttachmentsSource : []; - const modelBoundRequestAttachments = - this.getModelBoundAttachmentsForEndpoint(requestAttachments); const retainedHistoricalFileContexts = this.options.resendFiles === false ? orderedMessages @@ -2489,13 +2521,29 @@ class AgentClient extends BaseClient { bytes: Buffer.byteLength(message.fileContext, 'utf8'), })) : []; + const requestAttachments = await allocateTurnAttachmentsWithRetainedContext({ + agent: this.options.agent, + resendFiles: this.options.resendFiles, + retained: retainedHistoricalFileContexts, + current: requestAttachmentsSource ? await requestAttachmentsSource : [], + req: this.options.req, + endpoint: this.options.agent?.endpoint ?? this.options.endpoint, + endpointType: this.options.endpointType, + endpointsByAgentId, + onAllocated: (files) => { + this.options.attachments = files; + }, + signal: this.options.abortController?.signal, + }); + const modelBoundRequestAttachments = + this.getModelBoundAttachmentsForEndpoint(requestAttachments); const sharedAttachmentFiles = [ ...Object.values(this.message_file_map ?? {}).flat(), ...(this.modelBoundHistoricalSteerFiles ?? []), ...modelBoundRequestAttachments, ]; const sharedRunAttachmentIds = collectFileIds(sharedAttachmentFiles); - this.options.agentContextAttachmentsByAgentId = resolveScopedTurnAttachments({ + const scopedInputs = { agents: allAgents, sharedConversationAgentIds: [this.options.agent.id, ...(this.agentConfigs?.keys() ?? [])], resendFiles: this.options.resendFiles, @@ -2504,12 +2552,23 @@ class AgentClient extends BaseClient { requestAttachments, sharedRunAttachmentIds, attachmentsByAgentId: this.options.agentContextAttachmentsByAgentId, + }; + const scopedCandidates = await prepareScopedTurnCandidates({ + ...scopedInputs, + appConfig: this.options.req?.config, + endpointsByAgentId, + signal: this.options.abortController?.signal, + }); + this.options.agentContextAttachmentsByAgentId = resolveScopedTurnAttachments({ + ...scopedInputs, + ...scopedCandidates, }); const scopedAttachmentMap = buildAgentScopedAttachmentMap({ agentIds: allAgents.map(({ agentId }) => agentId), attachmentsByAgentId: this.options.agentContextAttachmentsByAgentId, sharedRunAttachmentIds, req: this.options.req, + consumers: this.options.agent?.fileConsumers, endpoint: this.options.agent?.endpoint ?? this.options.endpoint ?? EModelEndpoint.agents, endpointType: this.options.endpointType, endpointsByAgentId, @@ -2528,6 +2587,7 @@ class AgentClient extends BaseClient { historicalFileIds: this.turnHistoricalAttachmentIds, scopedAttachmentsByAgentId: scopedAttachmentMap, req: this.options.req, + consumers: this.options.agent?.fileConsumers, endpoint: this.options.agent?.endpoint ?? this.options.endpoint, endpointType: this.options.endpointType, endpointsByAgentId, @@ -2585,10 +2645,12 @@ class AgentClient extends BaseClient { }; } - const [, files] = await Promise.all([ - this.addFileContextToMessage(latestMessage, modelBoundRequestAttachments), - this.processAttachments(latestMessage, attachments), - ]); + const files = await this.processMessageAttachments( + latestMessage, + attachments, + undefined, + modelBoundRequestAttachments, + ); this.options.attachments = files; } else { @@ -5592,6 +5654,10 @@ class AgentClient extends BaseClient { checkpointNamespace: this.checkpointNamespace, agents, files: [...modelBoundAgentFiles, ...dynamicToolContexts], + delivery: { + routing: this.options.agent?.deliveryRouting, + consumers: this.options.agent?.fileConsumers, + }, }, { getAgentCheckpointer, diff --git a/api/server/controllers/agents/client.test.js b/api/server/controllers/agents/client.test.js index 3d30076aef9..a7fed44fc7d 100644 --- a/api/server/controllers/agents/client.test.js +++ b/api/server/controllers/agents/client.test.js @@ -10,6 +10,18 @@ const mockHasDurableAgentInterruptCheckpoint = jest.fn().mockResolvedValue(true) const mockBuildAgentScopedContext = jest.fn((...args) => jest.requireActual('@librechat/api').buildAgentScopedContext(...args), ); +const mockPrepareScopedTurnCandidates = jest.fn((...args) => + jest.requireActual('@librechat/api').prepareScopedTurnCandidates(...args), +); +const mockResolveScopedTurnAttachments = jest.fn((...args) => + jest.requireActual('@librechat/api').resolveScopedTurnAttachments(...args), +); +const mockAllocateTurnAttachmentsWithHistory = jest.fn((...args) => + jest.requireActual('@librechat/api').allocateTurnAttachmentsWithHistory(...args), +); +const mockAllocateTurnAttachmentsWithRetainedContext = jest.fn((...args) => + jest.requireActual('@librechat/api').allocateTurnAttachmentsWithRetainedContext(...args), +); const mockFormatAgentMessages = jest.fn(() => ({ messages: [], indexTokenCountMap: {}, @@ -276,6 +288,11 @@ jest.mock('@librechat/agents', () => ({ jest.mock('@librechat/api', () => ({ ...jest.requireActual('@librechat/api'), buildAgentScopedContext: (...args) => mockBuildAgentScopedContext(...args), + prepareScopedTurnCandidates: (...args) => mockPrepareScopedTurnCandidates(...args), + resolveScopedTurnAttachments: (...args) => mockResolveScopedTurnAttachments(...args), + allocateTurnAttachmentsWithHistory: (...args) => mockAllocateTurnAttachmentsWithHistory(...args), + allocateTurnAttachmentsWithRetainedContext: (...args) => + mockAllocateTurnAttachmentsWithRetainedContext(...args), checkAccess: jest.fn(), createRun: (...args) => mockCreateRun(...args), countFormattedMessageTokens: jest.fn(() => 42), @@ -5565,6 +5582,7 @@ describe('AgentClient - titleConvo', () => { null, {}, ); + await new Promise((resolve) => setImmediate(resolve)); expect(client.contextHandlers.createContext).toHaveBeenCalledTimes(1); expect(client.useMemory).toHaveBeenCalledTimes(1); @@ -5623,7 +5641,7 @@ describe('AgentClient - titleConvo', () => { expect(client.processAttachments).not.toHaveBeenCalled(); requestAttachments.resolve([requestFile]); - await Promise.resolve(); + await new Promise((resolve) => setImmediate(resolve)); expect(mockBuildAgentScopedContext).toHaveBeenCalledTimes(1); const scopedContextArgs = mockBuildAgentScopedContext.mock.calls[0][0]; @@ -6604,6 +6622,59 @@ describe('AgentClient - titleConvo', () => { expect(() => client.admitSteerAttachments([current], 'later-steer')).not.toThrow(); }); + it('rolls back prepared native fallback copies after steer extraction fails with File Search loaded but unreachable', async () => { + const { buildTurnReadingContext } = jest.requireActual('@librechat/api'); + mockReq.config.fileConfig = { endpoints: { openAI: { llmDeliveryPolicy: 'automatic' } } }; + /** Automatic reading needs a loaded file tool; text is the fallback once File Search + * cannot reach the file. */ + mockAgent.fileConsumers = { executeCode: false, fileSearch: true }; + mockAgent.deliveryRouting = jest.requireActual('@librechat/api').resolveTurnDeliveryRouting({ + agent: mockAgent, + config: mockReq.config, + }); + const context = buildTurnReadingContext({ + routing: mockAgent.deliveryRouting, + provider: EModelEndpoint.openAI, + fileTokenLimit: 1000, + configuredFileSizeLimit: undefined, + countTokens: (text) => text.length, + }); + context.setSearchEvidence({ queued: [], registered: [], preparation: new Map() }); + mockAgent.deliveryRouting.reading = context; + const file = { + ...makeUploadedFile('steered-pdf', 'steered.pdf', 'application/pdf'), + context: 'message_attachment', + llmDeliveryPath: 'provider', + text: 'Complete PDF fallback.', + metadata: { destinationChosen: false }, + }; + client.turnSharedAttachmentFiles = []; + client.turnHistoricalAttachmentIds = new Set(); + client.turnAttachmentEndpointsByAgentId = new Map([ + [mockAgent.id, { endpoint: mockAgent.endpoint }], + ]); + client.attachmentMemoryContext = { attachments: [] }; + client.admitSteerAttachments([file], 'failed-native-steer'); + context.recordRejections([{ file_id: file.file_id, reason: 'capacity' }]); + client.addFileContextToMessage = jest.fn().mockRejectedValue(new Error('extraction failed')); + + await expect(client.processMessageAttachments({}, [file])).rejects.toThrow( + 'extraction failed', + ); + expect(client.turnSharedAttachmentFiles[0]).toMatchObject({ llmDeliveryPath: 'text' }); + expect(client.admittedSteerAttachments.get('failed-native-steer')[0]).toBe( + client.turnSharedAttachmentFiles[0], + ); + client.rollbackSteerAttachmentAdmission('failed-native-steer'); + + expect(client.turnSharedAttachmentFiles).toEqual([]); + expect(client.attachmentMemoryContext.attachments).toEqual([]); + expect(client.admittedSteerAttachments.has('failed-native-steer')).toBe(false); + expect(() => + client.admitSteerAttachments([makeTextFile('later', 'later.txt', 'later')], 'later'), + ).not.toThrow(); + }); + it('keeps committed resubmissions counted when a later duplicate steer fails', () => { mockReq.config.fileConfig = { endpoints: { openAI: { fileLimit: 1 } } }; const historical = makeTextFile('history', 'history.txt', 'history'); @@ -6684,6 +6755,185 @@ describe('AgentClient - titleConvo', () => { expect(client.processAttachments).not.toHaveBeenCalled(); }); + describe('history allocation under the automatic policy', () => { + const historicalPdf = { + ...makeUploadedFile('history-1', 'one.pdf', 'application/pdf'), + bytes: 600_000, + context: 'message_attachment', + llmDeliveryPath: 'provider', + metadata: { destinationChosen: false }, + }; + const currentPdf = { + ...makeUploadedFile('current-1', 'two.pdf', 'application/pdf'), + bytes: 600_000, + context: 'message_attachment', + llmDeliveryPath: 'provider', + metadata: { destinationChosen: false }, + }; + const historyMessages = [ + { messageId: 'msg-1', isCreatedByUser: true, files: [{ file_id: historicalPdf.file_id }] }, + ]; + + const readAutomatically = (endpointConfig, deriveText) => { + const { resolveTurnDeliveryRouting, buildTurnReadingContext } = + jest.requireActual('@librechat/api'); + client.options.resendFiles = true; + mockReq.config.fileConfig = { endpoints: { openAI: endpointConfig } }; + mockAgent.fileConsumers = { executeCode: false, fileSearch: true }; + mockAgent.deliveryRouting = resolveTurnDeliveryRouting({ + agent: mockAgent, + config: mockReq.config, + }); + mockAgent.deliveryRouting.reading = buildTurnReadingContext({ + routing: mockAgent.deliveryRouting, + provider: EModelEndpoint.openAI, + model: 'gpt-4', + fileTokenLimit: 1000, + configuredFileSizeLimit: undefined, + countTokens: (text) => text.length, + deriveText, + }); + client.options.attachments = [currentPdf]; + require('~/models').getFiles.mockResolvedValue([historicalPdf]); + client.addFileContextToMessage = jest.fn(); + client.processAttachments = jest.fn(); + return mockAgent.deliveryRouting.reading; + }; + + it('allocates history beside the current attachments and adopts the copies it hands back', async () => { + readAutomatically({ fileLimit: 10, totalSizeLimit: 1, llmDeliveryPolicy: 'automatic' }); + + await expect(client.addPreviousAttachments(historyMessages)).resolves.toEqual( + expect.any(Array), + ); + + expect(mockAllocateTurnAttachmentsWithHistory).toHaveBeenCalledTimes(1); + expect(mockAllocateTurnAttachmentsWithHistory).toHaveBeenCalledWith( + expect.objectContaining({ + agent: mockAgent, + historical: [expect.objectContaining({ file_id: historicalPdf.file_id })], + current: [currentPdf], + onAllocated: expect.any(Function), + }), + ); + const [rerouted] = client.options.attachments; + expect(rerouted).not.toBe(currentPdf); + expect(rerouted).toEqual( + expect.objectContaining({ file_id: currentPdf.file_id, llmDeliveryPath: 'none' }), + ); + expect(client.processAttachments).toHaveBeenCalledWith( + expect.anything(), + [historicalPdf], + undefined, + ); + }); + + it('reserves repeated historical steer copies before native fallback is extracted with File Search loaded but unreachable', async () => { + const reading = readAutomatically( + { fileLimit: 10, llmDeliveryPolicy: 'automatic' }, + async () => ({ + status: 'derived', + text: 'complete', + textDerivation: { outcome: 'complete', extractor: 'document_parser' }, + }), + ); + mockReq.config.fileConfig.fileContextCharLimit = 10; + /** Automatic reading needs a loaded file tool; the text fallback follows once File + * Search cannot reach the replayed file. */ + reading.setSearchEvidence({ queued: [], registered: [], preparation: new Map() }); + const file = { + ...historicalPdf, + metadata: { destinationChosen: false, textDerivation: { outcome: 'deferred' } }, + }; + client.options.attachments = []; + require('~/models').getFiles.mockResolvedValue([file]); + client.processAttachments = jest.fn(async (_message, files) => { + expect(client.turnSharedAttachmentFiles).toHaveLength(3); + reading.recordRejections([{ file_id: file.file_id, reason: 'capacity' }]); + return files; + }); + + await client.addPreviousAttachments([ + { + messageId: 'repeated-history', + isCreatedByUser: true, + files: [{ file_id: file.file_id }], + content: [ + { type: ContentTypes.STEER, files: [{ file_id: file.file_id }] }, + { type: ContentTypes.STEER, files: [{ file_id: file.file_id }] }, + ], + }, + ]); + + expect(client.message_file_map['repeated-history']).toEqual([]); + expect(client.addFileContextToMessage).toHaveBeenCalledWith( + expect.anything(), + [expect.objectContaining({ file_id: file.file_id, llmDeliveryPath: 'none' })], + undefined, + ); + }); + + describe('without file replay', () => { + const retainedHistory = { + messageId: 'historical-context', + parentMessageId: null, + sender: 'User', + text: 'Earlier context.', + isCreatedByUser: true, + fileContext: 'r'.repeat(500), + }; + const latestMessage = { + messageId: 'msg-1', + parentMessageId: 'historical-context', + sender: 'User', + text: 'Continue.', + isCreatedByUser: true, + }; + const DOCX_TYPE = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'; + const notes = { + ...makeUploadedFile('notes', 'notes.docx', DOCX_TYPE), + context: 'message_attachment', + llmDeliveryPath: 'text', + text: 'n'.repeat(600), + metadata: { destinationChosen: false }, + }; + + it('allocates the current file against the retained text and adopts the copy it hands back', async () => { + readAutomatically({ llmDeliveryPolicy: 'automatic' }); + client.options.resendFiles = false; + mockReq.config.fileConfig.fileContextCharLimit = 1000; + client.options.attachments = [notes]; + client.processAttachments = jest.fn(async (_message, attachments) => attachments); + + await expect( + client.buildMessages([retainedHistory, latestMessage], 'msg-1', {}), + ).resolves.toEqual(expect.objectContaining({ prompt: expect.any(Array) })); + + expect(mockAllocateTurnAttachmentsWithRetainedContext).toHaveBeenCalledTimes(1); + expect(mockAllocateTurnAttachmentsWithRetainedContext).toHaveBeenCalledWith( + expect.objectContaining({ + agent: mockAgent, + resendFiles: false, + retained: [expect.objectContaining({ text: retainedHistory.fileContext })], + current: [notes], + onAllocated: expect.any(Function), + }), + ); + expect(client.options.attachments).toEqual([ + expect.objectContaining({ file_id: 'notes', llmDeliveryPath: 'none' }), + ]); + }); + }); + }); + + it('lists the primary and handoff agents as the conversation agents', () => { + const handoff = { id: 'handoff' }; + client.agentConfigs = new Map([['handoff', handoff]]); + + expect(client.getConversationAgents()).toEqual([client.options.agent, handoff]); + client.agentConfigs = undefined; + }); + it('rejects combined historical and scoped bytes before history is encoded', async () => { mockAgent.endpoint = 'Moonshot'; client.options.endpointType = EModelEndpoint.custom; @@ -6873,6 +7123,32 @@ describe('AgentClient - titleConvo', () => { expect(client.message_file_map['msg-1']).toEqual([toolOnly, admitted]); }); + it.each([{ disabled: true }, { supportedMimeTypes: ['^image/png$'] }])( + 'excludes rejected legacy text from current context (%j)', + async (endpointConfig) => { + mockReq.config.fileConfig = { endpoints: { openAI: endpointConfig } }; + const rejected = { + ...makeUploadedFile('rejected-legacy', 'rejected.txt', 'text/plain'), + text: 'Rejected text must stay out of the prompt.', + }; + client.options.attachments = [rejected]; + client.addFileContextToMessage = jest.fn(); + client.processAttachments = jest.fn(async (_message, files) => files); + + await client.buildMessages( + [{ messageId: 'msg-1', sender: 'User', text: 'Continue.', isCreatedByUser: true }], + 'msg-1', + {}, + ); + + expect(client.addFileContextToMessage).toHaveBeenCalledWith( + expect.objectContaining({ messageId: 'msg-1' }), + [], + ); + expect(client.processAttachments).toHaveBeenCalledWith(expect.anything(), []); + }, + ); + it('defers current attachment persistence until model admission', () => { client.modelBoundCurrentFiles = [makeTextFile('pending', 'pending.txt', 'context')]; @@ -7372,6 +7648,87 @@ describe('AgentClient - titleConvo', () => { }, ); + it('derives text a handoff receiver needs before resolving its scoped attachments', async () => { + const { resolveTurnDeliveryRouting, buildTurnReadingContext } = + jest.requireActual('@librechat/api'); + const workbook = { + ...makeUploadedFile( + 'deferred-xlsx', + 'quarterly.xlsx', + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + ), + context: 'message_attachment', + llmDeliveryPath: 'none', + metadata: { destinationChosen: false, textDerivation: { outcome: 'deferred', at: 1 } }, + }; + const derivedMarker = { outcome: 'complete', extractor: 'document_parser', at: 2 }; + const deriveText = jest.fn(async () => ({ + status: 'derived', + text: 'Q1 total 1200', + textDerivation: derivedMarker, + })); + client.options.req.config.fileConfig = { + endpoints: { default: { llmDeliveryPolicy: 'automatic' } }, + }; + mockAgent.deliveryRouting = resolveTurnDeliveryRouting({ + agent: mockAgent, + config: client.options.req.config, + }); + mockAgent.fileConsumers = { executeCode: true, fileSearch: false }; + const handoffAgent = { + id: 'handoff-agent', + endpoint: EModelEndpoint.openAI, + provider: EModelEndpoint.openAI, + instructions: 'Handoff instructions', + model_parameters: { model: 'gpt-4' }, + tools: [], + fileConsumers: { executeCode: false, fileSearch: false }, + }; + handoffAgent.deliveryRouting = resolveTurnDeliveryRouting({ + agent: handoffAgent, + config: client.options.req.config, + }); + handoffAgent.deliveryRouting.reading = buildTurnReadingContext({ + routing: handoffAgent.deliveryRouting, + provider: EModelEndpoint.openAI, + fileTokenLimit: 100000, + configuredFileSizeLimit: undefined, + countTokens: (text) => text.length, + deriveText, + }); + client.agentConfigs = new Map([['handoff-agent', handoffAgent]]); + client.options.resendFiles = true; + client.options.attachments = []; + client.authorizedHistoricalFiles = new Map([[workbook.file_id, workbook]]); + client.message_file_map = {}; + + await client.buildMessages( + [ + { + messageId: 'msg-1', + sender: 'User', + text: 'Total it', + isCreatedByUser: true, + files: [{ file_id: workbook.file_id }], + }, + ], + 'msg-1', + {}, + ); + + expect(mockPrepareScopedTurnCandidates).toHaveBeenCalledTimes(1); + expect(mockResolveScopedTurnAttachments).toHaveBeenCalledTimes(1); + expect(mockPrepareScopedTurnCandidates.mock.invocationCallOrder[0]).toBeLessThan( + mockResolveScopedTurnAttachments.mock.invocationCallOrder[0], + ); + const [{ candidates }] = mockResolveScopedTurnAttachments.mock.calls[0]; + expect(candidates.get(workbook.file_id)).toEqual( + expect.objectContaining({ text: 'Q1 total 1200' }), + ); + expect(handoffAgent.additional_instructions).toContain('Q1 total 1200'); + expect(mockAgent.additional_instructions ?? '').not.toContain('Q1 total 1200'); + }); + it('places request context inline and applies each agent context doc only once', async () => { const requestFile = makeTextFile('request-file', 'request.txt', 'Shared request context'); const primaryContext = makeTextFile( diff --git a/api/server/controllers/agents/openai.js b/api/server/controllers/agents/openai.js index c768bdefbcf..02ec6085399 100644 --- a/api/server/controllers/agents/openai.js +++ b/api/server/controllers/agents/openai.js @@ -1094,6 +1094,7 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => { agentIds: contextAgents.map(({ id }) => id), attachmentsByAgentId: buildAgentContextAttachmentsByAgentId(contextAgents), req, + consumers: primaryConfig.fileConsumers, endpoint: primaryConfig.endpoint, endpointsByAgentId: new Map( contextAgents.map((runAgent) => [runAgent.id, { endpoint: runAgent.endpoint }]), diff --git a/api/server/routes/files/files.agents.test.js b/api/server/routes/files/files.agents.test.js index d1afe540567..cb72ab1399f 100644 --- a/api/server/routes/files/files.agents.test.js +++ b/api/server/routes/files/files.agents.test.js @@ -973,6 +973,73 @@ describe('File Routes - Agent Files Endpoint', () => { readFile.mockRestore(); }); + describe('upload preflight under each delivery policy', () => { + const xlsx = { + originalname: 'quarterly.xlsx', + mimetype: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + bytes: Buffer.from([0x50, 0x4b, 0x03, 0x04, 0, 255, 0, 1, 2, 3]), + }; + /** [label, filters, status the opaque workbook bytes draw before processing] */ + const cases = [ + ['no filters', undefined, 200], + [ + 'uninspectable block on content', + { files: { pii: { fields: ['content'], uninspectable: 'block' } } }, + 400, + ], + [ + 'uninspectable block on extracted text only', + { files: { pii: { fields: ['extracted_text'], uninspectable: 'block' } } }, + 200, + ], + ]; + + it.each(cases)( + 'answers an xlsx message attachment under %s identically', + async (_label, filters, expectedStatus) => { + const { originalname, mimetype, bytes } = xlsx; + await createAgent({ + id: agentCustomId, + name: 'Test Agent', + provider: 'openai', + model: 'gpt-4', + author: authorId, + }); + const readFile = jest.spyOn(fs, 'readFile').mockResolvedValue(bytes); + const send = async (llmDeliveryPolicy) => { + processAgentFileUpload.mockClear(); + const testApp = createAppWithUser( + authorId, + SystemRoles.USER, + { + fileConfig: llmDeliveryPolicy ? { llmDeliveryPolicy } : {}, + ...(filters ? { filters } : {}), + }, + { originalname, mimetype }, + ); + const response = await request(testApp).post('/files').send({ + endpoint: 'agents', + agent_id: agentCustomId, + message_file: 'true', + file_id: uuidv4(), + }); + return { + status: response.status, + body: response.body, + processed: processAgentFileUpload.mock.calls.length, + }; + }; + + const classic = await send(undefined); + const automatic = await send('automatic'); + readFile.mockRestore(); + + expect(classic.status).toBe(expectedStatus); + expect(automatic).toEqual(classic); + }, + ); + }); + it('detects opaque content after a long printable file prefix', async () => { await createAgent({ id: agentCustomId, diff --git a/api/server/services/Endpoints/agents/addedConvo.js b/api/server/services/Endpoints/agents/addedConvo.js index e598fe46c4a..cf413f7a427 100644 --- a/api/server/services/Endpoints/agents/addedConvo.js +++ b/api/server/services/Endpoints/agents/addedConvo.js @@ -70,6 +70,8 @@ const loadAddedAgent = (params) => * Resolver for the added agent's own `instructionsPrompt` link, forwarded verbatim. * @param {boolean} [params.recordLinkedPromptUsage] - Forwarded to `initializeAgent`; * defaults to `true` there when omitted. + * @param {import('@librechat/api').FileTextDeriver} [params.deriveText] + * @param {import('@librechat/api').TextDerivationPersister} [params.persistDerivation] * @param {AbortSignal} [params.signal] - Owning run cancellation signal. * @returns {Promise<{userMCPAuthMap: Object|undefined}>} The updated userMCPAuthMap */ @@ -104,6 +106,8 @@ const processAddedConvo = async ({ toolIntentsAvailable, statefulSessionsAvailable, memoryAvailable, + deriveText, + persistDerivation, signal, }) => { const addedConvo = endpointOption.addedConvo; @@ -210,6 +214,8 @@ const processAddedConvo = async ({ memoryAvailable, skillStates, defaultActiveOnShare, + deriveText, + persistDerivation, signal, }, { diff --git a/api/server/services/Endpoints/agents/addedConvo.spec.js b/api/server/services/Endpoints/agents/addedConvo.spec.js index 8224eb4f5d3..dbe1fa8cbad 100644 --- a/api/server/services/Endpoints/agents/addedConvo.spec.js +++ b/api/server/services/Endpoints/agents/addedConvo.spec.js @@ -120,6 +120,21 @@ describe('processAddedConvo', () => { ); }); + it.each([ + ['present', { deriveText: jest.fn(), persistDerivation: jest.fn() }], + ['absent', { deriveText: undefined, persistDerivation: undefined }], + ])( + 'shares the request text deriver and persister with the parallel agent as supplied (%s)', + async (_label, derivation) => { + await processAddedConvo(baseParams(derivation)); + + expect(mockInitializeAgent).toHaveBeenCalledWith( + expect.objectContaining(derivation), + expect.anything(), + ); + }, + ); + /** The added convo re-hydrates the same conversation's prior-turn files, so a * denied `FILE_SEARCH` grant has to travel with it — otherwise the parallel * agent primes the search files the primary just skipped. `undefined` stays diff --git a/api/server/services/Endpoints/agents/initialize.js b/api/server/services/Endpoints/agents/initialize.js index b82704399ce..7f13eca06ec 100644 --- a/api/server/services/Endpoints/agents/initialize.js +++ b/api/server/services/Endpoints/agents/initialize.js @@ -9,6 +9,8 @@ const { loadSkillStates, resolveInitializationProjectContext, initializeAgent, + createFileTextDeriver, + createDerivationPersister, primeInvokedSkillsForProfiles, validateAgentModel, extractManualSkills, @@ -69,6 +71,7 @@ const { MAX_SUBAGENT_GRAPH_NODES, MAX_SUBAGENT_RUN_CONFIGS, isEphemeralAgentId, + mergeFileConfig, resolveAllowedStatefulCodeEnvironments, } = require('librechat-data-provider'); const { @@ -96,6 +99,7 @@ const { } = require('./skillDeps'); const { loadCodeApiKey, + openStoredFile, provisionToCodeEnv, provisionToVectorDB, checkSessionsAlive, @@ -771,6 +775,16 @@ const initializeClientWithProvider = async ({ skillsCapabilityEnabled, ephemeralSkillsToggle, }); + const deriveText = createFileTextDeriver({ + req, + openStoredFile, + filters: appConfig?.filters, + textMimeTypes: mergeFileConfig(appConfig?.fileConfig).text?.supportedMimeTypes, + }); + const persistDerivation = createDerivationPersister(db.saveFileTextDerivation, { + user: req.user.id, + tenantId: req.user.tenantId, + }); const primaryConfig = await initializeAgent( { useChatProjectContext: true, @@ -799,6 +813,8 @@ const initializeClientWithProvider = async ({ skillStates, defaultActiveOnShare, manualSkills, + deriveText, + persistDerivation, signal, }, { @@ -809,6 +825,7 @@ const initializeClientWithProvider = async ({ getConvoFiles: db.getConvoFiles, getAccessibleMcpServerNames, updateFilesUsage: db.updateFilesUsage, + saveFileTextDerivation: db.saveFileTextDerivation, getUserKeyValues: db.getUserKeyValues, getUserCodeFiles: db.getUserCodeFiles, getDeferredProvisionFiles: db.getDeferredProvisionFiles, @@ -891,6 +908,8 @@ const initializeClientWithProvider = async ({ statefulSessionsAvailable, allowedStatefulCodeEnvironments, memoryAvailable, + deriveText, + persistDerivation, }, { getAgent: db.getAgent, @@ -904,6 +923,7 @@ const initializeClientWithProvider = async ({ getConvoFiles: db.getConvoFiles, getAccessibleMcpServerNames, updateFilesUsage: db.updateFilesUsage, + saveFileTextDerivation: db.saveFileTextDerivation, getUserKeyValues: db.getUserKeyValues, getUserCodeFiles: db.getUserCodeFiles, getDeferredProvisionFiles: db.getDeferredProvisionFiles, @@ -979,6 +999,8 @@ const initializeClientWithProvider = async ({ toolIntentsAvailable, statefulSessionsAvailable, memoryAvailable, + deriveText, + persistDerivation, signal, }); @@ -1437,6 +1459,8 @@ const initializeClientWithProvider = async ({ memoryAvailable, skillStates, defaultActiveOnShare, + deriveText, + persistDerivation, signal: context.signal, }, { @@ -1447,6 +1471,7 @@ const initializeClientWithProvider = async ({ getConvoFiles: db.getConvoFiles, getAccessibleMcpServerNames, updateFilesUsage: db.updateFilesUsage, + saveFileTextDerivation: db.saveFileTextDerivation, getUserKeyValues: db.getUserKeyValues, getUserCodeFiles: db.getUserCodeFiles, getDeferredProvisionFiles: db.getDeferredProvisionFiles, diff --git a/api/server/services/Endpoints/agents/initialize.spec.js b/api/server/services/Endpoints/agents/initialize.spec.js index 9922136450d..fd759cdd36c 100644 --- a/api/server/services/Endpoints/agents/initialize.spec.js +++ b/api/server/services/Endpoints/agents/initialize.spec.js @@ -483,6 +483,24 @@ describe('initializeClient — processAgent ACL gate', () => { expect(resolveUpstreamTokenProvider).not.toHaveBeenCalled(); }); + it('passes the primary request derivation dependencies to parallel initialization', async () => { + mockInitializeAgent.mockResolvedValue(makePrimaryConfig([])); + + await initializeClient({ + req: makeReq(), + res: {}, + signal: new AbortController().signal, + endpointOption: makeEndpointOption(), + }); + + const { deriveText, persistDerivation } = mockInitializeAgent.mock.calls[0][0]; + expect(deriveText).toEqual(expect.any(Function)); + expect(persistDerivation).toEqual(expect.any(Function)); + expect(processAddedConvo).toHaveBeenCalledWith( + expect.objectContaining({ deriveText, persistDerivation }), + ); + }); + it('forwards the linked-instructions resolver and records usage by default', async () => { mockInitializeAgent.mockResolvedValue(makePrimaryConfig([])); diff --git a/api/server/services/Files/process.js b/api/server/services/Files/process.js index 35117ab248e..b59ce31513a 100644 --- a/api/server/services/Files/process.js +++ b/api/server/services/Files/process.js @@ -37,6 +37,13 @@ const { getFileExtractionLogDetails, getUploadExtractedTextPlan, resolveUploadFallbackText, + resolveUploadReading, + resolveUploadCodePossible, + orderToolsForReading, + acquireUploadText, + getUploadReadingMetadata, + logUploadReading, + ExtractorUnavailableError, UPLOAD_EXTRACTED_TEXT_PLANS, MAX_STORED_EXTRACTED_TEXT_BYTES, inspectContent, @@ -794,19 +801,6 @@ const processAgentFileUpload = async ({ req, res, metadata, sseStream }) => { } } - const llmDeliveryPath = resolveUploadLLMDeliveryPath({ - toolResource: tool_resource, - mimeType: file.mimetype, - endpointConfig, - fileConfig, - endpoint, - endpointProvider: getCustomEndpointProvider(appConfig?.endpoints?.custom, endpoint), - useResponsesApi: isResponsesApiUpload(metadata.useResponsesApi ?? req.body?.useResponsesApi), - sttConfigured: isSpeechProviderConfigured(appConfig?.speech?.stt), - }); - - /* Destination and acceptability are one decision, made by shared policy rather than - * rebuilt here. `agentTools` is undefined when no agent record backs the upload. */ /* Only a permanent agent upload can land on a context resource, so the capability is * looked up only there and the common attachment path pays nothing for it. */ const contextEnabled = @@ -819,18 +813,39 @@ const processAgentFileUpload = async ({ req, res, metadata, sseStream }) => { metadata.agentTools?.some( (tool) => tool === EToolResources.execute_code || tool === EToolResources.file_search, ); - const toolRoleGrants = hasRoleGatedConsumer - ? await resolveToolRoleGrants({ - req, - getRoleByName: db.getRoleByName, - context: 'fileUpload', - }) - : undefined; + const loadToolRoleGrants = () => + resolveToolRoleGrants({ req, getRoleByName: db.getRoleByName, context: 'fileUpload' }); + const toolRoleGrants = hasRoleGatedConsumer ? await loadToolRoleGrants() : undefined; + const enabledAgentTools = await filterEnabledConsumers(req, metadata.agentTools, toolRoleGrants); + + const uploadReading = await resolveUploadReading({ + toolResource: tool_resource, + mimeType: file.mimetype, + endpointConfig, + fileConfig, + endpoint, + endpointProvider: getCustomEndpointProvider(appConfig?.endpoints?.custom, endpoint), + useResponsesApi: isResponsesApiUpload(metadata.useResponsesApi ?? req.body?.useResponsesApi), + sttConfigured: isSpeechProviderConfigured(appConfig?.speech?.stt), + isMessageAttachment: messageAttachment, + filters: appConfig?.filters, + resolveCodePossible: () => + resolveUploadCodePossible({ + agentTools: enabledAgentTools, + checkCodeCapability: () => checkCapability(req, AgentCapabilities.execute_code), + getRunCodeGrant: async () => (await loadToolRoleGrants())?.runCode, + }), + }); + let llmDeliveryPath = uploadReading.path; + let textDerivation; + + /* Destination and acceptability are one decision, made by shared policy rather than + * rebuilt here. `agentTools` is undefined when no agent record backs the upload. */ const destination = resolveUploadDestination({ toolResource: tool_resource, deliveryPath: llmDeliveryPath, mimeType: file.mimetype, - agentTools: await filterEnabledConsumers(req, metadata.agentTools, toolRoleGrants), + agentTools: orderToolsForReading(uploadReading, enabledAgentTools), hasAgent: agent_id != null, isMessageAttachment: messageAttachment, /* A message attachment can acquire a file-tool consumer later in the same draft or @@ -1005,6 +1020,7 @@ const processAgentFileUpload = async ({ req, res, metadata, sseStream }) => { updatingUserId: req?.user?.id, }); } + logUploadReading(file_id, uploadReading); const result = await db.createFile(fileInfo, true); sendUploadSuccess(res, sseStream, 'Agent file uploaded and processed successfully', result); }; @@ -1052,92 +1068,118 @@ const processAgentFileUpload = async ({ req, res, metadata, sseStream }) => { } }; - if (shouldUseConfiguredOCR && !(await checkCapability(req, AgentCapabilities.ocr))) { - throw new Error('OCR capability is not enabled for Agents'); - } - - if (shouldUseOCR) { - const ocrResult = await extractInspectableFileText({ - filters: appConfig?.filters, - extract: resolveDocumentText, - }); - if (ocrResult) { - const { text } = ocrResult; - return await createTextFile({ text }); + const ocrCapable = + !shouldUseConfiguredOCR || (await checkCapability(req, AgentCapabilities.ocr)); + const acquireContextText = async () => { + if (!ocrCapable) { + throw new ExtractorUnavailableError( + 'OCR capability is not enabled for Agents', + 'extractor_unavailable', + ); } - throw new Error( - `Unable to extract text from "${file.originalname}". The document may be image-based and requires an OCR service to process.`, - ); - } - const shouldUseSTT = fileConfig.checkType( - file.mimetype, - fileConfig.stt?.supportedMimeTypes || [], - ); + if (shouldUseOCR) { + const ocrResult = await extractInspectableFileText({ + filters: appConfig?.filters, + extract: resolveDocumentText, + }); + if (ocrResult) { + const { text } = ocrResult; + return { text }; + } + throw new Error( + `Unable to extract text from "${file.originalname}". The document may be image-based and requires an OCR service to process.`, + ); + } - if (shouldUseSTT) { - const sttService = await STTService.getInstance(); - const { text } = await processAudioFile({ req, file, sttService }); - return await createTextFile({ text, isTranscript: true }); - } + const shouldUseSTT = fileConfig.checkType( + file.mimetype, + fileConfig.stt?.supportedMimeTypes || [], + ); - const shouldUseText = fileConfig.checkType( - file.mimetype, - fileConfig.text?.supportedMimeTypes || [], - ); + if (shouldUseSTT) { + const sttService = await STTService.getInstance(); + const { text } = await processAudioFile({ req, file, sttService }); + return { text, isTranscript: true }; + } - if (!shouldUseText) { - throw new Error(`File type ${file.mimetype} is not supported for text parsing.`); - } + const shouldUseText = fileConfig.checkType( + file.mimetype, + fileConfig.text?.supportedMimeTypes || [], + ); - /** - * A document type the admin routed to configured text extraction: prefer RAG `/text`, but fall - * back to the built-in document parser (not raw native text) when RAG is unavailable, so a - * transient outage doesn't degrade a docx/pdf to unreadable bytes. Only the RAG extraction is - * inside the fallback catch: a downstream persistence failure (size guard, DB, agent-resource - * mutation) must surface as itself, not trigger a second extraction attempt. - */ - if (shouldUseConfiguredText) { - let configuredText; - try { - configuredText = await parseText({ req, file, file_id, allowNativeFallback: false }); - } catch (err) { - const { errorMetadata } = getExtractionLogDetails(err); - logger.warn( - `[processAgentFileUpload] Configured RAG text extraction unavailable for ${extractionFileLabel}, using built-in document parser:`, - errorMetadata, + if (!shouldUseText) { + throw new ExtractorUnavailableError( + `File type ${file.mimetype} is not supported for text parsing.`, + 'no_extractor', ); - const documentText = await extractInspectableFileText({ - filters: appConfig?.filters, - extract: resolveDocumentText, - }); - if (!documentText) { - throw new Error( - `Unable to extract text from "${file.originalname}". RAG text extraction was unavailable and the built-in parser produced no result.`, + } + + /** + * A document type the admin routed to configured text extraction: prefer RAG `/text`, but fall + * back to the built-in document parser (not raw native text) when RAG is unavailable, so a + * transient outage doesn't degrade a docx/pdf to unreadable bytes. Only the RAG extraction is + * inside the fallback catch: a downstream persistence failure (size guard, DB, agent-resource + * mutation) must surface as itself, not trigger a second extraction attempt. + */ + if (shouldUseConfiguredText) { + let configuredText; + try { + configuredText = await parseText({ req, file, file_id, allowNativeFallback: false }); + } catch (err) { + const { errorMetadata } = getExtractionLogDetails(err); + logger.warn( + `[processAgentFileUpload] Configured RAG text extraction unavailable for ${extractionFileLabel}, using built-in document parser:`, + errorMetadata, ); + const documentText = await extractInspectableFileText({ + filters: appConfig?.filters, + extract: resolveDocumentText, + }); + if (!documentText) { + throw new Error( + `Unable to extract text from "${file.originalname}". RAG text extraction was unavailable and the built-in parser produced no result.`, + ); + } + const { text } = documentText; + return { text }; } - const { text } = documentText; - return await createTextFile({ text }); + return { text: configuredText.text }; } - return await createTextFile({ text: configuredText.text }); - } - /* The native reader decodes whatever bytes it is given as UTF-8, which is meaningful - * only for types that are already text. For anything else, a raster image on a - * deployment without OCR being the case in point, it would store mojibake as the - * file's text, so a real extractor is required and its absence surfaces as an error - * rather than as nonsense content. */ - const { text } = await extractInspectableFileText({ + /* The native reader decodes whatever bytes it is given as UTF-8, which is meaningful + * only for types that are already text. For anything else, a raster image on a + * deployment without OCR being the case in point, it would store mojibake as the + * file's text, so a real extractor is required and its absence surfaces as an error + * rather than as nonsense content. */ + const { text } = await extractInspectableFileText({ + filters: appConfig?.filters, + extract: () => + parseText({ + req, + file, + file_id, + allowNativeFallback: isNativelyReadableText(file.mimetype), + }), + }); + return { text }; + }; + + /* Only the acquisition is wrapped: inspection, the content policy, storage and the + * record in `createTextFile` fail the upload as before. A returned marker keeps the + * original as plain storage below. */ + const acquired = await acquireUploadText({ + reading: uploadReading, + acquire: acquireContextText, + fileId: file_id, filters: appConfig?.filters, - extract: () => - parseText({ - req, - file, - file_id, - allowNativeFallback: isNativelyReadableText(file.mimetype), - }), }); - return await createTextFile({ text }); + if (acquired.textDerivation == null) { + return await createTextFile(acquired); + } + textDerivation = acquired.textDerivation; + llmDeliveryPath = acquired.path; + effectiveToolResource = undefined; } /* Extracted before storage, which may move the temporary upload the extractors read. */ @@ -1149,6 +1191,8 @@ const processAgentFileUpload = async ({ req, res, metadata, sseStream }) => { isMessageAttachment: messageAttachment, endpointConfig, filters: appConfig?.filters, + reading: uploadReading, + textDerivation, }); // Dual storage pattern for RAG files: Storage + Vector DB @@ -1339,6 +1383,7 @@ const processAgentFileUpload = async ({ req, res, metadata, sseStream }) => { * it later. Conversion changes `type`, so without this the second answer is drawn * from a format the administrator never configured a route for. */ ...(storedType !== file.mimetype ? { routingMimeType: file.mimetype } : {}), + ...getUploadReadingMetadata(uploadReading, textDerivation), }, type: storedType, embedded, @@ -1352,6 +1397,7 @@ const processAgentFileUpload = async ({ req, res, metadata, sseStream }) => { ...retentionExpiry, }; + logUploadReading(file_id, uploadReading, textDerivation); const result = await db.createFile(fileInfo, true); sendUploadSuccess(res, sseStream, 'Agent file uploaded and processed successfully', result); diff --git a/api/server/services/Files/process.spec.js b/api/server/services/Files/process.spec.js index 33ec48b7861..59ffa3a4a3e 100644 --- a/api/server/services/Files/process.spec.js +++ b/api/server/services/Files/process.spec.js @@ -7,6 +7,7 @@ jest.mock('@librechat/data-schemas', () => ({ })); jest.mock('@librechat/agents', () => ({ + ...jest.requireActual('@librechat/agents'), Providers: { XAI: 'xai', DEEPSEEK: 'deepseek', @@ -120,7 +121,16 @@ jest.mock('@librechat/api', () => { return UPLOAD_EXTRACTED_TEXT_PLANS.documentParser; }, ); + const actualApi = jest.requireActual('@librechat/api'); return { + resolveUploadReading: jest.fn(actualApi.resolveUploadReading), + resolveUploadCodePossible: jest.fn(actualApi.resolveUploadCodePossible), + orderToolsForReading: jest.fn(actualApi.orderToolsForReading), + acquireUploadText: jest.fn(actualApi.acquireUploadText), + getUploadReadingMetadata: jest.fn(actualApi.getUploadReadingMetadata), + logUploadReading: jest.fn(actualApi.logUploadReading), + ExtractorUnavailableError: actualApi.ExtractorUnavailableError, + UninspectableFileError: actualApi.UninspectableFileError, sanitizeFilename: jest.fn((n) => n), /** Grants both; these specs vary the capability set, not the role. */ resolveToolRoleGrants: jest.fn(async () => ({ runCode: true, fileSearch: true })), @@ -3377,3 +3387,209 @@ describe('fallback text for uploads left to tools', () => { ); }); }); + +describe('automatic reading at upload', () => { + const { logUploadReading } = require('@librechat/api'); + const AUTOMATIC_ENDPOINT = 'Custom Provider'; + let parseDocument; + let storeFile; + + const useEndpointConfig = (endpointConfig) => + mergeFileConfig.mockReturnValue({ + ...makeFileConfig(), + endpoints: { [AUTOMATIC_ENDPOINT]: endpointConfig }, + }); + + beforeEach(() => { + jest.clearAllMocks(); + mockRes.status.mockReturnThis(); + mockRes.json.mockReturnValue({}); + checkCapability.mockResolvedValue(true); + inspectContent.mockReturnValue(null); + useEndpointConfig({ llmDeliveryPolicy: 'automatic' }); + parseDocument = jest.fn().mockResolvedValue({ text: 'region,total', bytes: 12 }); + storeFile = jest.fn().mockResolvedValue({ + bytes: 42, + filename: 'upload.bin', + filepath: '/uploads/upload.bin', + }); + getStrategyFunctions.mockImplementation((source) => ({ + handleFileUpload: source === FileSources.document_parser ? parseDocument : storeFile, + })); + }); + + const upload = ({ mimetype = XLSX_MIME, filters, metadata, ocrConfig } = {}) => { + const req = makeReq({ mimetype, filters, ocrConfig }); + req.body.endpoint = EModelEndpoint.agents; + return processAgentFileUpload({ + req, + res: mockRes, + metadata: { + message_file: 'true', + file_id: 'file-auto', + effectiveEndpoint: AUTOMATIC_ENDPOINT, + ...metadata, + }, + }); + }; + + const storedRecord = () => { + expect(db.createFile).toHaveBeenCalledTimes(1); + return db.createFile.mock.calls[0][0]; + }; + + test('leaves an ephemeral workbook to Run Code without parsing it', async () => { + const { resolveUploadFallbackText } = require('@librechat/api'); + + await upload(); + + storedRecord(); + expect(parseDocument).not.toHaveBeenCalled(); + expect(storeFile).toHaveBeenCalledTimes(1); + expect(checkCapability).toHaveBeenCalledWith(expect.anything(), AgentCapabilities.execute_code); + expect(resolveUploadFallbackText).toHaveBeenCalledWith( + expect.objectContaining({ + deliveryPath: 'none', + reading: expect.objectContaining({ codePreferred: true }), + textDerivation: undefined, + }), + ); + expect(logUploadReading).toHaveBeenCalledTimes(1); + expect(logUploadReading).toHaveBeenCalledWith( + 'file-auto', + expect.objectContaining({ policy: 'automatic', codePreferred: true }), + undefined, + ); + }); + + test('defers a workbook for a saved agent whose narrowed tools include Run Code', async () => { + const { orderToolsForReading } = require('@librechat/api'); + await upload({ + metadata: { + agent_id: 'agent-abc', + agentTools: [EToolResources.file_search, EToolResources.execute_code], + }, + }); + + storedRecord(); + expect(parseDocument).not.toHaveBeenCalled(); + expect(orderToolsForReading).toHaveLastReturnedWith([ + EToolResources.execute_code, + EToolResources.file_search, + ]); + expect(uploadVectors).not.toHaveBeenCalled(); + expect(db.addAgentResourceFile).not.toHaveBeenCalled(); + }); + + test('defers a document whose configured OCR capability is off, for a built-in reader later', async () => { + mergeFileConfig.mockReturnValue({ + ...makeFileConfig({ ocrSupportedMimeTypes: [DOCX_MIME] }), + endpoints: { [AUTOMATIC_ENDPOINT]: { llmDeliveryPolicy: 'automatic' } }, + }); + checkCapability.mockImplementation( + async (_req, capability) => capability !== AgentCapabilities.ocr, + ); + + await upload({ mimetype: DOCX_MIME, ocrConfig: { strategy: FileSources.mistral_ocr } }); + + storedRecord(); + expect(checkCapability).toHaveBeenCalledWith(expect.anything(), AgentCapabilities.ocr); + expect(parseDocument).not.toHaveBeenCalled(); + expect(storeFile).toHaveBeenCalledTimes(1); + }); + + test.each([ + [ + 'the Run Code capability is off', + () => + checkCapability.mockImplementation( + async (_req, capability) => capability !== AgentCapabilities.execute_code, + ), + {}, + ], + ['the endpoint stays classic', () => useEndpointConfig({}), {}], + ])('extracts the workbook at upload when %s', async (_label, arrange, options) => { + arrange(); + + await upload(options); + + const record = storedRecord(); + expect(parseDocument).toHaveBeenCalledTimes(1); + expect(record.llmDeliveryPath).toBe('text'); + expect(record.text).toBe('region,total'); + expect(record.metadata.textDerivation).toBeUndefined(); + expect(logUploadReading).toHaveBeenCalledTimes(1); + }); + + test.each([ + [ + 'keeps the original alone, deferred, when the parser fails unrecognized', + 'automatic', + 'Document parser crashed', + { outcome: 'deferred', reason: 'parser', at: expect.any(Number) }, + ], + [ + 'keeps the original alone, marked failed, when the parser names its failure', + 'automatic', + 'No text found in document', + { outcome: 'failed', extractor: 'document_parser', reason: 'empty', at: expect.any(Number) }, + ], + ['still fails a classic upload whose parser fails', 'classic', 'Document parser crashed', null], + ])('%s', async (_label, policy, failure, textDerivation) => { + const { resolveUploadFallbackText } = require('@librechat/api'); + if (policy === 'classic') { + useEndpointConfig({}); + } + parseDocument.mockRejectedValue(new Error(failure)); + + const result = upload({ mimetype: DOCX_MIME }); + + if (textDerivation === null) { + await expect(result).rejects.toThrow(failure); + expect(db.createFile).not.toHaveBeenCalled(); + return; + } + await result; + const record = storedRecord(); + expect(parseDocument).toHaveBeenCalledTimes(1); + expect(storeFile).toHaveBeenCalledWith(expect.objectContaining({ file_id: 'file-auto' })); + expect(record.llmDeliveryPath).toBe('none'); + expect(record.text).toBeUndefined(); + expect(record.metadata.textDerivation).toEqual(textDerivation); + expect(resolveUploadFallbackText).toHaveBeenCalledWith( + expect.objectContaining({ + deliveryPath: 'none', + textDerivation: record.metadata.textDerivation, + }), + ); + }); + + test('still fails the upload when storage fails after a kept original', async () => { + parseDocument.mockRejectedValue(new Error('Document parser crashed')); + storeFile.mockRejectedValue(new Error('storage unavailable')); + + await expect(upload({ mimetype: DOCX_MIME })).rejects.toThrow('storage unavailable'); + expect(db.createFile).not.toHaveBeenCalled(); + }); + + test('still answers a content-policy finding in extracted text with a 400', async () => { + inspectContent.mockReturnValue({ source: 'file', field: 'extracted_text' }); + + await upload({ + mimetype: DOCX_MIME, + filters: { + files: { + pii: { + fields: ['extracted_text'], + customPatterns: [{ id: 'region', label: 'region', regex: 'region' }], + }, + }, + }, + }); + + expect(parseDocument).toHaveBeenCalledTimes(1); + expect(mockRes.status).toHaveBeenCalledWith(400); + expect(storeFile).not.toHaveBeenCalled(); + expect(db.createFile).not.toHaveBeenCalled(); + }); +}); diff --git a/api/server/services/Files/provision.js b/api/server/services/Files/provision.js index 32311cc500e..8538ca10db6 100644 --- a/api/server/services/Files/provision.js +++ b/api/server/services/Files/provision.js @@ -16,6 +16,7 @@ const getService = () => module.exports = { loadCodeApiKey: (...args) => getService().loadCodeApiKey(...args), + openStoredFile: (...args) => getService().openStoredFile(...args), provisionToCodeEnv: (...args) => getService().provisionToCodeEnv(...args), provisionToVectorDB: (...args) => getService().provisionToVectorDB(...args), checkCodeEnvFileAlive: (...args) => getService().checkCodeEnvFileAlive(...args), diff --git a/api/server/services/ToolService.js b/api/server/services/ToolService.js index 632d884eca7..31a3cb21b29 100644 --- a/api/server/services/ToolService.js +++ b/api/server/services/ToolService.js @@ -1570,14 +1570,16 @@ async function loadToolDefinitionsWrapper({ } } + let primedSearchFiles; if (hasFileSearch && tool_resources) { try { - const { toolContext } = await primeSearchFiles({ + const { toolContext, files } = await primeSearchFiles({ req, tool_resources, agentId: agent.id, agentResourceType, }); + primedSearchFiles = files; if (toolContext) { dynamicToolContextMap[Tools.file_search] = toolContext; } @@ -1586,6 +1588,8 @@ async function loadToolDefinitionsWrapper({ '[loadToolDefinitionsWrapper] Error priming search files:', getSafeErrorMetadata(error), ); + /* Nothing was primed, so no resource file may be advertised as reachable by search. */ + primedSearchFiles = []; } } @@ -1630,6 +1634,7 @@ async function loadToolDefinitionsWrapper({ mcpToolAliases, actionsEnabled, primedCodeFiles, + primedSearchFiles, oauthActionToolNames, codeExecutionContext: resolvedCodeExecutionContext, repositoryInstructionSource: createRepositoryInstructionSource({ diff --git a/api/server/services/__tests__/ToolService.spec.js b/api/server/services/__tests__/ToolService.spec.js index b00271883a4..80093e79857 100644 --- a/api/server/services/__tests__/ToolService.spec.js +++ b/api/server/services/__tests__/ToolService.spec.js @@ -1040,6 +1040,54 @@ describe('ToolService - Action Capability Gating', () => { expect(JSON.stringify(errorSpy.mock.calls)).not.toContain(rawValue); errorSpy.mockRestore(); }); + + const primed = [{ file_id: 'file-indexed', filename: 'indexed.pdf', fromAgent: false }]; + const primedNote = `- Note: Use the ${Tools.file_search} tool to find relevant information within:`; + const emptyNote = `- Note: Semantic search is available through the ${Tools.file_search} tool but no files are currently loaded.`; + + it.each([ + [ + 'primed', + () => + mockPrimeSearchFiles.mockResolvedValueOnce({ files: primed, toolContext: primedNote }), + primed, + primedNote, + ], + [ + 'none', + () => mockPrimeSearchFiles.mockResolvedValueOnce({ files: [], toolContext: emptyNote }), + [], + emptyNote, + ], + [ + 'failed', + () => mockPrimeSearchFiles.mockRejectedValueOnce(new Error('search priming failed')), + [], + undefined, + ], + ])( + 'returns the primed search files beside the file search note when priming is %s', + async (_outcome, arrange, files, toolContext) => { + arrange(); + const req = createMockReq([AgentCapabilities.file_search]); + mockGetEndpointsConfig.mockResolvedValue( + createEndpointsConfig([AgentCapabilities.file_search]), + ); + + const result = await loadAgentTools({ + req, + res: {}, + agent: { id: 'agent_search', tools: [Tools.file_search] }, + tool_resources: { + [EToolResources.file_search]: { file_ids: ['file-indexed'] }, + }, + definitionsOnly: true, + }); + + expect(result.primedSearchFiles).toEqual(files); + expect(result.dynamicToolContextMap[Tools.file_search]).toBe(toolContext); + }, + ); }); describe('loadAgentTools (definitionsOnly=true) — action tool filtering', () => { diff --git a/api/server/services/__tests__/toolCapabilityGates.spec.js b/api/server/services/__tests__/toolCapabilityGates.spec.js index 1b8693ddc28..c947ab6d7b2 100644 --- a/api/server/services/__tests__/toolCapabilityGates.spec.js +++ b/api/server/services/__tests__/toolCapabilityGates.spec.js @@ -38,9 +38,11 @@ const CLASSIFIED = { /** `codeEnvAvailable` for the memory-agent initializer. */ 'api/server/controllers/agents/client.js': 1, /** Upload processing: two consumer-map entries, the role-gated consumer checks, - * and the code-environment/file-search branches. All consumer selection shares - * the role grant resolved before routing. */ - 'api/server/services/Files/process.js': 5, + * the code-environment/file-search branches, and the upload-time Run Code read + * that decides whether a spreadsheet is left to code, paired with the Run Code + * grant inside `resolveUploadCodePossible`. All consumer selection shares the + * role grant resolved before routing. */ + 'api/server/services/Files/process.js': 6, /** Agent-management upload purposes. */ 'api/server/routes/agents/management.js': 2, /** Two gates, each paired when the embedder wires `getRoleByName`; one in a diff --git a/api/test/__mocks__/logger.js b/api/test/__mocks__/logger.js index 699a94883f1..734102c5a19 100644 --- a/api/test/__mocks__/logger.js +++ b/api/test/__mocks__/logger.js @@ -24,6 +24,8 @@ jest.mock('winston', () => { warn: jest.fn(), debug: jest.fn(), error: jest.fn(), + /** Real winston derives `isEnabled` from its levels; the delivery log gates on it. */ + isDebugEnabled: jest.fn(() => false), }), transports: { Console: jest.fn(), diff --git a/client/src/components/Chat/Messages/Content/__tests__/FilePreviewDialog.test.ts b/client/src/components/Chat/Messages/Content/__tests__/FilePreviewDialog.test.ts index c8c0c9edb22..f76f7390706 100644 --- a/client/src/components/Chat/Messages/Content/__tests__/FilePreviewDialog.test.ts +++ b/client/src/components/Chat/Messages/Content/__tests__/FilePreviewDialog.test.ts @@ -22,6 +22,15 @@ describe('FilePreviewDialog text-source behavior', () => { expect(getDownloadFilename('report.pdf', 'file-3', FileSources.local)).toBe('report.pdf'); }); + it('does not decode an Office Open XML package as text', () => { + const xlsx = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'; + const docx = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'; + expect(getPreviewKind('quarterly.xlsx', xlsx, FileSources.local)).toBe(false); + expect(getPreviewKind('brief.docx', docx, FileSources.local)).toBe(false); + expect(getPreviewKind('feed.xml', 'application/xml', FileSources.local)).toBe('text'); + expect(getPreviewKind('logo.svg', 'image/svg+xml', FileSources.local)).toBe('text'); + }); + it('uses stored extracted text when the delivery path is text', () => { expect(shouldUseExtractedTextPreview('text')).toBe(true); expect(shouldUseExtractedTextPreview('provider')).toBe(false); diff --git a/client/src/components/Chat/Messages/Content/preview.ts b/client/src/components/Chat/Messages/Content/preview.ts index 45f64fa1d68..fdbe9dae61f 100644 --- a/client/src/components/Chat/Messages/Content/preview.ts +++ b/client/src/components/Chat/Messages/Content/preview.ts @@ -51,6 +51,9 @@ export function isExtractedTextPreviewLoading( return !isError && (isInitialLoading || status === 'pending'); } +/** XML proper and XML-based formats such as SVG; Office Open XML packages are zipped binaries. */ +const isXmlMime = (mime: string): boolean => mime === 'application/xml' || mime.endsWith('+xml'); + function getPreviewKindByMime(mime?: string): PreviewKind { if (!mime) { return false; @@ -61,7 +64,7 @@ function getPreviewKindByMime(mime?: string): PreviewKind { if ( mime.startsWith('text/') || mime.includes('json') || - mime.includes('xml') || + isXmlMime(mime) || mime.includes('javascript') || mime.includes('typescript') || mime.includes('yaml') || diff --git a/client/src/components/Messages/Content/Error/registry.ts b/client/src/components/Messages/Content/Error/registry.ts index 3f878637a03..727d872cff9 100644 --- a/client/src/components/Messages/Content/Error/registry.ts +++ b/client/src/components/Messages/Content/Error/registry.ts @@ -23,6 +23,7 @@ export const errorCopy: Record = { [ErrorTypes.MODELS_NOT_LOADED]: 'com_error_models_not_loaded', [ErrorTypes.GOOGLE_TOOL_CONFLICT]: 'com_error_google_tool_conflict', [ErrorTypes.GOOGLE_VIDEO_UNPROCESSABLE]: 'com_error_google_video_unprocessable', + file_search_preparation_failed: 'com_error_file_search_preparation_failed', [ErrorTypes.RESOURCE_RECOVERY_REQUIRED]: 'com_error_resource_recovery_required', [ErrorTypes.STATEFUL_CODE_ENVIRONMENT_NOT_ALLOWED]: 'com_error_stateful_code_environment_not_allowed', diff --git a/client/src/components/Messages/Content/__tests__/Error.spec.tsx b/client/src/components/Messages/Content/__tests__/Error.spec.tsx index 8212906edfe..3b97e59cc13 100644 --- a/client/src/components/Messages/Content/__tests__/Error.spec.tsx +++ b/client/src/components/Messages/Content/__tests__/Error.spec.tsx @@ -211,6 +211,18 @@ describe('Error — reader-facing provider and fallback copy', () => { expectReadable(); }); + it.each([ + '{"code":"file_search_preparation_failed"}', + 'File Search could not prepare the attached files. Try sending the message again.\n' + + '{"code":"file_search_preparation_failed"}', + ])('localizes a search preparation failure from the existing error boundary: %s', (payload) => { + renderError(payload, providerMessage); + + expect(screen.getByText(catalog.com_error_file_search_preparation_failed)).toBeInTheDocument(); + expect(document.body.textContent).not.toContain('{'); + expectReadable(); + }); + it.each([ ['MODEL_NOT_FOUND', 'com_error_model_not_found'], ['MODEL_RATE_LIMIT', 'com_error_model_rate_limit'], diff --git a/client/src/hooks/Files/__tests__/equivalence.spec.tsx b/client/src/hooks/Files/__tests__/equivalence.spec.tsx new file mode 100644 index 00000000000..d367f156270 --- /dev/null +++ b/client/src/hooks/Files/__tests__/equivalence.spec.tsx @@ -0,0 +1,449 @@ +import React from 'react'; +import { RecoilRoot } from 'recoil'; +import { useForm } from 'react-hook-form'; +import { ToastContext } from '@librechat/client'; +import { act, renderHook, waitFor } from '@testing-library/react'; +import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; +import { + Constants, + FileSources, + FileContext, + EModelEndpoint, + mergeFileConfig, +} from 'librechat-data-provider'; +import type { + FileConfig, + TFileUpload, + TConversation, + UploadMutationOptions, +} from 'librechat-data-provider'; +import type { SharePointFile } from '~/data-provider/Files/sharepoint'; +import type { AttachEntry } from '~/hooks/Input/useAttachItems'; +import type { ChatFormValues, ExtendedFile } from '~/common'; +import type { ChatContract } from '~/hooks/Chat/contract'; +import { UploadModalProvider, useUploadModalContext } from '~/Providers/UploadModalContext'; +import useAttachExisting from '~/hooks/Files/useAttachExisting'; +import { DragDropProvider } from '~/Providers/DragDropContext'; +import { ChatFormProvider } from '~/Providers/ChatFormContext'; +import useAttachTarget from '~/hooks/Input/useAttachTarget'; +import useAttachItems from '~/hooks/Input/useAttachItems'; +import { ChatContext } from '~/Providers/ChatContext'; +import useTextarea from '~/hooks/Input/useTextarea'; +import useDragHelpers from '../useDragHelpers'; +import { isUnifiedUploadMode } from '~/utils'; + +/** + * Every way a file reaches the composer in unified mode has to arrive at the server as the same + * request: no destination, marked as a message file, so the server decides how it is read. The + * hooks run for real down to the upload mutation; only the network, the browser's drag source and + * the hooks that have nothing to do with uploads are replaced. + */ + +type RawFileConfig = Parameters[0]; +type ChatFiles = Pick< + ChatContract, + 'index' | 'conversation' | 'isSubmitting' | 'files' | 'setFiles' | 'setFilesLoading' +>; +type Upload = { body: FormData; options: UploadMutationOptions }; +type TestChat = { chat: ChatFiles; staged: () => Map }; +type Attached = { body: FormData; staged: ExtendedFile | undefined; chat: TestChat }; + +const mockShowToast = jest.fn(); +let mockUploads: Upload[] = []; +let mockFileConfig: RawFileConfig; +let mockDrop: ((item: { files: File[] }) => void) | undefined; + +jest.mock('~/data-provider', () => ({ + useGetFileConfig: ({ select }: { select?: (data: RawFileConfig) => FileConfig } = {}) => ({ + data: select != null ? select(mockFileConfig) : mockFileConfig, + isSuccess: true, + isError: false, + isPaused: false, + }), + useGetEndpointsQuery: () => ({ data: undefined }), + useGetAgentByIdQuery: () => ({ data: undefined }), + useGetStartupConfig: () => ({ data: { sharePointFilePickerEnabled: true } }), + useGraphTokenQuery: () => ({ data: { access_token: 'graph-token' } }), + useInteractionHealthCheck: () => async () => true, + useUploadFileMutation: (options: UploadMutationOptions) => ({ + mutate: (body: FormData) => { + mockUploads.push({ body, options }); + }, + }), +})); + +/** The Graph download is an external API; the bytes it returns are what matter here. */ +jest.mock('~/data-provider/Files', () => ({ + useSharePointBatchDownload: () => ({ + mutateAsync: async ({ files }: { files: SharePointFile[] }) => + files.map((file) => new File(['%PDF-'], file.name, { type: 'application/pdf' })), + }), +})); + +/** jsdom cannot originate a native file drag, so the drop target's handler is taken directly. */ +jest.mock('react-dnd', () => ({ + useDrop: (spec: () => { drop: (item: { files: File[] }) => void }) => { + mockDrop = spec().drop; + return [{ canDrop: false, isOver: false }, () => undefined]; + }, +})); + +/* Barrels narrowed to the real modules these hooks use, so loading them does not pull in the + * whole application. */ +jest.mock('~/Providers', () => ({ + ...jest.requireActual( + '~/Providers/AgentsMapContext', + ), + ...jest.requireActual( + '~/Providers/ChatFormContext', + ), + ...jest.requireActual( + '~/Providers/DragDropContext', + ), + ...jest.requireActual( + '~/Providers/UploadModalContext', + ), +})); + +jest.mock('~/hooks/Agents', () => ({ + useAgentCapabilities: jest.requireActual( + '~/hooks/Agents/useAgentCapabilities', + ).default, + useGetAgentsConfig: jest.requireActual( + '~/hooks/Agents/useGetAgentsConfig', + ).default, +})); + +jest.mock('~/hooks', () => ({ + useLocalize: + jest.requireActual('~/hooks/useLocalize').default, + useAgentCapabilities: jest.requireActual( + '~/hooks/Agents/useAgentCapabilities', + ).default, + useGetAgentsConfig: jest.requireActual( + '~/hooks/Agents/useGetAgentsConfig', + ).default, + useAgentToolPermissions: jest.requireActual< + typeof import('~/hooks/Agents/useAgentToolPermissions') + >('~/hooks/Agents/useAgentToolPermissions').default, +})); + +jest.mock('~/hooks/Files', () => ({ + useFileHandlingNoChatContext: jest.requireActual( + '~/hooks/Files/useFileHandling', + ).useFileHandlingNoChatContext, +})); + +/* Composer concerns the paste path renders alongside, none of which touch an upload. */ +jest.mock('~/hooks/Messages/useLatestMessage', () => ({ useLatestMessageMeta: () => undefined })); +jest.mock('~/hooks/Input/useComposerBindings', () => ({ + __esModule: true, + default: () => ({ + shortcutsEnabled: false, + submitOverride: undefined, + yieldedChords: new Set(), + }), +})); +jest.mock('~/hooks/AuthContext', () => ({ useAuthContext: () => ({ user: undefined }) })); + +const conversation = { + conversationId: 'convo-1', + endpoint: EModelEndpoint.agents, + endpointType: EModelEndpoint.agents, + agent_id: Constants.EPHEMERAL_AGENT_ID, +} as TConversation; + +/** The fields every unified upload sends, whichever way the file arrived. */ +const UNIFIED_UPLOAD = { + endpoint: EModelEndpoint.agents, + endpointType: EModelEndpoint.agents, + conversationId: 'convo-1', + message_file: 'true', + agent_id: Constants.EPHEMERAL_AGENT_ID, +}; + +const report = () => new File(['%PDF-'], 'report.pdf', { type: 'application/pdf' }); + +/** What every entry point tells the user when the endpoint refuses the file's type. */ +const REFUSED_TYPE_TOAST = { + message: 'Unsupported file type: application/pdf', + status: 'error', + duration: 5000, +}; + +type Toast = { status?: string }; +const errorToasts = (): Toast[] => + mockShowToast.mock.calls.flatMap(([toast]: [Toast]) => (toast.status === 'error' ? [toast] : [])); + +function createChat(): TestChat { + let staged = new Map(); + const chat: ChatFiles = { + index: 0, + conversation, + isSubmitting: false, + files: staged, + setFiles: (action) => { + staged = typeof action === 'function' ? action(staged) : action; + }, + setFilesLoading: () => undefined, + }; + return { chat, staged: () => staged }; +} + +function providers({ chat }: TestChat) { + const queryClient = new QueryClient(); + return function Providers({ children }: { children: React.ReactNode }) { + const form = useForm(); + return ( + + + + + + + {children} + + + + + + + ); + }; +} + +/** The attach half of the composer palette, wired the way `Palette` wires it. */ +function usePalette({ chat }: TestChat) { + const target = useAttachTarget(conversation, false); + const attach = useAttachItems({ + agentId: conversation.agent_id, + endpoint: conversation.endpoint, + endpointType: target.endpointType, + endpointFileConfig: target.endpointFileConfig, + useResponsesApi: target.useResponsesApi, + conversationId: conversation.conversationId ?? Constants.NEW_CONVO, + conversation, + files: chat.files, + setFiles: chat.setFiles, + setFilesLoading: chat.setFilesLoading, + isUnifiedMode: isUnifiedUploadMode(target.endpointFileConfig, target.canAttach), + }); + const attachExisting = useAttachExisting({ + files: chat.files, + setFiles: chat.setFiles, + conversation, + endpoint: conversation.endpoint, + endpointType: target.endpointType, + endpointFileConfig: target.endpointFileConfig, + }); + return { attach, attachExisting }; +} + +/** What the server learns from an upload, less the per-upload id and the bytes. */ +const uploadFields = (body: FormData): Record => + Object.fromEntries( + Array.from(body.entries()).flatMap<[string, string]>(([key, value]) => + key === 'file_id' || key === 'file' ? [] : [[key, String(value)]], + ), + ); + +/** The fields the send payload copies off a staged file (`useChatFunctions`). */ +const messageFile = (file: ExtendedFile | undefined) => ({ + file_id: file?.file_id, + filepath: file?.filepath, + filename: file?.filename, + type: file?.type, + llmDeliveryPath: file?.llmDeliveryPath, + height: file?.height, + width: file?.width, +}); + +async function settled(test: TestChat): Promise { + await waitFor(() => expect(mockUploads).toHaveLength(1)); + const { body } = mockUploads[0]; + return { body, staged: test.staged().get(String(body.get('file_id'))), chat: test }; +} + +type Dropped = TestChat & { chooserOpen: () => boolean }; + +async function startDrop(file: File): Promise { + const test = createChat(); + const { result } = renderHook( + () => ({ drag: useDragHelpers(), modal: useUploadModalContext() }), + { wrapper: providers(test) }, + ); + act(() => mockDrop?.({ files: [file] })); + return { ...test, chooserOpen: () => result.current.modal.isVisible }; +} + +const pasteEvent = (files: File[]): React.ClipboardEvent => { + const event: Pick, 'preventDefault'> & { + clipboardData: Pick & { files: ArrayLike & Iterable }; + } = { clipboardData: { files, getData: () => '' }, preventDefault: () => undefined }; + return event as React.ClipboardEvent; +}; + +async function startPaste(file: File): Promise { + const test = createChat(); + const { result } = renderHook( + () => + useTextarea({ + textAreaRef: { current: document.createElement('textarea') }, + submitButtonRef: { current: document.createElement('button') }, + setIsScrollable: () => undefined, + enterToSend: true, + }), + { wrapper: providers(test) }, + ); + act(() => result.current.handlePaste(pasteEvent([file]))); + return test; +} + +/** Chooses a palette row, which has to be on offer for the pick to mean anything. */ +function choose(entries: AttachEntry[], id: string) { + const entry = entries.find((candidate) => candidate.id === id); + expect(entry).toBeDefined(); + act(() => entry?.onSelect()); +} + +const changeEvent = (input: HTMLInputElement): React.ChangeEvent => { + const event: Pick, 'target' | 'stopPropagation'> = { + target: input, + stopPropagation: () => undefined, + }; + return event as React.ChangeEvent; +}; + +async function startPick(file: File): Promise { + const test = createChat(); + const { result } = renderHook(() => usePalette(test), { wrapper: providers(test) }); + const input = document.createElement('input'); + input.click = () => undefined; + Object.defineProperty(result.current.attach.inputRef, 'current', { value: input }); + choose(result.current.attach.entries, 'local:unified'); + Object.defineProperty(input, 'files', { value: [file] }); + act(() => result.current.attach.onFileChange(changeEvent(input))); + return test; +} + +async function startSharePoint(file: File): Promise { + const test = createChat(); + const { result } = renderHook(() => usePalette(test), { wrapper: providers(test) }); + choose(result.current.attach.entries, 'sharepoint:unified'); + const picked: SharePointFile = { + id: 'sp-1', + name: file.name, + size: file.size, + webUrl: 'https://contoso.sharepoint.com/report.pdf', + downloadUrl: 'https://graph.microsoft.com/download/sp-1', + driveId: 'drive-1', + itemId: 'item-1', + sharePointItem: {}, + }; + await act(async () => { + await result.current.attach.onSharePointFilesSelected([picked]); + }); + return test; +} + +const drop = async (file: File): Promise => settled(await startDrop(file)); +const paste = async (file: File): Promise => settled(await startPaste(file)); +const pick = async (file: File): Promise => settled(await startPick(file)); +const pickFromSharePoint = async (file: File): Promise => + settled(await startSharePoint(file)); + +beforeAll(() => { + global.URL.createObjectURL = jest.fn(() => 'blob:preview'); + global.URL.revokeObjectURL = jest.fn(); +}); + +beforeEach(() => { + mockUploads = []; + mockDrop = undefined; + mockFileConfig = {}; + mockShowToast.mockClear(); + localStorage.clear(); +}); + +describe('unified upload entry points', () => { + it.each([ + ['drag and drop', drop], + ['a file paste', paste], + ['the local picker', pick], + ['the SharePoint picker', pickFromSharePoint], + ])('uploads through %s with no destination', async (_entry, attach) => { + const { body, staged } = await attach(report()); + expect(uploadFields(body)).toEqual(UNIFIED_UPLOAD); + expect(body.get('file')).toHaveProperty('name', expect.stringMatching(/report\.pdf$/)); + expect(staged).toBeDefined(); + expect(staged?.tool_resource).toBeUndefined(); + }); + + it.each([ + ['drag and drop', startDrop], + ['the SharePoint picker', startSharePoint], + ])( + 'refuses a type the endpoint does not accept through %s the same way', + async (_entry, start) => { + mockFileConfig = { + endpoints: { [EModelEndpoint.agents]: { supportedMimeTypes: ['^image/png$'] } }, + }; + const test = await start(report()); + await waitFor(() => expect(errorToasts()).toEqual([REFUSED_TYPE_TOAST])); + expect(mockUploads).toHaveLength(0); + expect(test.staged().size).toBe(0); + }, + ); + + it.each([ + [{}, false], + [{ legacyFileUploadUX: true }, true], + ])( + 'offers the destination chooser on a drop only where the deployment keeps the legacy menu (%o: %s)', + async (fileConfig, chooserOpen) => { + mockFileConfig = fileConfig; + const test = await startDrop(report()); + await waitFor(() => + expect({ chooserOpen: test.chooserOpen(), uploads: mockUploads.length }).toEqual({ + chooserOpen, + uploads: chooserOpen ? 0 : 1, + }), + ); + }, + ); + + it('stages a re-attached file exactly as the upload that produced it', async () => { + const uploaded = await pick(report()); + const tempId = String(uploaded.body.get('file_id')); + const record: TFileUpload = { + user: 'user-1', + file_id: 'server-file', + temp_file_id: tempId, + bytes: 5, + embedded: false, + filename: 'report.pdf', + filepath: '/uploads/user-1/server-file__report.pdf', + object: 'file', + type: 'application/pdf', + usage: 0, + source: FileSources.local, + context: FileContext.message_attachment, + llmDeliveryPath: 'none', + }; + act(() => mockUploads[0].options.onSuccess?.(record, uploaded.body)); + await waitFor(() => expect(uploaded.chat.staged().get(tempId)?.progress).toBe(1)); + + const reattached = createChat(); + const { result } = renderHook(() => usePalette(reattached), { + wrapper: providers(reattached), + }); + act(() => result.current.attachExisting(record)); + + const fromUpload = uploaded.chat.staged().get(tempId); + const fromLibrary = reattached.staged().get(record.file_id); + expect(fromLibrary).toBeDefined(); + expect(messageFile(fromLibrary)).toEqual(messageFile(fromUpload)); + expect(fromLibrary?.tool_resource).toBeUndefined(); + expect(fromUpload?.tool_resource).toBeUndefined(); + }); +}); diff --git a/client/src/locales/en/translation.json b/client/src/locales/en/translation.json index c5f7a04cafe..deb126a26f4 100644 --- a/client/src/locales/en/translation.json +++ b/client/src/locales/en/translation.json @@ -638,6 +638,7 @@ "com_error_file_upload_limit": "You have reached the upload limit of {{0}} files per {{1}}.", "com_error_files_dupe": "Duplicate file detected.", "com_error_files_empty": "Empty files are not allowed.", + "com_error_file_search_preparation_failed": "File Search couldn't prepare the attached files. Send this message again to retry.", "com_error_files_process": "An error occurred while processing the file.", "com_error_files_skipped_dupe": "Skipped duplicate file(s): {{0}}", "com_error_files_skipped_size": "Skipped file(s) over the {{0}} MB size limit: {{1}}", diff --git a/client/src/routes/Search.tsx b/client/src/routes/Search.tsx index 76e214f384c..15a09c0ee1d 100644 --- a/client/src/routes/Search.tsx +++ b/client/src/routes/Search.tsx @@ -3,6 +3,7 @@ import { useAtomValue } from 'jotai'; import throttle from 'lodash/throttle'; import { useRecoilValue } from 'recoil'; import { Search as SearchIcon, SearchX } from 'lucide-react'; +import { hydrateMessageFile } from 'librechat-data-provider'; import { List, CellMeasurer, CellMeasurerCache } from 'react-virtualized'; import { EmptyState, Spinner, useRemScale, useToastContext } from '@librechat/client'; import type { Index, ListRowProps } from 'react-virtualized'; @@ -119,7 +120,7 @@ export default function Search() { } return { ...message, - files: message.files.map((file) => fileMap[file.file_id ?? ''] ?? file), + files: message.files.map((file) => hydrateMessageFile(file, fileMap)), }; }), ) ?? [], diff --git a/client/src/utils/thread.ts b/client/src/utils/thread.ts index 1a53d2c33c0..529d51347ff 100644 --- a/client/src/utils/thread.ts +++ b/client/src/utils/thread.ts @@ -1,3 +1,4 @@ +import { hydrateMessageFile } from 'librechat-data-provider'; import type { TFile, TMessage } from 'librechat-data-provider'; /** Parent key under which parentless (and self-parented) messages are filed. */ @@ -53,7 +54,7 @@ function hydrateFiles(message: TMessage, fileMap?: Record): TMess } return { ...message, - files: message.files.map((file) => fileMap[file.file_id ?? ''] ?? file), + files: message.files.map((file) => hydrateMessageFile(file, fileMap)), }; } diff --git a/e2e/config/librechat.e2e.yaml b/e2e/config/librechat.e2e.yaml index be32ca709e9..e4cdd8d8820 100644 --- a/e2e/config/librechat.e2e.yaml +++ b/e2e/config/librechat.e2e.yaml @@ -99,6 +99,10 @@ mcpServers: # 'provider', so that test's CSV still reaches the model. # - Mock Provider B uses the unified single button with spreadsheet/csv routed # to llmDeliveryPath 'none' (unified-upload.spec.ts). +# - Mock Auto Provider and Mock Auto Small Provider select the automatic reading +# policy with no route overrides (automatic-upload.spec.ts). Auto Small's 1 MB +# fileSizeLimit is the hard limit for new uploads there and the native size +# limit for PDFs uploaded elsewhere. fileConfig: endpoints: 'Mock Run Files': @@ -117,6 +121,11 @@ fileConfig: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': 'none' # Routed to the provider: still delivered to the model AND shown in chat. 'text/markdown': 'provider' + 'Mock Auto Provider': + llmDeliveryPolicy: automatic + 'Mock Auto Small Provider': + llmDeliveryPolicy: automatic + fileSizeLimit: 1 endpoints: # Default capabilities plus run_in_background (off by default upstream) so the @@ -222,6 +231,28 @@ endpoints: titleConvo: false modelDisplayLabel: 'Mock Provider B' + # The automatic reading policy (see fileConfig above). Not mirrored by a model + # spec, so each stays a plain endpoint → model option in the selector. + - name: 'Mock Auto Provider' + apiKey: 'e2e-mock-key-auto' + baseURL: 'http://127.0.0.1:8889/v1' + models: + default: + - 'mock-auto-model' + fetch: false + titleConvo: false + modelDisplayLabel: 'Mock Auto Provider' + + - name: 'Mock Auto Small Provider' + apiKey: 'e2e-mock-key-auto-small' + baseURL: 'http://127.0.0.1:8889/v1' + models: + default: + - 'mock-auto-small-model' + fetch: false + titleConvo: false + modelDisplayLabel: 'Mock Auto Small Provider' + # No model spec mirrors this endpoint's label, so it stays unambiguous in the # selector and gives e2e tests a real ephemeral endpoint → model option. - name: 'Mock Provider C' @@ -299,6 +330,8 @@ modelSpecs: - 'Mock Provider D' - 'Mock Provider E' - 'Mock Provider F' + - 'Mock Auto Provider' + - 'Mock Auto Small Provider' - 'agents' list: - name: 'e2e-mock-provider-a' diff --git a/e2e/setup/fake-code-server.js b/e2e/setup/fake-code-server.js index d441ac6fe15..0c3f17e47e0 100644 --- a/e2e/setup/fake-code-server.js +++ b/e2e/setup/fake-code-server.js @@ -13,6 +13,16 @@ * - `GET /v1/files/:sid` — session liveness (`checkSessionsAlive`). * - `POST /v1/exec` — code execution (`execute_code` tool run). * + * Upload bytes are retained per `fileId` (64 MB in total, oldest evicted first) and + * each debug record carries the bytes' `sha256`. An exec whose code contains + * `E2E_WORKBOOK:` parses the retained upload the exec itself mounted + * under that name (an entry of its `files`, as the sandbox would see it in + * /mnt/data) with the hoisted SheetJS and prints `{ sheets, totals }` (column-B + * sums per sheet); `E2E_CSV:` prints `{ rows, total }` for a CSV or + * TSV upload. An exec that mounted no such file fails the analysis, as a real + * sandbox would find no file. Each exec record lists the files it mounted and + * the one it analyzed, so specs can tie an answer to the bytes the user attached. + * * Uploads are grouped into a deterministic `storage_session_id` per `kind:id`, * mirroring codeapi's sessionKey bucketing so liveness checks resolve. Every * request is recorded and surfaced at `GET /__debug/uploads` so specs can assert @@ -20,17 +30,27 @@ */ const http = require('http'); +const path = require('path'); const busboy = require('busboy'); -const { randomUUID } = require('crypto'); +const { createHash, randomUUID } = require('crypto'); +/** The repository's hoisted SheetJS, the same build LibreChat parses workbooks with. */ +const XLSX = require(path.resolve(__dirname, '../../node_modules/xlsx')); const PORT = parseInt(process.env.E2E_CODE_API_PORT || '8790', 10); const HOST = '127.0.0.1'; +const RETAINED_BYTES_LIMIT = 64 * 1024 * 1024; +const WORKBOOK_TOKEN = /E2E_WORKBOOK:(\S+)/; +const CSV_TOKEN = /E2E_CSV:(\S+)/; /** @type {Map>} */ const sessions = new Map(); -/** @type {Array<{ filename: string; kind: string; id: string; storage_session_id: string; fileId: string; apiKey: string; userId: string; bytes: number }>} */ +/** @type {Array<{ filename: string; kind: string; id: string; storage_session_id: string; fileId: string; apiKey: string; userId: string; bytes: number; sha256: string }>} */ const uploads = []; -/** @type {Array<{ lang: string; codeLength: number; fileCount: number }>} */ +/** Upload bytes by `fileId`, in arrival order so the oldest is evicted first. */ +/** @type {Map} */ +const retained = new Map(); +let retainedBytes = 0; +/** @type {Array<{ lang: string; codeLength: number; fileCount: number; files: Array<{ id: string; name: string }>; analyzedFileId?: string }>} */ const execs = []; /** @type {Map} */ const generated = new Map(); @@ -39,15 +59,45 @@ function sessionIdFor(kind, id) { return `sess-${kind || 'user'}-${id || 'anon'}`; } -function recordUpload({ kind, id, filename, bytes, apiKey, userId }) { - const storage_session_id = sessionIdFor(kind, id); +const sha256 = (content) => createHash('sha256').update(content).digest('hex'); + +function retainUpload(fileId, filename, content) { + retained.set(fileId, { filename, content }); + retainedBytes += content.length; + for (const [retainedId, entry] of retained) { + if (retainedBytes <= RETAINED_BYTES_LIMIT || retainedId === fileId) { + return; + } + retained.delete(retainedId); + retainedBytes -= entry.content.length; + } +} + +function clearRetained() { + retained.clear(); + retainedBytes = 0; +} + +/** Records one received file under its session and keeps its bytes for later execs. */ +function storeUpload({ kind, id, storage_session_id, file, apiKey, userId }) { const fileId = `fid-${randomUUID()}`; if (!sessions.has(storage_session_id)) { sessions.set(storage_session_id, []); } - sessions.get(storage_session_id).push({ fileId, filename }); - uploads.push({ filename, kind, id, storage_session_id, fileId, apiKey, userId, bytes }); - return { storage_session_id, fileId }; + sessions.get(storage_session_id).push({ fileId, filename: file.filename }); + uploads.push({ + filename: file.filename, + kind, + id, + storage_session_id, + fileId, + apiKey, + userId, + bytes: file.bytes, + sha256: sha256(file.content), + }); + retainUpload(fileId, file.filename, file.content); + return fileId; } function sendJson(res, status, body) { @@ -60,18 +110,20 @@ function parseMultipart(req) { const bb = busboy({ headers: req.headers }); /** @type {Record} */ const fields = {}; - /** @type {Array<{ field: string; filename: string; bytes: number }>} */ + /** @type {Array<{ field: string; filename: string; bytes: number; content: Buffer }>} */ const files = []; bb.on('field', (name, value) => { fields[name] = value; }); bb.on('file', (name, stream, info) => { - let bytes = 0; + /** @type {Buffer[]} */ + const chunks = []; stream.on('data', (chunk) => { - bytes += chunk.length; + chunks.push(chunk); }); stream.on('end', () => { - files.push({ field: name, filename: info.filename, bytes }); + const content = Buffer.concat(chunks); + files.push({ field: name, filename: info.filename, bytes: content.length, content }); }); stream.on('error', reject); }); @@ -104,15 +156,14 @@ async function handleUpload(req, res) { sendJson(res, 400, { message: 'error', error: 'no file provided' }); return; } - const apiKey = req.headers['x-api-key'] || ''; - const userId = req.headers['user-id'] || ''; - const { storage_session_id, fileId } = recordUpload({ + const storage_session_id = sessionIdFor(fields.kind, fields.id); + const fileId = storeUpload({ kind: fields.kind, id: fields.id, - filename: files[0].filename, - bytes: files[0].bytes, - apiKey, - userId, + storage_session_id, + file: files[0], + apiKey: req.headers['x-api-key'] || '', + userId: req.headers['user-id'] || '', }); sendJson(res, 200, { message: 'success', @@ -131,20 +182,13 @@ async function handleUploadBatch(req, res) { const userId = req.headers['user-id'] || ''; const storage_session_id = sessionIdFor(fields.kind, fields.id); const responseFiles = files.map((file) => { - const fileId = `fid-${randomUUID()}`; - if (!sessions.has(storage_session_id)) { - sessions.set(storage_session_id, []); - } - sessions.get(storage_session_id).push({ fileId, filename: file.filename }); - uploads.push({ - filename: file.filename, + const fileId = storeUpload({ kind: fields.kind, id: fields.id, storage_session_id, - fileId, + file, apiKey, userId, - bytes: file.bytes, }); return { status: 'success', fileId, filename: file.filename }; }); @@ -193,12 +237,84 @@ function handleRunFileVersion(body, res, label, operation) { }); } +/** The files an exec mounted, as `{ id, name }`. */ +const mountedFiles = (body) => + (Array.isArray(body.files) ? body.files : []).map((file) => ({ + id: String(file?.id ?? ''), + name: String(file?.name ?? ''), + })); + +/** The mounted file of that name whose bytes were uploaded here; nothing else is in the sandbox. */ +function findMountedFile(files, filename) { + const mounted = files.find((file) => file.name === filename && retained.has(file.id)); + return mounted ? { fileId: mounted.id, ...retained.get(mounted.id) } : undefined; +} + +const sumColumnB = (rows) => + rows.reduce((total, row) => (typeof row?.[1] === 'number' ? total + row[1] : total), 0); + +function summarizeWorkbook(content) { + const workbook = XLSX.read(content, { type: 'buffer' }); + const totals = Object.fromEntries( + workbook.SheetNames.map((name) => [ + name, + sumColumnB(XLSX.utils.sheet_to_json(workbook.Sheets[name], { header: 1, raw: true })), + ]), + ); + return { sheets: workbook.SheetNames, totals }; +} + +/** Data rows after the header, and the sum of their numeric column-B cells. */ +function summarizeDelimited(content, filename) { + const lines = content + .toString('utf8') + .split(/\r?\n/) + .filter((line) => line.trim() !== ''); + const separator = + filename.toLowerCase().endsWith('.tsv') || lines[0]?.includes('\t') ? '\t' : ','; + const rows = lines.slice(1).map((line) => line.split(separator)); + const total = rows.reduce((sum, cells) => { + const amount = Number(cells[1]); + return Number.isFinite(amount) ? sum + amount : sum; + }, 0); + return { rows: rows.length, total }; +} + +/** + * Answers the analysis tokens from the upload the exec mounted, or null when the code has none. + * The analyzed upload's id comes back with the output so the exec record can name it. + */ +function analyzeMountedFile(code, files) { + const workbookName = code.match(WORKBOOK_TOKEN)?.[1]; + const csvName = code.match(CSV_TOKEN)?.[1]; + const filename = workbookName ?? csvName; + if (!filename) { + return null; + } + const file = findMountedFile(files, filename); + if (!file) { + const names = files.map((entry) => entry.name).join(', ') || 'none'; + return { + stdout: `E2E file analysis failed: the exec mounted no upload named ${filename} (mounted: ${names})\n`, + }; + } + const summary = workbookName + ? summarizeWorkbook(file.content) + : summarizeDelimited(file.content, filename); + return { stdout: `${JSON.stringify(summary)}\n`, fileId: file.fileId }; +} + async function handleExec(req, res) { const body = await readJson(req); + const code = typeof body.code === 'string' ? body.code : ''; + const files = mountedFiles(body); + const analysis = analyzeMountedFile(code, files); execs.push({ lang: body.lang || '', - codeLength: typeof body.code === 'string' ? body.code.length : 0, - fileCount: Array.isArray(body.files) ? body.files.length : 0, + codeLength: code.length, + fileCount: files.length, + files, + ...(analysis?.fileId != null && { analyzedFileId: analysis.fileId }), }); const versionMarker = body.code?.match( /E2E_RUN_FILE_VERSION:([A-Za-z0-9-]+):(write-v1|inspect|write-v2)/, @@ -207,6 +323,10 @@ async function handleExec(req, res) { handleRunFileVersion(body, res, versionMarker[1], versionMarker[2]); return; } + if (analysis != null) { + sendJson(res, 200, { stdout: analysis.stdout, stderr: '', files: [] }); + return; + } const runFileLabel = body.code?.match(/E2E_RUN_FILE_ARTIFACT:([A-Za-z0-9-]+)/)?.[1]; if (runFileLabel) { const sessionId = `e2e-run-files-${runFileLabel}`; @@ -245,6 +365,7 @@ const server = http.createServer((req, res) => { if (pathname === '/__debug/reset' && req.method === 'POST') { sessions.clear(); generated.clear(); + clearRetained(); uploads.length = 0; execs.length = 0; sendJson(res, 200, { ok: true }); diff --git a/e2e/setup/fake-model.js b/e2e/setup/fake-model.js index 7035ba39328..14d3bae9715 100644 --- a/e2e/setup/fake-model.js +++ b/e2e/setup/fake-model.js @@ -8,7 +8,16 @@ * This exercises the real `Run.create` -> graph -> tool-node pipeline end to end * without a live provider or a standalone HTTP mock server: responses are decided * from the conversation and the agents' advertised tools. + * + * The hook runs inside the server process, so it has no HTTP surface of its own. + * Each model invocation's prompt (system instructions plus messages, inline file + * data left out and document parts listed by file name) is written to + * `e2e/.generated/last-request.json` (override: `E2E_MODEL_REQUEST_LOG`) for the + * most recent runs, which specs read to assert what the model was actually sent. */ +const fs = require('fs'); +const path = require('path'); +const { createHash, randomUUID } = require('crypto'); const { FakeChatModel } = require('@librechat/agents'); const { ChatGenerationChunk } = require('@langchain/core/outputs'); const { AIMessageChunk } = require('@langchain/core/messages'); @@ -185,8 +194,28 @@ const SKILL_DESCRIPTION = 'Use this skill to verify LibreChat skill file authoring in mock end-to-end tests.'; const EDITED_SKILL_DESCRIPTION = 'Use this edited skill to verify LibreChat skill file authoring in mock end-to-end tests.'; +/** + * Prompts that make the model read an attachment with Run Code: the model calls the code tool + * with `` for the file the instructions advertise to Run Code, then answers + * with the tool output verbatim. The fake code server parses the retained upload for the token. + */ +const FILE_ANALYSIS_TARGETS = [ + { marker: 'E2E_ANALYZE_WORKBOOK', token: 'E2E_WORKBOOK:', extensions: ['.xlsx', '.xls', '.ods'] }, + { marker: 'E2E_ANALYZE_CSV', token: 'E2E_CSV:', extensions: ['.csv', '.tsv'] }, +]; +const FILE_ANALYSIS_TOOLS = [BASH_TOOL_NAME, 'execute_code']; +/** A path the queued-file note or the file inventory gives Run Code for an attachment. */ +const ADVERTISED_CODE_PATH = /(?:\/mnt\/data|\$LIBRECHAT_CODE_DATA_DIR)\/(\S+)/g; +const MODEL_REQUEST_LOG_PATH = + process.env.E2E_MODEL_REQUEST_LOG || path.resolve(__dirname, '../.generated/last-request.json'); +const MODEL_REQUEST_LOG_RUNS = 8; +const MODEL_REQUEST_TEXT_LIMIT = 4_000_000; const countedReplies = new Map(); const slowCountedReplies = new Map(); +/** @type {Array<{ sequence: number; conversationId: string | null; userText: string; invocations: Array<{ systemText: string; promptText: string; documentFiles: string[] }> }>} */ +const modelRequestRuns = []; +let modelRequestSequence = 0; +let modelRequestWrite = Promise.resolve(); function messageType(message) { if (typeof message.getType === 'function') { @@ -815,6 +844,73 @@ function replyResponses(text) { return null; } +/** Text a provider would read from a message part; inline file data is left out. */ +function readableText(value, parts = []) { + if (value == null) { + return parts; + } + if (typeof value === 'string') { + if (!value.startsWith('data:')) { + parts.push(value); + } + return parts; + } + if (Array.isArray(value)) { + value.forEach((item) => readableText(item, parts)); + return parts; + } + if (typeof value === 'object' && !ArrayBuffer.isView(value)) { + Object.values(value).forEach((child) => readableText(child, parts)); + } + return parts; +} + +function describePromptMessage(message) { + const text = readableText(message?.content).join('\n'); + const toolCalls = message?.tool_calls?.length ? `\n${JSON.stringify(message.tool_calls)}` : ''; + return `${messageType(message)}: ${text}${toolCalls}`; +} + +function summarizeModelRequest(messages) { + const promptText = messages.map(describePromptMessage).join('\n'); + return { + systemText: collectSystemPromptText(messages), + promptText: + promptText.length > MODEL_REQUEST_TEXT_LIMIT + ? `${promptText.slice(0, MODEL_REQUEST_TEXT_LIMIT)}\n[truncated]` + : promptText, + documentFiles: [...collectProviderFileNames(messages.map((message) => message?.content))], + }; +} + +/** Writes the recent runs in order; a failed write is logged and never fails the run. */ +function persistModelRequestLog() { + const snapshot = JSON.stringify({ runs: modelRequestRuns }); + modelRequestWrite = modelRequestWrite + .then(() => fs.promises.mkdir(path.dirname(MODEL_REQUEST_LOG_PATH), { recursive: true })) + .then(() => fs.promises.writeFile(MODEL_REQUEST_LOG_PATH, snapshot)) + .catch((error) => { + console.warn(`[e2e] fake-model request log not written: ${error?.message ?? error}`); + }); +} + +/** Opens a run's entry in the request log and returns the per-invocation recorder. */ +function startModelRequestLog({ conversationId, userText }) { + modelRequestSequence += 1; + const run = { + sequence: modelRequestSequence, + conversationId: conversationId ?? null, + userText, + invocations: [], + }; + modelRequestRuns.push(run); + modelRequestRuns.splice(0, Math.max(0, modelRequestRuns.length - MODEL_REQUEST_LOG_RUNS)); + return (promptMessages) => { + run.invocations.push(summarizeModelRequest(promptMessages)); + persistModelRequestLog(); + }; +} + /** * Attaches synthetic usage_metadata on a final empty chunk (the OpenAI * streaming pattern) so token-usage SSE events flow end to end in mock runs. @@ -822,11 +918,12 @@ function replyResponses(text) { * instructions included — since the context snapshot calibrates against it. */ class UsageEmittingFakeChatModel extends FakeChatModel { - constructor({ graph, resolveInvocation, resolveOnStream, sleep, ...options }) { + constructor({ graph, resolveInvocation, resolveOnStream, recordRequest, sleep, ...options }) { super({ ...options, sleep }); this.graph = graph; this.resolveInvocation = resolveInvocation; this.resolveOnStream = resolveOnStream; + this.recordRequest = recordRequest; this.streamSleep = sleep ?? CHUNK_DELAY_MS; } @@ -921,6 +1018,13 @@ class UsageEmittingFakeChatModel extends FakeChatModel { async *_streamResponseChunks(messages, options, runManager) { let outputChars = 0; + const { messages: promptMessages } = await getStreamAgentView({ + graph: this.graph, + messages: messages ?? [], + options, + runManager, + }); + this.recordRequest?.(promptMessages); const scriptedResponse = await this.resolveInvocation?.(messages, options, runManager); const dynamicResponse = scriptedResponse ? null @@ -948,12 +1052,6 @@ class UsageEmittingFakeChatModel extends FakeChatModel { outputChars += typeof chunk.text === 'string' ? chunk.text.length : 0; yield chunk; } - const { messages: promptMessages } = await getStreamAgentView({ - graph: this.graph, - messages: messages ?? [], - options, - runManager, - }); const inputChars = promptMessages.reduce( (sum, message) => sum + getContentText(message?.content).length, 0, @@ -980,6 +1078,7 @@ function overrideModel({ disableHumanInTheLoop, resolveInvocation, resolveOnStream, + recordRequest, modelCallbacks, }) { /** The shared mock profile enables approval HITL for its dedicated specs. @@ -1012,6 +1111,7 @@ function overrideModel({ toolCalls, resolveInvocation, resolveOnStream, + recordRequest, }); model.callbacks = modelCallbacks; graph.overrideModel = model; @@ -3384,7 +3484,7 @@ function provisioningToolResponses({ text, toolNames }) { responses: ['', `${FILE_SEARCH_FINAL_TEXT}: ${searchLabel}`], toolCalls: [ { - id: FILE_SEARCH_TOOL_CALL_ID, + id: `${FILE_SEARCH_TOOL_CALL_ID}_${createHash('sha256').update(searchLabel).digest('hex').slice(0, 12)}`, name: FILE_SEARCH_TOOL_NAME, args: { query: `e2e ${searchLabel}` }, type: 'tool_call', @@ -3396,6 +3496,71 @@ function provisioningToolResponses({ text, toolNames }) { return null; } +/** The base name of the first advertised code path with one of `extensions`. */ +function findAdvertisedCodeFile(systemText, extensions) { + return ( + [...systemText.matchAll(ADVERTISED_CODE_PATH)] + .map(([, advertised]) => path.posix.basename(advertised.replace(/[.,;:)"'`]+$/, ''))) + .find((name) => extensions.some((extension) => name.toLowerCase().endsWith(extension))) ?? '' + ); +} + +const fileAnalysisArgs = (toolName, code) => + toolName === BASH_TOOL_NAME ? { command: `# ${code}` } : { lang: 'py', code: `# ${code}` }; + +/** + * Reads an attachment through Run Code. The file name comes from the prompt the model is sent, + * so the call names exactly the path the run advertised, and the answer is the tool output. + */ +function fileAnalysisResponses({ graph, text, toolNames }) { + const target = FILE_ANALYSIS_TARGETS.find(({ marker }) => text.includes(marker)); + if (!target) { + return null; + } + const toolName = FILE_ANALYSIS_TOOLS.find((name) => toolNames.has(name)); + if (!toolName) { + return { + responses: [ + `E2E file analysis unavailable: no code-execution tool advertised (saw ${JSON.stringify([ + ...toolNames, + ])}).`, + ], + }; + } + /** Unique per run, so an analysis earlier in the conversation is never read as this one. */ + const toolCallId = `call_e2e_file_analysis_${randomUUID().slice(0, 8)}`; + return { + responses: [''], + resolveInvocation: async (messages, options, runManager) => { + const result = findLastToolMessage(messages, toolCallId); + if (result) { + return { response: getContentText(result.content) }; + } + const view = await getStreamAgentView({ graph, messages, options, runManager }); + const filename = findAdvertisedCodeFile( + collectSystemPromptText(view.messages), + target.extensions, + ); + if (!filename) { + return { + response: `E2E file analysis failed: no ${target.extensions.join('/')} file was advertised to Run Code.`, + }; + } + return { + response: '', + toolCalls: [ + { + id: toolCallId, + name: toolName, + args: fileAnalysisArgs(toolName, `${target.token}${filename}`), + type: 'tool_call', + }, + ], + }; + }, + }; +} + function codeExecResponses({ filename, toolCallId, finalText, code }, toolNames) { if (!toolNames.has(BASH_TOOL_NAME)) { return { @@ -3514,6 +3679,11 @@ function resolveResponses({ graph, messages, text, toolNames }) { return mcpLegacyActionResponses(toolNames); } + const fileAnalysis = fileAnalysisResponses({ graph, text, toolNames }); + if (fileAnalysis) { + return fileAnalysis; + } + const provisioningTool = provisioningToolResponses({ text, toolNames }); if (provisioningTool) { return provisioningTool; @@ -3735,6 +3905,10 @@ module.exports = function fakeModelHook(run, context) { ) { return; } + const recordRequest = startModelRequestLog({ + conversationId: context?.conversationId, + userText: text, + }); const toolNames = collectToolNames(context?.agents); const handoffScript = parseHandoffScript(text); const { @@ -3777,6 +3951,7 @@ module.exports = function fakeModelHook(run, context) { approvalOutcomeResponses(streamMessages) ?? resolveOnStream?.(streamMessages, streamOptions, runManager) ?? null, + recordRequest, modelCallbacks: context?.modelCallbacks, }); }; diff --git a/e2e/setup/fake-rag-server.js b/e2e/setup/fake-rag-server.js index 20469df2e0b..71f5170acd9 100644 --- a/e2e/setup/fake-rag-server.js +++ b/e2e/setup/fake-rag-server.js @@ -15,6 +15,9 @@ * * Every embed is recorded and surfaced at `GET /__debug/embedded` so specs can * assert a file's bytes actually reached the RAG env, independent of the DB write. + * Embeds and queries share one sequence (`seq`, never reset): an embed takes its + * number once its upload is fully received and a query when it arrives, so a spec + * can show a file was indexed before it was searched. */ const http = require('http'); @@ -23,12 +26,16 @@ const busboy = require('busboy'); const PORT = parseInt(process.env.E2E_RAG_API_PORT || '8791', 10); const HOST = '127.0.0.1'; -/** @type {Array<{ file_id: string; filename: string; entity_id: string; bytes: number; auth: string }>} */ +/** @type {Array<{ seq: number; file_id: string; filename: string; entity_id: string; bytes: number; auth: string }>} */ const embedded = []; -/** @type {Array<{ file_id: string; query: string }>} */ +/** @type {Array<{ seq: number; file_id: string; query: string }>} */ const queries = []; +let sequence = 0; /** @type {string[]} */ const deleted = []; +/** Per-file failures let retry tests leave every other upload unaffected. */ +const embeddingFailures = new Set(); +const failedEmbeds = []; function sendJson(res, status, body) { res.writeHead(status, { 'Content-Type': 'application/json' }); @@ -80,7 +87,13 @@ function readJson(req) { async function handleEmbed(req, res) { const { fields, files } = await parseMultipart(req); + if (embeddingFailures.has(fields.file_id)) { + failedEmbeds.push({ seq: ++sequence, file_id: fields.file_id }); + sendJson(res, 503, { status: false, message: 'E2E embedding unavailable' }); + return; + } embedded.push({ + seq: ++sequence, file_id: fields.file_id || '', filename: files[0]?.filename || '', entity_id: fields.entity_id || '', @@ -91,8 +104,9 @@ async function handleEmbed(req, res) { } async function handleQuery(req, res) { + const seq = ++sequence; const body = await readJson(req); - queries.push({ file_id: body.file_id || '', query: body.query || '' }); + queries.push({ seq, file_id: body.file_id || '', query: body.query || '' }); sendJson(res, 200, []); } @@ -116,7 +130,7 @@ const server = http.createServer((req, res) => { } if (pathname === '/__debug/embedded' && req.method === 'GET') { - sendJson(res, 200, { embedded, queries, deleted }); + sendJson(res, 200, { embedded, queries, deleted, failedEmbeds }); return; } @@ -124,6 +138,23 @@ const server = http.createServer((req, res) => { embedded.length = 0; queries.length = 0; deleted.length = 0; + failedEmbeds.length = 0; + embeddingFailures.clear(); + sendJson(res, 200, { ok: true }); + return; + } + + if (pathname === '/__debug/embedding-failure' && req.method === 'POST') { + const body = await readJson(req); + if (typeof body.file_id !== 'string' || typeof body.enabled !== 'boolean') { + sendJson(res, 400, { message: 'file_id and enabled are required' }); + return; + } + if (body.enabled) { + embeddingFailures.add(body.file_id); + } else { + embeddingFailures.delete(body.file_id); + } sendJson(res, 200, { ok: true }); return; } diff --git a/e2e/setup/global-setup.ts b/e2e/setup/global-setup.ts index c417ebc3409..267a1c996b8 100644 --- a/e2e/setup/global-setup.ts +++ b/e2e/setup/global-setup.ts @@ -1,8 +1,10 @@ import { FullConfig } from '@playwright/test'; +import { writeUploadFixtures } from './uploads'; import authenticate from './authenticate'; import { getE2EUser } from './user'; async function globalSetup(config: FullConfig) { + writeUploadFixtures(); await authenticate(config, getE2EUser()); } diff --git a/e2e/setup/uploads.ts b/e2e/setup/uploads.ts new file mode 100644 index 00000000000..a59bcb38502 --- /dev/null +++ b/e2e/setup/uploads.ts @@ -0,0 +1,167 @@ +import fs from 'fs'; +import path from 'path'; +import * as XLSX from 'xlsx'; + +/** + * Upload fixtures for the automatic reading policy (automatic-upload.spec.ts). The mock lane's + * global setup writes them from the rows below; the totals a spec asserts are stated on their own + * and checked against those rows when the fixtures are written, so an edit to either fails there. + */ +export const UPLOAD_FIXTURE_DIR = path.resolve(__dirname, '../.generated/uploads'); + +/** Sits alone in `Notes!A1`, so it reaches the model only through extracted text. */ +export const WORKBOOK_SENTINEL = 'SENTINEL-7F3A'; + +export type SheetRow = readonly [item: string, amount: number]; + +/** Six-digit amounts, so none can be mistaken for an id, a port or a timestamp fragment. */ +export const Q1_ROWS: readonly SheetRow[] = [ + ['North', 731842], + ['South', 905317], + ['East', 648259], + ['West', 517093], +]; + +export const Q2_ROWS: readonly SheetRow[] = [ + ['North', 812406], + ['South', 693178], + ['East', 574921], + ['West', 786350], +]; + +/** Column-B totals the fake code server computes from the uploaded bytes, as known values. */ +export const WORKBOOK_TOTALS = { + Q1: 2_802_511, + Q2: 2_866_855, + Notes: 0, +} as const; + +const sumAmounts = (rows: readonly SheetRow[]): number => + rows.reduce((total, [, amount]) => total + amount, 0); + +function assertKnownTotals(): void { + const computed = { Q1: sumAmounts(Q1_ROWS), Q2: sumAmounts(Q2_ROWS) }; + if (computed.Q1 !== WORKBOOK_TOTALS.Q1 || computed.Q2 !== WORKBOOK_TOTALS.Q2) { + throw new Error( + `Fixture rows sum to ${JSON.stringify(computed)}, not the known totals ${JSON.stringify( + WORKBOOK_TOTALS, + )}`, + ); + } +} + +export type UploadFixture = { name: string; mimeType: string; path: string }; + +const fixture = (name: string, mimeType: string): UploadFixture => ({ + name, + mimeType, + path: path.join(UPLOAD_FIXTURE_DIR, name), +}); + +export const UPLOAD_FIXTURES = { + xlsx: fixture( + 'quarterly.xlsx', + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + ), + csv: fixture('q1.csv', 'text/csv'), + smallPdf: fixture('small.pdf', 'application/pdf'), + largePdf: fixture('large.pdf', 'application/pdf'), +} as const; + +const SHEET_HEADER = ['item', 'amount']; +const SMALL_PDF_BYTES = 50 * 1024; +const LARGE_PDF_BYTES = 3 * 1024 * 1024; + +const sheetRows = (rows: readonly SheetRow[]): (string | number)[][] => [ + SHEET_HEADER, + ...rows.map(([item, amount]) => [item, amount]), +]; + +function buildWorkbook(): XLSX.WorkBook { + const workbook = XLSX.utils.book_new(); + XLSX.utils.book_append_sheet(workbook, XLSX.utils.aoa_to_sheet(sheetRows(Q1_ROWS)), 'Q1'); + XLSX.utils.book_append_sheet(workbook, XLSX.utils.aoa_to_sheet(sheetRows(Q2_ROWS)), 'Q2'); + XLSX.utils.book_append_sheet(workbook, XLSX.utils.aoa_to_sheet([[WORKBOOK_SENTINEL]]), 'Notes'); + return workbook; +} + +const writeWorkbook = (workbook: XLSX.WorkBook, bookType: XLSX.BookType): Buffer => + XLSX.write(workbook, { type: 'buffer', bookType }) as Buffer; + +const delimited = (rows: readonly SheetRow[], separator: string): string => + `${sheetRows(rows) + .map((cells) => cells.join(separator)) + .join('\n')}\n`; + +/** Deterministic filler bytes for the PDF padding stream. */ +function fillerBytes(length: number): Buffer { + const bytes = Buffer.alloc(length); + let state = 0x2545f491; + for (let index = 0; index < length; index++) { + state = (Math.imul(state, 1103515245) + 12345) >>> 0; + bytes[index] = state >>> 24; + } + return bytes; +} + +const ascii = (value: string): Buffer => Buffer.from(value, 'latin1'); + +const streamBody = (data: Buffer): Buffer => + Buffer.concat([ascii(`<< /Length ${data.length} >>\nstream\n`), data, ascii('\nendstream')]); + +/** A one-page PDF with a correct cross-reference table; the extra stream sets its size. */ +function assemblePdf(title: string, padding: Buffer): Buffer { + const header = ascii('%PDF-1.4\n%\xe2\xe3\xcf\xd3\n'); + const bodies = [ + ascii('<< /Type /Catalog /Pages 2 0 R >>'), + ascii('<< /Type /Pages /Kids [3 0 R] /Count 1 >>'), + ascii( + '<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] ' + + '/Resources << /Font << /F1 5 0 R >> >> /Contents 4 0 R >>', + ), + streamBody(ascii(`BT /F1 18 Tf 72 720 Td (${title}) Tj ET`)), + ascii('<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>'), + streamBody(padding), + ]; + const objects = bodies.map((body, index) => + Buffer.concat([ascii(`${index + 1} 0 obj\n`), body, ascii('\nendobj\n')]), + ); + let cursor = header.length; + const offsets = objects.map((object) => { + const offset = cursor; + cursor += object.length; + return offset; + }); + const xref = [ + 'xref', + `0 ${objects.length + 1}`, + '0000000000 65535 f ', + ...offsets.map((offset) => `${String(offset).padStart(10, '0')} 00000 n `), + 'trailer', + `<< /Size ${objects.length + 1} /Root 1 0 R >>`, + 'startxref', + String(cursor), + '%%EOF', + '', + ].join('\n'); + return Buffer.concat([header, ...objects, ascii(xref)]); +} + +function buildPdf(title: string, targetBytes: number): Buffer { + const baseLength = assemblePdf(title, Buffer.alloc(0)).length; + return assemblePdf(title, fillerBytes(Math.max(0, targetBytes - baseLength))); +} + +/** Writes every upload fixture under `e2e/.generated/uploads/`, replacing earlier copies. */ +export function writeUploadFixtures(): void { + assertKnownTotals(); + fs.mkdirSync(UPLOAD_FIXTURE_DIR, { recursive: true }); + const workbook = buildWorkbook(); + const contents: ReadonlyArray = [ + [UPLOAD_FIXTURES.xlsx, writeWorkbook(workbook, 'xlsx')], + [UPLOAD_FIXTURES.csv, Buffer.from(delimited(Q1_ROWS, ','), 'utf8')], + [UPLOAD_FIXTURES.smallPdf, buildPdf('E2E small PDF', SMALL_PDF_BYTES)], + [UPLOAD_FIXTURES.largePdf, buildPdf('E2E large PDF', LARGE_PDF_BYTES)], + ]; + contents.forEach(([{ path: filePath }, content]) => fs.writeFileSync(filePath, content)); +} diff --git a/e2e/specs/mock/automatic-upload.spec.ts b/e2e/specs/mock/automatic-upload.spec.ts new file mode 100644 index 00000000000..d9973af47c6 --- /dev/null +++ b/e2e/specs/mock/automatic-upload.spec.ts @@ -0,0 +1,792 @@ +import path from 'path'; +import { expect, test } from '@playwright/test'; +import { createHash, randomUUID } from 'crypto'; +import type { TFile, TMessage, TextDerivation } from 'librechat-data-provider'; +import type { Page, Response } from '@playwright/test'; +import type { + AttachFile, + MockEndpoint, + ModelRunRecord, + RagQueryRecord, + RagEmbedRecord, + ModelRequestRecord, +} from './helpers'; +import type { UploadFixture } from '../../setup/uploads'; +import { + MOCK_ENDPOINTS, + NEW_CHAT_PATH, + fetchJson, + getModelRun, + uniqueName, + replyPrompt, + sendMessage, + getCodeExecs, + requestJson, + messagesView, + uploadViaDrop, + getRagQueries, + getAccessToken, + getRagEmbedded, + uploadViaPaste, + enableFileSearch, + attachFileBuffer, + resetProvisioning, + supportsFilePaste, + selectMockEndpoint, + enableCodeInterpreter, + uploadViaUnifiedButton, + getCodeProvisionedUploads, + sendMessageAndWaitForCompletion, +} from './helpers'; +import { + Q1_ROWS, + Q2_ROWS, + UPLOAD_FIXTURES, + WORKBOOK_TOTALS, + WORKBOOK_SENTINEL, +} from '../../setup/uploads'; +import { cleanupAgent, uniqueAgentName } from './agents.helpers'; +import { withMongo } from './db'; + +/** + * The automatic reading policy end to end. + * + * Mock Auto Provider and Mock Auto Small Provider (e2e/config/librechat.e2e.yaml) select + * `llmDeliveryPolicy: automatic` with no route overrides; Auto Small adds a 1 MB per-file limit. + * The fixtures come from e2e/setup/uploads.ts, written by the global setup; the totals asserted + * here are known constants checked against the rows the workbook was built from. Rows draw on up + * to three observers: + * + * - the file record: `/api/files` for the route and marker, MongoDB for the stored text the API + * leaves out, and the user message's files for the delivery path the turn used; + * - the fake model's request log (`e2e/.generated/last-request.json`): the system instructions + * with the file inventory, the message text, and the document parts the model was sent; + * - the fake code and RAG servers: the bytes Run Code received (by sha256) and the files each exec + * mounted, and the File Search embeds and queries in the order they arrived. + * + * `E2E_ANALYZE_WORKBOOK` and `E2E_ANALYZE_CSV` make the fake model run code on the file the + * instructions advertise to Run Code. The fake code server answers only from an upload the exec + * itself mounted, and records which one, so a total on screen means code read the attached bytes. + * + * Mock Provider C keeps the classic policy with no file configuration; A-29 shows classic still + * extracts a workbook at upload and writes none of the automatic policy's state. + */ + +const [, , AUTO_ENDPOINT, AUTO_SMALL_ENDPOINT] = MOCK_ENDPOINTS; +const CLASSIC_ENDPOINT: MockEndpoint = { label: 'Mock Provider C', model: 'mock-model-c' }; +const TEST_TIMEOUT = 120_000; +const MULTI_TURN_TIMEOUT = 180_000; +const TURN_TIMEOUT = 60_000; +const WORKBOOK_RESULT = JSON.stringify({ sheets: ['Q1', 'Q2', 'Notes'], totals: WORKBOOK_TOTALS }); +const CSV_RESULT = JSON.stringify({ rows: Q1_ROWS.length, total: WORKBOOK_TOTALS.Q1 }); +/** The error a turn ends with when File Search could not index its attachments. */ +const PREPARATION_FAILED_ERROR = /File Search couldn't prepare the attached files/; +/** The heading of the model's file inventory, which only the automatic policy writes. */ +const INVENTORY_HEADING = 'how you can read them on this turn'; + +type StoredText = { file_id: string; text?: string | null }; +type AgentRecord = { id: string }; +type ReadingRecord = Pick & + Partial>; +type MessageFileReading = Pick; + +const sha256 = (content: Buffer): string => createHash('sha256').update(content).digest('hex'); + +/** The fixture's bytes and type under a name no earlier upload has used. */ +const renamed = (fixture: UploadFixture, prefix: string): AttachFile => ({ + name: `${uniqueName(prefix)}${path.extname(fixture.name)}`, + mimeType: fixture.mimeType, + path: fixture.path, +}); + +const composerChip = (page: Page, filename: string) => + page.getByTestId('composer-tray').getByRole('button', { name: filename }); + +async function startChat(page: Page, endpoint: MockEndpoint) { + await page.goto(NEW_CHAT_PATH, { timeout: 10_000 }); + await selectMockEndpoint(page, endpoint); + await expect(page.getByRole('button', { name: 'Attach and tools', exact: true })).toBeVisible({ + timeout: 15_000, + }); + await resetProvisioning(page); +} + +/** Waits for an upload to land in the composer and returns the record it created. */ +async function confirmUpload(page: Page, upload: Promise): Promise { + const response = await upload; + expect(response.ok(), `upload returned ${response.status()}`).toBeTruthy(); + const uploaded = (await response.json()) as TFile; + await expect(composerChip(page, uploaded.filename)).toBeVisible({ timeout: 15_000 }); + return uploaded; +} + +async function getFileRecord(page: Page, fileId: string): Promise { + const token = await getAccessToken(page); + const files = await fetchJson(page, '/api/files', token); + const record = files.find((file) => file.file_id === fileId); + if (!record) { + throw new Error(`/api/files does not list ${fileId}`); + } + return record; +} + +/** The stored extracted text, which the file API never returns. */ +function getStoredText(fileId: string): Promise { + return withMongo(async (db) => { + const file = await db + .collection('files') + .findOne({ file_id: fileId }, { projection: { text: 1 } }); + return file?.text ?? undefined; + }); +} + +/** What decides how a record is read, without the derivation timestamp. */ +const readingRecord = ({ llmDeliveryPath, metadata }: TFile): ReadingRecord => ({ + llmDeliveryPath, + outcome: metadata?.textDerivation?.outcome, + extractor: metadata?.textDerivation?.extractor, + reason: metadata?.textDerivation?.reason, +}); + +/** Kept for Run Code at upload: no model route, no extracted text, a deferred marker. */ +async function expectDeferredRecord(page: Page, fileId: string) { + expect(readingRecord(await getFileRecord(page, fileId))).toMatchObject({ + llmDeliveryPath: 'none', + outcome: 'deferred', + }); + expect(await getStoredText(fileId), 'a deferred upload stores no extracted text').toBeFalsy(); +} + +function conversationIdFromPage(page: Page): string { + const conversationId = new URL(page.url()).pathname.split('/c/')[1] ?? ''; + expect(conversationId).toMatch(/^[0-9a-f-]{36}$/i); + return conversationId; +} + +/** The files the latest user message persisted, with the delivery path its turn gave each. */ +async function getLatestUserMessageFiles(page: Page): Promise[]> { + const token = await getAccessToken(page); + const conversationId = encodeURIComponent(conversationIdFromPage(page)); + const messages = await fetchJson(page, `/api/messages/${conversationId}`, token); + return messages.filter((message) => message.isCreatedByUser).pop()?.files ?? []; +} + +function messageFileReading(files: Partial[], fileId: string): MessageFileReading { + const file = files.find((entry) => entry.file_id === fileId); + expect(file, `the user message should carry ${fileId}`).toBeTruthy(); + return { llmDeliveryPath: file?.llmDeliveryPath }; +} + +/** Sends a turn, waits for the stored answer, and returns what the model was sent for it. */ +async function sendTurn(page: Page, prompt: string, minInvocations = 1): Promise { + await sendMessageAndWaitForCompletion(page, prompt, { timeout: TURN_TIMEOUT }); + return getModelRun(prompt, { minInvocations }); +} + +/** A turn where the model calls Run Code and then answers, so two model requests. */ +const sendAnalysisTurn = (page: Page, prompt: string): Promise => + sendTurn(page, prompt, 2); + +/** The turn's first model request: what the model had before any tool ran. */ +const initialRequest = (run: ModelRunRecord): ModelRequestRecord => run.invocations[0]; + +/** The inventory line naming `filename`; the queued-path note never quotes a name. */ +const inventoryLine = (request: ModelRequestRecord, filename: string): string => + request.systemText.split('\n').find((line) => line.includes(JSON.stringify(filename))) ?? ''; + +/** + * No request of the turn carried the workbook: not as a file part, and no cell as text (the + * sentinel or an amount). Inline file data never reaches `promptText`, so the parts are checked + * by name. + */ +function expectNoWorkbookCells(run: ModelRunRecord, filename: string) { + expect(run.invocations.flatMap(({ documentFiles }) => documentFiles)).not.toContain(filename); + const sent = run.invocations.map(({ promptText }) => promptText).join('\n'); + expect(sent).not.toContain(WORKBOOK_SENTINEL); + [...Q1_ROWS, ...Q2_ROWS].forEach(([, amount]) => expect(sent).not.toContain(String(amount))); +} + +/** + * Run Code received exactly this upload's bytes under its name, and an exec mounted that upload; + * with `analyzed`, the exec's analysis token read it. Renamed copies of a fixture share its + * bytes, so the name ties the receipt to this test's upload. + */ +async function expectCodeReceived( + page: Page, + file: AttachFile, + { analyzed = false }: { analyzed?: boolean } = {}, +) { + const expected = sha256(attachFileBuffer(file)); + let fileId: string | undefined; + await expect + .poll( + async () => { + const uploads = await getCodeProvisionedUploads(page); + fileId = uploads.find( + (upload) => upload.filename === file.name && upload.sha256 === expected, + )?.fileId; + return fileId != null; + }, + { timeout: 30_000, message: `Run Code should receive the bytes of ${file.name}` }, + ) + .toBe(true); + const execs = await getCodeExecs(page); + const mounted = execs.filter((exec) => exec.files.some(({ id }) => id === fileId)); + expect(mounted.length, `an exec should mount ${file.name}`).toBeGreaterThan(0); + if (analyzed) { + expect(mounted.map((exec) => exec.analyzedFileId)).toContain(fileId); + } +} + +/** The inventory line a deferred workbook gets while Run Code can still open it. */ +const expectQueuedForCode = (request: ModelRequestRecord, filename: string) => + expect(inventoryLine(request, filename)).toContain( + `read it with Run Code at /mnt/data/${filename}`, + ); + +const latestSeq = (records: ReadonlyArray): number => + records.reduce((latest, { seq }) => Math.max(latest, seq), 0); + +/** The assistant's visible answer carries the code server's result. */ +async function expectAnswer(page: Page, expected: string) { + await expect( + messagesView(page).getByText(expected).filter({ visible: true }).first(), + ).toBeVisible({ timeout: 20_000 }); +} + +async function createAutoAgent(page: Page, name: string, tools: string[]): Promise { + const token = await getAccessToken(page); + const agent = await requestJson(page, { + path: '/api/agents', + token, + method: 'POST', + body: { + name, + provider: AUTO_ENDPOINT.label, + model: AUTO_ENDPOINT.model, + model_parameters: {}, + tools, + }, + }); + return agent.id; +} + +async function setAgentTools(page: Page, agentId: string, tools: string[]) { + const token = await getAccessToken(page); + await requestJson(page, { + path: `/api/agents/${encodeURIComponent(agentId)}`, + token, + method: 'PATCH', + body: { tools }, + }); +} + +async function startAgentChat(page: Page, agentName: string) { + await page.goto(NEW_CHAT_PATH, { timeout: 10_000 }); + const trigger = page.getByRole('button', { name: 'Select a model' }).first(); + await trigger.click(); + await page.getByRole('option', { name: 'My Agents' }).click(); + await page.getByRole('option', { name: agentName }).click(); + await expect(trigger).toContainText(agentName); + await resetProvisioning(page); +} + +/** Reloads the open conversation, so the next turn starts from restored state. */ +async function reloadConversation(page: Page) { + await page.reload({ timeout: 20_000 }); + await expect(page.getByRole('textbox', { name: 'Message input' })).toBeVisible({ + timeout: 20_000, + }); +} + +/** Posts an upload straight to the server, past the client's own size check. */ +async function postUpload( + page: Page, + endpoint: MockEndpoint, + file: AttachFile, + toolResource?: 'execute_code' | 'file_search', +) { + const token = await getAccessToken(page); + return page.request.post('/api/files', { + headers: { Authorization: `Bearer ${token}` }, + multipart: { + endpoint: endpoint.label, + endpointType: 'custom', + file_id: randomUUID(), + message_file: 'true', + ...(toolResource != null && { tool_resource: toolResource }), + file: { name: file.name, mimeType: file.mimeType, buffer: attachFileBuffer(file) }, + }, + }); +} + +/** The stored original's bytes, as the download route serves them. */ +async function downloadOriginal(page: Page, record: TFile): Promise { + const token = await getAccessToken(page); + const response = await page.request.get( + `/api/files/download/${encodeURIComponent(record.user)}/${encodeURIComponent(record.file_id)}`, + { headers: { Authorization: `Bearer ${token}` } }, + ); + expect(response.ok(), `download returned ${response.status()}`).toBeTruthy(); + return response.body(); +} + +/** Attaches an earlier upload from the palette's file list, which re-stages the stored record. */ +async function attachExisting(page: Page, file: TFile) { + await page.getByRole('button', { name: 'Attach and tools', exact: true }).click(); + await page.getByTestId('composer-palette-search').fill(file.filename); + await page.locator(`[data-row-key="file:${file.file_id}"]`).getByRole('button').first().click(); + await expect(composerChip(page, file.filename)).toBeVisible({ timeout: 15_000 }); +} + +async function expectWorkbookTextPreview(page: Page, file: TFile) { + /* The latest message carrying the workbook; earlier turns in the conversation attached it too. */ + await messagesView(page).getByRole('button', { name: file.filename, exact: true }).last().click(); + const dialog = page.getByRole('dialog'); + await expect(dialog).toContainText(WORKBOOK_SENTINEL); + await expect(dialog).not.toContainText('[Content_Types].xml'); + await page.keyboard.press('Escape'); + await expect(dialog).not.toBeVisible(); +} + +/** Fail one original's external indexing until the retry explicitly restores the service. */ +async function setEmbeddingFailure(page: Page, fileId: string, enabled: boolean) { + const port = process.env.E2E_RAG_API_PORT || '8791'; + const response = await page.request.post(`http://127.0.0.1:${port}/__debug/embedding-failure`, { + data: { file_id: fileId, enabled }, + }); + expect(response.ok()).toBe(true); +} + +test.describe('automatic upload reading', () => { + test.describe.configure({ timeout: TEST_TIMEOUT }); + + const CODE_FORMATS = [ + { fixture: UPLOAD_FIXTURES.xlsx, prompt: 'E2E_ANALYZE_WORKBOOK', result: WORKBOOK_RESULT }, + { fixture: UPLOAD_FIXTURES.csv, prompt: 'E2E_ANALYZE_CSV', result: CSV_RESULT }, + ]; + + for (const { fixture, prompt, result } of CODE_FORMATS) { + test(`A-01/A-02: ${fixture.name} is kept for Run Code, parsed there, and never sent as cells`, async ({ + page, + }) => { + await startChat(page, AUTO_ENDPOINT); + await enableCodeInterpreter(page); + + const file = renamed(fixture, 'a02-quarterly'); + const uploaded = await confirmUpload(page, uploadViaUnifiedButton(page, file)); + await expectDeferredRecord(page, uploaded.file_id); + + const run = await sendAnalysisTurn(page, `${prompt} ${uniqueName('a02')}`); + expectQueuedForCode(initialRequest(run), file.name); + expectNoWorkbookCells(run, file.name); + await expectCodeReceived(page, file, { analyzed: true }); + await expectAnswer(page, result); + }); + } + + test('A-15: a workbook goes to Run Code and a small PDF to the provider in one message', async ({ + page, + }) => { + await startChat(page, AUTO_ENDPOINT); + await enableCodeInterpreter(page); + + const workbook = renamed(UPLOAD_FIXTURES.xlsx, 'a15-quarterly'); + const pdf = renamed(UPLOAD_FIXTURES.smallPdf, 'a15-small'); + await confirmUpload(page, uploadViaUnifiedButton(page, workbook)); + await confirmUpload(page, uploadViaUnifiedButton(page, pdf)); + + const run = await sendAnalysisTurn(page, `E2E_ANALYZE_WORKBOOK ${uniqueName('a15')}`); + const request = initialRequest(run); + expect(request.documentFiles).toContain(pdf.name); + expect(inventoryLine(request, pdf.name)).toContain('Run Code can also open it'); + expectQueuedForCode(request, workbook.name); + expectNoWorkbookCells(run, workbook.name); + await expectCodeReceived(page, workbook, { analyzed: true }); + await expectCodeReceived(page, pdf); + await expectAnswer(page, WORKBOOK_RESULT); + }); + + test('A-16: enabling Run Code later reads an extracted workbook with code, across reloads', async ({ + page, + }) => { + test.setTimeout(MULTI_TURN_TIMEOUT); + const agentName = uniqueAgentName('E2E Auto Later Code'); + let agentId: string | undefined; + try { + await page.goto(NEW_CHAT_PATH, { timeout: 10_000 }); + agentId = await createAutoAgent(page, agentName, []); + await startAgentChat(page, agentName); + + /* Without Run Code at upload, the workbook is extracted as it is under classic. */ + const workbook = renamed(UPLOAD_FIXTURES.xlsx, 'a16-quarterly'); + const uploaded = await confirmUpload(page, uploadViaUnifiedButton(page, workbook)); + const record = await getFileRecord(page, uploaded.file_id); + expect(record.llmDeliveryPath).toBe('text'); + expect(record.metadata?.textDerivation).toBeUndefined(); + expect(await getStoredText(uploaded.file_id)).toContain(WORKBOOK_SENTINEL); + + const firstTurn = await sendTurn(page, replyPrompt(uniqueName('a16-turn1'))); + expect(initialRequest(firstTurn).promptText).toContain(WORKBOOK_SENTINEL); + + await setAgentTools(page, agentId, ['execute_code']); + await reloadConversation(page); + await resetProvisioning(page); + const secondTurn = await sendAnalysisTurn( + page, + `E2E_ANALYZE_WORKBOOK ${uniqueName('a16-turn2')}`, + ); + expectNoWorkbookCells(secondTurn, workbook.name); + await expectCodeReceived(page, workbook, { analyzed: true }); + await expectAnswer(page, WORKBOOK_RESULT); + + await reloadConversation(page); + const thirdTurn = await sendTurn(page, replyPrompt(uniqueName('a16-turn3'))); + expect(initialRequest(thirdTurn).promptText).not.toContain(WORKBOOK_SENTINEL); + expect(readingRecord(await getFileRecord(page, uploaded.file_id))).toEqual( + readingRecord(record), + ); + } finally { + await cleanupAgent(page, agentId); + } + }); + + test('A-17: disabling Run Code derives the workbook text once for later turns, across reloads', async ({ + page, + }) => { + test.setTimeout(MULTI_TURN_TIMEOUT); + const agentName = uniqueAgentName('E2E Auto Drop Code'); + let agentId: string | undefined; + try { + await page.goto(NEW_CHAT_PATH, { timeout: 10_000 }); + agentId = await createAutoAgent(page, agentName, ['execute_code']); + await startAgentChat(page, agentName); + + const workbook = renamed(UPLOAD_FIXTURES.xlsx, 'a17-quarterly'); + const uploaded = await confirmUpload(page, uploadViaUnifiedButton(page, workbook)); + await expectDeferredRecord(page, uploaded.file_id); + + const firstTurn = await sendAnalysisTurn( + page, + `E2E_ANALYZE_WORKBOOK ${uniqueName('a17-turn1')}`, + ); + expectNoWorkbookCells(firstTurn, workbook.name); + await expectCodeReceived(page, workbook, { analyzed: true }); + await expectAnswer(page, WORKBOOK_RESULT); + + await setAgentTools(page, agentId, []); + await reloadConversation(page); + const secondTurn = await sendTurn(page, replyPrompt(uniqueName('a17-turn2'))); + const secondRequest = initialRequest(secondTurn); + expect(secondRequest.promptText).toContain(WORKBOOK_SENTINEL); + /* Run Code is gone, so nothing may still say the workbook can be opened there. */ + expect(secondRequest.systemText).not.toContain(`/mnt/data/${workbook.name}`); + expect(inventoryLine(secondRequest, workbook.name)).not.toContain('Run Code can'); + await expect + .poll(async () => readingRecord(await getFileRecord(page, uploaded.file_id)).outcome, { + timeout: 15_000, + message: 'the derived text should be saved onto the deferred record', + }) + .toBe('complete'); + const derivation = (await getFileRecord(page, uploaded.file_id)).metadata?.textDerivation; + expect(derivation?.at, 'the derivation should be stamped when it was saved').toEqual( + expect.any(Number), + ); + expect(await getStoredText(uploaded.file_id)).toContain(WORKBOOK_SENTINEL); + + /* Derived once: a later turn reads the saved text and leaves the derivation, stamp and + * all, as it was. */ + await reloadConversation(page); + await attachExisting(page, uploaded); + const thirdTurn = await sendTurn(page, replyPrompt(uniqueName('a17-turn3'))); + expect(initialRequest(thirdTurn).promptText).toContain(WORKBOOK_SENTINEL); + expect((await getFileRecord(page, uploaded.file_id)).metadata?.textDerivation).toEqual( + derivation, + ); + expect(messageFileReading(await getLatestUserMessageFiles(page), uploaded.file_id)).toEqual({ + llmDeliveryPath: 'text', + }); + await expectWorkbookTextPreview(page, uploaded); + expect(await downloadOriginal(page, uploaded)).toEqual(attachFileBuffer(workbook)); + } finally { + await cleanupAgent(page, agentId); + } + }); + + test('A-06/A-13: a PDF over the small endpoint limit is searched there, and refused as a new upload', async ({ + page, + }) => { + test.setTimeout(MULTI_TURN_TIMEOUT); + await startChat(page, AUTO_ENDPOINT); + await sendTurn(page, replyPrompt(uniqueName('a06-open'))); + + /* Uploaded where it fits natively; the conversation keeps it across the model switch. */ + const pdf = renamed(UPLOAD_FIXTURES.largePdf, 'a06-large'); + const uploaded = await confirmUpload(page, uploadViaUnifiedButton(page, pdf)); + expect(uploaded.llmDeliveryPath).toBe('provider'); + + await selectMockEndpoint(page, AUTO_SMALL_ENDPOINT); + await expect(composerChip(page, pdf.name)).toBeVisible(); + await enableFileSearch(page); + + /* Read as late as possible before the turn, so an index built at upload has had time to + * land: nothing may hold the PDF yet, and everything after this mark happened in the turn. */ + const beforeTurn = await getRagEmbedded(page); + expect(beforeTurn.map((entry) => entry.file_id)).not.toContain(uploaded.file_id); + const mark = Math.max(latestSeq(beforeTurn), latestSeq(await getRagQueries(page))); + + const run = await sendTurn(page, `E2E_FILE_SEARCH:${uniqueName('a06')}`); + const request = initialRequest(run); + expect(request.documentFiles).not.toContain(pdf.name); + const line = inventoryLine(request, pdf.name); + expect(line).toContain('too large to send directly'); + expect(line).toContain('file_search'); + expect(line).toContain('it is indexed when you first search'); + + /* Embedded once, during the turn, and before any search of it. */ + let embeds: RagEmbedRecord[] = []; + await expect + .poll( + async () => { + embeds = (await getRagEmbedded(page)).filter( + (entry) => entry.file_id === uploaded.file_id, + ); + return embeds.length; + }, + { timeout: 30_000, message: 'the PDF should be indexed for File Search during the turn' }, + ) + .toBe(1); + let searches: RagQueryRecord[] = []; + await expect + .poll(async () => { + searches = (await getRagQueries(page)).filter( + (query) => query.file_id === uploaded.file_id, + ); + return searches.length; + }) + .toBeGreaterThan(0); + const [embed] = embeds; + expect(embed.seq).toBeGreaterThan(mark); + expect(embed.seq).toBeLessThan(Math.min(...searches.map(({ seq }) => seq))); + + /* A fresh conversation receives this indexed original through message resources.files, + * not an agent's configured file_ids. It must remain searchable without another embed. */ + await page.goto(NEW_CHAT_PATH); + await selectMockEndpoint(page, AUTO_ENDPOINT); + await sendTurn(page, replyPrompt(uniqueName('a06-reuse-open'))); + await attachExisting(page, uploaded); + await selectMockEndpoint(page, AUTO_SMALL_ENDPOINT); + if ( + !(await page.getByRole('button', { name: 'Remove File Search', exact: true }).isVisible()) + ) { + await enableFileSearch(page); + } + const reuseMark = latestSeq(await getRagQueries(page)); + const reusedRun = await sendTurn(page, `E2E_FILE_SEARCH:${uniqueName('a06-reuse')}`); + expect(initialRequest(reusedRun).documentFiles).not.toContain(pdf.name); + expect(inventoryLine(initialRequest(reusedRun), pdf.name)).toContain('file_search'); + expect(messageFileReading(await getLatestUserMessageFiles(page), uploaded.file_id)).toEqual({ + llmDeliveryPath: 'none', + }); + expect( + (await getRagEmbedded(page)).filter((entry) => entry.file_id === uploaded.file_id), + ).toHaveLength(1); + expect( + (await getRagQueries(page)).some( + (query) => query.file_id === uploaded.file_id && query.seq > reuseMark, + ), + ).toBe(true); + await reloadConversation(page); + expect(messageFileReading(await getLatestUserMessageFiles(page), uploaded.file_id)).toEqual({ + llmDeliveryPath: 'none', + }); + + /* Refused as before this policy existed, on every route: the size check throws inside the + * upload handler, which answers with its generic error, and no tool receives the bytes. The + * small PDF on the same route shows size decides. */ + const direct = renamed(UPLOAD_FIXTURES.largePdf, 'a13-large'); + const largeSha = sha256(attachFileBuffer(direct)); + for (const toolResource of [undefined, 'execute_code', 'file_search'] as const) { + const rejected = await postUpload(page, AUTO_SMALL_ENDPOINT, direct, toolResource); + const route = toolResource ?? 'message file'; + expect(rejected.status(), `${route} upload status`).toBe(500); + expect(await rejected.json(), `${route} upload body`).toEqual({ + message: 'Error processing file', + }); + } + expect((await getCodeProvisionedUploads(page)).map((upload) => upload.sha256)).not.toContain( + largeSha, + ); + expect((await getRagEmbedded(page)).map((entry) => entry.filename)).not.toContain(direct.name); + const control = await postUpload( + page, + AUTO_SMALL_ENDPOINT, + renamed(UPLOAD_FIXTURES.smallPdf, 'a13-small'), + ); + expect(control.ok(), await control.text()).toBe(true); + const token = await getAccessToken(page); + const files = await fetchJson(page, '/api/files', token); + expect(files.map((file) => file.filename)).not.toContain(direct.name); + }); + + test('failed indexing fails the turn with a retryable error, and retry reuses the original', async ({ + page, + }) => { + test.setTimeout(MULTI_TURN_TIMEOUT); + await startChat(page, AUTO_ENDPOINT); + await sendTurn(page, replyPrompt(uniqueName('search-failure-open'))); + const pdf = renamed(UPLOAD_FIXTURES.largePdf, 'search-retry'); + const uploaded = await confirmUpload(page, uploadViaUnifiedButton(page, pdf)); + await selectMockEndpoint(page, AUTO_SMALL_ENDPOINT); + await enableFileSearch(page); + await setEmbeddingFailure(page, uploaded.file_id, true); + try { + const response = await sendMessage(page, `E2E_FILE_SEARCH:${uniqueName('search-failure')}`); + expect(response.ok()).toBe(true); + /* The turn ends with the retryable preparation error rather than a search over an index + * that was never built, and the message keeps its attachment. */ + await expect(messagesView(page).getByText(PREPARATION_FAILED_ERROR)).toBeVisible({ + timeout: TURN_TIMEOUT, + }); + expect(messageFileReading(await getLatestUserMessageFiles(page), uploaded.file_id)).toEqual({ + llmDeliveryPath: 'none', + }); + expect( + (await getRagQueries(page)).filter((query) => query.file_id === uploaded.file_id), + ).toHaveLength(0); + const port = process.env.E2E_RAG_API_PORT || '8791'; + const debugResponse = await page.request.get(`http://127.0.0.1:${port}/__debug/embedded`); + const debug = (await debugResponse.json()) as { failedEmbeds: { file_id: string }[] }; + expect(debug.failedEmbeds.some((entry) => entry.file_id === uploaded.file_id)).toBe(true); + await reloadConversation(page); + await expect(messagesView(page).getByText(PREPARATION_FAILED_ERROR)).toBeVisible(); + + await setEmbeddingFailure(page, uploaded.file_id, false); + await selectMockEndpoint(page, AUTO_ENDPOINT); + await attachExisting(page, uploaded); + await selectMockEndpoint(page, AUTO_SMALL_ENDPOINT); + const recovered = await sendTurn(page, `E2E_FILE_SEARCH:${uniqueName('search-recovery')}`); + expect(inventoryLine(initialRequest(recovered), pdf.name)).toContain('file_search'); + expect(messageFileReading(await getLatestUserMessageFiles(page), uploaded.file_id)).toEqual({ + llmDeliveryPath: 'none', + }); + const embeds = (await getRagEmbedded(page)).filter( + (entry) => entry.file_id === uploaded.file_id, + ); + const queries = (await getRagQueries(page)).filter( + (entry) => entry.file_id === uploaded.file_id, + ); + expect(embeds).toHaveLength(1); + expect(queries.length).toBeGreaterThan(0); + expect(embeds[0].seq).toBeLessThan(Math.min(...queries.map(({ seq }) => seq))); + } finally { + await setEmbeddingFailure(page, uploaded.file_id, false); + } + }); + + test('A-10: with no file tool loaded, a spreadsheet reads as it does under classic routing', async ({ + page, + }) => { + await startChat(page, AUTO_ENDPOINT); + + const csv = renamed(UPLOAD_FIXTURES.csv, 'a10-sheet'); + const uploaded = await confirmUpload(page, uploadViaUnifiedButton(page, csv)); + await expectDeferredRecord(page, uploaded.file_id); + + /* Neither Run Code nor File Search is loaded, so the automatic policy stands aside: the + * text is derived from the stored original and sent the way classic routing sends extracted + * text, under the same limits, and no inventory tells the model anything about the file. */ + const run = await sendTurn(page, replyPrompt(uniqueName('a10'))); + const request = initialRequest(run); + expect(request.promptText).toContain(`${Q1_ROWS[0][0]},${Q1_ROWS[0][1]}`); + expect(request.systemText).not.toContain(INVENTORY_HEADING); + expect(inventoryLine(request, csv.name)).toBe(''); + expect(messageFileReading(await getLatestUserMessageFiles(page), uploaded.file_id)).toEqual({ + llmDeliveryPath: 'text', + }); + await expect + .poll(async () => readingRecord(await getFileRecord(page, uploaded.file_id)).outcome, { + timeout: 15_000, + message: 'the derived text should be saved onto the deferred record', + }) + .toBe('complete'); + + /* The record and its stored original are untouched by the derivation. */ + const record = await getFileRecord(page, uploaded.file_id); + expect(sha256(await downloadOriginal(page, record))).toBe(sha256(attachFileBuffer(csv))); + }); + + test('A-26: palette, drop, paste and attach-existing give the same automatic reading', async ({ + page, + }) => { + await startChat(page, AUTO_ENDPOINT); + await enableCodeInterpreter(page); + + const picked = await confirmUpload( + page, + uploadViaUnifiedButton(page, renamed(UPLOAD_FIXTURES.xlsx, 'a26-palette')), + ); + const dropped = await confirmUpload( + page, + uploadViaDrop(page, renamed(UPLOAD_FIXTURES.xlsx, 'a26-drop')), + ); + /* A browser that exposes no files on a synthetic paste event skips the third chip. */ + const pasted = (await supportsFilePaste(page)) + ? await confirmUpload(page, uploadViaPaste(page, renamed(UPLOAD_FIXTURES.xlsx, 'a26-paste'))) + : undefined; + if (pasted) { + expect(pasted.filename).toMatch(/^clipboard_\d+_a26-paste-/); + } + const uploads = [picked, dropped, ...(pasted ? [pasted] : [])]; + const pickedRecord = readingRecord(await getFileRecord(page, picked.file_id)); + expect(pickedRecord).toMatchObject({ llmDeliveryPath: 'none', outcome: 'deferred' }); + for (const { file_id } of uploads) { + expect(readingRecord(await getFileRecord(page, file_id))).toEqual(pickedRecord); + } + + const firstRun = await sendTurn(page, replyPrompt(uniqueName('a26-first'))); + const firstFiles = await getLatestUserMessageFiles(page); + const pickedReading = messageFileReading(firstFiles, picked.file_id); + expect(pickedReading).toEqual({ llmDeliveryPath: 'none' }); + for (const { filename, file_id } of uploads) { + expect(inventoryLine(initialRequest(firstRun), filename)).toContain('read it with Run Code'); + expect(messageFileReading(firstFiles, file_id)).toEqual(pickedReading); + } + + await attachExisting(page, picked); + const existingRun = await sendTurn(page, replyPrompt(uniqueName('a26-existing'))); + expect(inventoryLine(initialRequest(existingRun), picked.filename)).toContain( + 'read it with Run Code', + ); + const existingFiles = await getLatestUserMessageFiles(page); + expect(messageFileReading(existingFiles, picked.file_id)).toEqual(pickedReading); + expect(readingRecord(await getFileRecord(page, picked.file_id))).toEqual(pickedRecord); + }); + + test('A-29: a classic endpoint still extracts at upload and stores no marker or inventory', async ({ + page, + }) => { + test.setTimeout(MULTI_TURN_TIMEOUT); + /* Run Code is on, so a classic upload would be deferred if the policy gate broke. */ + await startChat(page, CLASSIC_ENDPOINT); + await enableCodeInterpreter(page); + const workbook = renamed(UPLOAD_FIXTURES.xlsx, 'a29-quarterly'); + const uploaded = await confirmUpload(page, uploadViaUnifiedButton(page, workbook)); + const record = await getFileRecord(page, uploaded.file_id); + expect(record.llmDeliveryPath).toBe('text'); + expect(record.metadata?.textDerivation).toBeUndefined(); + expect(await getStoredText(uploaded.file_id)).toContain(WORKBOOK_SENTINEL); + + const run = await sendTurn(page, replyPrompt(uniqueName('a29'))); + const request = initialRequest(run); + expect(request.promptText).toContain(WORKBOOK_SENTINEL); + expect(request.systemText).not.toContain(INVENTORY_HEADING); + expect(inventoryLine(request, workbook.name)).toBe(''); + expect(messageFileReading(await getLatestUserMessageFiles(page), uploaded.file_id)).toEqual({ + llmDeliveryPath: 'text', + }); + const after = await getFileRecord(page, uploaded.file_id); + expect(readingRecord(after)).toEqual(readingRecord(record)); + expect(after.metadata?.textDerivation).toBeUndefined(); + }); +}); diff --git a/e2e/specs/mock/file-provisioning.spec.ts b/e2e/specs/mock/file-provisioning.spec.ts index 47950e4caa9..4a10deda681 100644 --- a/e2e/specs/mock/file-provisioning.spec.ts +++ b/e2e/specs/mock/file-provisioning.spec.ts @@ -155,19 +155,28 @@ test.describe('file provisioning — lazy (unified upload, at tool-execute)', () ).toBeVisible({ timeout: 15000 }); // The first tool call lazily embeds the attachment and searches it. - await sendMessageAndWaitForCompletion(page, `E2E_FILE_SEARCH:${uniqueName('first')}`); + const firstSearch = uniqueName('first'); + await sendMessageAndWaitForCompletion(page, `E2E_FILE_SEARCH:${firstSearch}`); await expect .poll(async () => (await getRagEmbedded(page)).map((e) => e.file_id), { timeout: 30000 }) .toContain(fileId); await expect - .poll(async () => (await getRagQueries(page)).filter((q) => q.file_id === fileId).length) - .toBe(1); + .poll(async () => + (await getRagQueries(page)).filter((q) => q.file_id === fileId).map((q) => q.query), + ) + .toEqual([`e2e ${firstSearch}`]); // A completed, persisted turn must still expose the embedded file to the next call. - await sendMessageAndWaitForCompletion(page, `E2E_FILE_SEARCH:${uniqueName('second')}`); + const secondSearch = uniqueName('second'); + const secondMessage = `E2E_FILE_SEARCH:${secondSearch}`; + await sendMessageAndWaitForCompletion(page, secondMessage); + // Classic mode adds context from the now-embedded file before invoking the model; + // the fresh tool call must then perform its own search, without another embedding. await expect - .poll(async () => (await getRagQueries(page)).filter((q) => q.file_id === fileId).length) - .toBe(2); + .poll(async () => + (await getRagQueries(page)).filter((q) => q.file_id === fileId).map((q) => q.query), + ) + .toEqual([`e2e ${firstSearch}`, secondMessage, `e2e ${secondSearch}`]); expect((await getRagEmbedded(page)).filter((e) => e.file_id === fileId)).toHaveLength(1); }); }); diff --git a/e2e/specs/mock/helpers.ts b/e2e/specs/mock/helpers.ts index 9c16acef008..1589d47dddb 100644 --- a/e2e/specs/mock/helpers.ts +++ b/e2e/specs/mock/helpers.ts @@ -1,3 +1,5 @@ +import fs from 'fs'; +import path from 'path'; import { expect } from '@playwright/test'; import { ContentTypes } from 'librechat-data-provider'; import type { TMessage } from 'librechat-data-provider'; @@ -10,6 +12,10 @@ export const MOCK_REPLY_TEXT = 'E2E mock reply'; export const MOCK_ENDPOINTS = [ { label: 'Mock Provider A', model: 'mock-model-a' }, { label: 'Mock Provider B', model: 'mock-model-b' }, + /** Automatic reading policy, no route overrides (automatic-upload.spec.ts). */ + { label: 'Mock Auto Provider', model: 'mock-auto-model' }, + /** Automatic reading policy with a 1 MB per-file limit. */ + { label: 'Mock Auto Small Provider', model: 'mock-auto-small-model' }, ] as const; export type MockEndpoint = { label: string; model: string }; @@ -406,10 +412,22 @@ export type CodeProvisionRecord = { id: string; storage_session_id: string; fileId: string; + bytes: number; + /** Hex digest of the bytes the code server received. */ + sha256: string; }; -export type RagEmbedRecord = { file_id: string; filename: string; entity_id: string }; -export type RagQueryRecord = { file_id: string; query: string }; +/** One exec the fake code server ran: the files it mounted, and the one an analysis token read. */ +export type CodeExecRecord = { + lang: string; + fileCount: number; + files: Array<{ id: string; name: string }>; + analyzedFileId?: string; +}; + +/** `seq` orders embeds and queries against each other across both logs. */ +export type RagEmbedRecord = { seq: number; file_id: string; filename: string; entity_id: string }; +export type RagQueryRecord = { seq: number; file_id: string; query: string }; /** Every `/query` the fake RAG service received from file_search. */ export async function getRagQueries(page: Page): Promise { @@ -419,14 +437,24 @@ export async function getRagQueries(page: Page): Promise { return body.queries; } -/** Files the fake code server received via /upload (proof they reached the code env). */ -export async function getCodeProvisionedUploads(page: Page): Promise { +/** The fake code server's record of what it received and ran since the last reset. */ +async function getCodeDebug(page: Page): Promise<{ + uploads: CodeProvisionRecord[]; + execs: CodeExecRecord[]; +}> { const response = await page.request.get(`${CODE_API_BASE}/__debug/uploads`); expect(response.ok(), 'fake code server /__debug/uploads should respond').toBeTruthy(); - const body = (await response.json()) as { uploads: CodeProvisionRecord[] }; - return body.uploads; + return response.json(); } +/** Files the fake code server received via /upload (proof they reached the code env). */ +export const getCodeProvisionedUploads = async (page: Page): Promise => + (await getCodeDebug(page)).uploads; + +/** Every exec the fake code server ran since the last reset. */ +export const getCodeExecs = async (page: Page): Promise => + (await getCodeDebug(page)).execs; + /** Files the fake RAG server embedded via /embed (proof they reached the vector DB). */ export async function getRagEmbedded(page: Page): Promise { const response = await page.request.get(`${RAG_API_BASE}/__debug/embedded`); @@ -437,10 +465,78 @@ export async function getRagEmbedded(page: Page): Promise { /** Clear both fake servers' recorded provisioning (call at test start for isolation). */ export async function resetProvisioning(page: Page): Promise { - await Promise.all([ + const [code, rag] = await Promise.all([ page.request.post(`${CODE_API_BASE}/__debug/reset`), page.request.post(`${RAG_API_BASE}/__debug/reset`), ]); + expect(code.ok(), 'fake code server /__debug/reset should succeed').toBeTruthy(); + expect(rag.ok(), 'fake RAG server /__debug/reset should succeed').toBeTruthy(); +} + +/** + * Where the in-process fake model writes the prompts of its most recent runs + * (e2e/setup/fake-model.js). It has no HTTP surface, so the file is its debug endpoint. + */ +export const MODEL_REQUEST_LOG_PATH = + process.env.E2E_MODEL_REQUEST_LOG ?? + path.resolve(__dirname, '../../.generated/last-request.json'); + +/** One model invocation as the provider would have received it. */ +export type ModelRequestRecord = { + /** The system instructions, including the queued code files and the file inventory. */ + systemText: string; + /** Every message's readable text, system first; inline file data is left out. */ + promptText: string; + /** File names of the document and file parts sent to the model. */ + documentFiles: string[]; +}; + +/** One run of the fake model, keyed by the user text that started it. */ +export type ModelRunRecord = { + sequence: number; + conversationId: string | null; + userText: string; + invocations: ModelRequestRecord[]; +}; + +function readModelRuns(): ModelRunRecord[] { + try { + const log = JSON.parse(fs.readFileSync(MODEL_REQUEST_LOG_PATH, 'utf8')) as { + runs?: ModelRunRecord[]; + }; + return log.runs ?? []; + } catch { + return []; + } +} + +/** + * The latest fake-model run whose user text contains `label`, once it has recorded at least + * `minInvocations` model requests. + */ +export async function getModelRun( + label: string, + { minInvocations = 1, timeout = 15_000 }: { minInvocations?: number; timeout?: number } = {}, +): Promise { + let match: ModelRunRecord | undefined; + await expect + .poll( + () => { + match = readModelRuns() + .filter((run) => run.userText.includes(label)) + .pop(); + return match?.invocations.length ?? 0; + }, + { + timeout, + message: `the fake model should record ${minInvocations} request(s) for "${label}"`, + }, + ) + .toBeGreaterThanOrEqual(minInvocations); + if (!match) { + throw new Error(`No fake-model request was recorded for "${label}"`); + } + return match; } /** Shape of a file record as returned by POST /api/files and GET /api/files. */ @@ -453,7 +549,14 @@ export type UploadedFile = { metadata?: { codeEnvRef?: { storage_session_id?: string; file_id?: string } }; }; -export type AttachFile = { name: string; mimeType: string; content: string }; +/** A file to attach: inline text, or a fixture read from disk under the given name. */ +export type AttachFile = + | { name: string; mimeType: string; content: string } + | { name: string; mimeType: string; path: string }; + +/** The bytes the browser receives for an attach helper's file. */ +export const attachFileBuffer = (file: AttachFile): Buffer => + 'path' in file ? fs.readFileSync(file.path) : Buffer.from(file.content, 'utf8'); /** Unique, filesystem-safe name so tests never collide on accumulated fake-server state. */ export const uniqueName = (prefix: string) => @@ -482,8 +585,59 @@ export async function uploadViaUnifiedButton(page: Page, file: AttachFile) { await fileChooser.setFiles({ name: file.name, mimeType: file.mimeType, - buffer: Buffer.from(file.content, 'utf8'), + buffer: attachFileBuffer(file), + }); + /* The palette lists the new record while it closes, which would duplicate the chip's name. */ + await expect(palette).toBeHidden(); + return uploadResponse; +} + +/** A DataTransfer in the page holding one file, as a native drag or a file paste carries it. */ +function createFileTransfer(page: Page, file: AttachFile) { + return page.evaluateHandle( + ({ name, mimeType, base64 }) => { + const bytes = Uint8Array.from(atob(base64), (char) => char.charCodeAt(0)); + const transfer = new DataTransfer(); + transfer.items.add(new File([bytes], name, { type: mimeType })); + return transfer; + }, + { name: file.name, mimeType: file.mimeType, base64: attachFileBuffer(file).toString('base64') }, + ); +} + +/** Drop a file onto the composer with the event sequence a native file drag produces. */ +export async function uploadViaDrop(page: Page, file: AttachFile) { + const input = page.getByRole('textbox', { name: 'Message input' }); + await expect(input).toBeVisible(); + const dataTransfer = await createFileTransfer(page, file); + const uploadResponse = waitForUpload(page); + await input.dispatchEvent('dragenter', { dataTransfer }); + await input.dispatchEvent('dragover', { dataTransfer }); + await input.dispatchEvent('drop', { dataTransfer }); + return uploadResponse; +} + +/** Whether the browser exposes files on a synthetic paste event, which file paste needs. */ +export function supportsFilePaste(page: Page): Promise { + return page.evaluate(() => { + const transfer = new DataTransfer(); + transfer.items.add(new File(['probe'], 'probe.txt', { type: 'text/plain' })); + const event = new ClipboardEvent('paste', { clipboardData: transfer }); + return event.clipboardData?.files.length === 1; }); +} + +/** Paste a file into the composer as the clipboard delivers a copied file. */ +export async function uploadViaPaste(page: Page, file: AttachFile) { + const input = page.getByRole('textbox', { name: 'Message input' }); + await input.click(); + const clipboardData = await createFileTransfer(page, file); + const uploadResponse = waitForUpload(page); + await input.evaluate((element, transfer) => { + element.dispatchEvent( + new ClipboardEvent('paste', { clipboardData: transfer, bubbles: true, cancelable: true }), + ); + }, clipboardData); return uploadResponse; } @@ -525,7 +679,7 @@ export async function uploadViaLegacyOption(page: Page, optionName: string, file await fileChooser.setFiles({ name: file.name, mimeType: file.mimeType, - buffer: Buffer.from(file.content, 'utf8'), + buffer: attachFileBuffer(file), }); return uploadResponse; } diff --git a/e2e/specs/mock/run-files-delivery.spec.ts b/e2e/specs/mock/run-files-delivery.spec.ts index 77d5d04a9c7..715572c15ae 100644 --- a/e2e/specs/mock/run-files-delivery.spec.ts +++ b/e2e/specs/mock/run-files-delivery.spec.ts @@ -7,9 +7,9 @@ import type { AgentDetail } from './agents.helpers'; import { fetchJson, uniqueName, - RAG_API_BASE, messagesView, requestJson, + getRagQueries, getAccessToken, getRagEmbedded, resetProvisioning, @@ -352,12 +352,9 @@ test.describe('run-file delivery and authorization', () => { ).toEqual([input.file_id, file.file_id].sort()); expect(output(messages, label, 'reader_search')).toContain('No content found in the files.'); expect((await getRagEmbedded(page)).map((entry) => entry.file_id)).toContain(file.file_id); - const queryResponse = await page.request.get(`${RAG_API_BASE}/__debug/embedded`); - expect(queryResponse.ok()).toBeTruthy(); - const { queries } = (await queryResponse.json()) as { - queries: Array<{ file_id: string; query: string }>; - }; - expect(queries).toContainEqual({ file_id: file.file_id, query: file.filename }); + expect(await getRagQueries(page)).toContainEqual( + expect.objectContaining({ file_id: file.file_id, query: file.filename }), + ); expect( messages.some((message) => message.attachments?.some((attachment) => diff --git a/librechat.example.yaml b/librechat.example.yaml index 982f7fd9d1c..a8ba84b6faa 100644 --- a/librechat.example.yaml +++ b/librechat.example.yaml @@ -1373,6 +1373,12 @@ endpoints: # # Existing files without extracted text are not backfilled. Paused runs retain encoded text # # and conservatively count referenced fallback text against current limits on resume. # textFallbackWithoutTools: false +# # "automatic" lets a turn that loads Run Code or File Search choose each attachment's reader: +# # spreadsheets go to Run Code, documents to the provider within its limits, then text, File +# # Search and Run Code. Without a file tool, and under "classic" (default), routing is unchanged. +# # Explicit destinations and defaultLLMDeliveryPath still win. Set back to classic before a +# # downgrade; deferred workbooks stay readable only by Run Code on older versions. +# llmDeliveryPolicy: classic # fileContextSizeLimit: 128 # Maximum aggregate model-bound attachment size in one agent turn (MB) # fileContextCharLimit: 1000000 # Maximum aggregate extracted-text characters in one agent turn # endpoints: diff --git a/packages/api/src/agents/__tests__/initialize.reading.integration.test.ts b/packages/api/src/agents/__tests__/initialize.reading.integration.test.ts new file mode 100644 index 00000000000..3e87b6a6220 --- /dev/null +++ b/packages/api/src/agents/__tests__/initialize.reading.integration.test.ts @@ -0,0 +1,330 @@ +import os from 'os'; +import fs from 'fs'; +import path from 'path'; +import mongoose from 'mongoose'; +import { Providers } from '@librechat/agents'; +import { MongoMemoryServer } from 'mongodb-memory-server'; +import { createModels, createMethods } from '@librechat/data-schemas'; +import { + Tools, + FileContext, + FileSources, + EModelEndpoint, + decideFileReading, +} from 'librechat-data-provider'; +import type { AppConfig, IMongoFile } from '@librechat/data-schemas'; +import type { Agent, TextDerivation } from 'librechat-data-provider'; +import type { LCTool } from '@librechat/agents'; +import type { ProvisionService } from '~/files/provision/service'; +import type { InitializeAgentDbMethods } from '../initialize'; +import type { ServerRequest } from '~/types'; +import { getTurnReadingContext, createFileTextDeriver } from '~/files/reading'; +import { extractFileContext } from '~/files/context'; +import { initializeAgent } from '../initialize'; + +const XLSX_TYPE = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'; +const SHEET_TEXT = 'Quarter,Total\nQ1,1200\nQ2,1400\nSENTINEL-7F3A'; +const SAMPLE_XLSX = path.join(__dirname, '../../files/documents/sample.xlsx'); +const SAMPLE_TEXT = 'Sheet One:\nData,on,first,sheet\nSecond Sheet:\nData,On\nSecond,Sheet\n'; + +type FileConfigInput = NonNullable; +type OpenStoredFile = ProvisionService['openStoredFile']; +type RoutingFields = Pick; + +interface TurnOptions { + fileIds: string[]; + policy?: 'automatic'; + tools?: string[]; + /** Tools the loader returns definitions for; Run Code registers itself when enabled. */ + loaded?: string[]; + fileConfig?: Omit; + checkSessionsAlive?: InitializeAgentDbMethods['checkSessionsAlive']; + /** Wires the request's text deriver over this storage, as the host does. */ + openStoredFile?: OpenStoredFile; +} + +let mongoServer: MongoMemoryServer; +let methods: ReturnType; +let userId: string; +let storageRoot: string; +const originalApiKey = process.env.OPENAI_API_KEY; + +beforeAll(async () => { + mongoServer = await MongoMemoryServer.create(); + await mongoose.connect(mongoServer.getUri()); + createModels(mongoose); + methods = createMethods(mongoose); + storageRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'reading-integration-')); + process.env.OPENAI_API_KEY = 'sk-reading-integration'; +}, 60000); + +afterAll(async () => { + process.env.OPENAI_API_KEY = originalApiKey; + fs.rmSync(storageRoot, { recursive: true, force: true }); + await mongoose.disconnect(); + await mongoServer.stop(); +}); + +beforeEach(() => { + userId = new mongoose.Types.ObjectId().toString(); +}); + +afterEach(async () => { + await mongoose.connection.db?.dropDatabase(); +}); + +const baseFile = (fileId: string) => ({ + file_id: fileId, + user: new mongoose.Types.ObjectId(userId), + object: 'file' as const, + source: FileSources.local, + context: FileContext.message_attachment, + usage: 0, +}); + +/** + * A spreadsheet the automatic upload path deferred to Run Code: no text and a `deferred` + * marker. Written to the collection directly because the marker is a later commit's schema + * field; reads return it as stored. + */ +async function seedDeferredXlsx(fileId = 'xlsx-deferred'): Promise { + const textDerivation: TextDerivation = { outcome: 'deferred', at: Date.now() }; + await mongoose.models.File.collection.insertOne({ + ...baseFile(fileId), + filename: 'deferred.xlsx', + filepath: `/uploads/${userId}/deferred.xlsx`, + type: XLSX_TYPE, + bytes: 4096, + llmDeliveryPath: 'none', + metadata: { destinationChosen: false, textDerivation }, + }); + return fileId; +} + +/** Writes the real sample workbook where local storage keeps the record's original. */ +async function storeSampleOriginal(fileId: string): Promise { + const [file] = (await methods.getFiles({ file_id: fileId }, null, {})) ?? []; + const target = path.join(storageRoot, file?.filepath ?? fileId); + fs.mkdirSync(path.dirname(target), { recursive: true }); + fs.copyFileSync(SAMPLE_XLSX, target); +} + +/** Local storage: streams a record's original from under the storage root, observed. */ +const localStorage = () => + jest.fn, Parameters>(async (file) => + fs.createReadStream(path.join(storageRoot, file.filepath ?? file.file_id)), + ); + +async function seedPdf(fileId: string, bytes: number): Promise { + await methods.createFile({ + ...baseFile(fileId), + filename: `${fileId}.pdf`, + filepath: `/uploads/${userId}/${fileId}.pdf`, + type: 'application/pdf', + bytes, + llmDeliveryPath: 'provider', + metadata: { destinationChosen: false }, + }); + return fileId; +} + +async function readRoutingFields(fileId: string): Promise { + const [file] = (await methods.getFiles({ file_id: fileId }, null, {})) ?? []; + if (file == null) { + return undefined; + } + const { llmDeliveryPath, text, type, bytes, metadata } = file; + return { llmDeliveryPath, text, type, bytes, metadata }; +} + +function buildDb(options: TurnOptions): InitializeAgentDbMethods { + return { + getFiles: methods.getFiles as InitializeAgentDbMethods['getFiles'], + updateFilesUsage: methods.updateFilesUsage, + getToolFilesByIds: methods.getToolFilesByIds as InitializeAgentDbMethods['getToolFilesByIds'], + getProjectFiles: methods.getProjectFiles as InitializeAgentDbMethods['getProjectFiles'], + getConvoFiles: async () => [], + getUserKey: async () => '', + getUserKeyValues: async () => ({}), + checkSessionsAlive: options.checkSessionsAlive, + saveFileTextDerivation: methods.saveFileTextDerivation, + }; +} + +async function runTurn(options: TurnOptions) { + const { policy, tools = [], loaded = [], fileConfig, openStoredFile } = options; + const appConfig: Pick = { + fileConfig: { + ...fileConfig, + endpoints: { + [EModelEndpoint.openAI]: { ...(policy != null && { llmDeliveryPolicy: policy }) }, + }, + }, + }; + const req = { + user: { id: userId, role: 'USER' }, + config: appConfig, + body: {}, + } as ServerRequest; + const agent = { + id: 'agent-reading', + provider: Providers.OPENAI, + model: 'gpt-4o', + tools, + model_parameters: { model: 'gpt-4o' }, + } as Agent; + const loadTools = jest.fn(async ({ tools: requested }: { tools: string[] }) => ({ + tools: [], + toolContextMap: {}, + toolDefinitions: loaded + .filter((name) => requested.includes(name)) + .map( + (name): LCTool => ({ + name, + description: name, + parameters: { type: 'object', properties: {} }, + }), + ), + })); + const result = await initializeAgent( + { + req, + agent, + loadTools, + requestFiles: options.fileIds.map((file_id) => ({ file_id })) as IMongoFile[], + codeEnvAvailable: true, + fileSearchAvailable: true, + endpointOption: { endpoint: EModelEndpoint.agents }, + allowedProviders: new Set([Providers.OPENAI]), + isInitialAgent: true, + ...(openStoredFile != null && { deriveText: createFileTextDeriver({ req, openStoredFile }) }), + }, + buildDb(options), + ); + return { result, req }; +} + +const pathsById = (files: ReadonlyArray>) => + Object.fromEntries(files.map((file) => [file.file_id, file.llmDeliveryPath])); + +type InitializedAgentResult = Awaited>; + +/** The reader the final decision picked for each request attachment. */ +const readersById = (result: InitializedAgentResult) => + Object.fromEntries( + result.requestAttachments.map((file) => [ + file.file_id, + decideFileReading({ + routing: result.deliveryRouting, + file, + consumers: result.fileConsumers, + }).reader, + ]), + ); + +const injectedText = (files: IMongoFile[], req: ServerRequest) => + extractFileContext({ attachments: files, req, tokenCountFn: (text) => text.length }); + +describe('initializeAgent reading under the automatic policy (MongoDB)', () => { + it('keeps the Run Code reader and injects no text when provisioning fails', async () => { + const fileId = 'xlsx-sandboxed'; + await methods.createFile({ + ...baseFile(fileId), + filename: 'sandboxed.xlsx', + filepath: `/uploads/${userId}/sandboxed.xlsx`, + type: XLSX_TYPE, + bytes: 4096, + text: SHEET_TEXT, + llmDeliveryPath: 'text', + metadata: { + destinationChosen: false, + codeEnvRef: { + kind: 'user', + id: userId, + storage_session_id: 'session-1', + file_id: 'sandbox-file-1', + }, + }, + }); + const checkSessionsAlive = jest.fn().mockRejectedValue(new Error('Code API unreachable')); + + const { result, req } = await runTurn({ + fileIds: [fileId], + policy: 'automatic', + tools: [Tools.execute_code], + checkSessionsAlive, + }); + + expect(checkSessionsAlive).toHaveBeenCalled(); + expect(result.provisionState).toBeUndefined(); + expect(result.fileConsumers?.executeCode).toBe(true); + expect(pathsById(result.requestAttachments)).toEqual({ [fileId]: 'none' }); + expect(await injectedText(result.requestAttachments, req)).toBeFalsy(); + }); + + it('sends the PDFs that fit the context allowance natively and the third to File Search', async () => { + const fileIds = [ + await seedPdf('pdf-a', 400 * 1024), + await seedPdf('pdf-b', 400 * 1024), + await seedPdf('pdf-c', 400 * 1024), + ]; + + const { result } = await runTurn({ + fileIds, + policy: 'automatic', + tools: [Tools.file_search], + loaded: [Tools.file_search], + fileConfig: { fileContextSizeLimit: 1 }, + }); + + expect(pathsById(result.requestAttachments)).toEqual({ + 'pdf-a': 'provider', + 'pdf-b': 'provider', + 'pdf-c': 'none', + }); + expect(readersById(result)).toEqual({ + 'pdf-a': 'provider', + 'pdf-b': 'provider', + 'pdf-c': 'search', + }); + const context = getTurnReadingContext(result.deliveryRouting); + expect(context?.stats().overflow).toBe(1); + const queued = result.provisionState?.vectorDBFiles.map((file) => file.file_id) ?? []; + expect(queued).toContain('pdf-c'); + }); +}); + +describe('initializeAgent text derivation from stored originals (MongoDB)', () => { + it('derives a deferred spreadsheet without Run Code once and keeps the text on the record', async () => { + const fileId = await seedDeferredXlsx(); + await storeSampleOriginal(fileId); + const openStoredFile = localStorage(); + + const { result, req } = await runTurn({ + fileIds: [fileId], + policy: 'automatic', + openStoredFile, + }); + + expect(result.fileConsumers).toEqual({ executeCode: false, fileSearch: false }); + expect(pathsById(result.requestAttachments)).toEqual({ [fileId]: 'text' }); + expect(readersById(result)).toEqual({ [fileId]: 'text' }); + expect(result.requestAttachments[0].text).toBe(SAMPLE_TEXT); + expect(await injectedText(result.requestAttachments, req)).toContain('Data,on,first,sheet'); + expect(openStoredFile).toHaveBeenCalledTimes(1); + expect(await readRoutingFields(fileId)).toMatchObject({ + llmDeliveryPath: 'none', + text: SAMPLE_TEXT, + metadata: { + destinationChosen: false, + textDerivation: { outcome: 'complete', extractor: 'document_parser' }, + }, + }); + + const next = await runTurn({ fileIds: [fileId], policy: 'automatic', openStoredFile }); + + expect(openStoredFile).toHaveBeenCalledTimes(1); + expect(pathsById(next.result.requestAttachments)).toEqual({ [fileId]: 'text' }); + expect(next.result.requestAttachments[0].text).toBe(SAMPLE_TEXT); + }); +}); diff --git a/packages/api/src/agents/__tests__/initialize.test.ts b/packages/api/src/agents/__tests__/initialize.test.ts index 9ac8f6bcf50..3e1b3fe9117 100644 --- a/packages/api/src/agents/__tests__/initialize.test.ts +++ b/packages/api/src/agents/__tests__/initialize.test.ts @@ -50,10 +50,17 @@ import { configSchema, } from 'librechat-data-provider'; import type { IMongoFile } from '@librechat/data-schemas'; -import type { Agent, TFile, FiltersConfig, AgentInstructionsPrompt } from 'librechat-data-provider'; +import type { + Agent, + TFile, + FiltersConfig, + AgentToolResources, + AgentInstructionsPrompt, +} from 'librechat-data-provider'; import type { ServerRequest, InitializeResultBase, EndpointTokenConfig } from '~/types'; import type { ResolveLinkedInstructions } from '../instructions/linked'; import type { InitializeAgentDbMethods } from '../initialize'; +import type { ProvisionState } from '../resources'; import type { CodeExecutionContext } from '../execution'; import type { GraphSubagentHostConfig } from '../discovery'; import type { ProjectFileRecord } from '../../projects/resources'; @@ -143,6 +150,12 @@ jest.mock('../../middleware/modelBoundContent', () => { import { initializeAgent } from '../initialize'; import { primeResources } from '../resources'; import { isFatalAgentInitializationError } from '../errors'; +import { filterFilesByEndpointRuntimeConfig as filterByEndpointPolicy } from '~/files/filter'; +import { assertModelBoundContent } from '../../middleware/modelBoundContent'; +import * as attachments from '../attachments'; +import { ContentFilterError } from '../../middleware/contentFilter'; +import { getTurnReadingContext, createDerivationPersister } from '~/files/reading'; +import type { FileTextDeriver, TextDerivationPersister } from '~/files/reading'; const realUtils = jest.requireActual('~/utils'); @@ -249,6 +262,7 @@ function createMocks(overrides?: { getUserKey: jest.fn().mockResolvedValue('user-1'), getUserKeyValues: jest.fn().mockResolvedValue([]), getToolFilesByIds: jest.fn().mockResolvedValue([]), + saveFileTextDerivation: jest.fn().mockResolvedValue(true), }; return { agent, req, res, loadTools, db }; @@ -6214,3 +6228,625 @@ describe('initializeAgent turn delivery routing', () => { expect(filesOrder).toBeLessThan(toolsOrder); }); }); + +describe('initializeAgent automatic delivery policy', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + const XLSX_TYPE = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'; + + /** A spreadsheet uploaded under classic routing: extracted at upload, route inferred. */ + const classicEraXlsx = () => + ({ + file_id: 'xlsx-file', + filename: 'quarterly.xlsx', + filepath: '/uploads/user-1/quarterly.xlsx', + type: XLSX_TYPE, + bytes: 2048, + source: FileSources.local, + context: FileContext.message_attachment, + text: 'Q1,1200\nQ2,1400', + llmDeliveryPath: 'text', + metadata: { destinationChosen: false }, + }) as IMongoFile; + + const oversizedPdf = () => + ({ + file_id: 'pdf-file', + filename: 'annual.pdf', + filepath: '/uploads/user-1/annual.pdf', + type: 'application/pdf', + bytes: 3 * 1024 * 1024, + source: FileSources.local, + context: FileContext.message_attachment, + llmDeliveryPath: 'provider', + metadata: { destinationChosen: false }, + }) as IMongoFile; + + /** The queued copy priming hands the planner. */ + const queued = (file: IMongoFile): TFile => ({ + file_id: file.file_id, + filename: file.filename, + filepath: file.filepath, + type: file.type, + bytes: file.bytes, + user: 'user-1', + object: 'file', + embedded: false, + usage: 0, + source: FileSources.local, + context: FileContext.message_attachment, + metadata: { destinationChosen: false }, + }); + + const provisionStateFor = ({ + code = [], + search = [], + }: { + code?: IMongoFile[]; + search?: IMongoFile[]; + }): ProvisionState => ({ + codeEnvFiles: code.map(queued), + vectorDBFiles: search.map(queued), + aliveFileIds: new Set(), + agentScopedFileIds: new Set(), + }); + + /** A spreadsheet uploaded under the automatic policy, its extraction left for a later turn. */ + const deferredXlsx = () => + ({ + ...classicEraXlsx(), + text: undefined, + llmDeliveryPath: 'none', + metadata: { destinationChosen: false, textDerivation: { outcome: 'deferred' } }, + }) as IMongoFile; + + async function initializeWith({ + file, + policy, + tools = [], + skillTools, + endpointConfig = {}, + fileContextCharLimit, + contextFiles = [], + deriveText, + saveFileTextDerivation, + persistDerivation, + loaded = {}, + toolResources = {}, + provisionState, + filters, + }: { + file: IMongoFile; + /** What the loader returns beyond the tools the agent lists. */ + loaded?: { + toolNames?: string[]; + dynamicToolContextMap?: Record; + primedSearchFiles?: Array<{ file_id: string; filename: string; fromAgent: boolean }>; + }; + /** The provisioning queues priming hands back. */ + provisionState?: ProvisionState; + toolResources?: AgentToolResources; + filters?: FiltersConfig; + /** The host's text deriver, which also builds a derive-only context under classic. */ + deriveText?: FileTextDeriver; + saveFileTextDerivation?: InitializeAgentDbMethods['saveFileTextDerivation']; + /** The host's request-scoped persister, shared by every agent it initializes. */ + persistDerivation?: TextDerivationPersister; + /** Persistent agent files priming adds to the delivered and agent-context sets. */ + contextFiles?: IMongoFile[]; + policy?: 'automatic' | 'classic'; + tools?: string[]; + /** Tools a manual skill adds; the loader loads none of them. */ + skillTools?: string[]; + endpointConfig?: { fileSizeLimit?: number }; + fileContextCharLimit?: number; + }) { + const { filterFilesByEndpointRuntimeConfig } = jest.requireMock('~/files') as { + filterFilesByEndpointRuntimeConfig: jest.Mock; + }; + filterFilesByEndpointRuntimeConfig.mockImplementation(filterByEndpointPolicy); + const { agent, req, res, loadTools, db } = createMocks(); + agent.tools = tools; + loadTools.mockResolvedValue({ + tools: [], + toolContextMap: {}, + toolDefinitions: (loaded.toolNames ?? []).map((name) => ({ name })), + dynamicToolContextMap: loaded.dynamicToolContextMap, + primedSearchFiles: loaded.primedSearchFiles, + }); + (primeResources as jest.Mock).mockImplementationOnce(async ({ attachments }) => { + const files = await attachments; + return { + attachments: [...files, ...contextFiles], + requestAttachments: files, + agentContextAttachments: contextFiles, + tool_resources: toolResources, + provisionState, + }; + }); + req.config = { + filters, + fileConfig: { + fileContextCharLimit, + endpoints: { + [Providers.OPENAI]: { + ...(policy != null && { llmDeliveryPolicy: policy }), + ...endpointConfig, + }, + }, + }, + } as unknown as ServerRequest['config']; + (db.getFiles as jest.Mock).mockResolvedValueOnce([file]); + + const { Types } = await import('mongoose'); + const skillId = new Types.ObjectId(); + const skillParams = + skillTools == null + ? {} + : { accessibleSkillIds: [skillId], manualSkills: ['spreadsheet-analysis'] }; + const skillDb = + skillTools == null + ? {} + : { + listSkillsByAccess: async () => ({ skills: [], has_more: false, after: null }), + getSkillByName: jest.fn().mockResolvedValue({ + _id: skillId, + name: 'spreadsheet-analysis', + body: 'Analyze spreadsheets.', + author: { toString: () => 'user-1' } as unknown as import('mongoose').Types.ObjectId, + allowedTools: skillTools, + }) as InitializeAgentDbMethods['getSkillByName'], + }; + + return initializeAgent( + { + req, + res, + agent, + loadTools, + requestFiles: [file], + codeEnvAvailable: true, + fileSearchAvailable: true, + endpointOption: { endpoint: EModelEndpoint.agents }, + allowedProviders: new Set([Providers.OPENAI]), + isInitialAgent: true, + deriveText, + persistDerivation, + ...skillParams, + }, + { ...db, ...skillDb, ...(saveFileTextDerivation != null && { saveFileTextDerivation }) }, + ); + } + + /** The delivery routes content inspection saw for one file, per call that inspected it. */ + const inspectionsOf = (fileId: string) => + (assertModelBoundContent as jest.Mock).mock.calls + .map(([input]: [{ files?: IMongoFile[] }]) => + (input.files ?? []) + .filter((file) => file.file_id === fileId) + .map((file) => file.llmDeliveryPath), + ) + .filter((routes) => routes.length > 0); + + it('leaves a classic-era spreadsheet to Run Code, inspecting its classic route without counting or rewriting it', async () => { + const xlsx = classicEraXlsx(); + const stored = structuredClone(xlsx); + + const result = await initializeWith({ + file: xlsx, + policy: 'automatic', + tools: [Tools.execute_code], + fileContextCharLimit: 3, + }); + + expect(result.fileConsumers).toEqual({ executeCode: true, fileSearch: false }); + expect(result.requestAttachments).toEqual([{ ...stored, llmDeliveryPath: 'none' }]); + expect(result.currentRequestAttachments.map((file) => file.llmDeliveryPath)).toEqual(['none']); + expect(xlsx).toEqual(stored); + expect(result.deliveryRouting.reading).toBeDefined(); + expect([...new Set(inspectionsOf(xlsx.file_id).flat())]).toEqual(['text']); + }); + + it('delivers the spreadsheet text, re-running admission and inspection, once the loader drops the Run Code tool', async () => { + const xlsx = classicEraXlsx(); + const admission = jest.spyOn(attachments, 'assertAgentAttachmentLimits'); + + const result = await initializeWith({ + file: xlsx, + policy: 'automatic', + skillTools: [Tools.execute_code], + }); + + expect(result.fileConsumers).toEqual({ executeCode: false, fileSearch: false }); + expect(result.requestAttachments).toEqual([{ ...xlsx, llmDeliveryPath: 'text' }]); + expect(xlsx.llmDeliveryPath).toBe('text'); + expect(admission).toHaveBeenCalledTimes(2); + expect(inspectionsOf(xlsx.file_id)).toHaveLength(2); + }); + + it('leaves the text out rather than failing the turn when it no longer fits the allowance', async () => { + /* File Search is loaded but nothing was provisioned for it, so the overflow finds no + * reader; the policy still leaves the text out instead of failing the turn. */ + const xlsx = classicEraXlsx(); + + const result = await initializeWith({ + file: xlsx, + policy: 'automatic', + tools: [Tools.file_search], + loaded: { toolNames: [Tools.file_search] }, + fileContextCharLimit: 3, + }); + + expect(result.fileConsumers).toEqual({ executeCode: false, fileSearch: true }); + expect(result.requestAttachments).toEqual([{ ...xlsx, llmDeliveryPath: 'none' }]); + expect(getTurnReadingContext(result.deliveryRouting)?.stats()).toMatchObject({ + overflow: 1, + dropped: 0, + }); + }); + + it('fails the turn on the text allowance as classic routing does once the loader drops the only file tool', async () => { + const xlsx = classicEraXlsx(); + + await expect( + initializeWith({ + file: xlsx, + policy: 'automatic', + skillTools: [Tools.execute_code], + fileContextCharLimit: 3, + }), + ).rejects.toMatchObject({ name: 'AgentAttachmentLimitError' }); + }); + + it.each([ + { record: 'classic-era', file: classicEraXlsx, policy: undefined }, + { record: 'classic-era', file: classicEraXlsx, policy: 'classic' as const }, + { record: 'deferred', file: deferredXlsx, policy: undefined }, + ])( + 'keeps classic routing and attaches no reading evidence for a $record spreadsheet when the policy is $policy', + async ({ file, policy }) => { + const xlsx = file(); + + const result = await initializeWith({ file: xlsx, policy, tools: [Tools.execute_code] }); + + expect(result.deliveryRouting.reading).toBeUndefined(); + expect(result.requestAttachments).toEqual([xlsx]); + }, + ); + + it('keeps an oversized PDF the automatic policy reads past the endpoint size limit', async () => { + const pdf = oversizedPdf(); + + const result = await initializeWith({ + file: pdf, + policy: 'automatic', + tools: [Tools.file_search], + loaded: { toolNames: [Tools.file_search] }, + endpointConfig: { fileSizeLimit: 1 }, + provisionState: provisionStateFor({ search: [pdf] }), + }); + + expect(result.fileConsumers).toEqual({ executeCode: false, fileSearch: true }); + expect(result.requestAttachments.map((file) => file.file_id)).toEqual([pdf.file_id]); + expect(result.requestAttachments[0].llmDeliveryPath).toBe('none'); + expect(getTurnReadingContext(result.deliveryRouting)?.stats().dropped).toBe(0); + expect(result.dynamicToolContextMap?.file_inventory).toContain( + `"annual.pdf" (PDF): too large to send directly. Search it with ${Tools.file_search}; it is indexed when you first search.`, + ); + }); + + const refusedImage = () => + ({ + ...oversizedPdf(), + file_id: 'image-file', + filename: 'scan.png', + type: 'image/png', + }) as IMongoFile; + + it.each([ + { + refused: 'an oversized PDF', + file: oversizedPdf, + policy: 'automatic' as const, + dropped: ['pdf-file'], + }, + { refused: 'an oversized PDF', file: oversizedPdf, policy: undefined, dropped: undefined }, + { + refused: 'an image', + file: refusedImage, + policy: 'automatic' as const, + dropped: ['image-file'], + }, + ])( + 'drops $refused the endpoint refuses as classic routing does when no file tool is loaded (policy $policy, recorded $dropped)', + async ({ file, policy, dropped }) => { + const result = await initializeWith({ + file: file(), + policy, + endpointConfig: { fileSizeLimit: 1 }, + }); + + expect(result.fileConsumers).toEqual({ executeCode: false, fileSearch: false }); + expect(result.requestAttachments).toEqual([]); + const context = getTurnReadingContext(result.deliveryRouting); + expect(context?.dropped().map((file) => file.file_id)).toEqual(dropped); + }, + ); + + it('leaves persistent agent files on their stored route when no reading changed', async () => { + /* Agent files are never read by the automatic policy. Re-resolving this stored text route + * to its classic provider route every turn would re-run the full-set checks classic skips. */ + const agentFile = { + file_id: 'agent-pdf', + filename: 'handbook.pdf', + filepath: '/uploads/user-1/handbook.pdf', + type: 'application/pdf', + bytes: 2048, + source: FileSources.local, + context: FileContext.agents, + text: 'handbook text', + llmDeliveryPath: 'text', + metadata: {}, + } as IMongoFile; + + const result = await initializeWith({ + file: classicEraXlsx(), + policy: 'automatic', + contextFiles: [agentFile], + }); + + expect(result.agentContextAttachments).toEqual([agentFile]); + const inspectedAgentFile = (assertModelBoundContent as jest.Mock).mock.calls + .flatMap(([input]: [{ files?: IMongoFile[] }]) => input.files ?? []) + .filter((file) => file.file_id === agentFile.file_id); + expect(inspectedAgentFile).toEqual([]); + }); + + describe('file inventory', () => { + const handbook = { file_id: 'kb', filename: 'handbook.pdf', fromAgent: true }; + const filters: FiltersConfig = { + files: { + pii: { + fields: ['content'], + starterPatterns: [], + customPatterns: [{ id: 'private', label: 'private value', regex: 'PRIVATE-[A-Z]+' }], + }, + }, + }; + it.each([ + ['as the loader primed them', [handbook], ['kb']], + ['as none when the loader primed none', [], []], + ['as unknown when the loader did not report them', undefined, undefined], + ])('records the File Search files %s', async (_case, primedSearchFiles, expected) => { + const result = await initializeWith({ + file: classicEraXlsx(), + policy: 'automatic', + loaded: { primedSearchFiles }, + }); + + expect(result.primedSearchFileIds).toEqual(expected); + }); + + it('reuses an indexed user PDF through the primed search files instead of its cached text', async () => { + const pdf = { + ...oversizedPdf(), + embedded: true, + text: 'Complete cached text', + } as IMongoFile; + const result = await initializeWith({ + file: pdf, + policy: 'automatic', + tools: [Tools.file_search], + endpointConfig: { fileSizeLimit: 1 }, + loaded: { + toolNames: [Tools.file_search], + primedSearchFiles: [{ file_id: pdf.file_id, filename: pdf.filename, fromAgent: false }], + }, + toolResources: { + [EToolResources.file_search]: { files: [queued(pdf)], file_ids: [] }, + }, + }); + + expect(result.provisionState).toBeUndefined(); + expect(result.requestAttachments[0].llmDeliveryPath).toBe('none'); + expect(getTurnReadingContext(result.deliveryRouting)?.judge(pdf).search).toBe('reachable'); + expect(getTurnReadingContext(result.deliveryRouting)?.searchState(pdf.file_id)).toBe('ready'); + expect(result.dynamicToolContextMap?.file_inventory).toContain('Search it with file_search.'); + expect(result.dynamicToolContextMap?.file_inventory).not.toContain('text is included'); + }); + + it('uses both resource representations when an older loader does not report primed files', async () => { + const pdf = { ...oversizedPdf(), embedded: true } as IMongoFile; + const result = await initializeWith({ + file: pdf, + policy: 'automatic', + tools: [Tools.file_search], + endpointConfig: { fileSizeLimit: 1 }, + loaded: { toolNames: [Tools.file_search] }, + toolResources: { + [EToolResources.file_search]: { files: [queued(pdf)], file_ids: ['knowledge-base'] }, + }, + }); + + expect(getTurnReadingContext(result.deliveryRouting)?.searchState(pdf.file_id)).toBe('ready'); + expect(getTurnReadingContext(result.deliveryRouting)?.searchState('knowledge-base')).toBe( + 'ready', + ); + expect(result.requestAttachments[0].llmDeliveryPath).toBe('none'); + }); + + it('honors a loader that explicitly primed no search files', async () => { + const pdf = { ...oversizedPdf(), embedded: true } as IMongoFile; + const result = await initializeWith({ + file: pdf, + policy: 'automatic', + tools: [Tools.file_search], + endpointConfig: { fileSizeLimit: 1 }, + loaded: { toolNames: [Tools.file_search], primedSearchFiles: [] }, + toolResources: { [EToolResources.file_search]: { files: [queued(pdf)] } }, + }); + + expect(getTurnReadingContext(result.deliveryRouting)?.judge(pdf).search).toBe('unreachable'); + expect( + getTurnReadingContext(result.deliveryRouting)?.searchState(pdf.file_id), + ).toBeUndefined(); + expect(result.dynamicToolContextMap?.file_inventory).toContain('cannot be read on this turn'); + }); + + it('plans the queued spreadsheet destination and checks both advertisements', async () => { + const xlsx = classicEraXlsx(); + + const result = await initializeWith({ + file: xlsx, + policy: 'automatic', + tools: [Tools.execute_code], + provisionState: provisionStateFor({ code: [xlsx] }), + filters, + }); + + expect(result.provisionState?.codeEnvDestinations?.get(xlsx.file_id)).toBe('quarterly.xlsx'); + expect(assertModelBoundContent).toHaveBeenCalledWith({ + filters, + files: [{ content: result.dynamicToolContextMap?.queued_code_files }], + }); + expect(assertModelBoundContent).toHaveBeenCalledWith({ + filters, + files: [{ content: result.dynamicToolContextMap?.file_inventory }], + }); + }); + + it('fails the turn when the content policy refuses the inventory', async () => { + const xlsx = () => ({ ...classicEraXlsx(), filename: 'PRIVATE-PLAN.xlsx' }) as IMongoFile; + const params = { tools: [Tools.execute_code], filters }; + + await expect( + initializeWith({ ...params, file: xlsx(), policy: 'automatic' }), + ).rejects.toBeInstanceOf(ContentFilterError); + await expect(initializeWith({ ...params, file: xlsx() })).resolves.toBeDefined(); + }); + }); + + describe('with a host that derives text', () => { + const derived = { + status: 'derived' as const, + text: 'Q1,1200\nQ2,1400', + textDerivation: { + outcome: 'complete' as const, + extractor: 'document_parser' as const, + at: 1, + }, + }; + const deriver = () => + jest.fn, Parameters>(async () => derived); + const saver = () => + jest.fn< + ReturnType>, + Parameters> + >(async () => true); + + it.each([ + ['classic', undefined], + ['automatic', 'automatic' as const], + ])( + 'derives a deferred spreadsheet with no Run Code under %s routing and saves it for the owner', + async (policyName, policy) => { + const xlsx = deferredXlsx(); + const stored = structuredClone(xlsx); + const deriveText = deriver(); + const saveFileTextDerivation = saver(); + + const result = await initializeWith({ + file: xlsx, + policy, + deriveText, + saveFileTextDerivation, + }); + + expect(getTurnReadingContext(result.deliveryRouting)?.policy).toBe(policyName); + expect(deriveText).toHaveBeenCalledTimes(1); + expect(deriveText).toHaveBeenCalledWith( + expect.objectContaining({ file_id: xlsx.file_id }), + undefined, + ); + expect(result.requestAttachments).toEqual([ + { + ...stored, + llmDeliveryPath: 'text', + text: derived.text, + metadata: { destinationChosen: false, textDerivation: derived.textDerivation }, + }, + ]); + expect(saveFileTextDerivation).toHaveBeenCalledTimes(1); + expect(saveFileTextDerivation).toHaveBeenCalledWith( + { file_id: xlsx.file_id, text: derived.text, textDerivation: derived.textDerivation }, + { user: 'user-1', tenantId: undefined }, + ); + expect(xlsx).toEqual(stored); + }, + ); + + it.each([ + ['with stored text', classicEraXlsx()], + ['without stored text', { ...classicEraXlsx(), text: undefined } as IMongoFile], + ])( + 'never derives a record the automatic policy did not mark under classic routing (%s)', + async (_shape, file) => { + const deriveText = deriver(); + const saveFileTextDerivation = saver(); + + const result = await initializeWith({ file, deriveText, saveFileTextDerivation }); + + expect(getTurnReadingContext(result.deliveryRouting)?.policy).toBe('classic'); + expect(deriveText).not.toHaveBeenCalled(); + expect(saveFileTextDerivation).not.toHaveBeenCalled(); + expect(result.requestAttachments).toEqual([file]); + }, + ); + + it('saves through the host persister, once for every agent sharing it', async () => { + const deriveText = deriver(); + const hostSave = saver(); + const ownSave = saver(); + const persistDerivation = createDerivationPersister(hostSave, { user: 'user-1' }); + + for (let agent = 0; agent < 2; agent++) { + await initializeWith({ + file: deferredXlsx(), + policy: 'automatic', + deriveText, + saveFileTextDerivation: ownSave, + persistDerivation, + }); + } + + expect(deriveText).toHaveBeenCalledTimes(2); + expect(ownSave).not.toHaveBeenCalled(); + expect(hostSave).toHaveBeenCalledTimes(1); + expect(hostSave).toHaveBeenCalledWith( + expect.objectContaining({ text: derived.text, textDerivation: derived.textDerivation }), + { user: 'user-1' }, + ); + }); + + it('leaves a deferred spreadsheet to Run Code without deriving it', async () => { + const xlsx = deferredXlsx(); + const deriveText = deriver(); + const saveFileTextDerivation = saver(); + + const result = await initializeWith({ + file: xlsx, + policy: 'automatic', + tools: [Tools.execute_code], + deriveText, + saveFileTextDerivation, + }); + + expect(result.fileConsumers).toEqual({ executeCode: true, fileSearch: false }); + expect(deriveText).not.toHaveBeenCalled(); + expect(saveFileTextDerivation).not.toHaveBeenCalled(); + expect(result.requestAttachments).toEqual([xlsx]); + }); + }); +}); diff --git a/packages/api/src/agents/__tests__/run-codeTools.test.ts b/packages/api/src/agents/__tests__/run-codeTools.test.ts index 286735db7d7..a59ef88e173 100644 --- a/packages/api/src/agents/__tests__/run-codeTools.test.ts +++ b/packages/api/src/agents/__tests__/run-codeTools.test.ts @@ -1,13 +1,16 @@ import { FileContext } from 'librechat-data-provider'; import type { AgentInputs, SubagentTaskConfig, SubagentResolveContext } from '@librechat/agents'; -import type { TFile } from 'librechat-data-provider'; +import type { TFile, FiltersConfig } from 'librechat-data-provider'; +import type { AppConfig } from '@librechat/data-schemas'; import type { HostSubagentTaskConfig } from '~/agents/subagentDelivery'; import type { ProvisionToolContext } from '~/files/provision/callback'; import type { ServerRequest } from '~/types'; import { createProvisionFilesCallback } from '~/files/provision/callback'; import { SUBAGENT_COMPLETION_DELIVERY } from '~/agents/subagentDelivery'; import { mergeCodeFilesIntoContext } from '~/agents/codeFilesSession'; +import { resolveTurnDeliveryRouting } from '~/agents/files/delivery'; import { CHECK_BACKGROUND_TASK_NAME } from '~/agents/background'; +import { ContentFilterError } from '~/middleware/contentFilter'; import { createRun } from '~/agents/run'; /** @@ -98,6 +101,7 @@ async function captureRunConfig( async function captureAgentsRunConfig( agents: Array>, subagentTasks?: SubagentTaskConfig, + appConfig?: AppConfig, ): Promise> { await createRun({ agents: agents as never, @@ -105,6 +109,7 @@ async function captureAgentsRunConfig( streaming: true, streamUsage: true, subagentTasks, + appConfig, }); const createMock = Run.create as jest.Mock; expect(createMock).toHaveBeenCalledTimes(1); @@ -217,6 +222,206 @@ describe('createRun code-tool eager/session wiring', () => { expect(childInput.additional_instructions?.match(/\/mnt\/data\/data\.csv/g)).toHaveLength(1); }); + describe('file inventory', () => { + const INVENTORY_HEADER = 'Attached files and how you can read them on this turn'; + const runsCode = { executeCode: true, fileSearch: false }; + const routingFor = (policy?: 'automatic' | 'classic') => + resolveTurnDeliveryRouting({ + agent: { provider: 'openAI', endpoint: 'openAI' }, + config: { + fileConfig: { + endpoints: { openAI: policy == null ? {} : { llmDeliveryPolicy: policy } }, + }, + }, + }); + const upload = (overrides: Partial = {}): TFile => ({ + file_id: 'shared', + filename: 'data.csv', + filepath: '/uploads/data.csv', + type: 'text/csv', + user: 'user-1', + object: 'file', + bytes: 10, + embedded: false, + usage: 0, + context: FileContext.message_attachment, + llmDeliveryPath: 'none', + metadata: { destinationChosen: false }, + createdAt: '2026-09-01', + ...overrides, + }); + /** Every code path a text advertises, in order. */ + const advertisedPaths = (text: unknown): string[] => + typeof text === 'string' ? (text.match(/\/mnt\/data\/\S+?(?=\.?(?:\s|$))/g) ?? []) : []; + const plannedPaths = (destinations?: Map): string[] => + [...(destinations?.values() ?? [])].map((name) => `/mnt/data/${name}`); + + /** A parent and the lazy child it spawns, both holding colliding queued uploads. */ + function setupRun(policy?: 'automatic' | 'classic', requestFiles: TFile[] = []) { + const shared = upload(); + const fresh = upload({ file_id: 'new', createdAt: '2026-10-01' }); + const child = { + ...makeAgent({ + id: 'child', + deliveryRouting: routingFor(policy), + currentRequestAttachments: [shared, fresh, ...requestFiles], + fileConsumers: runsCode, + }), + dynamicToolContextMap: { file_inventory: 'stale /mnt/data/draft-alias.csv' } as Record< + string, + unknown + >, + provisionState: { + codeEnvFiles: [{ ...shared }, { ...fresh }], + vectorDBFiles: [], + aliveFileIds: new Set(), + agentScopedFileIds: new Set(), + codeEnvDestinations: new Map([ + ['shared', 'draft-alias.csv'], + ['new', 'data.csv'], + ]), + }, + }; + const parent = { + ...makeAgent({ + deliveryRouting: routingFor(policy), + currentRequestAttachments: [shared, ...requestFiles], + fileConsumers: runsCode, + subagents: { enabled: true, allowSelf: false, agent_ids: ['child'] }, + lazySubagentConfigs: [ + { id: 'child', configId: 'child:1', resolve: jest.fn().mockResolvedValue(child) }, + ], + }), + dynamicToolContextMap: {} as Record, + provisionState: { + codeEnvFiles: [{ ...shared }], + vectorDBFiles: [], + aliveFileIds: new Set(), + agentScopedFileIds: new Set(), + codeEnvDestinations: undefined as Map | undefined, + }, + }; + return { parent, child }; + } + + async function runWithChild(policy?: 'automatic' | 'classic') { + const { parent, child } = setupRun(policy); + const config = await captureRunConfig(parent); + const [parentInput] = (config.graphConfig as { agents: AgentInputs[] }).agents; + const resolveInputs = parentInput.subagentConfigs?.[0]?.resolveAgentInputs; + if (!resolveInputs) throw new Error('Missing lazy subagent resolver'); + const childInput = await resolveInputs({ + signal: new AbortController().signal, + } as SubagentResolveContext); + return { parent, child, parentInput, childInput }; + } + + it('re-renders a lazy child’s inventory at the paths its re-plan advertises', async () => { + const { parent, child, parentInput, childInput } = await runWithChild('automatic'); + const childInventory = child.dynamicToolContextMap.file_inventory; + const childPlan = plannedPaths(child.provisionState.codeEnvDestinations); + + expect(childPlan).toHaveLength(2); + expect(childPlan).toContain('/mnt/data/data.csv'); + expect(advertisedPaths(childInventory).sort()).toEqual([...childPlan].sort()); + expect(advertisedPaths(child.dynamicToolContextMap.queued_code_files).sort()).toEqual( + [...childPlan].sort(), + ); + expect(childInventory).not.toContain('draft-alias.csv'); + expect(childInput.additional_instructions).toContain(childInventory); + + const parentInventory = parent.dynamicToolContextMap.file_inventory; + expect(advertisedPaths(parentInventory)).toEqual( + plannedPaths(parent.provisionState.codeEnvDestinations), + ); + expect(parentInput.additional_instructions).toContain(parentInventory); + }); + + it('never tells a child that a request file was sent or included with its task', async () => { + const brief = upload({ + file_id: 'brief', + filename: 'brief.pdf', + type: 'application/pdf', + llmDeliveryPath: 'provider', + }); + const { parent, child } = setupRun('automatic', [brief]); + const eagerChild = { + ...makeAgent({ + id: 'eager-child', + deliveryRouting: routingFor('automatic'), + currentRequestAttachments: [brief], + fileConsumers: runsCode, + }), + dynamicToolContextMap: {} as Record, + }; + Object.assign(parent, { subagentAgentConfigs: [eagerChild] }); + + const config = await captureRunConfig(parent); + const [parentInput] = (config.graphConfig as { agents: AgentInputs[] }).agents; + const resolveInputs = parentInput.subagentConfigs?.find( + (subagent) => subagent.resolveAgentInputs != null, + )?.resolveAgentInputs; + if (!resolveInputs) throw new Error('Missing lazy subagent resolver'); + const childInput = await resolveInputs({ + signal: new AbortController().signal, + } as SubagentResolveContext); + + const sentOrIncluded = /sent with this message|included in this message/; + for (const inventory of [ + child.dynamicToolContextMap.file_inventory, + eagerChild.dynamicToolContextMap.file_inventory, + ]) { + expect(inventory).toContain('"brief.pdf" (PDF): not sent with your task.'); + expect(inventory).not.toMatch(sentOrIncluded); + } + expect(childInput.additional_instructions).not.toMatch(sentOrIncluded); + expect(parent.dynamicToolContextMap.file_inventory).toContain( + '"brief.pdf" (PDF): sent with this message.', + ); + }); + + it.each([undefined, 'classic' as const])( + 'writes no inventory for either agent under %p routing', + async (policy) => { + const { parent, child, parentInput, childInput } = await runWithChild(policy); + + expect(parent.dynamicToolContextMap).not.toHaveProperty('file_inventory'); + expect(child.dynamicToolContextMap).not.toHaveProperty('file_inventory'); + expect(parentInput.additional_instructions).not.toContain(INVENTORY_HEADER); + expect(childInput.additional_instructions).not.toContain(INVENTORY_HEADER); + expect(childInput.additional_instructions).toContain('/mnt/data/data.csv'); + }, + ); + + it('content-checks the inventory with the run’s filters', async () => { + const filters: FiltersConfig = { + files: { + pii: { + fields: ['content'], + starterPatterns: [], + customPatterns: [{ id: 'private', label: 'private value', regex: 'PRIVATE-[A-Z]+' }], + }, + }, + }; + const file = upload({ filename: 'PRIVATE-PLAN.csv' }); + const agent = makeAgent({ + deliveryRouting: routingFor('automatic'), + currentRequestAttachments: [file], + fileConsumers: runsCode, + provisionState: { + codeEnvFiles: [{ ...file }], + vectorDBFiles: [], + aliveFileIds: new Set(), + agentScopedFileIds: new Set(), + }, + }); + + await expect( + captureAgentsRunConfig([agent], undefined, { filters } as AppConfig), + ).rejects.toBeInstanceOf(ContentFilterError); + }); + }); + it('settles a lazy child only after the run has its inputs, so a late cancel is given back', async () => { const controller = new AbortController(); const outcomes: string[] = []; diff --git a/packages/api/src/agents/attachments.test.ts b/packages/api/src/agents/attachments.test.ts index e4a7678143d..29f6df9555e 100644 --- a/packages/api/src/agents/attachments.test.ts +++ b/packages/api/src/agents/attachments.test.ts @@ -1,5 +1,10 @@ import { logger } from '@librechat/data-schemas'; -import { EModelEndpoint, FileContext, FileSources } from 'librechat-data-provider'; +import { + FileContext, + FileSources, + EModelEndpoint, + allocateDirectContent, +} from 'librechat-data-provider'; import type { IMongoFile } from '@librechat/data-schemas'; import type { ServerRequest } from '~/types'; @@ -22,6 +27,8 @@ import { buildAgentContextAttachmentsByAgentId, isModelBoundAttachmentFile, isToolOwnedAttachment, + measureAttachment, + resolveAgentAttachmentLimits, } from './attachments'; import { applyTurnDelivery, resolveTurnDeliveryRouting } from './files/delivery'; @@ -689,6 +696,54 @@ describe('agent attachment helpers', () => { ).rejects.toBeInstanceOf(AgentAttachmentPolicyError); }); + it('holds a shared provider copy to each automatic receiver size limit', async () => { + /* The primary's automatic decision sent this PDF natively within its own 10 MB limit; the + * receiver reads the same encoded message, so its smaller limit must still refuse it. */ + const shared = { + ...makeTextFile('shared-pdf', 'shared.pdf', ''), + source: FileSources.local, + type: 'application/pdf', + bytes: 3 * 1024 * 1024, + context: FileContext.message_attachment, + llmDeliveryPath: 'provider', + metadata: { destinationChosen: false }, + } as IMongoFile; + const textCopy = { + ...shared, + file_id: 'shared-text', + llmDeliveryPath: 'text', + text: 'extracted', + } as IMongoFile; + const fileConfig: NonNullable['fileConfig'] = { + endpoints: { + openAI: { fileSizeLimit: 10, llmDeliveryPolicy: 'automatic' }, + Moonshot: { fileSizeLimit: 1, llmDeliveryPolicy: 'automatic' }, + }, + }; + const req = { body: { fileTokenLimit: 1000 }, config: { fileConfig } } as ServerRequest; + const searches = { executeCode: false, fileSearch: true }; + const shareWith = (sharedAttachments: IMongoFile[], consumers = searches) => + buildAgentScopedContext({ + agentIds: ['primary', 'secondary'], + attachmentsByAgentId: new Map(), + sharedAttachments, + endpointsByAgentId: new Map([ + ['primary', { endpoint: 'openAI' }], + ['secondary', { endpoint: 'Moonshot' }], + ]), + req, + consumers, + }); + + await expect(shareWith([shared])).rejects.toBeInstanceOf(AgentAttachmentPolicyError); + await expect(shareWith([textCopy])).resolves.toBeInstanceOf(Map); + /* Without a file tool the automatic policy reads nothing, so the size limit is admission + * for the text copy as well, as classic routing applies it. */ + await expect( + shareWith([textCopy], { executeCode: false, fileSearch: false }), + ).rejects.toBeInstanceOf(AgentAttachmentPolicyError); + }); + it('does not apply the primary file count limit across disjoint private scopes', async () => { const req = { body: { fileTokenLimit: 1000 }, @@ -890,3 +945,168 @@ describe('files that belong to a tool', () => { ).not.toThrow(); }); }); + +describe('resolveAgentAttachmentLimits', () => { + const MB = 1024 * 1024; + + it('resolves the count, aggregate byte and extracted-text limits of an endpoint', () => { + expect( + resolveAgentAttachmentLimits({ + fileConfig: { + fileContextCharLimit: 40, + endpoints: { agents: { fileLimit: 3, totalSizeLimit: 2 } }, + }, + }), + ).toEqual({ count: 3, bytes: 2 * MB, textChars: 40 }); + }); + + it('reads the request config when no file config is passed, and prefers a passed one', () => { + const req = { config: { fileConfig: { endpoints: { agents: { fileLimit: 4 } } } } }; + expect(resolveAgentAttachmentLimits({ req }).count).toBe(4); + expect( + resolveAgentAttachmentLimits({ + req, + fileConfig: { endpoints: { agents: { fileLimit: 2 } } }, + }).count, + ).toBe(2); + }); + + it('leaves the count unlimited when the count is not enforced', () => { + expect( + resolveAgentAttachmentLimits({ + fileConfig: { endpoints: { agents: { fileLimit: 3 } } }, + enforceAttachmentCount: false, + }).count, + ).toBeUndefined(); + }); + + it('uses the global context size instead of an endpoint aggregate when asked to', () => { + const fileConfig = { + fileContextSizeLimit: 5, + endpoints: { [EModelEndpoint.openAI]: { totalSizeLimit: 1 } }, + }; + expect( + resolveAgentAttachmentLimits({ fileConfig, endpoint: EModelEndpoint.openAI }).bytes, + ).toBe(MB); + expect( + resolveAgentAttachmentLimits({ + fileConfig, + endpoint: EModelEndpoint.openAI, + useGlobalContextSizeLimit: true, + }).bytes, + ).toBe(5 * MB); + }); + + it.each<{ name: string; params: Parameters[0] }>([ + { + name: 'a provider-backed agent on the agents fallback', + params: { + fileConfig: { fileContextCharLimit: 7, endpoints: { agents: { fileLimit: 1 } } }, + endpoint: EModelEndpoint.openAI, + }, + }, + { + name: 'a named custom endpoint on the generic custom config', + params: { + fileConfig: { endpoints: { custom: { fileLimit: 2, totalSizeLimit: 1 } } }, + endpoint: 'Moonshot', + }, + }, + { + name: 'an explicit backing-endpoint aggregate', + params: { + fileConfig: { endpoints: { openAI: { totalSizeLimit: 3, fileLimit: 2 } } }, + endpoint: EModelEndpoint.openAI, + }, + }, + ])('is the limit the assertion enforces for $name', ({ params }) => { + const limits = resolveAgentAttachmentLimits(params); + const over = (limit: number | undefined): number => (limit ?? 0) + 1; + const files = Array.from({ length: over(limits.count) }, (_, index) => ({ + file_id: `file-${index}`, + })); + + expect(() => assertAgentAttachmentLimits({ ...params, attachments: files })).toThrow( + expect.objectContaining({ limitType: 'count', limit: limits.count }), + ); + expect(() => + assertAgentAttachmentLimits({ + ...params, + attachments: [{ file_id: 'large', bytes: over(limits.bytes) }], + }), + ).toThrow(expect.objectContaining({ limitType: 'bytes', limit: limits.bytes })); + expect(() => + assertAgentAttachmentLimits({ + ...params, + attachments: [{ file_id: 'long', text: 'x'.repeat(over(limits.textChars)) }], + }), + ).toThrow(expect.objectContaining({ limitType: 'extracted_text', limit: limits.textChars })); + }); +}); + +describe('measureAttachment', () => { + it('measures bytes, extracted text and whether the file counts', () => { + expect(measureAttachment({ file_id: 'doc', bytes: 12, text: 'hello' })).toEqual({ + fileId: 'doc', + counts: true, + bytes: 12, + textChars: 5, + }); + }); + + it('does not count a replayed historical file toward the attachment count', () => { + const historicalFileIds = new Set(['history']); + expect(measureAttachment({ file_id: 'history' }, { historicalFileIds }).counts).toBe(false); + expect(measureAttachment({ file_id: 'current' }, { historicalFileIds }).counts).toBe(true); + expect(measureAttachment({}, { historicalFileIds })).toEqual({ + fileId: '', + counts: true, + bytes: 0, + textChars: 0, + }); + }); + + it.each([null, undefined, -1, Number.NaN, Number.POSITIVE_INFINITY])( + 'measures unusable byte count %p as zero', + (bytes) => { + expect(measureAttachment({ file_id: 'odd', bytes }).bytes).toBe(0); + }, + ); + + it('matches the totals the attachment stats report', () => { + const files = [ + { file_id: 'history', bytes: 10, text: 'context' }, + { file_id: 'current', bytes: 5, text: 'new' }, + { bytes: -3, text: 'unidentified' }, + ]; + const historicalFileIds = new Set(['history']); + const entries = files.map((file) => measureAttachment(file, { historicalFileIds })); + + expect(collectAgentAttachmentStats(files, { historicalFileIds })).toMatchObject({ + attachmentCount: entries.filter((entry) => entry.counts).length, + totalKnownBytes: entries.reduce((sum, entry) => sum + entry.bytes, 0), + extractedTextChars: entries.reduce((sum, entry) => sum + entry.textChars, 0), + }); + }); + + it('lets a first-fit allocation and the assertion agree on what fits', () => { + const fileConfig = { fileContextCharLimit: 10, endpoints: { agents: { fileLimit: 3 } } }; + const files = [ + { file_id: 'a', bytes: 1, text: 'abcd' }, + { file_id: 'b', bytes: 1, text: 'efghijk' }, + { file_id: 'c', bytes: 1, text: 'lmn' }, + ]; + const overflow = allocateDirectContent( + [], + files.map((file) => measureAttachment(file)), + resolveAgentAttachmentLimits({ fileConfig }), + ); + const admitted = files.filter((file) => !overflow.has(file.file_id)); + + expect(overflow).toEqual(new Set(['b'])); + expect(() => assertAgentAttachmentLimits({ attachments: admitted, fileConfig })).not.toThrow(); + expect(() => assertAgentAttachmentLimits({ attachments: files, fileConfig })).toThrow( + expect.objectContaining({ limitType: 'extracted_text' }), + ); + }); +}); diff --git a/packages/api/src/agents/attachments.ts b/packages/api/src/agents/attachments.ts index 04885b05c2b..43509637066 100644 --- a/packages/api/src/agents/attachments.ts +++ b/packages/api/src/agents/attachments.ts @@ -6,13 +6,18 @@ import { mergeFileConfig, getEndpointFileConfig, } from 'librechat-data-provider'; +import type { + DirectContentEntry, + DirectContentLimits, + TurnFileConsumers, +} from 'librechat-data-provider'; import type { IMongoFile } from '@librechat/data-schemas'; import type { TokenCountFn } from '~/utils/text'; import type { ServerRequest } from '~/types'; import { filterFilesByEndpointRuntimeConfig } from '~/files/filter'; import { AGENT_ATTACHMENT_LIMIT_EXCEEDED } from './errors'; +import { extractFileContext } from '~/files/context'; import { countTokens } from '~/utils/tokenizer'; -import { extractFileContext } from '~/files'; type FileWithId = { file_id?: string | null; @@ -236,6 +241,32 @@ export function isAgentAttachmentLimitError( return error instanceof AgentAttachmentLimitError || error instanceof AgentAttachmentPolicyError; } +const toNonNegativeFinite = (value: number | null | undefined): number | undefined => + typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : undefined; + +/** + * One file's direct-content footprint as the attachment limits measure it. A replayed historical + * file spends the context budgets but not the current submission's count allowance. + */ +export function measureAttachment( + file: Pick, + { historicalFileIds }: { historicalFileIds?: ReadonlySet } = {}, +): DirectContentEntry { + return { + fileId: file.file_id ?? '', + counts: !file.file_id || !historicalFileIds?.has(file.file_id), + bytes: toNonNegativeFinite(file.bytes) ?? 0, + textChars: typeof file.text === 'string' ? file.text.length : 0, + }; +} + +/** A file's direct content as admission charges it on the current request: none unless model-bound. */ +export function measureModelBoundAttachment(file: AttachmentTelemetryFile): DirectContentEntry { + return isModelBoundAttachmentFile(file) + ? measureAttachment(file) + : { fileId: file.file_id ?? '', counts: false, bytes: 0, textChars: 0 }; +} + export function collectAgentAttachmentStats( attachments?: Iterable | null, options: { @@ -256,14 +287,13 @@ export function collectAgentAttachmentStats( if (!file) { continue; } - const extractedTextChars = typeof file.text === 'string' ? file.text.length : 0; + const { counts, bytes, textChars } = measureAttachment(file, options); if (file.file_id && seenFileIds.has(file.file_id)) { if (options.countRepeatedExtractedText === true) { - stats.extractedTextChars += extractedTextChars; + stats.extractedTextChars += textChars; } if (options.countRepeatedBytes === true) { - stats.totalKnownBytes += - Number.isFinite(file.bytes) && Number(file.bytes) >= 0 ? Number(file.bytes) : 0; + stats.totalKnownBytes += bytes; } continue; } @@ -271,21 +301,17 @@ export function collectAgentAttachmentStats( seenFileIds.add(file.file_id); } - const bytes = Number.isFinite(file.bytes) && Number(file.bytes) >= 0 ? Number(file.bytes) : 0; - const pageCount = - Number.isFinite(file.metadata?.pageCount) && Number(file.metadata?.pageCount) >= 0 - ? Number(file.metadata?.pageCount) - : undefined; - if (!file.file_id || !options.historicalFileIds?.has(file.file_id)) { + const pageCount = toNonNegativeFinite(file.metadata?.pageCount); + if (counts) { stats.attachmentCount += 1; } stats.totalKnownBytes += bytes; - stats.extractedTextChars += extractedTextChars; + stats.extractedTextChars += textChars; stats.files.push({ ...(file.file_id && { fileId: file.file_id }), ...(file.type && { mimeType: file.type }), ...(bytes > 0 && { bytes }), - ...(extractedTextChars > 0 && { extractedTextChars }), + ...(textChars > 0 && { extractedTextChars: textChars }), ...(pageCount != null && { pageCount }), }); } @@ -293,35 +319,26 @@ export function collectAgentAttachmentStats( return stats; } -export function assertAgentAttachmentLimits({ - attachments, - req, +/** + * The direct-content limits a turn's attachments are held to on an endpoint: the count, aggregate + * byte and extracted-text limits {@link assertAgentAttachmentLimits} enforces, so an allocation + * against them and the assertion cannot disagree. + */ +export function resolveAgentAttachmentLimits({ fileConfig: providedFileConfig, + req, endpoint = EModelEndpoint.agents, endpointType, - countRepeatedExtractedText = false, - countRepeatedBytes = countRepeatedExtractedText, enforceAttachmentCount = true, - historicalFileIds, useGlobalContextSizeLimit = false, }: { - attachments?: Iterable | null; + fileConfig?: DynamicFileConfig; req?: AgentAttachmentLimitRequest; - fileConfig?: Parameters[0]; endpoint?: string | null; endpointType?: string | null; - countRepeatedExtractedText?: boolean; - countRepeatedBytes?: boolean; enforceAttachmentCount?: boolean; - /** Replayed files consume context budgets, not the current submission's count allowance. */ - historicalFileIds?: ReadonlySet; useGlobalContextSizeLimit?: boolean; -}): AgentAttachmentStats { - const stats = collectAgentAttachmentStats(attachments, { - countRepeatedExtractedText, - countRepeatedBytes, - historicalFileIds, - }); +}): DirectContentLimits { const dynamicFileConfig = providedFileConfig ?? req?.config?.fileConfig; const fileConfig = mergeFileConfig(dynamicFileConfig); const endpointConfig = getEndpointFileConfig({ fileConfig, endpoint, endpointType }); @@ -339,18 +356,57 @@ export function assertAgentAttachmentLimits({ ? undefined : getExplicitEndpointAggregateLimit(dynamicFileConfig, endpoint, endpointType)) ?? fileConfig.fileContextSizeLimit; - const configuredContextCharLimit = dynamicFileConfig?.fileContextCharLimit; - const limits: Array<[AgentAttachmentLimit, number, number | undefined]> = [ - ['count', stats.attachmentCount, enforceAttachmentCount ? configuredFileLimit : undefined], - ['bytes', stats.totalKnownBytes, configuredContextSizeLimit], - [ - 'extracted_text', - stats.extractedTextChars, - configuredContextCharLimit ?? fileConfig.fileContextCharLimit, - ], + return { + count: enforceAttachmentCount ? configuredFileLimit : undefined, + bytes: configuredContextSizeLimit, + textChars: dynamicFileConfig?.fileContextCharLimit ?? fileConfig.fileContextCharLimit, + }; +} + +export function assertAgentAttachmentLimits({ + attachments, + req, + fileConfig, + endpoint, + endpointType, + countRepeatedExtractedText = false, + countRepeatedBytes = countRepeatedExtractedText, + enforceAttachmentCount, + historicalFileIds, + useGlobalContextSizeLimit, +}: { + attachments?: Iterable | null; + req?: AgentAttachmentLimitRequest; + fileConfig?: DynamicFileConfig; + endpoint?: string | null; + endpointType?: string | null; + countRepeatedExtractedText?: boolean; + countRepeatedBytes?: boolean; + enforceAttachmentCount?: boolean; + /** Replayed files consume context budgets, not the current submission's count allowance. */ + historicalFileIds?: ReadonlySet; + useGlobalContextSizeLimit?: boolean; +}): AgentAttachmentStats { + const stats = collectAgentAttachmentStats(attachments, { + countRepeatedExtractedText, + countRepeatedBytes, + historicalFileIds, + }); + const limits = resolveAgentAttachmentLimits({ + fileConfig, + req, + endpoint, + endpointType, + enforceAttachmentCount, + useGlobalContextSizeLimit, + }); + const checks: Array<[AgentAttachmentLimit, number, number | undefined]> = [ + ['count', stats.attachmentCount, limits.count], + ['bytes', stats.totalKnownBytes, limits.bytes], + ['extracted_text', stats.extractedTextChars, limits.textChars], ]; - for (const [limitType, observed, limit] of limits) { + for (const [limitType, observed, limit] of checks) { if (limit != null && limit > 0 && observed > limit) { throw new AgentAttachmentLimitError(limitType, observed, limit); } @@ -367,6 +423,7 @@ export function assertAgentAttachmentTopology({ endpointType, endpointsByAgentId, historicalFileIds, + consumers, }: { sharedAttachments?: IMongoFile[]; scopedAttachmentsByAgentId?: Map; @@ -375,24 +432,23 @@ export function assertAgentAttachmentTopology({ endpointType?: string | null; endpointsByAgentId?: AgentAttachmentEndpointsByAgentId; historicalFileIds?: ReadonlySet; + /** The requesting agent's loaded file tools, which decide whether an oversized record stays. */ + consumers?: TurnFileConsumers | null; }): void { const agentIds = new Set([ ...scopedAttachmentsByAgentId.keys(), - ...(endpointsByAgentId instanceof Map - ? endpointsByAgentId.keys() - : Object.keys(endpointsByAgentId ?? {})), + ...listAgentIds(endpointsByAgentId), ]); for (const agentId of agentIds) { - const agentEndpoint = - endpointsByAgentId instanceof Map - ? endpointsByAgentId.get(agentId) - : endpointsByAgentId?.[agentId]; + const agentEndpoint = getAgentEntry(endpointsByAgentId, agentId); const compatibleSharedAttachments = filterFilesByEndpointRuntimeConfig(req?.config, { files: sharedAttachments, endpoint: agentEndpoint?.endpoint ?? endpoint ?? EModelEndpoint.agents, endpointType: agentEndpoint?.endpointType ?? endpointType, skipTotalSizeLimit: true, preserveTextSources: true, + bindProviderCopies: true, + consumers, }); if (compatibleSharedAttachments.length !== sharedAttachments.length) { throw new AgentAttachmentPolicyError(); @@ -524,6 +580,23 @@ export type AgentAttachmentEndpointsByAgentId = | Map | Record; +/** A collection keyed by agent id, which callers supply as a Map or a plain record. */ +export type AgentKeyed = Map | Record | null | undefined; + +export function getAgentEntry(entries: AgentKeyed, agentId: string): T | undefined { + return entries instanceof Map ? entries.get(agentId) : entries?.[agentId]; +} + +export function listAgentIds(entries: AgentKeyed): string[] { + return entries instanceof Map ? [...entries.keys()] : Object.keys(entries ?? {}); +} + +/** The entries of an agent-keyed collection, without the gaps a record may carry. */ +export function listAgentEntries(entries: AgentKeyed): T[] { + const values = entries instanceof Map ? [...entries.values()] : Object.values(entries ?? {}); + return values.filter((value): value is T => value != null); +} + export function collectFileIds( files?: Array | null, ): Set { @@ -659,10 +732,7 @@ export function getAgentContextAttachments({ return []; } - const attachments: TFile[] = - attachmentsByAgentId instanceof Map - ? (attachmentsByAgentId.get(agentId) ?? []) - : (attachmentsByAgentId[agentId] ?? []); + const attachments: TFile[] = getAgentEntry(attachmentsByAgentId, agentId) ?? []; if (!excludeFileIds || excludeFileIds.size === 0) { return attachments; @@ -679,6 +749,7 @@ export function buildAgentScopedAttachmentMap({ endpoint, endpointType, endpointsByAgentId, + consumers, }: { agentIds: string[]; attachmentsByAgentId: AgentContextAttachmentsByAgentId; @@ -687,12 +758,11 @@ export function buildAgentScopedAttachmentMap({ endpoint?: string | null; endpointType?: string | null; endpointsByAgentId?: AgentAttachmentEndpointsByAgentId; + /** The requesting agent's loaded file tools, which decide whether an oversized record stays. */ + consumers?: TurnFileConsumers | null; }): Map { const entries = Array.from(new Set(agentIds.filter(Boolean))).map((agentId) => { - const agentEndpoint = - endpointsByAgentId instanceof Map - ? endpointsByAgentId.get(agentId) - : endpointsByAgentId?.[agentId]; + const agentEndpoint = getAgentEntry(endpointsByAgentId, agentId); const attachments = getAgentContextAttachments({ agentId, attachmentsByAgentId, @@ -706,6 +776,7 @@ export function buildAgentScopedAttachmentMap({ endpointType: agentEndpoint?.endpointType ?? endpointType, skipTotalSizeLimit: true, preserveTextSources: true, + consumers, }), ] as const; }); @@ -723,6 +794,7 @@ export async function buildAgentScopedContext({ endpoint, endpointType, endpointsByAgentId, + consumers, }: { agentIds: string[]; attachmentsByAgentId: AgentContextAttachmentsByAgentId; @@ -734,6 +806,7 @@ export async function buildAgentScopedContext({ endpoint?: string | null; endpointType?: string | null; endpointsByAgentId?: AgentAttachmentEndpointsByAgentId; + consumers?: TurnFileConsumers | null; }): Promise> { const attachmentEntries = [ ...buildAgentScopedAttachmentMap({ @@ -744,6 +817,7 @@ export async function buildAgentScopedContext({ endpoint, endpointType, endpointsByAgentId, + consumers, }), ]; assertAgentAttachmentTopology({ @@ -754,6 +828,7 @@ export async function buildAgentScopedContext({ endpoint, endpointType, endpointsByAgentId, + consumers, }); const entries = await Promise.all( attachmentEntries.map(async ([agentId, attachments]) => { diff --git a/packages/api/src/agents/discovery.spec.ts b/packages/api/src/agents/discovery.spec.ts index a7fbb6e36ca..75cf4440272 100644 --- a/packages/api/src/agents/discovery.spec.ts +++ b/packages/api/src/agents/discovery.spec.ts @@ -310,6 +310,37 @@ describe('discoverConnectedAgents', () => { /** A handoff agent's provider config can turn native web search on as well, so * dropping the resolver here would leave it ungated one hop later. */ + it('forwards the text deriver and its persister to every handoff initializeAgent call', async () => { + const primaryConfig = makeConfig('A', [{ from: 'A', to: 'B', edgeType: 'handoff' }]); + const getAgent = jest.fn(async () => makeAgent('B', [])); + const deriveText = jest.fn(); + const persistDerivation = jest.fn(); + + await discoverConnectedAgents( + { + req: makeReq(), + res: makeRes(), + primaryConfig, + allowedProviders: new Set(), + modelsConfig: { openai: ['gpt-4o'] }, + loadTools: jest.fn(), + deriveText, + persistDerivation, + }, + { + getAgent, + checkPermission: jest.fn().mockResolvedValue(true), + logViolation: jest.fn(), + db: {} as never, + }, + ); + + expect(mockInitializeAgent).toHaveBeenCalledWith( + expect.objectContaining({ deriveText, persistDerivation }), + expect.anything(), + ); + }); + it('forwards resolveWebSearchGrant to every handoff initializeAgent call', async () => { const primaryConfig = makeConfig('A', [{ from: 'A', to: 'B', edgeType: 'handoff' }]); const getAgent = jest.fn(async () => makeAgent('B', [])); diff --git a/packages/api/src/agents/discovery.ts b/packages/api/src/agents/discovery.ts index 18b9cb4f775..a171b8f6e32 100644 --- a/packages/api/src/agents/discovery.ts +++ b/packages/api/src/agents/discovery.ts @@ -115,6 +115,10 @@ export interface DiscoverConnectedAgentsParams { * the terms its parent did. */ fileSearchAvailable?: InitializeAgentParams['fileSearchAvailable']; + /** The request's text deriver, forwarded so a handoff receiver can read a deferred record as text. */ + deriveText?: InitializeAgentParams['deriveText']; + /** The request's shared derivation persister, so each derived file is written once across agents. */ + persistDerivation?: InitializeAgentParams['persistDerivation']; /** * The caller's `WEB_SEARCH` grant resolver, forwarded so a handoff or subagent * whose provider config turns native web search on is authorized by the same @@ -287,6 +291,8 @@ async function initializeReferencedAgent( statefulSessionsAvailable: params.statefulSessionsAvailable, allowedStatefulCodeEnvironments: params.allowedStatefulCodeEnvironments, memoryAvailable: params.memoryAvailable, + deriveText: params.deriveText, + persistDerivation: params.persistDerivation, signal: params.signal, }, deps.db, diff --git a/packages/api/src/agents/files/delivery.spec.ts b/packages/api/src/agents/files/delivery.spec.ts index 181501f375d..522c8d1406f 100644 --- a/packages/api/src/agents/files/delivery.spec.ts +++ b/packages/api/src/agents/files/delivery.spec.ts @@ -1,9 +1,29 @@ -import type { TurnFileConsumers, TurnDeliveryFile } from 'librechat-data-provider'; +import { FileContext, FileSources } from 'librechat-data-provider'; +import type { + TurnReadingInputs, + TurnFileConsumers, + TurnDeliveryFile, + ReadingEvidence, + FiltersConfig, +} from 'librechat-data-provider'; +import type { AppConfig } from '@librechat/data-schemas'; +import type { + DerivedText, + FileTextDeriver, + TurnReadingFile, + TextDerivationPersister, +} from '~/files/reading/turn'; import { applyTurnDelivery as materializeTurnDelivery, - resolveTurnDeliveryRouting, + prepareScopedTurnCandidates, resolveScopedTurnAttachments, + resolveTurnDeliveryRouting, + toClassicInspectionView, + applyCheckpointDelivery, } from './delivery'; +import * as modelBoundContent from '~/middleware/modelBoundContent'; +import { buildTurnReadingContext } from '~/files/reading/turn'; +import { UninspectableFileError } from '~/protection/files'; function applyTurnDelivery( files: T[], @@ -322,3 +342,453 @@ describe('resolveScopedTurnAttachments', () => { ).toEqual([]); }); }); + +describe('the automatic policy', () => { + const agent = { provider: 'openAI' }; + const automatic = { + fileConfig: { endpoints: { openAI: { llmDeliveryPolicy: 'automatic' as const } } }, + }; + /** The turn's evidence seam, answering every file with the same verdicts. */ + const withEvidence = (evidence: ReadingEvidence) => ({ + ...resolveTurnDeliveryRouting({ agent, config: automatic }), + reading: { judge: () => evidence, canDerive: false } satisfies TurnReadingInputs, + }); + const attachment = { + source: FileSources.local, + context: FileContext.message_attachment, + metadata: { destinationChosen: false }, + }; + /** A PDF too large for the provider, with complete extracted text that fits the prompt. */ + const largePdf = { + ...attachment, + file_id: 'large-pdf', + type: 'application/pdf', + bytes: 40 * 1024 * 1024, + text: 'quarterly totals', + llmDeliveryPath: 'provider', + }; + /** A spreadsheet extracted at upload under classic routing. */ + const csv = { + ...attachment, + file_id: 'csv', + type: 'text/csv', + bytes: 20, + text: 'region,total\nwest,4', + llmDeliveryPath: 'text', + }; + + describe('applyTurnDelivery', () => { + it('gives a classic-era spreadsheet to Run Code and back to text without changing it', () => { + const routing = withEvidence({}); + const files = [csv]; + + expect(materializeTurnDelivery(files, { routing, consumers: runsCode })).toEqual([ + { ...csv, llmDeliveryPath: 'none' }, + ]); + expect(materializeTurnDelivery(files, { routing, consumers: noReader })).toBe(files); + }); + }); + + describe('toClassicInspectionView', () => { + const routing = withEvidence({}); + const strictContent = { + files: { pii: { fields: ['content'], starterPatterns: [], uninspectable: 'block' } }, + } as FiltersConfig; + + it('shows a spreadsheet Run Code reads with the text route classic routing gives it', () => { + const copies = materializeTurnDelivery([csv], { routing, consumers: runsCode }); + + const view = toClassicInspectionView(copies, routing, runsCode); + + expect(view).toEqual([{ ...csv, llmDeliveryPath: 'text' }]); + expect(view).not.toBe(copies); + expect(copies).toEqual([{ ...csv, llmDeliveryPath: 'none' }]); + }); + + it('keeps the inspected content of a rerouted file under a fail-closed content filter', () => { + /* Coverage reads the text route, so the code-routed copy alone would read as + * uninspectable although classic routing inspects and delivers the same text. */ + const copies = materializeTurnDelivery([csv], { routing, consumers: runsCode }); + + expect(() => + modelBoundContent.assertModelBoundContent({ filters: strictContent, files: copies }), + ).toThrow(UninspectableFileError); + expect(() => + modelBoundContent.assertModelBoundContent({ + filters: strictContent, + files: toClassicInspectionView(copies, routing, runsCode), + }), + ).not.toThrow(); + }); + + it('returns its input when no copy differs from its classic route', () => { + const pdf = { ...largePdf, bytes: 20 }; + const copies = materializeTurnDelivery([pdf, csv], { routing, consumers: noReader }); + + expect(toClassicInspectionView(copies, routing, noReader)).toBe(copies); + }); + + it('returns its input under classic routing', () => { + const classic = resolveTurnDeliveryRouting({ agent, config }); + const copies = [{ ...csv, llmDeliveryPath: 'none' }]; + + expect(toClassicInspectionView(copies, classic, runsCode)).toBe(copies); + expect(toClassicInspectionView(copies, undefined, runsCode)).toBe(copies); + }); + }); +}); + +describe('applyCheckpointDelivery', () => { + const agent = { provider: 'openAI' }; + const routing = resolveTurnDeliveryRouting({ + agent, + config: { fileConfig: { endpoints: { openAI: { llmDeliveryPolicy: 'automatic' } } } }, + }); + const csv = { + file_id: 'csv', + type: 'text/csv', + bytes: 20, + source: FileSources.local, + context: FileContext.message_attachment, + text: 'region,total\nwest,4', + llmDeliveryPath: 'none', + metadata: { destinationChosen: false }, + }; + /** A record written before uploads marked whether a destination was chosen. */ + const unmarked = { ...csv, file_id: 'unmarked', metadata: {} }; + + it('charges retained fallback text for every inferred tool file under classic routing', () => { + const classic = resolveTurnDeliveryRouting({ agent, config }); + + expect(applyCheckpointDelivery([csv, unmarked])).toEqual([ + { ...csv, llmDeliveryPath: 'text' }, + { ...unmarked, llmDeliveryPath: 'text' }, + ]); + expect(applyCheckpointDelivery([csv], { routing: classic, consumers: runsCode })).toEqual([ + { ...csv, llmDeliveryPath: 'text' }, + ]); + }); + + it('charges the automatic decision for files it reads and the fallback for the rest', () => { + /* Text the policy withheld for Run Code never reached the checkpoint, so charging it would + * spend the resumed turn's budget on content the model never received. */ + const charged = applyCheckpointDelivery([csv, unmarked], { routing, consumers: runsCode }); + + expect(charged[0]).toBe(csv); + expect(charged[1]).toEqual({ ...unmarked, llmDeliveryPath: 'text' }); + expect(applyCheckpointDelivery([csv], { routing, consumers: noReader })).toEqual([ + { ...csv, llmDeliveryPath: 'text' }, + ]); + expect(csv.llmDeliveryPath).toBe('none'); + }); +}); + +describe('prepareScopedTurnCandidates', () => { + const XLSX = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'; + const automatic = { + fileConfig: { endpoints: { openAI: { llmDeliveryPolicy: 'automatic' as const } } }, + }; + const derived: DerivedText = { + status: 'derived', + text: 'quarter,total\nQ1,42', + textDerivation: { outcome: 'complete', extractor: 'document_parser', at: 1 }, + }; + /** A workbook uploaded under the automatic policy with extraction left for a later turn. */ + const workbook = (file_id: string): TurnReadingFile => ({ + file_id, + filename: `${file_id}.xlsx`, + type: XLSX, + bytes: 2048, + source: FileSources.local, + context: FileContext.message_attachment, + llmDeliveryPath: 'none', + metadata: { destinationChosen: false, textDerivation: { outcome: 'deferred' } }, + }); + const historical = workbook('history-xlsx'); + const requested = workbook('request-xlsx'); + const messages = [{ files: [{ file_id: historical.file_id }] }]; + + /** A receiver whose turn carries a reading context over the given deriver. */ + function receiver( + agentId: string, + { + consumers = noReader, + deriveText, + persistDerivation, + endpoint = 'openAI', + config = automatic, + }: { + consumers?: TurnFileConsumers; + deriveText?: FileTextDeriver; + persistDerivation?: TextDerivationPersister; + endpoint?: string; + config?: Pick; + } = {}, + ) { + const deliveryRouting = resolveTurnDeliveryRouting({ + agent: { provider: endpoint }, + config, + }); + const context = buildTurnReadingContext({ + routing: deliveryRouting, + provider: endpoint, + fileTokenLimit: 100_000, + configuredFileSizeLimit: undefined, + countTokens: (text) => text.length, + deriveText, + persistDerivation, + }); + if (context == null) { + throw new Error('expected a reading context'); + } + deliveryRouting.reading = context; + return { agentId, agent: { deliveryRouting, fileConsumers: consumers }, context }; + } + + const deriver = () => + jest.fn, Parameters>(async () => derived); + + /** The app config a receiver's endpoint file policy is read from. */ + const appConfigWith = (fileConfig: AppConfig['fileConfig']): AppConfig => ({ + config: {}, + fileStrategy: FileSources.local, + imageOutputType: 'png', + fileConfig, + }); + + it('collects nothing when no receiver has a reading context', async () => { + const historicalFiles = new Map([[historical.file_id, historical]]); + const requestAttachments = [requested]; + const routing = resolveTurnDeliveryRouting({ + agent: { provider: 'openAI' }, + config: automatic, + }); + + const prepared = await prepareScopedTurnCandidates({ + agents: [ + { agentId: 'handoff', agent: { deliveryRouting: routing, fileConsumers: noReader } }, + ], + historicalFiles, + requestAttachments, + }); + + expect(prepared).toEqual({}); + }); + + it('derives each candidate once for every receiver that needs its text', async () => { + const firstDeriver = deriver(); + const secondDeriver = deriver(); + const persistDerivation = jest.fn< + ReturnType, + Parameters + >(async () => true); + const first = receiver('first', { deriveText: firstDeriver, persistDerivation }); + const second = receiver('second', { deriveText: secondDeriver }); + const flushSpy = jest.spyOn(first.context, 'flush'); + const inputs = { + agents: [first, second], + sharedConversationAgentIds: ['first', 'second'], + messages, + historicalFiles: new Map([[historical.file_id, historical]]), + requestAttachments: [requested], + sharedRunAttachmentIds: new Set(), + }; + + const prepared = await prepareScopedTurnCandidates(inputs); + + expect(firstDeriver).toHaveBeenCalledTimes(2); + expect(secondDeriver).not.toHaveBeenCalled(); + expect(flushSpy).toHaveBeenCalledTimes(1); + expect(persistDerivation.mock.calls.map(([update]) => update.file_id).sort()).toEqual([ + historical.file_id, + requested.file_id, + ]); + expect(prepared.candidates?.get(historical.file_id)).toMatchObject({ + text: derived.text, + metadata: { textDerivation: derived.textDerivation }, + }); + expect(prepared.candidates?.get(requested.file_id)).toMatchObject({ text: derived.text }); + expect(historical.text).toBeUndefined(); + expect(requested.text).toBeUndefined(); + + const historyWalk = jest.spyOn(modelBoundContent, 'collectModelBoundHistoricalFileIdState'); + const scoped = resolveScopedTurnAttachments({ ...inputs, ...prepared }); + expect(historyWalk).not.toHaveBeenCalled(); + historyWalk.mockRestore(); + for (const agentId of ['first', 'second']) { + expect(scoped.get(agentId)?.map((file) => [file.file_id, file.llmDeliveryPath])).toEqual([ + [historical.file_id, 'text'], + [requested.file_id, 'text'], + ]); + } + }); + + it('derives nothing for a receiver that reads the candidates with Run Code', async () => { + const deriveText = deriver(); + const historicalFiles = new Map([[historical.file_id, historical]]); + const requestAttachments = [requested]; + + const prepared = await prepareScopedTurnCandidates({ + agents: [receiver('handoff', { consumers: runsCode, deriveText })], + historicalFiles, + requestAttachments, + }); + + expect(deriveText).not.toHaveBeenCalled(); + expect(prepared.candidates?.get(historical.file_id)).toBe(historical); + expect(prepared.candidates?.get(requested.file_id)).toBe(requested); + }); + + it('derives the workbook once for a parallel text reader while the primary uses code', async () => { + const deriveText = deriver(); + const persistDerivation = jest.fn< + ReturnType, + Parameters + >(async () => true); + const primary = receiver('primary', { consumers: runsCode, deriveText, persistDerivation }); + const parallel = receiver('parallel', { deriveText, persistDerivation }); + const secondParallel = receiver('second-parallel', { deriveText, persistDerivation }); + const inputs = { + agents: [primary, parallel, secondParallel], + sharedConversationAgentIds: ['primary', 'parallel', 'second-parallel'], + messages: [], + requestAttachments: [requested], + sharedRunAttachmentIds: new Set(), + }; + const prepared = await prepareScopedTurnCandidates(inputs); + const scoped = resolveScopedTurnAttachments({ ...inputs, ...prepared }); + + expect(scoped.get('primary')).toEqual([]); + for (const agentId of ['parallel', 'second-parallel']) { + expect(scoped.get(agentId)).toEqual([ + expect.objectContaining({ + file_id: requested.file_id, + llmDeliveryPath: 'text', + text: derived.text, + }), + ]); + } + expect(deriveText).toHaveBeenCalledTimes(1); + expect(persistDerivation).toHaveBeenCalledTimes(1); + expect(persistDerivation).toHaveBeenCalledWith( + expect.objectContaining({ + file_id: requested.file_id, + text: derived.text, + textDerivation: derived.textDerivation, + }), + ); + expect(requested.text).toBeUndefined(); + expect(requested.metadata?.textDerivation?.outcome).toBe('deferred'); + }); + + it('decides only what each receiver can be offered', async () => { + const deriveText = deriver(); + const outsideDeriver = deriver(); + const scopedFile = workbook('scoped-xlsx'); + const stale = workbook('stale-xlsx'); + + const prepared = await prepareScopedTurnCandidates({ + agents: [ + receiver('handoff', { deriveText }), + receiver('subagent', { deriveText: outsideDeriver }), + ], + sharedConversationAgentIds: ['handoff'], + messages, + historicalFiles: new Map([ + [historical.file_id, historical], + [stale.file_id, stale], + ]), + requestAttachments: [requested, scopedFile], + sharedRunAttachmentIds: new Set([historical.file_id]), + attachmentsByAgentId: { handoff: [scopedFile] }, + }); + + expect(outsideDeriver).not.toHaveBeenCalled(); + expect(deriveText.mock.calls.map(([file]) => file.file_id)).toEqual([requested.file_id]); + expect(prepared.candidates?.get(requested.file_id)?.text).toBe(derived.text); + expect(prepared.candidates?.get(scopedFile.file_id)).toBe(scopedFile); + }); + + it('derives nothing for a receiver whose endpoint refuses the candidate', async () => { + const fileConfig = { + endpoints: { + openAI: { llmDeliveryPolicy: 'automatic' as const, disabled: true }, + anthropic: { + llmDeliveryPolicy: 'automatic' as const, + supportedMimeTypes: ['^application/pdf$'], + }, + }, + }; + const appConfig = appConfigWith(fileConfig); + const disabledDeriver = jest.fn, Parameters>( + async () => ({ status: 'blocked', error: new UninspectableFileError('extracted_text') }), + ); + const allowlistDeriver = deriver(); + + const prepared = await prepareScopedTurnCandidates({ + agents: [ + receiver('disabled', { deriveText: disabledDeriver, config: { fileConfig } }), + receiver('allowlist', { + deriveText: allowlistDeriver, + endpoint: 'anthropic', + config: { fileConfig }, + }), + ], + appConfig, + endpointsByAgentId: new Map([['allowlist', { endpointType: null }]]), + requestAttachments: [requested], + }); + + expect(disabledDeriver).not.toHaveBeenCalled(); + expect(allowlistDeriver).not.toHaveBeenCalled(); + expect(prepared.candidates?.get(requested.file_id)).toBe(requested); + }); + + it('still derives for a receiver whose endpoint accepts the candidate', async () => { + const fileConfig = { + endpoints: { + openAI: { + llmDeliveryPolicy: 'automatic' as const, + supportedMimeTypes: [`^${XLSX}$`], + }, + }, + }; + const deriveText = deriver(); + + const prepared = await prepareScopedTurnCandidates({ + agents: [receiver('handoff', { deriveText, config: { fileConfig } })], + appConfig: appConfigWith(fileConfig), + requestAttachments: [requested], + }); + + expect(deriveText).toHaveBeenCalledTimes(1); + expect(prepared.candidates?.get(requested.file_id)?.text).toBe(derived.text); + }); + + it('marks the text failed on every receiver that needed it, so each moves on', async () => { + const first = receiver('first', { + deriveText: async () => ({ + status: 'failed', + textDerivation: { outcome: 'failed', reason: 'parser' }, + persist: true, + }), + }); + const second = receiver('second', { deriveText: deriver() }); + const inputs = { + agents: [first, second], + sharedConversationAgentIds: ['first', 'second'], + messages: [], + requestAttachments: [requested], + sharedRunAttachmentIds: new Set(), + }; + + const prepared = await prepareScopedTurnCandidates(inputs); + + expect(prepared.candidates?.get(requested.file_id)).toBe(requested); + expect(first.context.judge(requested).textFailed).toBe(true); + expect(second.context.judge(requested).textFailed).toBe(true); + const scoped = resolveScopedTurnAttachments({ ...inputs, ...prepared }); + expect(scoped.get('first')).toEqual([]); + expect(scoped.get('second')).toEqual([]); + }); +}); diff --git a/packages/api/src/agents/files/delivery.ts b/packages/api/src/agents/files/delivery.ts index 97dc85e80ac..30ee430f137 100644 --- a/packages/api/src/agents/files/delivery.ts +++ b/packages/api/src/agents/files/delivery.ts @@ -1,19 +1,31 @@ import { mergeFileConfig, + decideFileReading, getEndpointFileConfig, resolveUseResponsesApi, + resolveLLMDeliveryPolicy, getCustomEndpointProvider, isSpeechProviderConfigured, resolveTurnLLMDeliveryPath, hasInferredLLMDeliveryPath, } from 'librechat-data-provider'; import type { + TDefaultLLMDeliveryPath, TurnDeliveryRouting, TurnDeliveryFile, TurnFileConsumers, } from 'librechat-data-provider'; import type { AppConfig } from '@librechat/data-schemas'; +import type { TurnReadingFile, TextRequest, TurnReader } from '~/files/reading/turn'; +import { + withDerivedText, + needsDerivedText, + deriveRequestedText, + getTurnReadingContext, +} from '~/files/reading/turn'; import { collectModelBoundHistoricalFileIdState } from '~/middleware/modelBoundContent'; +import { filterFilesByEndpointRuntimeConfig } from '~/files/filter'; +import { getAgentEntry } from '~/agents/attachments'; /** The app config a turn's attachment routing reads. */ export type TurnDeliveryConfig = Pick; @@ -56,66 +68,114 @@ export function resolveTurnDeliveryRouting({ }; } +/** The routing an agent's turn delivers by and the tools judged as its readers. */ +export interface TurnDeliveryInputs { + routing?: TurnDeliveryRouting; + consumers?: TurnFileConsumers; +} + +const isAutomaticRouting = (routing?: TurnDeliveryRouting): boolean => + resolveLLMDeliveryPolicy(routing?.endpointConfig) === 'automatic'; + +const withDeliveryPath = ( + file: T, + llmDeliveryPath: TDefaultLLMDeliveryPath | undefined, +): T => + llmDeliveryPath == null || llmDeliveryPath === file.llmDeliveryPath + ? file + : { ...file, llmDeliveryPath }; + +/** Maps each file, returning the input array itself when no file was replaced. */ +function mapTurnCopies(files: T[], map: (file: T) => T): T[] { + let changed = false; + const result = files.map((file) => { + const next = map(file); + changed ||= next !== file; + return next; + }); + return changed ? result : files; +} + /** Materialize the exact turn route before admission without mutating stored records. */ export function applyTurnDelivery( files: T[], - { routing, consumers }: { routing?: TurnDeliveryRouting; consumers?: TurnFileConsumers }, + { routing, consumers }: TurnDeliveryInputs, ): T[] { if (routing == null) { return files; } - let changed = false; - const result = files.map((file) => { - if (!hasInferredLLMDeliveryPath(file)) { - return file; - } - const llmDeliveryPath = resolveTurnLLMDeliveryPath(routing, file, consumers); - if (llmDeliveryPath == null || llmDeliveryPath === file.llmDeliveryPath) { - return file; - } - changed = true; - return { ...file, llmDeliveryPath }; + return mapTurnCopies(files, (file) => + hasInferredLLMDeliveryPath(file) + ? withDeliveryPath(file, resolveTurnLLMDeliveryPath(routing, file, consumers)) + : file, + ); +} + +/** + * The turn copies as classic routing would deliver them, for the full-set content checks. + * + * Inspection coverage keys on the text route, so a file the automatic policy hands to a tool + * instead of the prompt would otherwise lose the text coverage classic routing gives it, and + * rerouting must not change what is inspected. Returns its input under classic routing and + * whenever no copy differs; stored records and the input copies are never changed. + */ +export function toClassicInspectionView( + files: T[], + routing?: TurnDeliveryRouting, + consumers?: TurnFileConsumers, +): T[] { + if (!isAutomaticRouting(routing)) { + return files; + } + return mapTurnCopies(files, (file) => { + const { automatic, classicPath } = decideFileReading({ routing, file, consumers }); + return automatic ? withDeliveryPath(file, classicPath) : file; }); - return changed ? result : files; } +/** Charges text a referenced inferred tool file may have been delivered as fallback. */ +const chargeRetainedFallbackText = (file: T): T => + hasInferredLLMDeliveryPath(file) && file.llmDeliveryPath === 'none' && file.text + ? { ...file, llmDeliveryPath: 'text' } + : file; + /** * Checkpoints replay already encoded content; current tools and policy cannot remove it. * A referenced inferred tool file carrying extracted text may have been delivered as fallback. * Charge that text conservatively even if fallback was since disabled. Explicit destinations * and records predating routing keep their existing accounting; no stored record is changed. + * + * Under the automatic policy a file it reads is charged by the resuming agent's own decision, + * so text that policy withheld for a tool is not charged; the conservative charge still covers + * every file the decision leaves on its classic route. */ -export function applyCheckpointDelivery(files: T[]): T[] { - return files.map((file) => - hasInferredLLMDeliveryPath(file) && file.llmDeliveryPath === 'none' && file.text - ? { ...file, llmDeliveryPath: 'text' } - : file, - ); +export function applyCheckpointDelivery( + files: T[], + { routing, consumers }: TurnDeliveryInputs = {}, +): T[] { + if (!isAutomaticRouting(routing)) { + return files.map(chargeRetainedFallbackText); + } + return files.map((file) => { + const reading = decideFileReading({ routing, file, consumers }); + return reading.automatic + ? withDeliveryPath(file, reading.path) + : chargeRetainedFallbackText(file); + }); } -/** - * A primary agent's tool route cannot decide whether a handoff receives text. Keep - * owner-hydrated candidates until each receiver resolves them, then add only text absent - * from the shared prompt to its scoped context. The existing scoped-context pipeline owns - * endpoint filtering, aggregate admission, inspection and extraction for these copies. - */ -export function resolveScopedTurnAttachments({ - agents, - sharedConversationAgentIds, - resendFiles, - messages, - historicalFiles, - requestAttachments, - sharedRunAttachmentIds, - attachmentsByAgentId, -}: { - agents: readonly { - agentId: string; - agent: { - deliveryRouting?: TurnDeliveryRouting; - fileConsumers?: TurnFileConsumers; - }; - }[]; +/** A receiver of the conversation's scoped attachments, by the routing its turn settled. */ +export interface ScopedTurnAgent { + agentId: string; + agent: { + deliveryRouting?: TurnDeliveryRouting; + fileConsumers?: TurnFileConsumers; + }; +} + +/** The inputs each handoff receiver's scoped attachment context is resolved from. */ +export interface ScopedTurnAttachmentParams { + agents: readonly ScopedTurnAgent[]; /** Only the primary/handoff graph shares root conversation files. */ sharedConversationAgentIds: readonly string[]; resendFiles?: boolean; @@ -124,23 +184,228 @@ export function resolveScopedTurnAttachments; attachmentsByAgentId?: Map | Record; -}): Map { + /** + * The candidates {@link prepareScopedTurnCandidates} already collected from these same inputs, + * so the retained messages are not walked a second time. + */ + candidates?: ReadonlyMap; +} + +/** The inputs that pick the candidates, with the narrowing ones optional. */ +type ScopedCandidateInputs = Pick< + ScopedTurnAttachmentParams, + 'resendFiles' | 'historicalFiles' | 'requestAttachments' +> & + Partial, 'messages' | 'sharedRunAttachmentIds'>>; + +/** The endpoint type a receiver's attachments are filtered under, where the caller knows it. */ +export interface ScopedReceiverEndpoint { + endpointType?: string | null; +} + +/** Each receiver's endpoint type by agent id. */ +export type ScopedReceiverEndpoints = + | Map + | Record; + +/** + * The scoped attachment inputs whose text {@link prepareScopedTurnCandidates} derives. Without the + * narrowing inputs every receiver with a reading context decides every historical and request file. + */ +export type ScopedTurnCandidateParams = ScopedCandidateInputs & + Pick, 'agents' | 'attachmentsByAgentId'> & + Partial, 'sharedConversationAgentIds'>> & { + /** The app config whose endpoint file policy each receiver's scoped files must pass. */ + appConfig?: AppConfig; + /** Each receiver's endpoint type; its endpoint is the one its routing settled. */ + endpointsByAgentId?: ScopedReceiverEndpoints; + signal?: AbortSignal; + }; + +/** + * The scoped attachment candidates by file id, as copies carrying the text the receivers derived; + * absent when no receiver has a reading context, so none were collected. + */ +export interface ScopedTurnCandidates { + candidates?: ReadonlyMap; +} + +/** Historical files still model-bound in the retained messages; every one when none are given. */ +function selectHistoricalFileIds({ + resendFiles, + messages, + historicalFiles, +}: ScopedCandidateInputs): Iterable { + if (resendFiles === false) { + return []; + } + if (messages == null) { + return historicalFiles?.keys() ?? []; + } + return collectModelBoundHistoricalFileIdState(messages).fileIds; +} + +/** Hydrated history and request files no shared prompt carries, once each by file id. */ +function collectScopedCandidates( + inputs: ScopedCandidateInputs, +): Map { + const { historicalFiles, requestAttachments, sharedRunAttachmentIds } = inputs; + const isShared = (fileId: string): boolean => sharedRunAttachmentIds?.has(fileId) === true; const candidates = new Map(); - for (const fileId of collectModelBoundHistoricalFileIdState(resendFiles === false ? [] : messages) - .fileIds) { + for (const fileId of selectHistoricalFileIds(inputs)) { const file = historicalFiles?.get(fileId); - if (file && !sharedRunAttachmentIds.has(fileId)) candidates.set(fileId, file); + if (file && !isShared(fileId)) candidates.set(fileId, file); } for (const file of requestAttachments) { - if (!sharedRunAttachmentIds.has(file.file_id)) candidates.set(file.file_id, file); + if (!isShared(file.file_id)) candidates.set(file.file_id, file); + } + return candidates; +} + +/** + * A receiver that can derive text, with the files its own scoped context already carries and the + * candidates its endpoint's file policy accepts. + */ +interface ScopedTurnReader extends TurnReader { + scopedFileIds: ReadonlySet; + acceptedFileIds: ReadonlySet; +} + +/** A candidate as the endpoint file policy reads it: a missing type or size reads as empty. */ +type PolicyFile = TurnReadingFile & { type: string; bytes: number }; + +const toPolicyFile = (file: TurnReadingFile): PolicyFile => ({ + ...file, + type: file.type ?? '', + bytes: file.bytes ?? 0, +}); + +/** + * The candidates the receiver's endpoint accepts, by the policy the scoped pipeline applies to + * them later: a disabled endpoint, a MIME allowlist and the per-file size limit. A file refused + * there is never derived for that receiver. + */ +function selectAcceptedFileIds( + candidates: PolicyFile[], + routing: TurnDeliveryRouting, + agentId: string, + consumers: TurnFileConsumers | undefined, + { appConfig, endpointsByAgentId }: ScopedTurnCandidateParams, +): ReadonlySet { + const accepted = filterFilesByEndpointRuntimeConfig(appConfig, { + files: candidates, + endpoint: routing.endpoint, + endpointType: getAgentEntry(endpointsByAgentId, agentId)?.endpointType, + skipTotalSizeLimit: true, + preserveTextSources: true, + consumers, + }); + return new Set(accepted.map((file) => file.file_id)); +} + +function selectScopedReaders( + params: ScopedTurnCandidateParams, + candidates: PolicyFile[], +): ScopedTurnReader[] { + const { agents, sharedConversationAgentIds, attachmentsByAgentId } = params; + const sharedAgents = sharedConversationAgentIds && new Set(sharedConversationAgentIds); + return agents.flatMap(({ agentId, agent }) => { + const routing = agent.deliveryRouting; + const context = getTurnReadingContext(routing); + if (routing == null || context == null || sharedAgents?.has(agentId) === false) { + return []; + } + const scoped = getAgentEntry(attachmentsByAgentId, agentId) ?? []; + return [ + { + routing, + consumers: agent.fileConsumers, + context, + scopedFileIds: new Set(scoped.map((file) => file.file_id)), + acceptedFileIds: selectAcceptedFileIds( + candidates, + routing, + agentId, + agent.fileConsumers, + params, + ), + }, + ]; + }); +} + +const needsScopedText = (file: TurnReadingFile, reader: ScopedTurnReader): boolean => + !reader.scopedFileIds.has(file.file_id) && + reader.acceptedFileIds.has(file.file_id) && + needsDerivedText(file, reader); + +/** The candidate with every receiver whose decision needs its text, or nothing when none does. */ +function requestScopedText( + file: T, + readers: readonly ScopedTurnReader[], +): TextRequest[] { + const [first, ...rest] = readers + .filter((reader) => needsScopedText(file, reader)) + .map(({ context }) => context); + return first == null ? [] : [{ file, contexts: [first, ...rest] }]; +} + +/** + * Derives, before handoff receivers resolve their scoped context, the text a receiver's reading + * needs. Each candidate the receiver's endpoint accepts is decided with that receiver's routing + * and readers, and its text is derived once, through the first receiver that needs it, then saved + * where the record allows. + * Returns the candidates, as copies carrying the text, for {@link resolveScopedTurnAttachments} + * to resolve synchronously without collecting them again; nothing when no receiver has a + * reading context. + */ +export async function prepareScopedTurnCandidates( + params: ScopedTurnCandidateParams, +): Promise> { + const { signal } = params; + if (!params.agents.some(({ agent }) => getTurnReadingContext(agent.deliveryRouting) != null)) { + return {}; } + const candidates = collectScopedCandidates(params); + const readers = selectScopedReaders(params, [...candidates.values()].map(toPolicyFile)); + if (readers.length === 0) { + return {}; + } + const requests = [...candidates.values()].flatMap((file) => requestScopedText(file, readers)); + if (requests.length === 0) { + return { candidates }; + } + signal?.throwIfAborted(); + const texts = await deriveRequestedText(requests, signal); + const derivers = new Set(requests.map(({ contexts }) => contexts[0])); + await Promise.all([...derivers].map((context) => context.flush())); + if (texts.size === 0) { + return { candidates }; + } + for (const [fileId, derived] of texts) { + const file = candidates.get(fileId); + if (file != null) { + candidates.set(fileId, withDerivedText(file, derived)); + } + } + return { candidates }; +} + +/** + * A primary agent's tool route cannot decide whether a handoff receives text. Keep + * owner-hydrated candidates until each receiver resolves them, then add only text absent + * from the shared prompt to its scoped context. The existing scoped-context pipeline owns + * endpoint filtering, aggregate admission, inspection and extraction for these copies. + */ +export function resolveScopedTurnAttachments( + params: ScopedTurnAttachmentParams, +): Map { + const { agents, sharedConversationAgentIds, attachmentsByAgentId } = params; + const candidates = params.candidates ?? collectScopedCandidates(params); const sharedAgents = new Set(sharedConversationAgentIds); const result = new Map(); for (const { agentId, agent } of agents) { - const scoped = - attachmentsByAgentId instanceof Map - ? (attachmentsByAgentId.get(agentId) ?? []) - : (attachmentsByAgentId?.[agentId] ?? []); + const scoped = getAgentEntry(attachmentsByAgentId, agentId) ?? []; const files = new Map(scoped.map((file) => [file.file_id, file])); for (const [fileId, file] of sharedAgents.has(agentId) ? candidates : []) { if (files.has(fileId)) continue; diff --git a/packages/api/src/agents/files/encode.spec.ts b/packages/api/src/agents/files/encode.spec.ts index 10c29025be9..9310b55140e 100644 --- a/packages/api/src/agents/files/encode.spec.ts +++ b/packages/api/src/agents/files/encode.spec.ts @@ -1,8 +1,10 @@ import { FileContext, FileSources, ImageDetail } from 'librechat-data-provider'; import type { TFile } from 'librechat-data-provider'; import type { RunFileEncodingAgent, RunFileMessageEncoderDeps } from './encode'; +import type { FileTextDeriver } from '~/files/reading'; import type { ServerRequest } from '~/types'; import { AgentAttachmentLimitError, AgentAttachmentPolicyError } from '../attachments'; +import { buildTurnReadingContext, getTurnReadingContext } from '~/files/reading'; import { resolveTurnDeliveryRouting } from './delivery'; import { createRunFileMessageEncoder } from './encode'; @@ -56,7 +58,10 @@ function setup({ ]), ); const encodeImages = jest.fn(async () => ({ image_urls: [nativeImage] })); - const encodeDocuments = jest.fn(async () => ({ documents: [nativeDocument] })); + const encodeDocuments = jest.fn< + ReturnType, + Parameters + >(async () => ({ documents: [nativeDocument] })); const encodeAudios = jest.fn(async () => ({ audios: [{ type: 'media', data: 'audio' }] })); const encodeVideos = jest.fn(async () => ({ videos: [{ type: 'media', data: 'video' }] })); const extractText = jest.fn< @@ -73,7 +78,7 @@ function setup({ getStrategyFunctions: jest.fn(), extractText, }; - return { ...deps, ...createRunFileMessageEncoder(deps) }; + return { ...deps, encodeDocuments, ...createRunFileMessageEncoder(deps) }; } describe('createRunFileMessageEncoder', () => { @@ -506,4 +511,384 @@ describe('createRunFileMessageEncoder', () => { expect(harness.encodeDocuments).not.toHaveBeenCalled(); expect(harness.extractText).not.toHaveBeenCalled(); }); + + describe('under the automatic policy', () => { + const runsCode = { executeCode: true, fileSearch: false }; + const searches = { executeCode: false, fileSearch: true }; + const noReader = { executeCode: false, fileSearch: false }; + const attachment = { + context: FileContext.message_attachment, + metadata: { destinationChosen: false }, + }; + /** + * File Search is loaded but queued and registered none of the child's files, as a failed + * provisioning leaves it: the walk passes search by, no code tool is loaded, and a document + * the provider rejects falls back to its text or, without any, to unavailable. + */ + const withoutSearchFiles = ( + context: ReturnType, + ): ReturnType => { + context?.setSearchEvidence({ queued: [], registered: [] }); + return context; + }; + + it('admits an oversized file the child reads with Run Code and never counts it', async () => { + /* The endpoint size limit is the provider's capacity for a file the policy reads, and a + * file left to Run Code never reaches the provider, so it passes the limit and spends none + * of the count, byte or text budgets. Classic routing still refuses the same file. */ + const workbook: TFile = { + ...pdf, + ...attachment, + file_id: 'input-xlsx', + filename: 'quarterly.xlsx', + filepath: '/files/quarterly.xlsx', + type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + bytes: 2 * 1024 * 1024, + text: 'quarter,total\nQ1,4', + llmDeliveryPath: 'text', + }; + const limits = { fileSizeLimit: 1, fileLimit: 1 }; + const harness = setup({ + agents: { + automatic: { provider: 'openAI', endpoint: 'automatic', fileConsumers: runsCode }, + classic: { provider: 'openAI', endpoint: 'classic', fileConsumers: runsCode }, + }, + fileConfig: { + fileContextSizeLimit: 1, + fileContextCharLimit: 5, + endpoints: { + automatic: { ...limits, llmDeliveryPolicy: 'automatic' }, + classic: limits, + }, + }, + }); + + await expect( + harness.encode([workbook, { ...workbook, file_id: 'second-xlsx' }], 'automatic'), + ).resolves.toEqual([]); + await expect(harness.encode([workbook], 'classic')).rejects.toBeInstanceOf( + AgentAttachmentPolicyError, + ); + expect(harness.encodeDocuments).not.toHaveBeenCalled(); + expect(harness.extractText).not.toHaveBeenCalled(); + expect(workbook.llmDeliveryPath).toBe('text'); + }); + + it('inspects a file the child reads with Run Code as classic routing would deliver it', async () => { + const csv: TFile = { + ...pdf, + ...attachment, + file_id: 'input-csv', + filename: 'sales.csv', + filepath: '/files/sales.csv', + type: 'text/csv', + text: 'region,total\nwest,4', + llmDeliveryPath: 'text', + }; + const harness = setup({ + agents: { child: { provider: 'openAI', fileConsumers: runsCode } }, + fileConfig: { endpoints: { openAI: { llmDeliveryPolicy: 'automatic' } } }, + }); + harness.req.config!.filters = { + files: { pii: { fields: ['content'], starterPatterns: [], uninspectable: 'block' } }, + }; + + await expect(harness.encode([csv], 'child')).resolves.toEqual([]); + expect(harness.extractText).not.toHaveBeenCalled(); + }); + + it('leaves out a document the child cannot accept and records it for the next decision', async () => { + const brief: TFile = { ...pdf, ...attachment, text: undefined }; + const harness = setup({ + agents: { + automatic: { provider: 'openAI', endpoint: 'automatic', fileConsumers: searches }, + classic: { provider: 'openAI', endpoint: 'classic', fileConsumers: searches }, + }, + fileConfig: { endpoints: { automatic: { llmDeliveryPolicy: 'automatic' } } }, + }); + const child = harness.getAgent('automatic'); + const context = + child && + withoutSearchFiles( + buildTurnReadingContext({ + routing: child.deliveryRouting, + provider: 'openAI', + fileTokenLimit: 100_000, + configuredFileSizeLimit: undefined, + countTokens: (text) => text.length, + }), + ); + if (child == null || context == null) { + throw new Error('expected an automatic reading context'); + } + child.deliveryRouting.reading = context; + harness.encodeDocuments.mockResolvedValueOnce({ + documents: [], + rejected: [{ file_id: brief.file_id, reason: 'capacity' }], + }); + + const [message] = await harness.encode([brief], 'automatic'); + await harness.encode([brief], 'classic'); + + expect(harness.encodeDocuments.mock.calls.map(([, , params]) => params)).toEqual([ + expect.objectContaining({ endpoint: 'automatic', onValidationFailure: 'skip' }), + expect.objectContaining({ endpoint: 'classic', onValidationFailure: 'throw' }), + ]); + expect(context.judge(brief).rejected).toBe('capacity'); + expect(context.stats().rejected).toBe(1); + expect(getTurnReadingContext(harness.getAgent('classic')?.deliveryRouting)).toBeUndefined(); + expect(message.content).toBe( + [ + '- Shared files left out of this message (file contents are not listed here):', + '\t- "report.pdf" (PDF): cannot be read on this turn (too large to send directly). Say so if asked; do not claim to have read it.', + ].join('\n'), + ); + }); + + const capacityHarness = ({ + fileConfig = {}, + deriveText, + }: { + fileConfig?: NonNullable['fileConfig']; + deriveText?: FileTextDeriver; + } = {}) => { + const harness = setup({ + agents: { child: { provider: 'anthropic', fileConsumers: searches } }, + fileConfig: { + ...fileConfig, + endpoints: { + ...fileConfig.endpoints, + anthropic: { ...fileConfig.endpoints?.anthropic, llmDeliveryPolicy: 'automatic' }, + }, + }, + }); + const child = harness.getAgent('child'); + if (child == null) { + throw new Error('expected a child agent'); + } + const context = withoutSearchFiles( + buildTurnReadingContext({ + routing: child.deliveryRouting, + provider: 'anthropic', + fileTokenLimit: 100_000, + configuredFileSizeLimit: undefined, + countTokens: (text) => text.length, + deriveText, + }), + ); + child.deliveryRouting.reading = context; + return { ...harness, context, child }; + }; + + it('actually includes cached text after encode-time capacity rejection, once', async () => { + const harness = capacityHarness(); + const file: TFile = { ...pdf, ...attachment, text: 'Complete cached PDF text.' }; + harness.encodeDocuments.mockResolvedValueOnce({ + documents: [], + rejected: [{ file_id: file.file_id, reason: 'capacity' }], + }); + + const [message] = await harness.encode([file], 'child'); + + const content = JSON.stringify(message.content); + expect(content.match(/Complete cached PDF text\./g)).toHaveLength(1); + expect(content).not.toContain('file_data'); + expect(content).not.toContain('left out'); + expect(harness.extractText).toHaveBeenCalledTimes(1); + expect(harness.context?.judge(file).rejected).toBe('capacity'); + expect(file.llmDeliveryPath).toBe('provider'); + }); + + it('charges a fallback shared with permanent agent context once', async () => { + const harness = capacityHarness({ + fileConfig: { endpoints: { anthropic: { fileLimit: 1 } } }, + }); + const file: TFile = { ...pdf, ...attachment, text: 'Complete overlapping PDF text.' }; + harness.child.agentContextAttachments = [file]; + harness.encodeDocuments.mockResolvedValueOnce({ + documents: [], + rejected: [{ file_id: file.file_id, reason: 'capacity' }], + }); + + const [message] = await harness.encode([file], 'child'); + + expect( + JSON.stringify(message.content).match(/Complete overlapping PDF text\./g), + ).toHaveLength(1); + expect(harness.context?.judge(file).overflow).not.toBe(true); + expect(harness.extractText).toHaveBeenCalledTimes(1); + }); + + it('derives the capacity fallback after native validation and sends the complete result', async () => { + const deriveText = jest.fn, Parameters>(); + deriveText.mockResolvedValue({ + status: 'derived', + text: 'Complete derived PDF text.', + textDerivation: { outcome: 'complete', extractor: 'document_parser', at: 2 }, + }); + const harness = capacityHarness({ deriveText }); + const file: TFile = { + ...pdf, + ...attachment, + text: undefined, + metadata: { destinationChosen: false, textDerivation: { outcome: 'deferred', at: 1 } }, + }; + harness.encodeDocuments.mockResolvedValueOnce({ + documents: [], + rejected: [{ file_id: file.file_id, reason: 'capacity' }], + }); + + const [message] = await harness.encode([file], 'child'); + + expect(deriveText).toHaveBeenCalledTimes(1); + expect(JSON.stringify(message.content).match(/Complete derived PDF text\./g)).toHaveLength(1); + expect(harness.extractText).toHaveBeenCalledTimes(1); + expect(file.text).toBeUndefined(); + }); + + it('omits complete text after an integrity rejection', async () => { + const harness = capacityHarness(); + const file: TFile = { ...pdf, ...attachment, text: 'Cached text must not bypass integrity.' }; + harness.encodeDocuments.mockResolvedValueOnce({ + documents: [], + rejected: [{ file_id: file.file_id, reason: 'integrity' }], + }); + + const [message] = await harness.encode([file], 'child'); + + expect(JSON.stringify(message.content)).not.toContain(file.text); + expect(harness.extractText).not.toHaveBeenCalled(); + expect(message.content).toEqual(expect.stringContaining('could not accept it')); + }); + + it('moves an aggregate-overflowed derived fallback to unavailable before extraction', async () => { + const harness = capacityHarness({ + fileConfig: { fileContextCharLimit: 8 }, + deriveText: async () => ({ + status: 'derived', + text: 'This complete text fits per-file but exceeds the aggregate.', + textDerivation: { outcome: 'complete', extractor: 'document_parser' }, + }), + }); + const file: TFile = { ...pdf, ...attachment, text: undefined }; + harness.encodeDocuments.mockResolvedValueOnce({ + documents: [], + rejected: [{ file_id: file.file_id, reason: 'capacity' }], + }); + + const [message] = await harness.encode([file], 'child'); + + expect(harness.extractText).not.toHaveBeenCalled(); + expect(harness.context?.judge(file).overflow).toBe(true); + expect(JSON.stringify(message.content)).not.toContain('This complete text fits'); + }); + + it('derives the text a file needs inside encode, and validates it without the text', async () => { + /* No file tool is loaded, so the deferred workbook keeps its classic text route, which + * has no text yet and needs it derived. Validation stays synchronous: the copy is not + * model-bound until its text exists, so nothing is counted. */ + const workbook: TFile = { + ...pdf, + ...attachment, + file_id: 'input-xlsx', + filename: 'quarterly.xlsx', + filepath: '/files/quarterly.xlsx', + type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + text: undefined, + llmDeliveryPath: 'none', + metadata: { destinationChosen: false, textDerivation: { outcome: 'deferred' } }, + }; + const harness = setup({ + agents: { child: { provider: 'openAI', fileConsumers: noReader } }, + fileConfig: { endpoints: { openAI: { llmDeliveryPolicy: 'automatic' } } }, + }); + const deriveText = jest.fn, Parameters>( + async () => ({ + status: 'derived', + text: 'quarter,total\nQ1,42', + textDerivation: { outcome: 'complete', extractor: 'document_parser', at: 1 }, + }), + ); + const child = harness.getAgent('child'); + const context = + child && + buildTurnReadingContext({ + routing: child.deliveryRouting, + provider: 'openAI', + fileTokenLimit: 100_000, + configuredFileSizeLimit: undefined, + countTokens: (text) => text.length, + deriveText, + }); + if (child == null || context == null) { + throw new Error('expected an automatic reading context'); + } + child.deliveryRouting.reading = context; + + expect(() => harness.validate([workbook], 'child')).not.toThrow(); + expect(deriveText).not.toHaveBeenCalled(); + + const [message] = await harness.encode([workbook], 'child'); + + expect(deriveText).toHaveBeenCalledTimes(1); + expect(deriveText).toHaveBeenCalledWith( + expect.objectContaining({ file_id: 'input-xlsx' }), + undefined, + ); + expect(harness.extractText).toHaveBeenCalledWith( + expect.objectContaining({ + attachments: [ + expect.objectContaining({ file_id: 'input-xlsx', text: 'quarter,total\nQ1,42' }), + ], + }), + ); + expect(message.content).toEqual(expect.stringContaining('Q1,42')); + expect(workbook.text).toBeUndefined(); + }); + + it('reuses the token count the reading judged and marks any truncation', async () => { + /* The byte length exceeds the 10-token limit, so the judge counts (3 tokens) and the + * text fits. Recounting with the host tokenizer (one per character) would truncate it. */ + const csv: TFile = { + ...pdf, + ...attachment, + file_id: 'input-csv', + filename: 'sales.csv', + filepath: '/files/sales.csv', + type: 'text/csv', + text: 'region,total\nwest,4', + llmDeliveryPath: 'text', + }; + const harness = setup({ + agents: { child: { provider: 'openAI', fileConsumers: searches } }, + fileConfig: { endpoints: { openAI: { llmDeliveryPolicy: 'automatic' } } }, + }); + harness.req.body = { fileTokenLimit: 10 }; + const child = harness.getAgent('child'); + const countTokens = jest.fn(() => 3); + const context = + child && + buildTurnReadingContext({ + routing: child.deliveryRouting, + provider: 'openAI', + fileTokenLimit: 10, + configuredFileSizeLimit: undefined, + countTokens, + }); + if (child == null || context == null) { + throw new Error('expected an automatic reading context'); + } + child.deliveryRouting.reading = context; + + const [message] = await harness.encode([csv], 'child'); + + expect(harness.extractText).toHaveBeenCalledWith( + expect.objectContaining({ markTruncation: true, knownTokenCount: context.knownTokenCount }), + ); + expect(countTokens).toHaveBeenCalledTimes(1); + expect(context.knownTokenCount(csv)).toBe(3); + expect(message.content).toEqual(expect.stringContaining('west,4')); + expect(message.content).toEqual(expect.not.stringContaining('[Truncated')); + }); + }); }); diff --git a/packages/api/src/agents/files/encode.ts b/packages/api/src/agents/files/encode.ts index e04d6cbed86..936e51a79e4 100644 --- a/packages/api/src/agents/files/encode.ts +++ b/packages/api/src/agents/files/encode.ts @@ -13,15 +13,32 @@ import type { TurnFileConsumers, } from 'librechat-data-provider'; import type { BaseMessage } from '@librechat/agents/langchain'; -import type { ServerRequest, StrategyFunctions } from '~/types'; +import type { + ServerRequest, + StrategyFunctions, + DocumentRejection, + NativeValidationMode, +} from '~/types'; +import type { TurnTextOptions } from '~/files/reading'; import type { TokenCountFn } from '~/utils/text'; import { isToolOwnedAttachment, isModelBoundAttachmentFile, + measureModelBoundAttachment, assertAgentAttachmentLimits, + resolveAgentAttachmentLimits, AgentAttachmentPolicyError, } from '../attachments'; +import { + prepareTurnFiles, + settleTurnFiles, + renderLeftOutFiles, + getTurnTextOptions, + encodeNativeDocuments, + recordNativeRejections, +} from '~/files/reading'; import { assertModelBoundContent } from '~/middleware/modelBoundContent'; +import { toClassicInspectionView, applyTurnDelivery } from './delivery'; import { filterFilesByEndpointRuntimeConfig } from '~/files/filter'; import { countTokens } from '~/utils/tokenizer'; @@ -47,10 +64,15 @@ export interface RunFileEncodingParams { imageDetail?: ImageDetail; } -type MediaEncoder = ( +/** The document encoder also learns whether to leave out a file it cannot send. */ +export interface RunFileDocumentEncodingParams extends RunFileEncodingParams { + onValidationFailure?: NativeValidationMode; +} + +type MediaEncoder = ( req: ServerRequest, files: TFile[], - params: RunFileEncodingParams, + params: P, getStrategyFunctions: (source: string) => StrategyFunctions, ) => Promise; @@ -59,15 +81,20 @@ export interface RunFileMessageEncoderDeps { req: ServerRequest; getAgent: (agentId: string) => RunFileEncodingAgent | undefined; encodeImages: MediaEncoder<{ image_urls: ContentBlock[] }>; - encodeDocuments: MediaEncoder<{ documents: ContentBlock[] }>; + encodeDocuments: MediaEncoder< + { documents: ContentBlock[]; rejected?: readonly DocumentRejection[] }, + RunFileDocumentEncodingParams + >; encodeAudios: MediaEncoder<{ audios: ContentBlock[] }>; encodeVideos: MediaEncoder<{ videos: ContentBlock[] }>; getStrategyFunctions: (source: string) => StrategyFunctions; - extractText: (params: { - attachments: TFile[]; - req: ServerRequest; - tokenCountFn: TokenCountFn; - }) => Promise; + extractText: ( + params: { + attachments: TFile[]; + req: ServerRequest; + tokenCountFn: TokenCountFn; + } & TurnTextOptions, + ) => Promise; } export interface RunFileMessageEncoder { @@ -79,11 +106,15 @@ export interface RunFileMessageEncoder { export function createRunFileMessageEncoder( deps: RunFileMessageEncoderDeps, ): RunFileMessageEncoder { - function prepare(files: TFile[], agentId: string) { + function resolveAgent(agentId: string): RunFileEncodingAgent { const agent = deps.getAgent(agentId); if (!agent) { throw new Error('The target agent is not available for shared file delivery.'); } + return agent; + } + + function prepare(files: TFile[], agent: RunFileEncodingAgent) { const { deliveryRouting } = agent; const { endpoint, fileConfig, endpointConfig } = deliveryRouting; const params: RunFileEncodingParams = { @@ -111,6 +142,7 @@ export function createRunFileMessageEncoder( endpoint, skipTotalSizeLimit: true, preserveTextSources: true, + consumers: agent.fileConsumers, }); if (compatibleFiles.length !== sharedFiles.length) { throw new AgentAttachmentPolicyError(); @@ -127,17 +159,26 @@ export function createRunFileMessageEncoder( req: deps.req, endpoint, }); - assertModelBoundContent({ filters: deps.req.config?.filters, files: sharedFiles }); + assertModelBoundContent({ + filters: deps.req.config?.filters, + files: toClassicInspectionView(sharedFiles, deliveryRouting, agent.fileConsumers), + }); return { agent, params, sharedFiles, fileConfig, endpointConfig }; } function validate(files: TFile[], agentId: string): void { - if (files.length > 0) prepare(files, agentId); + if (files.length > 0) prepare(files, resolveAgent(agentId)); } async function encode(files: TFile[], agentId: string): Promise { if (files.length === 0) return []; - const { agent, params, sharedFiles, fileConfig, endpointConfig } = prepare(files, agentId); + const target = resolveAgent(agentId); + const turnFiles = await prepareTurnFiles({ + routing: target.deliveryRouting, + files, + consumers: target.fileConsumers, + }); + const { agent, params, sharedFiles, fileConfig, endpointConfig } = prepare(turnFiles, target); const images: TFile[] = []; const documents: TFile[] = []; const audios: TFile[] = []; @@ -183,21 +224,88 @@ export function createRunFileMessageEncoder( } } - const encodeMedia = (encoder: MediaEncoder, inputs: TFile[], empty: T): Promise => + const encodeMedia = ( + encoder: MediaEncoder, + inputs: TFile[], + empty: T, + encoderParams: P, + ): Promise => inputs.length > 0 - ? encoder(deps.req, inputs, params, deps.getStrategyFunctions) + ? encoder(deps.req, inputs, encoderParams, deps.getStrategyFunctions) : Promise.resolve(empty); + const encodeDocuments = (inputs: TFile[], onValidationFailure: NativeValidationMode) => + encodeMedia( + deps.encodeDocuments, + inputs, + { documents: [] }, + { ...params, onValidationFailure }, + ); const [imageResult, documentResult, audioResult, videoResult, text] = await Promise.all([ - encodeMedia(deps.encodeImages, images, { image_urls: [] }), - encodeMedia(deps.encodeDocuments, documents, { documents: [] }), - encodeMedia(deps.encodeAudios, audios, { audios: [] }), - encodeMedia(deps.encodeVideos, videos, { videos: [] }), + encodeMedia(deps.encodeImages, images, { image_urls: [] }, params), + encodeNativeDocuments(documents, agent, encodeDocuments), + encodeMedia(deps.encodeAudios, audios, { audios: [] }, params), + encodeMedia(deps.encodeVideos, videos, { videos: [] }, params), textFiles.length > 0 - ? deps.extractText({ attachments: textFiles, req: deps.req, tokenCountFn: countTokens }) + ? deps.extractText({ + attachments: textFiles, + req: deps.req, + tokenCountFn: countTokens, + ...getTurnTextOptions(agent.deliveryRouting), + }) : Promise.resolve(undefined), ]); + recordNativeRejections([agent], documentResult.rejected); + const rejectedIds = new Set((documentResult.rejected ?? []).map(({ file_id }) => file_id)); + let fallbackText: string | undefined; + let fallbackFiles: TFile[] = []; + if (rejectedIds.size > 0) { + const sharedIds = new Set(sharedFiles.map((file) => file.file_id)); + const settled = await settleTurnFiles({ + routing: agent.deliveryRouting, + files: sharedFiles, + consumers: agent.fileConsumers, + allocation: { + requestFileIds: sharedFiles + .filter((file) => rejectedIds.has(file.file_id)) + .map((file) => file.file_id), + limits: resolveAgentAttachmentLimits({ req: deps.req, endpoint: params.endpoint }), + measure: measureModelBoundAttachment, + committedExtra: applyTurnDelivery( + (agent.agentContextAttachments ?? []).filter((file) => !sharedIds.has(file.file_id)), + { routing: agent.deliveryRouting, consumers: agent.fileConsumers }, + ), + committedEntries: sharedFiles + .filter((file) => !rejectedIds.has(file.file_id)) + .map(measureModelBoundAttachment), + scope: 'request', + }, + flush: true, + }); + const prepared = prepare(settled, agent).sharedFiles; + fallbackFiles = prepared.filter( + (file) => rejectedIds.has(file.file_id) && file.llmDeliveryPath === 'text', + ); + assertModelBoundContent({ + filters: deps.req.config?.filters, + files: fallbackFiles, + }); + if (fallbackFiles.length > 0) { + fallbackText = await deps.extractText({ + attachments: fallbackFiles, + req: deps.req, + tokenCountFn: countTokens, + ...getTurnTextOptions(agent.deliveryRouting), + }); + } + } + const deliveredIds = new Set(fallbackFiles.map(({ file_id }) => file_id)); + const leftOut = renderLeftOutFiles( + agent, + documents.filter((file) => rejectedIds.has(file.file_id) && !deliveredIds.has(file.file_id)), + ); + const body = [text, fallbackText, leftOut].filter(Boolean).join('\n\n'); if ( - !text && + !body && imageResult.image_urls.length === 0 && documentResult.documents.length === 0 && audioResult.audios.length === 0 && @@ -208,7 +316,7 @@ export function createRunFileMessageEncoder( const formatted = formatMessage({ message: { role: 'user', - content: text ?? 'Read-only files shared for this task.', + content: body || 'Read-only files shared for this task.', image_urls: imageResult.image_urls, documents: documentResult.documents, audios: audioResult.audios, diff --git a/packages/api/src/agents/hitl/protection.spec.ts b/packages/api/src/agents/hitl/protection.spec.ts index 86f0220a26f..dbff26961d4 100644 --- a/packages/api/src/agents/hitl/protection.spec.ts +++ b/packages/api/src/agents/hitl/protection.spec.ts @@ -1,3 +1,4 @@ +import { FileContext } from 'librechat-data-provider'; import type { AssertResumeRuntimeContentAllowedInput, ResumeContentProtectionDependencies, @@ -9,6 +10,7 @@ import { isModelBoundAttachmentFile, } from '../attachments'; import { assertResumeContentAllowed, assertResumeRuntimeContentAllowed } from './protection'; +import { resolveTurnDeliveryRouting } from '../files/delivery'; const user = { id: 'user-1', @@ -146,6 +148,111 @@ describe('assertResumeRuntimeContentAllowed', () => { expect(dependencies.getFiles).toHaveBeenCalledTimes(1); }); + it('charges checkpoint files by the resuming agent automatic decisions', async () => { + /* The automatic policy withheld this file's text for Run Code, so the checkpoint never + * carried it and the resumed turn must not spend its text budget on it. */ + const dependencies = createDependencies(); + dependencies.getAgentCheckpointer.mockResolvedValue({ + getTuple: jest.fn().mockResolvedValue({ + checkpoint: { + channel_values: { + messages: [{ role: 'human', additional_kwargs: { sourceMessageId: 'source-message' } }], + }, + }, + }), + }); + dependencies.getMessages.mockResolvedValue([ + { messageId: 'source-message', files: [{ file_id: 'code-file' }] }, + ]); + const storedFile = { + file_id: 'code-file', + filename: 'sales.csv', + type: 'text/csv', + source: 'local', + context: FileContext.message_attachment, + bytes: 20, + text: 'region,total', + llmDeliveryPath: 'none', + metadata: { destinationChosen: false }, + }; + dependencies.getFiles.mockResolvedValue([storedFile]); + const routing = resolveTurnDeliveryRouting({ + agent: { provider: 'openAI' }, + config: { fileConfig: { endpoints: { openAI: { llmDeliveryPolicy: 'automatic' } } } }, + }); + const input = { ...createInput({}), isTemporary: false }; + + const [runsCode, noReader] = await Promise.all([ + assertResumeRuntimeContentAllowed( + { ...input, delivery: { routing, consumers: { executeCode: true, fileSearch: false } } }, + dependencies, + ), + assertResumeRuntimeContentAllowed( + { ...input, delivery: { routing, consumers: { executeCode: false, fileSearch: false } } }, + dependencies, + ), + ]); + + expect(runsCode.checkpointFiles).toEqual([storedFile]); + expect(noReader.checkpointFiles).toEqual([{ ...storedFile, llmDeliveryPath: 'text' }]); + expect(storedFile.llmDeliveryPath).toBe('none'); + }); + + it('charges checkpoint files by the primary agent when the host passes only its agents', async () => { + /* AgentClient lists its own agent first and passes no delivery; the resume budget must + * still follow that agent's automatic decision, not charge the withheld text. */ + const dependencies = createDependencies(); + dependencies.getAgentCheckpointer.mockResolvedValue({ + getTuple: jest.fn().mockResolvedValue({ + checkpoint: { + channel_values: { + messages: [{ role: 'human', additional_kwargs: { sourceMessageId: 'source-message' } }], + }, + }, + }), + }); + dependencies.getMessages.mockResolvedValue([ + { messageId: 'source-message', files: [{ file_id: 'workbook' }] }, + ]); + const storedFile = { + file_id: 'workbook', + filename: 'quarterly.xlsx', + type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + source: 'local', + context: FileContext.message_attachment, + bytes: 2048, + text: 'quarter,total\nQ1,42', + llmDeliveryPath: 'none', + metadata: { destinationChosen: false }, + }; + dependencies.getFiles.mockResolvedValue([storedFile]); + const deliveryRouting = resolveTurnDeliveryRouting({ + agent: { provider: 'openAI' }, + config: { fileConfig: { endpoints: { openAI: { llmDeliveryPolicy: 'automatic' } } } }, + }); + const primary = { + name: 'primary', + deliveryRouting, + fileConsumers: { executeCode: true, fileSearch: false }, + }; + const handoff = { name: 'handoff', fileConsumers: { executeCode: false, fileSearch: false } }; + + const projection = await assertResumeRuntimeContentAllowed( + { ...createInput({}), isTemporary: false, agents: [primary, handoff] }, + dependencies, + ); + + expect(projection.checkpointFiles).toEqual([storedFile]); + expect(projection.checkpointFiles.filter(isModelBoundAttachmentFile)).toEqual([]); + expect(() => + assertAgentAttachmentLimits({ + attachments: projection.checkpointFiles.filter(isModelBoundAttachmentFile), + fileConfig: { fileContextCharLimit: 3 }, + endpoint: 'openAI', + }), + ).not.toThrow(); + }); + it('hydrates checkpoint-bound files when content filters are inactive', async () => { const dependencies = createDependencies(); dependencies.getAgentCheckpointer.mockResolvedValue({ diff --git a/packages/api/src/agents/hitl/protection.ts b/packages/api/src/agents/hitl/protection.ts index 88d27200342..da436b1afc4 100644 --- a/packages/api/src/agents/hitl/protection.ts +++ b/packages/api/src/agents/hitl/protection.ts @@ -40,6 +40,8 @@ import type { import type { LocatorTraversalReporter } from '../../protection/diagnostics'; import type { TextContentFragment } from '~/protection/types'; import type { CheckAccessParams } from '~/middleware/access'; +import type { TurnDeliveryInputs } from '../files/delivery'; +import type { TurnReadingAgent } from '~/files/reading'; import { assertModelBoundContent, collectModelBoundHistoricalFileIdState, @@ -217,10 +219,26 @@ export interface AssertResumeRuntimeContentAllowedInput | 'isTemporary' | 'checkpointNamespace' > { - readonly agents: NonNullable; + /** The run's reachable agents, the primary agent first. */ + readonly agents: readonly (ResumeAgentContentInput | null | undefined)[]; readonly files: NonNullable; + /** + * The resuming agent's routing and readers, so checkpoint files are charged by its decisions. + * Defaults to the primary agent's `deliveryRouting` and `fileConsumers`. + */ + readonly delivery?: TurnDeliveryInputs; } +/** An initialized agent as the resume check reads it: its content plus how it receives files. */ +type ResumeAgentContentInput = AgentContentInput & + Pick; + +const resolveResumeDelivery = ({ + delivery, + agents, +}: Pick): TurnDeliveryInputs => + delivery ?? { routing: agents[0]?.deliveryRouting, consumers: agents[0]?.fileConsumers }; + export type ResumeRuntimeContentProtectionDependencies = Pick< ResumeContentProtectionDependencies, 'getAgentCheckpointer' | 'getMessages' | 'getFiles' | 'onTraversalFailure' @@ -1148,7 +1166,10 @@ export async function assertResumeRuntimeContentAllowed( ); throw new AttachmentObjectNotFoundError(unresolvedFileId ?? 'unknown'); } - return { resolvedFiles, checkpointFiles: applyCheckpointDelivery(checkpointFiles) }; + return { + resolvedFiles, + checkpointFiles: applyCheckpointDelivery(checkpointFiles, resolveResumeDelivery(input)), + }; } export function getUserFacingResumeError( diff --git a/packages/api/src/agents/initialize.ts b/packages/api/src/agents/initialize.ts index 4eb34948a1e..8ca944c8caa 100644 --- a/packages/api/src/agents/initialize.ts +++ b/packages/api/src/agents/initialize.ts @@ -14,6 +14,7 @@ import { hasActivePiiPatterns, replaceSpecialVars, providerEndpointMap, + resolveLLMDeliveryPolicy, } from 'librechat-data-provider'; import type { AgentToolResources, @@ -28,8 +29,8 @@ import type { TUser, TurnFileConsumers, } from 'librechat-data-provider'; +import type { AppConfig, IMongoFile, FileMethods, FileOwnerScope } from '@librechat/data-schemas'; import type { GenericTool, LCToolRegistry, ToolMap, LCTool } from '@librechat/agents'; -import type { AppConfig, IMongoFile, FileOwnerScope } from '@librechat/data-schemas'; import type { Request, Response as ServerResponse } from 'express'; import type { TFileUpdate, @@ -40,6 +41,13 @@ import type { TCheckSessionsAlive, TLoadCodeApiKey, } from './resources'; +import type { + FileTextDeriver, + PrimedSearchFile, + TurnReadingContext, + DirectContentAllocation, + TextDerivationPersister, +} from '~/files/reading'; import type { ResolvedManualSkill, ResolvedAlwaysApplySkill, @@ -75,6 +83,12 @@ import { unionPrimeAllowedTools, MAX_PRIMED_SKILLS_PER_TURN, } from './skills'; +import { + isModelBoundAttachmentFile, + assertAgentAttachmentLimits, + measureModelBoundAttachment, + resolveAgentAttachmentLimits, +} from './attachments'; import { resolveChatProjectFiles, resolveChatProjectPolicyFiles, @@ -109,6 +123,17 @@ import { resolveAttachedWorkspaceCommandTimeoutMax, resolveAttachedWorkspaceCommandTimeoutDefault, } from '~/code/command'; +import { + logTurnReading, + settleTurnFiles, + buildTurnReadingContext, + prepareAgentFileContext, +} from '~/files/reading'; +import { + applyTurnDelivery, + toClassicInspectionView, + resolveTurnDeliveryRouting, +} from './files/delivery'; import { formatChatProjectInstructions, hydrateChatProjectContextResources, @@ -116,29 +141,28 @@ import { import { assertChatProjectInstructions, ChatProjectResourcesChangedError } from '../projects/turn'; import { extractAgentContent, extractSkillContent } from '../protection/adapters/submissions'; import { createConfiguredContentInspector, inspectContent } from '../protection/runtime'; -import { assertAgentAttachmentLimits, isModelBoundAttachmentFile } from './attachments'; import { assertModelBoundContent } from '../middleware/modelBoundContent'; import { resolveAttachedWorkspaceReadFileLines } from '~/code/workspace'; +import { resolveConfiguredFileSizeLimit } from '~/files/encode/utils'; import { isValidInstructionsPromptLink } from './instructions/linked'; import { PARTIAL_RESOLVED_CONVERSATION } from './conversationSymbols'; import { registerMemoryTools, memoryToolUsageGuard } from './memory'; import { isImplicitStatefulCodeRouteAvailable } from '~/code/config'; import { applyIntentLabels, sanitizeIntentLabels } from './intent'; -import { prepareQueuedCodeFileContext } from '~/files/code/queued'; import { ContentFilterError } from '../middleware/contentFilter'; import { resolveToolRoleGrants } from '~/tools/rolePermissions'; import { createRequestAgentExecutionContext } from './runtime'; -import { resolveTurnDeliveryRouting } from './files/delivery'; import { filterFilesByEndpointRuntimeConfig } from '~/files'; import { isFatalAgentInitializationError } from './errors'; import { hasActiveFilePolicy } from '../protection/files'; import { resolveAgentCodeExecution } from '~/code/agent'; +import { resolveFileTokenLimit } from '~/files/context'; import { hasActiveFileFieldPolicy } from '~/protection'; import { applyBackgroundToolCalls } from './background'; -import { applyTurnDelivery } from './files/delivery'; import { generateArtifactsPrompt } from '~/prompts'; import { getProviderConfig } from '~/endpoints'; import { primeResources } from './resources'; +import Tokenizer from '~/utils/tokenizer'; /** * Fraction of context budget reserved as headroom when no explicit maxContextTokens is set. @@ -194,6 +218,91 @@ function sumUniqueBytes(files: Array<{ file_id?: string; bytes?: number }>): num return total; } +type TurnAttachment = IMongoFile | TFile; + +/** The request attachments the endpoint runtime policy removed from the delivered set. */ +function selectDroppedRequestFiles( + delivered: readonly T[], + kept: readonly T[], + requestFileIds: ReadonlySet, +): T[] { + const keptFiles = new Set(kept); + return delivered.filter((file) => !keptFiles.has(file) && requestFileIds.has(file.file_id)); +} + +/** The encoding the prompt's file text budget counts with (`extractFileContext`). */ +const READING_TOKEN_ENCODING = 'o200k_base'; + +interface TurnReadingSetup { + routing: TurnDeliveryRouting; + /** The provider the document encoder receives for this agent. */ + provider: string; + model?: string; + endpoint?: string; + fileConfig?: AppConfig['fileConfig']; + req?: ServerRequest; + deriveText?: FileTextDeriver; + persistDerivation?: TextDerivationPersister; + signal?: AbortSignal; +} + +/** Saves a derivation under the request's file owner, when the host can save one. */ +function bindDerivationPersister( + save: InitializeAgentDbMethods['saveFileTextDerivation'], + scope: FileOwnerScope | undefined, +): TextDerivationPersister | undefined { + if (save == null || scope == null) { + return undefined; + } + return (update) => save(update, { user: scope.userId, tenantId: scope.tenantId }); +} + +/** + * The reading evidence for one agent's turn: the automatic policy's, or under classic with a text + * deriver a context that only derives text for records the automatic policy marked. Nothing + * otherwise, so a classic turn runs none of the reading steps. The automatic policy judges text + * fit synchronously, so the tokenizer is loaded first; if it cannot load, counts fall back to + * the tokenizer's estimate. + */ +async function prepareTurnReading({ + routing, + provider, + model, + endpoint, + fileConfig, + req, + deriveText, + persistDerivation, + signal, +}: TurnReadingSetup): Promise { + const automatic = resolveLLMDeliveryPolicy(routing.endpointConfig) === 'automatic'; + if (!automatic && deriveText == null) { + return undefined; + } + if (automatic) { + await Tokenizer.initEncoding(READING_TOKEN_ENCODING).catch((error: unknown) => { + logger.warn( + '[initializeAgent] Tokenizer unavailable; judging text fit with estimated counts', + getSafeErrorMetadata(error), + ); + }); + } + return buildTurnReadingContext({ + routing, + provider, + model, + fileTokenLimit: resolveFileTokenLimit(req), + configuredFileSizeLimit: resolveConfiguredFileSizeLimit(fileConfig, { + provider: provider as Providers, + endpoint, + }), + countTokens: (text) => Tokenizer.getTokenCount(text, READING_TOKEN_ENCODING), + deriveText, + persistDerivation, + signal, + }); +} + const temporalSpecialVarRegex = /{{\s*(current_date|current_datetime|iso_datetime)\s*}}/i; const geminiModelVersionRegex = /^gemini-(\d+)(?:\.(\d+))?(?:-|$)/; const googleToolCombinationTextModels = [ @@ -869,6 +978,8 @@ export type InitializedAgent = Agent & { * call #1 the sandbox can't see the files at all. */ primedCodeFiles?: import('@librechat/agents').CodeEnvFile[]; + /** Files the File Search advert listed on this turn, as the loader primed them. */ + primedSearchFileIds?: string[]; /** * Resolved token/pricing config for this agent's endpoint (admin static * `tokenConfig` and/or fetched custom-endpoint config). Surfaced from the @@ -955,6 +1066,8 @@ export interface InitializeAgentParams { * artifacts don't reach the sandbox. */ primedCodeFiles?: import('@librechat/agents').CodeEnvFile[]; + /** Files the loader listed in the File Search advert (`tool_resources.file_search`). */ + primedSearchFiles?: PrimedSearchFile[]; /** Live workspace binding resolved by the execution-side loader. */ codeExecutionContext?: CodeExecutionContext; repositoryInstructionSource?: RepositoryInstructionSource; @@ -1041,6 +1154,18 @@ export interface InitializeAgentParams { * meta user messages before the LLM call. */ manualSkills?: string[]; + /** + * Derives an attachment's text from its retained original on this request. With it, a record + * the automatic policy marked for later extraction can still be read as text, under either + * policy; without it the turn reads only text stored at upload. + */ + deriveText?: FileTextDeriver; + /** + * The request's derivation persister (`createDerivationPersister`), shared by every agent + * so a derived file is written once per request. Without it each agent saves its own + * derivations through `saveFileTextDerivation` under the request's file owner. + */ + persistDerivation?: TextDerivationPersister; } /** @@ -1146,6 +1271,11 @@ export interface InitializeAgentDbMethods extends EndpointDbMethods { loadCodeApiKey?: TLoadCodeApiKey; /** Optional: persist file metadata updates after provisioning */ updateFile?: (data: TFileUpdate) => Promise; + /** + * Optional: saves text derived on this turn onto a record whose upload deferred extraction. + * Without it derived text stays on the turn copies and is derived again on a later turn. + */ + saveFileTextDerivation?: FileMethods['saveFileTextDerivation']; /** Resolves a role by name for the tool role-permission grants. Optional: when * absent the role half of the web-search gate is not applied. */ getRoleByName?: CheckAccessParams['getRoleByName']; @@ -1589,6 +1719,22 @@ export async function initializeAgent( }, config: appConfig, }); + const readingContext = await prepareTurnReading({ + routing: deliveryRouting, + provider: agent.provider, + model: typeof llmConfig.model === 'string' ? llmConfig.model : (agent.model ?? undefined), + endpoint: agent.endpoint ?? undefined, + fileConfig: appConfig?.fileConfig, + req: params.req, + deriveText: params.deriveText, + persistDerivation: + params.persistDerivation ?? + bindDerivationPersister(db.saveFileTextDerivation, requestFileOwnerScope), + signal: params.signal, + }); + if (readingContext != null) { + deliveryRouting.reading = readingContext; + } /** Resolve the per-agent Code API route before resource/tool priming. A * stateful agent must perform freshness checks and recovery uploads against @@ -1872,8 +2018,24 @@ export async function initializeAgent( if (!paramEndpoints.has(agent.endpoint ?? '')) { endpointFileType = EModelEndpoint.custom; } + const admissionFileIds = new Set( + (authorizedRunFiles ?? requestUsageFiles).map((file) => file.file_id), + ); + /** First-fit allocation of the request's direct content against the limits admission holds + * it to, so allocation and the admission check cannot disagree. */ + const requestAllocation = (): DirectContentAllocation => ({ + requestFileIds: [...admissionFileIds], + limits: resolveAgentAttachmentLimits({ + fileConfig: appConfig?.fileConfig, + endpoint: agent.endpoint ?? '', + endpointType: endpointFileType, + }), + measure: measureModelBoundAttachment, + agentId: agent.id, + }); if ((currentFiles && currentFiles.length) || deferredProvisionFiles.length > 0) { const endpointType = endpointFileType; + const deliveredFiles = currentFiles ?? []; currentFiles = filterFilesByEndpointRuntimeConfig(appConfig, { files: currentFiles, @@ -1881,13 +2043,27 @@ export async function initializeAgent( endpointType, skipTotalSizeLimit: true, preserveTextSources: true, + consumers: fileConsumers, }); - const requestUsageFileIds = new Set( - (authorizedRunFiles ?? requestUsageFiles).map((file) => file.file_id), - ); + /* Only filter survivors are allocated: a file the endpoint refuses never spends the + * request's direct-content allowance, and the reading records it as dropped. */ + if (readingContext != null) { + if (readingContext.policy === 'automatic') { + readingContext.recordDropped( + selectDroppedRequestFiles(deliveredFiles, currentFiles, admissionFileIds), + ); + } + currentFiles = await settleTurnFiles({ + routing: deliveryRouting, + consumers: fileConsumers, + files: currentFiles, + allocation: requestAllocation(), + signal: params.signal, + }); + } assertAgentAttachmentLimits({ attachments: currentFiles.filter( - (file) => requestUsageFileIds.has(file.file_id) && isModelBoundAttachmentFile(file), + (file) => admissionFileIds.has(file.file_id) && isModelBoundAttachmentFile(file), ), fileConfig: appConfig?.fileConfig, endpoint: agent.endpoint ?? '', @@ -1895,8 +2071,9 @@ export async function initializeAgent( }); /* The same endpoint configuration governs both paths. A file this endpoint refuses - * by size, MIME type, or a files-disabled setting must not reach the Code API or - * RAG through provisioning just because it left the delivery set. */ + * by MIME type, a files-disabled setting, or (outside the automatic reading policy, + * where size is delivery capacity) size must not reach the Code API or RAG through + * provisioning just because it left the delivery set. */ if (deferredProvisionFiles.length > 0) { /* One request, one total-size allowance. Filtering each set from zero would let a * delivery attachment and a provisioning candidate that each fit alone exceed the @@ -1915,6 +2092,7 @@ export async function initializeAgent( files: deferredProvisionFiles, endpoint: agent.endpoint ?? '', endpointType, + consumers: fileConsumers, /* The deferred pass charges its own list as it walks it, so a file in both sets * is counted there. Only what delivery spends on files the deferred pass will * not see is carried in. */ @@ -1929,7 +2107,7 @@ export async function initializeAgent( assertModelBoundContent({ filters: appConfig?.filters, - files: currentFiles, + files: currentFiles && toClassicInspectionView(currentFiles, deliveryRouting, fileConsumers), }); /* Provisioning candidates are inspected under the same policy before their bytes can @@ -2000,6 +2178,7 @@ export async function initializeAgent( * and current content policy checks. Ignore returned rows so priming cannot * observe a different post-inspection snapshot. */ + const readingSettled = readingContext?.flush(); if (requestFileOwnerId && requestUsageFiles.length > 0) { await db.updateFilesUsage(requestUsageFiles, undefined, { user: requestFileOwnerId, @@ -2012,6 +2191,7 @@ export async function initializeAgent( tenantId: user?.tenantId, }); } + await readingSettled; let runtimeToolResources = agent.tool_resources; if (attachedEnvironmentOptOut && runtimeToolResources != null) { @@ -2058,6 +2238,7 @@ export async function initializeAgent( endpoint: agent.endpoint ?? '', endpointType: endpointFileType, consumedBytes: sumUniqueBytes(committedFiles), + consumers: fileConsumers, }) as unknown as TFile[]; /* Dropped rather than fatal, matching the deferred candidates: these were not @@ -2183,6 +2364,7 @@ export async function initializeAgent( oauthActionToolNames, tools: structuredTools, primedCodeFiles, + primedSearchFiles, codeExecutionContext: loadedCodeExecutionContext, repositoryInstructionSource, } = loadToolsResult ?? { @@ -2199,6 +2381,7 @@ export async function initializeAgent( actionsEnabled: undefined, oauthActionToolNames: undefined, primedCodeFiles: undefined, + primedSearchFiles: undefined, codeExecutionContext: undefined, repositoryInstructionSource: undefined, }; @@ -2684,30 +2867,60 @@ export async function initializeAgent( fileConsumers.executeCode !== finalFileConsumers.executeCode || fileConsumers.fileSearch !== finalFileConsumers.fileSearch; Object.assign(fileConsumers, finalFileConsumers); - const finalizeAttachments = (files: Array | undefined): IMongoFile[] => { - const hydrated = toMongoFiles(files); - return consumersChanged - ? applyTurnDelivery(hydrated, { routing: deliveryRouting, consumers: fileConsumers }) - : hydrated; - }; - const finalAttachments = finalizeAttachments(primedAttachments); - const finalRequestAttachments = consumersChanged - ? applyTurnDelivery( - (primedRequestAttachments ?? []).filter((file): file is TFile => file != null), - { - routing: deliveryRouting, - consumers: fileConsumers, - }, - ) - : primedRequestAttachments; + const automaticReading = readingContext?.policy === 'automatic'; + if (automaticReading) { + const searchResource = runtimeToolResources?.[EToolResources.file_search]; + const registeredSearchFileIds = primedSearchFiles?.map((file) => file.file_id) ?? [ + ...(searchResource?.file_ids ?? []), + ...(searchResource?.files ?? []).flatMap((file) => (file == null ? [] : [file.file_id])), + ]; + if (provisionState != null) { + provisionState.searchPreparation ??= new Map(); + } + readingContext?.setSearchEvidence({ + queued: (provisionState?.vectorDBFiles ?? []).map((file) => file.file_id), + registered: registeredSearchFileIds, + preparation: provisionState?.searchPreparation, + }); + } + const primedRequestFiles = (primedRequestAttachments ?? []).filter( + (file): file is TFile => file != null, + ); + const primedFiles = toMongoFiles(primedAttachments); + const primedContextFiles = toMongoFiles(primedAgentContextAttachments); + const redeliver = (files: T[]): T[] => + consumersChanged + ? applyTurnDelivery(files, { routing: deliveryRouting, consumers: fileConsumers }) + : files; + /* The automatic reading decides the request again whether or not a reader dropped out: + * search evidence is known only now, so a file left for a search that never received it moves + * on. A derive-only classic reading settles again only when a reader dropped out, since only + * then can a marked record newly need text. Request files are the only ones the reading + * settles, so the other sets take the settled request copies and keep classic routing. */ + const settlesFinalRequest = automaticReading || (readingContext != null && consumersChanged); + const finalRequestAttachments = settlesFinalRequest + ? await settleTurnFiles({ + routing: deliveryRouting, + consumers: fileConsumers, + files: primedRequestFiles, + allocation: requestAllocation(), + signal: params.signal, + }) + : redeliver(primedRequestFiles); const requestAttachments = toMongoFiles(finalRequestAttachments); - const agentContextAttachments = finalizeAttachments(primedAgentContextAttachments); - if (consumersChanged) { + const readingChanged = settlesFinalRequest && finalRequestAttachments !== primedRequestFiles; + const settledRequestCopies = new Map( + readingChanged ? requestAttachments.map((file) => [file.file_id, file] as const) : [], + ); + const finalizeShared = (files: IMongoFile[]): IMongoFile[] => + settledRequestCopies.size === 0 + ? redeliver(files) + : redeliver(files).map((file) => settledRequestCopies.get(file.file_id) ?? file); + const finalAttachments = finalizeShared(primedFiles); + const agentContextAttachments = finalizeShared(primedContextFiles); + if (consumersChanged || readingChanged) { /* A loader may drop a reader, including a skill extra on retry. Newly model-bound text * must pass admission before it can escape initialization, just like initial fallback. */ - const admissionFileIds = new Set( - (authorizedRunFiles ?? requestUsageFiles).map((file) => file.file_id), - ); assertAgentAttachmentLimits({ attachments: requestAttachments.filter( (file) => admissionFileIds.has(file.file_id) && isModelBoundAttachmentFile(file), @@ -2718,9 +2931,14 @@ export async function initializeAgent( }); assertModelBoundContent({ filters: appConfig?.filters, - files: [...finalAttachments, ...requestAttachments, ...agentContextAttachments], + files: toClassicInspectionView( + [...finalAttachments, ...requestAttachments, ...agentContextAttachments], + deliveryRouting, + fileConsumers, + ), }); } + await readingContext?.flush(); const currentRequestFileIds = new Set(requestFileIds); const currentRequestAttachments: TFile[] = (finalRequestAttachments ?? []) .filter((file): file is TFile => file != null && currentRequestFileIds.has(file.file_id)) @@ -2807,11 +3025,14 @@ export async function initializeAgent( ? maxContextTokens : Math.max(1024, Math.round(baseContextTokens * (1 - DEFAULT_RESERVE_RATIO))), primedCodeFiles, + primedSearchFileIds: primedSearchFiles?.map((file) => file.file_id), endpointTokenConfig: options.endpointTokenConfig, instructionsPromptFacts, }; - prepareQueuedCodeFileContext(initializedAgent, [initializedAgent], user?.id); + prepareAgentFileContext(initializedAgent, [initializedAgent], user?.id, false, { + filters: appConfig?.filters, + }); const queuedFileContext = initializedAgent.dynamicToolContextMap?.queued_code_files; if (typeof queuedFileContext === 'string') { assertModelBoundContent({ @@ -2819,6 +3040,7 @@ export async function initializeAgent( files: [{ content: queuedFileContext }], }); } + logTurnReading(initializedAgent, 'init'); /** * Usage is recorded only once initialization has fully succeeded — every diff --git a/packages/api/src/agents/resources.test.ts b/packages/api/src/agents/resources.test.ts index 9e3530be7db..ea9718491f2 100644 --- a/packages/api/src/agents/resources.test.ts +++ b/packages/api/src/agents/resources.test.ts @@ -3016,6 +3016,7 @@ describe('primeResources', () => { bytes: 4096, embedded: false, usage: 0, + metadata: { destinationChosen: false }, } as TFile; const result = await primeResources({ @@ -3046,6 +3047,7 @@ describe('primeResources', () => { bytes: 2048, embedded: false, usage: 0, + metadata: { destinationChosen: false }, } as TFile; const result = await primeResources({ diff --git a/packages/api/src/agents/resources.ts b/packages/api/src/agents/resources.ts index 743b2e6a042..e9746c2c8eb 100644 --- a/packages/api/src/agents/resources.ts +++ b/packages/api/src/agents/resources.ts @@ -18,9 +18,9 @@ import type { } from 'librechat-data-provider'; import type { IMongoFile, AppConfig, IUser } from '@librechat/data-schemas'; import type { FilterQuery, QueryOptions, ProjectionType } from 'mongoose'; +import type { SearchPreparationState } from '~/files/reading/turn'; import type { CodeExecutionContext } from './execution'; import type { ServerRequest } from '~/types'; - import { TOOL_RESOURCE_KEYS } from './orphans'; /** Removes runtime-only file records before persisted Agent resources enter tool initialization. */ @@ -97,6 +97,8 @@ export type ProvisionState = { codeEnvDestinations?: Map; /** Files that need embedding into the vector DB for file_search */ vectorDBFiles: TFile[]; + /** Request-scoped indexing outcomes, shared with the turn's reading notices. */ + searchPreparation?: Map; /** Set of file_ids confirmed alive in code env (from staleness check) */ aliveFileIds: Set; /** The active agent's own resource files, which are the only ones provisioned under its diff --git a/packages/api/src/agents/run.ts b/packages/api/src/agents/run.ts index 0867a483e1d..8b8e2da3a53 100644 --- a/packages/api/src/agents/run.ts +++ b/packages/api/src/agents/run.ts @@ -64,6 +64,7 @@ import type { LangfuseTraceContext } from '~/langfuse/identity'; import type { ResolvedAlwaysApplySkill } from '~/agents/skills'; import type { CodeExecutionContext } from '~/agents/execution'; import type { SubagentCodeHostArgSpecs } from '~/code/targets'; +import type { ReadingAgent } from '~/files/reading/inventory'; import type { MCPToolAlias } from '~/tools/classification'; import type { ReviewedToolApprovals } from './hitl/modes'; import type { SubagentUsageEvent } from '~/agents/usage'; @@ -125,7 +126,7 @@ import { buildAgentInitialToolSessions } from '~/agents/codeFilesSession'; import { getDirectDispatcher, getProxyDispatcher } from '~/utils/proxy'; import { getAzureCredentials, constructAzureURL } from '~/utils/azure'; import { buildEffectiveToolApprovalPolicy } from '~/agents/hitl/allow'; -import { prepareQueuedCodeFileContext } from '~/files/code/queued'; +import { prepareAgentFileContext } from '~/files/reading/inventory'; import { getBuiltInBaseURL } from '~/endpoints/openai/initialize'; import { getProviderConfig } from '~/endpoints/config/providers'; import { buildToolApprovalHooks } from '~/agents/hitl/hooks'; @@ -435,82 +436,83 @@ export function shouldReplayReasoningContent( return isDeepSeekReasoningProvider(agent.provider, agent.model_parameters?.model ?? agent.model); } -type RunAgent = Omit & { - provisionState?: ProvisionState; - fileConsumers?: TurnFileConsumers; - azureOptions?: t.AzureOptions; - tools?: GenericTool[]; - maxContextTokens?: number; - /** Pre-ratio context budget from initializeAgent. */ - baseContextTokens?: number; - useLegacyContent?: boolean; - imageDetail?: ImageDetail; - toolContextMap?: Record; - dynamicToolContextMap?: Record; - toolRegistry?: LCToolRegistry; - /** Serializable tool definitions for event-driven execution */ - toolDefinitions?: LCTool[]; - /** Precomputed flag indicating if any tools have defer_loading enabled */ - hasDeferredTools?: boolean; - /** Both-direction identity aliases for MCP tools whose key spelling changed */ - mcpToolAliases?: MCPToolAlias[]; - /** Names of tools injected with the `run_in_background` param (excluded from eager execution). */ - backgroundToolNames?: string[]; - /** Names of tools with the host-injected `intent` param (stripped from self-spawn inputs). */ - intentToolNames?: string[]; - /** Marker-verified tool names whose intent labels are safe compaction guidance. */ - semanticIntentToolNames?: string[]; - /** - * Per-agent codeenv gate set by `initializeAgent`: admin-level - * `execute_code` capability AND the agent actually requested - * `execute_code` in its tools. Used here to enable - * `RunConfig.toolOutputReferences` only on runs where the bash tool - * is actually registered. - */ - codeEnvAvailable?: boolean; - /** - * Per-agent stateful-session gate set by `initializeAgent`: the admin - * `stateful_code_sessions` capability AND the agent's builder opt-in AND - * `codeEnvAvailable`. Carried into per-agent tool loading and prewarming. - */ - statefulCodeSessions?: boolean; - /** Per-agent stateful workspace sharing scope. */ - statefulCodeEnvironment?: Agent['stateful_code_environment']; - /** Trusted partition for transient code session ids and file references. */ - codeSessionKey?: string; - /** Trusted Code API route selected during initialization. */ - codeExecutionContext?: CodeExecutionContext; - /** Whether this initialized agent can route skills/ writes to persistent skill storage. */ - skillAuthoringAvailable?: boolean; - /** Optional per-agent summarization overrides */ - summarization?: SummarizationConfig; - /** Response field to read model reasoning from for custom OpenAI-compatible endpoints. */ - reasoningKey?: ReasoningResponseKey; - /** Whether to reconstruct `reasoning_content` from persisted history across turns. */ - includeReasoningHistory?: boolean; - /** - * Maximum characters allowed in a single tool result before truncation. - * Overrides the default computed from maxContextTokens. - */ - maxToolResultChars?: number; - /** Initialized subagent configs (loaded by initialize.js from agent.subagents.agent_ids). */ - subagentAgentConfigs?: RunAgent[]; - /** - * Inert, VIEW-checked descriptors for explicit children that are initialized - * only after the SDK selects them. These resolvers are request-scoped: they - * may use the active request's authorization and tool-loading context. - */ - lazySubagentConfigs?: LazySubagentAgent[]; - /** All-or-nothing saved-agent teams resolved by initialize.js. */ - subagentGraphConfigs?: Array<{ - definition: AgentSubagentGraph; - memberConfigs: RunAgent[]; - }>; - /** Member-scoped always-apply skills resolved during agent initialization. */ - alwaysApplySkillPrimes?: ResolvedAlwaysApplySkill[]; - /** Source subagent spawning configuration (enabled / allowSelf / agent_ids). */ - subagents?: AgentSubagentsConfig; -}; +type RunAgent = Omit & + Pick & { + provisionState?: ProvisionState; + fileConsumers?: TurnFileConsumers; + azureOptions?: t.AzureOptions; + tools?: GenericTool[]; + maxContextTokens?: number; + /** Pre-ratio context budget from initializeAgent. */ + baseContextTokens?: number; + useLegacyContent?: boolean; + imageDetail?: ImageDetail; + toolContextMap?: Record; + dynamicToolContextMap?: Record; + toolRegistry?: LCToolRegistry; + /** Serializable tool definitions for event-driven execution */ + toolDefinitions?: LCTool[]; + /** Precomputed flag indicating if any tools have defer_loading enabled */ + hasDeferredTools?: boolean; + /** Both-direction identity aliases for MCP tools whose key spelling changed */ + mcpToolAliases?: MCPToolAlias[]; + /** Names of tools injected with the `run_in_background` param (excluded from eager execution). */ + backgroundToolNames?: string[]; + /** Names of tools with the host-injected `intent` param (stripped from self-spawn inputs). */ + intentToolNames?: string[]; + /** Marker-verified tool names whose intent labels are safe compaction guidance. */ + semanticIntentToolNames?: string[]; + /** + * Per-agent codeenv gate set by `initializeAgent`: admin-level + * `execute_code` capability AND the agent actually requested + * `execute_code` in its tools. Used here to enable + * `RunConfig.toolOutputReferences` only on runs where the bash tool + * is actually registered. + */ + codeEnvAvailable?: boolean; + /** + * Per-agent stateful-session gate set by `initializeAgent`: the admin + * `stateful_code_sessions` capability AND the agent's builder opt-in AND + * `codeEnvAvailable`. Carried into per-agent tool loading and prewarming. + */ + statefulCodeSessions?: boolean; + /** Per-agent stateful workspace sharing scope. */ + statefulCodeEnvironment?: Agent['stateful_code_environment']; + /** Trusted partition for transient code session ids and file references. */ + codeSessionKey?: string; + /** Trusted Code API route selected during initialization. */ + codeExecutionContext?: CodeExecutionContext; + /** Whether this initialized agent can route skills/ writes to persistent skill storage. */ + skillAuthoringAvailable?: boolean; + /** Optional per-agent summarization overrides */ + summarization?: SummarizationConfig; + /** Response field to read model reasoning from for custom OpenAI-compatible endpoints. */ + reasoningKey?: ReasoningResponseKey; + /** Whether to reconstruct `reasoning_content` from persisted history across turns. */ + includeReasoningHistory?: boolean; + /** + * Maximum characters allowed in a single tool result before truncation. + * Overrides the default computed from maxContextTokens. + */ + maxToolResultChars?: number; + /** Initialized subagent configs (loaded by initialize.js from agent.subagents.agent_ids). */ + subagentAgentConfigs?: RunAgent[]; + /** + * Inert, VIEW-checked descriptors for explicit children that are initialized + * only after the SDK selects them. These resolvers are request-scoped: they + * may use the active request's authorization and tool-loading context. + */ + lazySubagentConfigs?: LazySubagentAgent[]; + /** All-or-nothing saved-agent teams resolved by initialize.js. */ + subagentGraphConfigs?: Array<{ + definition: AgentSubagentGraph; + memberConfigs: RunAgent[]; + }>; + /** Member-scoped always-apply skills resolved during agent initialization. */ + alwaysApplySkillPrimes?: ResolvedAlwaysApplySkill[]; + /** Source subagent spawning configuration (enabled / allowSelf / agent_ids). */ + subagents?: AgentSubagentsConfig; + }; type LazySubagentAgent = Pick< RunAgent, @@ -2390,8 +2392,13 @@ export async function createRun({ visitedCodeFileAgents.add(agent.id); enqueueSubagentChildren(agent, pendingCodeFileAgents, visitedCodeFileAgents, false, false); } + /** Only the run's own agents receive the request; a subagent child receives its task. */ + const requestAgents = new Set(agents); for (const agent of codeFileAgents.values()) { - prepareQueuedCodeFileContext(agent, codeFileAgents.values(), user?.id); + prepareAgentFileContext(agent, codeFileAgents.values(), user?.id, false, { + filters: appConfig?.filters, + receivesRequest: requestAgents.has(agent), + }); } const preparedCodeFileAgents = new WeakSet(codeFileAgents.values()); @@ -2399,7 +2406,10 @@ export async function createRun({ if (!preparedCodeFileAgents.has(agent)) { if (agent.provisionState) agent.provisionState.codeEnvDestinations = undefined; codeFileAgents.set(agent.id, agent); - prepareQueuedCodeFileContext(agent, codeFileAgents.values(), user?.id, true); + prepareAgentFileContext(agent, codeFileAgents.values(), user?.id, true, { + filters: appConfig?.filters, + receivesRequest: requestAgents.has(agent), + }); preparedCodeFileAgents.add(agent); } const isSubagent = opts.isSubagent === true; diff --git a/packages/api/src/agents/steering/__tests__/media.spec.ts b/packages/api/src/agents/steering/__tests__/media.spec.ts index 460f162aa11..7c8f455e92b 100644 --- a/packages/api/src/agents/steering/__tests__/media.spec.ts +++ b/packages/api/src/agents/steering/__tests__/media.spec.ts @@ -1,8 +1,14 @@ -import type { IMongoFile } from '@librechat/data-schemas'; +import { FileContext, resolveTurnLLMDeliveryPath } from 'librechat-data-provider'; +import type { AppConfig, IMongoFile } from '@librechat/data-schemas'; +import type { SteerMediaClient, SteerReadingAgent } from '../media'; +import type { FileTextDeriver } from '~/files/reading'; import type { SteerFileFetcher } from '../request'; -import type { SteerMediaClient } from '../media'; +import type { ServerRequest } from '~/types'; import { buildSteerMedia, collectSteerStampTargets, stampSteerPartMedia } from '../media'; +import { applyTurnDelivery, resolveTurnDeliveryRouting } from '../../files/delivery'; +import { buildTurnReadingContext, prepareTurnFiles } from '~/files/reading'; import { AttachmentObjectNotFoundError } from '~/files/encode/utils'; +import { extractFileContext } from '~/files/context'; jest.spyOn(console, 'log').mockImplementation(); @@ -37,6 +43,8 @@ function createClient({ } const user = { id: 'user-1' }; +/** A stored record carrying only the fields a test reads. */ +const asStoredFile = (fields: Partial): IMongoFile => fields as IMongoFile; const imagePart = { type: 'image_url', image_url: { url: 'data:image/png;base64,abc', detail: 'auto' }, @@ -169,6 +177,33 @@ describe('buildSteerMedia', () => { }); }); + it('checks and encodes the copies a host that derives text prepares', async () => { + const storedXlsx = { file_id: 'xlsx', type: 'text/csv', llmDeliveryPath: 'none' }; + const preparedXlsx = { ...storedXlsx, llmDeliveryPath: 'text', text: 'derived' }; + const getFiles: SteerFileFetcher = jest.fn(async () => [asStoredFile(storedXlsx)]); + const client = createClient(); + const prepareTurnAttachments = jest.fn(async () => [asStoredFile(preparedXlsx)]); + const assertFilesAllowed = jest.fn(); + + await buildSteerMedia({ + client: { ...client, prepareTurnAttachments }, + user, + item: steerItem([{ file_id: 'xlsx' }]), + getFiles, + assertFilesAllowed, + }); + + const consumers = { executeCode: false, fileSearch: false }; + expect(prepareTurnAttachments).toHaveBeenCalledWith([storedXlsx], consumers); + expect(client.resolveTurnAttachments).not.toHaveBeenCalled(); + expect(assertFilesAllowed).toHaveBeenCalledWith([preparedXlsx]); + expect(client.processAttachments).toHaveBeenCalledWith( + expect.anything(), + [preparedXlsx], + consumers, + ); + }); + it('prepends extracted file context to the steer text', async () => { const getFiles: SteerFileFetcher = jest.fn(async () => [ { file_id: 'f2', type: 'text/plain' } as unknown as IMongoFile, @@ -518,3 +553,231 @@ describe('stampSteerPartMedia', () => { expect((message.content as Array>)[1]).toBe(filesOnlyPart); }); }); + +describe('buildSteerMedia file reading', () => { + const steerText = 'use the sheet'; + const holdNote = + '"quarterly.xlsx" is attached; Run Code can open it starting with your next message.'; + /** A workbook extracted at upload, so classic routing reads it as text. */ + const workbook = { + file_id: 'xlsx', + filename: 'quarterly.xlsx', + type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + source: 'local', + context: FileContext.message_attachment, + bytes: 2048, + text: 'SENTINEL-7F3A', + llmDeliveryPath: 'text', + metadata: { destinationChosen: false }, + } as Partial as IMongoFile; + const routingFor = (fileConfig: AppConfig['fileConfig']) => + resolveTurnDeliveryRouting({ agent: { provider: 'openAI' }, config: { fileConfig } }); + const automatic = routingFor({ endpoints: { openAI: { llmDeliveryPolicy: 'automatic' } } }); + const classic = routingFor({}); + + /** + * The AgentClient surface over the real turn routing and text extraction: like BaseClient, it + * applies the running agent's routing and exposes the agent only through `options`. + */ + function createRoutedClient( + agent: SteerReadingAgent | undefined, + ): SteerMediaClient & { processAttachments: jest.Mock } { + const req = { body: {}, config: {} } as ServerRequest; + const routing = agent?.deliveryRouting; + return { + options: { agent }, + resolveTurnAttachments: (files, consumers = agent?.fileConsumers) => + applyTurnDelivery(files, { routing, consumers }), + addFileContextToMessage: async (message, files, consumers = agent?.fileConsumers) => { + const fileContext = await extractFileContext({ + attachments: files.filter((file) => { + const path = resolveTurnLLMDeliveryPath(routing, file, consumers); + return path == null || path === 'text'; + }), + req, + tokenCountFn: (text) => text.length, + }); + if (fileContext) { + message.fileContext = fileContext; + } + }, + processAttachments: jest.fn( + async (_message: Record, files: IMongoFile[]) => files, + ), + }; + } + + const steer = (client: SteerMediaClient) => + buildSteerMedia({ + client, + user, + item: steerItem([{ file_id: 'xlsx' }], steerText), + getFiles: jest.fn(async () => [workbook]), + }); + const textOf = (result: Awaited>) => + (result?.content ?? []).map((part) => part.text).join('\n'); + + it('holds a spreadsheet for Run Code under the automatic policy and says so', async () => { + /* The run's code queue cannot take a mid-run file, so the workbook waits for the next + * message's provisioning rather than reaching the prompt as text. */ + const client = createRoutedClient({ + deliveryRouting: automatic, + fileConsumers: { executeCode: true, fileSearch: true }, + }); + + const result = await steer(client); + + expect(result?.content).toEqual([{ type: 'text', text: `${steerText}\n\n${holdNote}` }]); + expect(textOf(result)).not.toContain('SENTINEL-7F3A'); + expect(client.processAttachments).toHaveBeenCalledWith( + expect.anything(), + [{ ...workbook, llmDeliveryPath: 'none' }], + { executeCode: true, fileSearch: false }, + ); + }); + + it('notes a newly held Run Code file after native validation rejects it', async () => { + const deliveryRouting = routingFor({ + endpoints: { openAI: { llmDeliveryPolicy: 'automatic' } }, + }); + const reading = buildTurnReadingContext({ + routing: deliveryRouting, + provider: 'openAI', + fileTokenLimit: 100_000, + configuredFileSizeLimit: undefined, + countTokens: (text) => text.length, + }); + deliveryRouting.reading = reading; + const client = createRoutedClient({ + deliveryRouting, + fileConsumers: { executeCode: true, fileSearch: true }, + }); + const file = asStoredFile({ + ...workbook, + file_id: 'steered-pdf', + filename: 'steered.pdf', + type: 'application/pdf', + llmDeliveryPath: 'provider', + }); + client.processMessageAttachments = async (_message, files, consumers) => { + expect(files[0].llmDeliveryPath).toBe('provider'); + reading?.recordRejections([{ file_id: file.file_id, reason: 'capacity' }]); + return applyTurnDelivery(files, { routing: deliveryRouting, consumers }); + }; + + const result = await buildSteerMedia({ + client, + user, + item: steerItem([{ file_id: file.file_id }], steerText), + getFiles: jest.fn(async () => [file]), + }); + + expect(result?.content).toEqual([ + { + type: 'text', + text: `${steerText}\n\n"steered.pdf" is attached; Run Code can open it starting with your next message.`, + }, + ]); + expect(textOf(result)).not.toContain('SENTINEL-7F3A'); + }); + + it('delivers the text of a spreadsheet when the automatic run has no Run Code', async () => { + const client = createRoutedClient({ + deliveryRouting: automatic, + fileConsumers: { executeCode: false, fileSearch: true }, + }); + + const result = await steer(client); + + expect(textOf(result)).toContain('SENTINEL-7F3A'); + expect(textOf(result)).not.toContain(holdNote); + }); + + describe('a workbook whose upload deferred extraction', () => { + const deferred = asStoredFile({ + ...workbook, + text: undefined, + llmDeliveryPath: 'none', + metadata: { destinationChosen: false, textDerivation: { outcome: 'deferred' } }, + }); + const derivedSentinel = 'DERIVED-7F3A'; + + /** An automatic run without Run Code whose reading derives text through the given stub. */ + function deferredRun(deriveText: FileTextDeriver) { + const deliveryRouting = routingFor({ + endpoints: { openAI: { llmDeliveryPolicy: 'automatic' } }, + }); + const context = buildTurnReadingContext({ + routing: deliveryRouting, + provider: 'openAI', + fileTokenLimit: 100_000, + configuredFileSizeLimit: undefined, + countTokens: (text) => text.length, + deriveText, + }); + deliveryRouting.reading = context; + return createRoutedClient({ + deliveryRouting, + fileConsumers: { executeCode: false, fileSearch: true }, + }); + } + const steerDeferred = (client: SteerMediaClient) => + buildSteerMedia({ + client, + user, + item: steerItem([{ file_id: 'xlsx' }], steerText), + getFiles: jest.fn(async () => [deferred]), + }); + const deriver = () => + jest.fn, Parameters>(async () => ({ + status: 'derived', + text: derivedSentinel, + textDerivation: { outcome: 'complete', extractor: 'document_parser', at: 1 }, + })); + + it('steers with the text the host derives for it', async () => { + const deriveText = deriver(); + const client = deferredRun(deriveText); + const routing = client.options?.agent?.deliveryRouting; + + const result = await steerDeferred({ + ...client, + prepareTurnAttachments: (files, consumers) => + prepareTurnFiles({ routing, files, consumers }), + }); + + expect(deriveText).toHaveBeenCalledTimes(1); + expect(textOf(result)).toContain(derivedSentinel); + expect(textOf(result)).not.toContain(holdNote); + expect(deferred.text).toBeUndefined(); + }); + + it('leaves it out of the steer when the host cannot prepare it', async () => { + const deriveText = deriver(); + + const result = await steerDeferred(deferredRun(deriveText)); + + expect(deriveText).not.toHaveBeenCalled(); + expect(textOf(result)).not.toContain(derivedSentinel); + expect(result?.content).toEqual([{ type: 'text', text: steerText }]); + }); + }); + + it('steers exactly as before under classic routing, even with Run Code loaded', async () => { + const withCode = createRoutedClient({ + deliveryRouting: classic, + fileConsumers: { executeCode: true, fileSearch: false }, + }); + const withoutAgent = createRoutedClient(undefined); + + const [coded, bare] = await Promise.all([steer(withCode), steer(withoutAgent)]); + + expect(coded).toEqual(bare); + expect(textOf(coded)).toContain('SENTINEL-7F3A'); + expect(textOf(coded)).not.toContain(holdNote); + expect(withCode.processAttachments).toHaveBeenCalledWith(expect.anything(), [workbook], { + executeCode: false, + fileSearch: false, + }); + }); +}); diff --git a/packages/api/src/agents/steering/media.ts b/packages/api/src/agents/steering/media.ts index 977549a59f8..a4415fcbb27 100644 --- a/packages/api/src/agents/steering/media.ts +++ b/packages/api/src/agents/steering/media.ts @@ -1,9 +1,10 @@ import { logger } from '@librechat/data-schemas'; import { formatMessage } from '@librechat/agents'; -import { ContentTypes } from 'librechat-data-provider'; +import { ContentTypes, decideFileReading, resolveLLMDeliveryPolicy } from 'librechat-data-provider'; import type { TFile, TurnFileConsumers } from 'librechat-data-provider'; import type { IMongoFile } from '@librechat/data-schemas'; import type { SteerQueueItem } from '~/stream/interfaces/IJobStore'; +import type { TurnReadingAgent } from '~/files/reading'; import type { SteerFileFetcher } from './request'; import type { SteerMediaResult } from './runtime'; import type { SteerRequestUser } from './refs'; @@ -16,6 +17,14 @@ import { prependFileContext } from '../client'; export interface SteerMediaClient { /** The turn's view of stored records, which every check and encode below must share. */ resolveTurnAttachments(files: IMongoFile[], consumers?: TurnFileConsumers): IMongoFile[]; + /** + * The same view after deriving the text its reading needs, when the host can derive it; the + * steer then checks and encodes the copies carrying that text. + */ + prepareTurnAttachments?( + files: IMongoFile[], + consumers?: TurnFileConsumers, + ): Promise; addFileContextToMessage( message: Record, files: IMongoFile[], @@ -26,8 +35,22 @@ export interface SteerMediaClient { files: IMongoFile[], consumers?: TurnFileConsumers, ): Promise; + processMessageAttachments?( + message: Record, + files: IMongoFile[], + consumers?: TurnFileConsumers, + ): Promise; + /** + * The host's running agent. `resolveTurnAttachments` applies its `deliveryRouting`, so the + * steer judges its files by the same routing, with Run Code as a reader exactly when the + * agent's final `fileConsumers` loaded it. + */ + readonly options?: { readonly agent?: SteerReadingAgent | null }; } +/** The fields of the running agent a steer reads its files by. */ +export type SteerReadingAgent = Pick; + interface PseudoMessage { messageId: string; fileContext?: string; @@ -67,6 +90,44 @@ function mergeSteerModelText(text: string, quotes?: string[] | null): string { return normalized != null ? mergeQuotedText(text, normalized) : text; } +const codeHoldNote = (filename: string): string => + `"${filename}" is attached; Run Code can open it starting with your next message.`; + +function appendCodeHoldNotes(text: string, heldFilenames: readonly string[]): string { + if (heldFilenames.length === 0) { + return text; + } + const notes = heldFilenames.map(codeHoldNote).join('\n'); + return text.length > 0 ? `${text}\n\n${notes}` : notes; +} + +/** + * How a mid-run steer reads its files. Its files arrived after resource priming, so no loaded + * tool can reach them before a new turn provisions them. Classic routing therefore judges them + * with no reader and delivers the opted-in text fallback; the automatic policy keeps a file Run + * Code reads for the next message and says so in the steer text. + */ +function resolveSteerReading( + client: SteerMediaClient, + docs: IMongoFile[], +): { fileConsumers: TurnFileConsumers; heldFilenames: string[] } { + const agent = client.options?.agent; + const routing = agent?.deliveryRouting; + if (resolveLLMDeliveryPolicy(routing?.endpointConfig) !== 'automatic') { + return { fileConsumers: { executeCode: false, fileSearch: false }, heldFilenames: [] }; + } + const fileConsumers: TurnFileConsumers = { + executeCode: agent?.fileConsumers?.executeCode === true, + fileSearch: false, + }; + const heldFilenames = docs.flatMap((file) => + decideFileReading({ routing, file, consumers: fileConsumers }).reader === 'code' + ? [file.filename] + : [], + ); + return { fileConsumers, heldFilenames }; +} + /** * Encodes authorized file docs for one steer and assembles the multimodal * content array, reusing the exact pipeline regular user turns go through: @@ -83,6 +144,7 @@ async function encodeSteerContent({ steerId, fileDocs, fileConsumers, + heldFilenames = [], }: { client: SteerMediaClient; text: string; @@ -90,11 +152,21 @@ async function encodeSteerContent({ steerId: string; fileDocs: IMongoFile[]; fileConsumers?: TurnFileConsumers; + /** Files kept for Run Code on the next message, each noted after the steer body. */ + heldFilenames?: readonly string[]; }): Promise { - const modelText = mergeSteerModelText(text, quotes); const pseudo: PseudoMessage = { messageId: `steer:${steerId}` }; - await client.addFileContextToMessage(pseudo, fileDocs, fileConsumers); - const validated = await client.processAttachments(pseudo, fileDocs, fileConsumers); + let validated: IMongoFile[] | undefined; + if (client.processMessageAttachments != null) { + validated = await client.processMessageAttachments(pseudo, fileDocs, fileConsumers); + } else { + await client.addFileContextToMessage(pseudo, fileDocs, fileConsumers); + validated = await client.processAttachments(pseudo, fileDocs, fileConsumers); + } + const settledFiles = validated ?? fileDocs; + const finalHeldFilenames = + fileConsumers == null ? heldFilenames : resolveSteerReading(client, settledFiles).heldFilenames; + const modelText = appendCodeHoldNotes(mergeSteerModelText(text, quotes), finalHeldFilenames); const formatted = formatMessage({ message: { role: 'user', @@ -148,13 +220,10 @@ export async function buildSteerMedia({ return undefined; } const docsById = new Map(rawDocs.map((file) => [file.file_id, file])); - /* These files arrived after resource priming. A loaded tool is not evidence that - * it can access them; use the opted-in text fallback until a new turn provisions them. */ - const fileConsumers: TurnFileConsumers = { executeCode: false, fileSearch: false }; - const fileDocs = client.resolveTurnAttachments( - ids.map((id) => docsById.get(id)).filter((doc): doc is IMongoFile => doc != null), - fileConsumers, - ); + const docs = ids.map((id) => docsById.get(id)).filter((doc): doc is IMongoFile => doc != null); + const { fileConsumers, heldFilenames } = resolveSteerReading(client, docs); + const fileDocs = await (client.prepareTurnAttachments?.(docs, fileConsumers) ?? + client.resolveTurnAttachments(docs, fileConsumers)); assertFilesAllowed?.(fileDocs); return encodeSteerContent({ client, @@ -163,6 +232,7 @@ export async function buildSteerMedia({ steerId: item.steerId, fileDocs, fileConsumers, + heldFilenames, }); } diff --git a/packages/api/src/files/code/priming.ts b/packages/api/src/files/code/priming.ts index 0dba3389f0f..7eecaa9d056 100644 --- a/packages/api/src/files/code/priming.ts +++ b/packages/api/src/files/code/priming.ts @@ -208,7 +208,8 @@ export type CodeFileLocation = 'sandbox' | 'programmatic'; type CodeContextFile = Pick; -const CODE_FILE_CONTEXT_ROOTS: Record = { +/** The directory primed code files mount under on each execution route. */ +export const CODE_FILE_CONTEXT_ROOTS: Readonly> = { sandbox: '/mnt/data', programmatic: '$LIBRECHAT_CODE_DATA_DIR', }; diff --git a/packages/api/src/files/context.spec.ts b/packages/api/src/files/context.spec.ts index 56d09ff5f44..2927c107db7 100644 --- a/packages/api/src/files/context.spec.ts +++ b/packages/api/src/files/context.spec.ts @@ -1,5 +1,7 @@ +import { FileSources, mergeFileConfig } from 'librechat-data-provider'; import type { TFile } from 'librechat-data-provider'; -import { getAttachmentTitleText } from './context'; +import type { ServerRequest } from '~/types'; +import { extractFileContext, getAttachmentTitleText, resolveFileTokenLimit } from './context'; const file = (filename?: string): TFile => ({ filename }) as TFile; @@ -28,3 +30,131 @@ describe('getAttachmentTitleText', () => { expect(getAttachmentTitleText([file(), file()])).toBe(''); }); }); + +type TextAttachment = { + file_id: string; + filename: string; + text: string; + source: string; + llmDeliveryPath: string; +}; + +const textFile = (file_id: string, text: string): TextAttachment => ({ + file_id, + filename: `${file_id}.txt`, + text, + source: FileSources.local, + llmDeliveryPath: 'text', +}); + +const requestWithLimit = (fileTokenLimit?: number, configLimit?: number): ServerRequest => + ({ + body: { fileTokenLimit }, + config: { fileConfig: { fileTokenLimit: configLimit } }, + }) as ServerRequest; + +describe('resolveFileTokenLimit', () => { + it('prefers the request override over the configured limit', () => { + expect(resolveFileTokenLimit(requestWithLimit(25, 50))).toBe(25); + }); + + it('falls back to the configured limit', () => { + expect(resolveFileTokenLimit(requestWithLimit(undefined, 50))).toBe(50); + }); + + it('falls back to the default limit without a request', () => { + expect(resolveFileTokenLimit()).toBe(mergeFileConfig(undefined).fileTokenLimit); + }); +}); + +describe('extractFileContext options', () => { + const tokenLimit = 10; + + it.each<{ + name: string; + text: string; + known: number | undefined; + counted: 'never' | 'only for the truncation' | 'in full'; + rendered: string; + }>([ + { + name: 'reuses a known token count instead of counting the text again', + text: 'fits', + known: 4, + counted: 'never', + rendered: '# "note.txt"\nfits\n', + }, + { + name: 'never counts the full text of a truncated file whose count is known', + text: 'x'.repeat(50), + known: 50, + counted: 'only for the truncation', + rendered: '# "note.txt"\nx', + }, + { + name: 'counts the text when the known count is unavailable', + text: 'fits', + known: undefined, + counted: 'in full', + rendered: '# "note.txt"\nfits\n', + }, + ])('$name', async ({ text, known, counted, rendered }) => { + const tokenCountFn = jest.fn((value: string): number => value.length); + const attachment = textFile('note', text); + const knownTokenCount = jest.fn((_file: TextAttachment): number | undefined => known); + + const result = await extractFileContext({ + attachments: [attachment], + req: requestWithLimit(tokenLimit), + tokenCountFn, + knownTokenCount, + }); + + expect(result).toContain(rendered); + expect(knownTokenCount).toHaveBeenCalledWith(attachment); + if (counted === 'never') { + expect(tokenCountFn).not.toHaveBeenCalled(); + } else if (counted === 'only for the truncation') { + expect(tokenCountFn).toHaveBeenCalled(); + expect(tokenCountFn).not.toHaveBeenCalledWith(text); + } else { + expect(tokenCountFn).toHaveBeenCalledWith(text); + } + }); + + it('follows truncated text with a notice only when asked to', async () => { + const tokenCountFn = (text: string): number => text.length; + const attachments = [textFile('long', 'x'.repeat(50)), textFile('short', 'fits')]; + const notice = '[Truncated: only the beginning of "long.txt" fits; the rest is omitted.]'; + + const marked = await extractFileContext({ + attachments, + req: requestWithLimit(tokenLimit), + tokenCountFn, + markTruncation: true, + }); + const unmarked = await extractFileContext({ + attachments, + req: requestWithLimit(tokenLimit), + tokenCountFn, + }); + + expect(marked).toMatch(/# "long\.txt"\nx+\n\[Truncated: [^\n]+\]\n/); + expect(marked).toContain(notice); + expect(marked?.split('[Truncated:')).toHaveLength(2); + expect(unmarked).not.toContain('[Truncated:'); + expect(marked?.replace(`\n${notice}`, '')).toBe(unmarked); + }); + + it('keeps the classic output byte-for-byte without options', async () => { + const result = await extractFileContext({ + attachments: [textFile('alpha', 'first'), textFile('beta', 'y'.repeat(20))], + req: requestWithLimit(tokenLimit), + tokenCountFn: (text: string): number => text.length, + }); + + expect(result).toBe( + 'Attached document(s):\n```md# "alpha.txt"\nfirst\n\n\n---\n\n# "beta.txt"\nyyyyyyyyy\n\n```', + ); + }); +}); diff --git a/packages/api/src/files/context.ts b/packages/api/src/files/context.ts index 211f5a07834..389d8aad745 100644 --- a/packages/api/src/files/context.ts +++ b/packages/api/src/files/context.ts @@ -29,6 +29,20 @@ export function getAttachmentTitleText(files?: TFile[] | null): string { return filenames.length > 0 ? `Attached file(s): ${filenames.join(', ')}` : ''; } +/** The per-file token limit for attachment text: the request's override, else the merged config's. */ +export function resolveFileTokenLimit(req?: ServerRequest): number | undefined { + return req?.body?.fileTokenLimit ?? mergeFileConfig(req?.config?.fileConfig).fileTokenLimit; +} + +type FileContextAttachment = Pick & { + file_id?: string; + source?: string; + llmDeliveryPath?: string; +}; + +const truncationNotice = (filename: string): string => + `[Truncated: only the beginning of "${filename}" fits; the rest is omitted.]`; + /** * Extracts text context from attachments and returns formatted text. * This handles text that was already extracted from files (OCR, transcriptions, document text, etc.) @@ -36,26 +50,28 @@ export function getAttachmentTitleText(files?: TFile[] | null): string { * @param params.attachments - Array of file attachments * @param params.req - Express request object for config access * @param params.tokenCountFn - Function to count tokens in text + * @param params.knownTokenCount - A file's already-measured text token count, so it is not recounted + * @param params.markTruncation - Follow truncated text with a line saying the rest was omitted * @returns The formatted file context text, or undefined if no text found */ -export async function extractFileContext({ +export async function extractFileContext({ attachments, req, tokenCountFn, + knownTokenCount, + markTruncation = false, }: { - attachments: readonly (Pick & { - source?: string; - llmDeliveryPath?: string; - })[]; + attachments: readonly T[]; req?: ServerRequest; tokenCountFn: TokenCountFn; + knownTokenCount?: (file: T) => number | undefined; + markTruncation?: boolean; }): Promise { if (!attachments || attachments.length === 0) { return undefined; } - const fileConfig = mergeFileConfig(req?.config?.fileConfig); - const fileTokenLimit = req?.body?.fileTokenLimit ?? fileConfig.fileTokenLimit; + const fileTokenLimit = resolveFileTokenLimit(req); if (!fileTokenLimit) { // If no token limit, return undefined (no processing) @@ -71,21 +87,23 @@ export async function extractFileContext({ } const hasTextDelivery = file.llmDeliveryPath === 'text' || source === FileSources.text; - if (hasTextDelivery && file.text) { - const { text: limitedText, wasTruncated } = await processTextWithTokenLimit({ - text: file.text, - tokenLimit: fileTokenLimit, - tokenCountFn, - }); - - if (wasTruncated) { - logger.debug( - `[extractFileContext] Text content truncated for file: ${file.filename} due to token limits`, - ); - } - - resultText += `${!resultText ? 'Attached document(s):\n```md' : '\n\n---\n\n'}# "${file.filename}"\n${limitedText}\n`; + if (!hasTextDelivery || !file.text) { + continue; } + + const { text: limitedText, wasTruncated } = await processTextWithTokenLimit({ + text: file.text, + tokenLimit: fileTokenLimit, + tokenCountFn, + knownTokenCount: knownTokenCount?.(file), + }); + + if (wasTruncated) { + logger.debug(`[extractFileContext] text truncated file_id=${file.file_id ?? 'unknown'}`); + } + + const notice = markTruncation && wasTruncated ? `\n${truncationNotice(file.filename)}` : ''; + resultText += `${!resultText ? 'Attached document(s):\n```md' : '\n\n---\n\n'}# "${file.filename}"\n${limitedText}${notice}\n`; } if (resultText) { diff --git a/packages/api/src/files/documents/crud.ts b/packages/api/src/files/documents/crud.ts index 4457a30b3d6..c2721b8adf0 100644 --- a/packages/api/src/files/documents/crud.ts +++ b/packages/api/src/files/documents/crud.ts @@ -7,7 +7,8 @@ import { assertSafeZipSize } from './zipSafety'; type FileParseFn = (file: Express.Multer.File) => Promise; -const DOCUMENT_PARSER_MAX_FILE_SIZE = 15 * megabyte; +/** The largest file the built-in document parser accepts, checked before it reads anything. */ +export const DOCUMENT_PARSER_MAX_FILE_SIZE: number = 15 * megabyte; const ODT_MAX_DECOMPRESSED_SIZE = 50 * megabyte; /** diff --git a/packages/api/src/files/encode/document.spec.ts b/packages/api/src/files/encode/document.spec.ts index f58d591ed67..fe234cdc312 100644 --- a/packages/api/src/files/encode/document.spec.ts +++ b/packages/api/src/files/encode/document.spec.ts @@ -1,59 +1,47 @@ import { Providers } from '@librechat/agents'; -import { mbToBytes } from 'librechat-data-provider'; +import { mbToBytes, fileConfig as baseFileConfig } from 'librechat-data-provider'; import type { AppConfig, IMongoFile } from '@librechat/data-schemas'; -import type { ServerRequest } from '~/types'; +import type { + NativeValidationMode, + DocumentRejection, + DocumentResult, + ServerRequest, +} from '~/types'; import { encodeAndFormatDocuments } from './document'; /** Mock the validation module */ jest.mock('~/files/validation', () => ({ + ...jest.requireActual('~/files/validation'), validatePdf: jest.fn(), validateBedrockDocument: jest.fn(), })); -/** Mock the utils module */ +/** Mock storage reads; the file size limit resolves through the real fileConfig merge */ jest.mock('./utils', () => ({ + ...jest.requireActual('./utils'), getFileStream: jest.fn(), - getConfiguredFileSizeLimit: jest.fn(), - isAttachmentObjectNotFoundError: jest.fn(() => false), })); import { validatePdf, validateBedrockDocument } from '~/files/validation'; -import { getFileStream, getConfiguredFileSizeLimit } from './utils'; +import { getFileStream, AttachmentObjectNotFoundError } from './utils'; import { Types } from 'mongoose'; +const realValidation = + jest.requireActual('~/files/validation'); + const mockedValidatePdf = validatePdf as jest.MockedFunction; const mockedValidateBedrockDocument = validateBedrockDocument as jest.MockedFunction< typeof validateBedrockDocument >; const mockedGetFileStream = getFileStream as jest.MockedFunction; -const mockedGetConfiguredFileSizeLimit = getConfiguredFileSizeLimit as jest.MockedFunction< - typeof getConfiguredFileSizeLimit ->; describe('encodeAndFormatDocuments - fileConfig integration', () => { const mockStrategyFunctions = jest.fn(); + const inheritedLimit = baseFileConfig.endpoints.default.fileSizeLimit; + beforeEach(() => { - jest.clearAllMocks(); - /** Default mock implementation for getConfiguredFileSizeLimit */ - mockedGetConfiguredFileSizeLimit.mockImplementation((req, params) => { - if (!req.config?.fileConfig) { - return undefined; - } - const { provider, endpoint } = params; - const lookupKey = endpoint ?? provider; - const fileConfig = req.config.fileConfig; - const endpoints = fileConfig.endpoints; - if (endpoints?.[lookupKey]) { - const limit = endpoints[lookupKey].fileSizeLimit; - return limit !== undefined ? mbToBytes(limit) : undefined; - } - if (endpoints?.default) { - const limit = endpoints.default.fileSizeLimit; - return limit !== undefined ? mbToBytes(limit) : undefined; - } - return undefined; - }); + jest.resetAllMocks(); }); /** Helper to create a mock request with file config */ @@ -168,7 +156,7 @@ describe('encodeAndFormatDocuments - fileConfig integration', () => { ); }); - it('should pass undefined when fileConfig.endpoints is not defined', async () => { + it('should pass the inherited default limit for an empty fileConfig', async () => { const req = { config: { fileConfig: {}, @@ -176,13 +164,11 @@ describe('encodeAndFormatDocuments - fileConfig integration', () => { } as ServerRequest; const file = createMockFile(10); - const mockContent = Buffer.from('test-pdf-content').toString('base64'); mockedGetFileStream.mockResolvedValue({ file, - content: mockContent, + content: Buffer.from('test-pdf-content').toString('base64'), metadata: file, }); - mockedValidatePdf.mockResolvedValue({ isValid: true }); await encodeAndFormatDocuments( @@ -192,12 +178,11 @@ describe('encodeAndFormatDocuments - fileConfig integration', () => { mockStrategyFunctions, ); - /** When fileConfig has no endpoints, getConfiguredFileSizeLimit returns undefined */ expect(mockedValidatePdf).toHaveBeenCalledWith( expect.any(Buffer), expect.any(Number), Providers.OPENAI, - undefined, + inheritedLimit, undefined, ); }); @@ -282,12 +267,11 @@ describe('encodeAndFormatDocuments - fileConfig integration', () => { ); }); - it('should pass undefined when provider-specific config not found and no default', async () => { + it('should pass the inherited default limit when only another provider is configured', async () => { const req = { config: { fileConfig: { endpoints: { - /** Only configure a different provider, not OpenAI */ [Providers.ANTHROPIC]: { fileSizeLimit: 25, }, @@ -297,13 +281,11 @@ describe('encodeAndFormatDocuments - fileConfig integration', () => { } as unknown as ServerRequest; const file = createMockFile(20); - const mockContent = Buffer.from('test-pdf-content').toString('base64'); mockedGetFileStream.mockResolvedValue({ file, - content: mockContent, + content: Buffer.from('test-pdf-content').toString('base64'), metadata: file, }); - mockedValidatePdf.mockResolvedValue({ isValid: true }); await encodeAndFormatDocuments( @@ -313,12 +295,11 @@ describe('encodeAndFormatDocuments - fileConfig integration', () => { mockStrategyFunctions, ); - /** When provider-specific config not found and no default, returns undefined */ expect(mockedValidatePdf).toHaveBeenCalledWith( expect.any(Buffer), expect.any(Number), Providers.OPENAI, - undefined, + inheritedLimit, undefined, ); }); @@ -1202,4 +1183,431 @@ describe('encodeAndFormatDocuments - fileConfig integration', () => { expect(result.files).toHaveLength(6); }); }); + + describe('onValidationFailure', () => { + const pdfBytes = (size: number): Buffer => { + const buffer = Buffer.alloc(size); + buffer.write('%PDF-1.4\n', 0); + return buffer; + }; + + /** Serves each file its own bytes so the real validators see what production would. */ + const serveContents = (contents: Map): void => { + mockedGetFileStream.mockImplementation(async (_req, file) => ({ + file, + content: (contents.get(file.file_id) ?? Buffer.alloc(0)).toString('base64'), + metadata: file, + })); + }; + + type ServedFile = { file: IMongoFile; bytes?: Buffer; reason?: DocumentRejection['reason'] }; + + /** Serves the bytes each row declares and returns its files in order. */ + const serveRows = (rows: ServedFile[]): IMongoFile[] => { + const contents = new Map(); + for (const { file, bytes } of rows) { + if (bytes != null) { + contents.set(file.file_id, bytes); + } + } + serveContents(contents); + return rows.map(({ file }) => file); + }; + const rejectedOf = (rows: ServedFile[]): DocumentRejection[] => + rows.flatMap(({ file, reason }) => + reason == null ? [] : [{ file_id: file.file_id, reason }], + ); + const keptOf = (rows: ServedFile[]): IMongoFile[] => + rows.filter(({ reason }) => reason == null).map(({ file }) => file); + const DOCX = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'; + + type UnreadableSet = Record<'unreadable' | 'unstored' | 'empty' | 'readable', IMongoFile>; + + /** A failed read, a record with no stored object, a zero-byte object and a readable file. */ + const serveUnreadableSet = (): UnreadableSet => { + const files: UnreadableSet = { + unreadable: createMockDocFile(0.01, 'text/plain', 'unreadable.txt'), + unstored: createMockDocFile(0.01, 'text/plain', 'unstored.txt'), + empty: createMockDocFile(0.01, 'text/plain', 'empty.txt'), + readable: createMockDocFile(0.01, 'text/plain', 'readable.txt'), + }; + mockedGetFileStream.mockImplementation(async (_req, file) => { + if (file.file_id === files.unreadable.file_id) { + throw new Error('storage unavailable'); + } + if (file.file_id === files.unstored.file_id) { + return null; + } + const content = + file.file_id === files.empty.file_id ? '' : Buffer.from('notes').toString('base64'); + return { file, content, metadata: file }; + }); + return files; + }; + + const encodeUnreadableSet = async ( + files: UnreadableSet, + onValidationFailure?: NativeValidationMode, + ): Promise => { + const consoleError = jest.spyOn(console, 'error').mockImplementation(() => undefined); + try { + const result = await encodeAndFormatDocuments( + createMockRequest(15) as ServerRequest, + [files.unreadable, files.unstored, files.empty, files.readable], + { provider: Providers.OPENAI, onValidationFailure }, + mockStrategyFunctions, + ); + expect(consoleError).toHaveBeenCalledWith( + 'Document processing failed:', + expect.objectContaining({ message: 'storage unavailable' }), + ); + return result; + } finally { + consoleError.mockRestore(); + } + }; + + /** Passes the case-insensitive provider gate but matches no case-sensitive block format. */ + const providerWithoutBlockFormat = 'OpenRouter' as Providers; + + beforeEach(() => { + mockedValidatePdf.mockImplementation(realValidation.validatePdf); + mockedValidateBedrockDocument.mockImplementation(realValidation.validateBedrockDocument); + }); + + describe('skip', () => { + it.each<{ + name: string; + request: Partial; + provider: Providers; + serve: () => ServedFile[]; + document: unknown; + }>([ + { + name: 'a PDF over the configured limit as capacity', + request: createMockRequest(1), + provider: Providers.OPENAI, + serve: () => [ + { file: createMockFile(2), bytes: pdfBytes(mbToBytes(2)), reason: 'capacity' }, + { file: createMockFile(0.01), bytes: pdfBytes(1024) }, + ], + document: expect.anything(), + }, + { + name: 'an encrypted Anthropic PDF as integrity', + request: createMockRequest(30, Providers.ANTHROPIC), + provider: Providers.ANTHROPIC, + serve: () => { + const encryptedBytes = pdfBytes(1024); + encryptedBytes.write('/Encrypt ', 100); + return [ + { file: createMockFile(0.01), bytes: encryptedBytes, reason: 'integrity' }, + { file: createMockFile(0.01), bytes: pdfBytes(1024) }, + ]; + }, + document: expect.objectContaining({ + type: 'document', + source: expect.objectContaining({ media_type: 'application/pdf' }), + }), + }, + { + name: 'Bedrock capacity and integrity failures with their own reasons', + request: createMockRequest(), + provider: Providers.BEDROCK, + serve: () => { + const brokenBytes = Buffer.alloc(1024); + brokenBytes.write('INVALID', 0); + return [ + { + file: createMockDocFile(5, 'text/csv', 'large.csv'), + bytes: Buffer.alloc(mbToBytes(5)), + reason: 'capacity', + }, + { + file: createMockDocFile(0.01, 'application/pdf', 'broken.pdf'), + bytes: brokenBytes, + reason: 'integrity', + }, + { + file: createMockDocFile(0.01, 'text/csv', 'small.csv'), + bytes: Buffer.from('a,b\n1,2'), + }, + ]; + }, + document: expect.objectContaining({ + type: 'document', + document: expect.objectContaining({ format: 'csv', name: 'small_csv' }), + }), + }, + { + name: 'a generic file over the configured limit as capacity', + request: createMockRequest(1, Providers.ANTHROPIC), + provider: Providers.ANTHROPIC, + serve: () => [ + { + file: createMockDocFile(2, 'text/plain', 'large.txt'), + bytes: Buffer.alloc(mbToBytes(2)), + reason: 'capacity', + }, + { + file: createMockDocFile(0.01, 'text/plain', 'notes.txt'), + bytes: Buffer.from('notes'), + }, + ], + document: expect.anything(), + }, + ])( + 'records $name, omitting the failures and keeping the rest', + async ({ request, provider, serve, document }) => { + const rows = serve(); + + const result = await encodeAndFormatDocuments( + request as ServerRequest, + serveRows(rows), + { provider, onValidationFailure: 'skip' }, + mockStrategyFunctions, + ); + + expect(result.rejected).toEqual(rejectedOf(rows)); + expect(result.documents).toEqual([document]); + expect(result.files).toEqual(keptOf(rows)); + }, + ); + + it.each<{ + name: string; + request: Partial; + provider: Providers; + serve: () => ServedFile[]; + reads?: number; + }>([ + { + name: 'every file when the provider has no document path', + request: createMockRequest(15), + provider: Providers.AZURE, + serve: () => [ + { file: createMockFile(1), reason: 'unsupported' }, + { file: createMockDocFile(0.01, 'text/plain', 'notes.txt'), reason: 'unsupported' }, + ], + reads: 0, + }, + { + name: 'a type with no Bedrock document format', + request: createMockRequest(), + provider: Providers.BEDROCK, + serve: () => [ + { + file: createMockDocFile(0.01, 'application/zip', 'archive.zip'), + reason: 'unsupported', + }, + { + file: createMockDocFile(0.01, 'text/csv', 'data.csv'), + bytes: Buffer.from('a,b\n1,2'), + }, + ], + reads: 1, + }, + { + name: 'a type Claude document input cannot take', + request: createMockRequest(30, Providers.ANTHROPIC), + provider: Providers.ANTHROPIC, + serve: () => [ + { file: createMockDocFile(0.01, DOCX, 'report.docx'), reason: 'unsupported' }, + { + file: createMockDocFile(0.01, 'text/markdown', 'readme.md'), + bytes: Buffer.from('# heading'), + }, + ], + reads: 1, + }, + { + name: 'a file no block format exists for', + request: createMockRequest(), + provider: providerWithoutBlockFormat, + serve: () => [ + { + file: createMockDocFile(0.01, 'text/plain', 'notes.txt'), + bytes: Buffer.from('notes'), + reason: 'unsupported', + }, + ], + }, + ])('reports $name as unsupported', async ({ request, provider, serve, reads }) => { + const rows = serve(); + + const result = await encodeAndFormatDocuments( + request as ServerRequest, + serveRows(rows), + { provider, onValidationFailure: 'skip' }, + mockStrategyFunctions, + ); + + expect(result.rejected).toEqual(rejectedOf(rows)); + expect(result.files).toEqual(keptOf(rows)); + expect(result.documents).toHaveLength(keptOf(rows).length); + if (reads != null) { + expect(mockedGetFileStream).toHaveBeenCalledTimes(reads); + } + }); + + it('returns an empty rejected list when every file encodes', async () => { + const req = createMockRequest(15) as ServerRequest; + const file = createMockFile(0.01); + serveContents(new Map([[file.file_id, pdfBytes(1024)]])); + + const result = await encodeAndFormatDocuments( + req, + [file], + { provider: Providers.OPENAI, onValidationFailure: 'skip' }, + mockStrategyFunctions, + ); + + expect(result.rejected).toEqual([]); + expect(result.files).toEqual([file]); + }); + + it('records unreadable, unstored and empty files as integrity and keeps the readable one', async () => { + const files = serveUnreadableSet(); + + const result = await encodeUnreadableSet(files, 'skip'); + + expect(result.rejected).toEqual([ + { file_id: files.unreadable.file_id, reason: 'integrity' }, + { file_id: files.unstored.file_id, reason: 'integrity' }, + { file_id: files.empty.file_id, reason: 'integrity' }, + ]); + expect(result.files).toEqual([files.readable]); + expect(result.documents).toEqual([expect.objectContaining({ type: 'file' })]); + }); + + it('still rethrows a missing storage object', async () => { + const req = createMockRequest(15) as ServerRequest; + const missing = createMockFile(0.01); + const present = createMockFile(0.01); + mockedGetFileStream.mockImplementation(async (_req, file) => { + if (file.file_id === missing.file_id) { + throw new AttachmentObjectNotFoundError(missing.file_id); + } + return { file, content: pdfBytes(1024).toString('base64'), metadata: file }; + }); + + await expect( + encodeAndFormatDocuments( + req, + [missing, present], + { provider: Providers.OPENAI, onValidationFailure: 'skip' }, + mockStrategyFunctions, + ), + ).rejects.toMatchObject({ code: 'ATTACHMENT_OBJECT_NOT_FOUND', fileId: missing.file_id }); + }); + }); + + describe('throw', () => { + it.each<{ + name: string; + request: Partial; + provider: Providers; + onValidationFailure?: NativeValidationMode; + serve: () => ServedFile; + message: string; + }>([ + { + name: 'PDF capacity error when the mode is omitted', + request: createMockRequest(1), + provider: Providers.OPENAI, + serve: () => ({ file: createMockFile(2), bytes: pdfBytes(mbToBytes(2)) }), + message: 'PDF validation failed: PDF file size (2MB) exceeds the 1MB limit', + }, + { + name: 'PDF capacity error when the mode is throw', + request: createMockRequest(1), + provider: Providers.OPENAI, + onValidationFailure: 'throw', + serve: () => ({ file: createMockFile(2), bytes: pdfBytes(mbToBytes(2)) }), + message: 'PDF validation failed: PDF file size (2MB) exceeds the 1MB limit', + }, + { + name: 'Bedrock capacity error', + request: createMockRequest(), + provider: Providers.BEDROCK, + serve: () => ({ + file: createMockDocFile(5, 'text/csv', 'large.csv'), + bytes: Buffer.alloc(mbToBytes(5)), + }), + message: + 'Document validation failed: File size (5.0MB) exceeds the 4.5MB limit for Bedrock', + }, + { + name: 'generic capacity error', + request: createMockRequest(1, Providers.ANTHROPIC), + provider: Providers.ANTHROPIC, + serve: () => ({ + file: createMockDocFile(2, 'text/plain', 'large.txt'), + bytes: Buffer.alloc(mbToBytes(2)), + }), + message: 'File size (~2.0MB) exceeds the configured limit for anthropic', + }, + ])( + 'throws the unchanged $name', + async ({ request, provider, onValidationFailure, serve, message }) => { + await expect( + encodeAndFormatDocuments( + request as ServerRequest, + serveRows([serve()]), + { provider, onValidationFailure }, + mockStrategyFunctions, + ), + ).rejects.toThrow(message); + }, + ); + + it('logs and leaves out unreadable and unstored files and keeps an empty one in files', async () => { + const files = serveUnreadableSet(); + + const result = await encodeUnreadableSet(files); + + expect(result).not.toHaveProperty('rejected'); + expect(result.files).toEqual([files.empty, files.readable]); + expect(result.documents).toEqual([expect.objectContaining({ type: 'file' })]); + }); + + it('leaves rejected off the result, leaving out files the provider cannot take', async () => { + const req = createMockRequest(30, Providers.ANTHROPIC) as ServerRequest; + const notes = createMockDocFile(0.01, 'text/plain', 'notes.txt'); + const docx = createMockDocFile(0.01, DOCX, 'report.docx'); + const readme = createMockDocFile(0.01, 'text/markdown', 'readme.md'); + serveContents( + new Map([ + [notes.file_id, Buffer.from('notes')], + [readme.file_id, Buffer.from('# heading')], + ]), + ); + + await expect( + encodeAndFormatDocuments( + createMockRequest() as ServerRequest, + [notes], + { provider: providerWithoutBlockFormat }, + mockStrategyFunctions, + ), + ).resolves.toEqual({ documents: [], files: [] }); + const anthropicResult = await encodeAndFormatDocuments( + req, + [docx, readme], + { provider: Providers.ANTHROPIC }, + mockStrategyFunctions, + ); + expect(anthropicResult).not.toHaveProperty('rejected'); + expect(anthropicResult.files).toEqual([readme]); + await expect( + encodeAndFormatDocuments( + req, + [readme], + { provider: Providers.AZURE }, + mockStrategyFunctions, + ), + ).resolves.toEqual({ documents: [], files: [] }); + }); + }); + }); }); diff --git a/packages/api/src/files/encode/document.ts b/packages/api/src/files/encode/document.ts index 57ef72db9a3..01a1303a9e3 100644 --- a/packages/api/src/files/encode/document.ts +++ b/packages/api/src/files/encode/document.ts @@ -9,18 +9,26 @@ import { } from 'librechat-data-provider'; import type { IMongoFile } from '@librechat/data-schemas'; import type { - DocumentBlock, + ValidationFailureReason, AnthropicDocumentBlock, + NativeValidationMode, + DocumentRejection, StrategyFunctions, DocumentResult, + ProcessedFile, + DocumentBlock, ServerRequest, } from '~/types'; +import { + validatePdf, + validateBedrockDocument, + getNativeDocumentSizeLimit, +} from '~/files/validation'; import { getFileStream, getConfiguredFileSizeLimit, isAttachmentObjectNotFoundError, } from './utils'; -import { validatePdf, validateBedrockDocument } from '~/files/validation'; import { runGuardedEncode } from './memoryGuard'; /** Anthropic only accepts PDFs as base64 documents; textual types must use a text source */ @@ -51,7 +59,7 @@ function getAnthropicDocumentSource( * Whether the model behind this provider is Claude, which accepts only PDFs as base64 * documents. OpenAI-compatible gateways report an OpenAI-like provider for Claude models. */ -function usesAnthropicDocumentCapabilities(provider: Providers, model?: string): boolean { +export function usesAnthropicDocumentCapabilities(provider: Providers, model?: string): boolean { return ( provider === Providers.ANTHROPIC || (isOpenAILikeProvider(provider) && (model?.toLowerCase().includes('claude') ?? false)) @@ -133,8 +141,29 @@ function formatDocumentBlock( return null; } +interface ProviderDocumentFiles { + processable: IMongoFile[]; + unsupported: IMongoFile[]; +} + +function partitionDocumentFiles( + files: IMongoFile[], + isSupported: (mimeType?: string) => boolean, +): ProviderDocumentFiles { + const processable: IMongoFile[] = []; + const unsupported: IMongoFile[] = []; + for (const file of files) { + if (isSupported(file.type)) { + processable.push(file); + } else { + unsupported.push(file); + } + } + return { processable, unsupported }; +} + /** - * Filters out files the provider's document path cannot send to the model. + * Separates out files the provider's document path cannot send to the model. * Claude rejects non-PDF binary documents with a 400 that recurs on every retry, * including when it is reached through an OpenAI-compatible gateway. Unsupported * types are skipped instead of bricking the conversation. @@ -143,32 +172,33 @@ function filterProviderDocumentFiles( provider: Providers, files: IMongoFile[], model?: string, -): IMongoFile[] { +): ProviderDocumentFiles { if (provider === Providers.BEDROCK) { - return files.filter((file) => isBedrockDocumentType(file.type)); + return partitionDocumentFiles(files, isBedrockDocumentType); } if (!usesAnthropicDocumentCapabilities(provider, model)) { - return files; - } - - const processable: IMongoFile[] = []; - const skipped: string[] = []; - for (const file of files) { - if (isAnthropicDocumentType(file.type)) { - processable.push(file); - } else { - skipped.push(`"${file.filename}" (${file.type})`); - } + return { processable: files, unsupported: [] }; } - if (skipped.length) { + const partition = partitionDocumentFiles(files, isAnthropicDocumentType); + if (partition.unsupported.length) { + const skipped = partition.unsupported.map((file) => `"${file.filename}" (${file.type})`); console.warn( `Skipping attachment(s) unsupported by Claude document input: ${skipped.join(', ')}`, ); } - return processable; + return partition; +} + +/** Records files left out in skip mode; a no-op in throw mode, where `rejected` is absent. */ +function recordRejections( + result: DocumentResult, + files: IMongoFile[], + reason: DocumentRejection['reason'], +): void { + result.rejected?.push(...files.map((file) => ({ file_id: file.file_id, reason }))); } function getBase64DecodedByteCount(content: string): number { @@ -194,29 +224,73 @@ function getBase64DecodedByteCount(content: string): number { * all others are skipped. * - **PDF**: Validated via `validatePdf` before encoding. * - **Generic types**: Encoded with a provider-specific size check. + * + * A validation failure throws by default. With `onValidationFailure: 'skip'` every file + * that yields no block is listed in `rejected` with its reason instead: a failed + * validation, a type the provider cannot take, or bytes that could not be read. A missing + * storage object still throws. */ export async function encodeAndFormatDocuments( req: ServerRequest, files: IMongoFile[], - params: { provider: Providers; endpoint?: string; useResponsesApi?: boolean; model?: string }, + params: { + provider: Providers; + endpoint?: string; + useResponsesApi?: boolean; + model?: string; + onValidationFailure?: NativeValidationMode; + }, getStrategyFunctions: (source: string) => StrategyFunctions, ): Promise { - const { provider, endpoint, useResponsesApi, model } = params; + const { provider, endpoint, useResponsesApi, model, onValidationFailure = 'throw' } = params; + const skipInvalid = onValidationFailure === 'skip'; + const result: DocumentResult = skipInvalid + ? { documents: [], files: [], rejected: [] } + : { documents: [], files: [] }; if (!files?.length) { - return { documents: [], files: [] }; + return result; } const encodingMethods: Record = {}; - const result: DocumentResult = { documents: [], files: [] }; + + const rejectInvalid = ( + file: IMongoFile, + reason: ValidationFailureReason, + message: string, + ): void => { + if (!skipInvalid) { + throw new Error(message); + } + recordRejections(result, [file], reason); + }; + + const addBlock = ( + file: IMongoFile, + block: DocumentBlock | null, + metadata: ProcessedFile['metadata'], + ): void => { + if (!block) { + recordRejections(result, [file], 'unsupported'); + return; + } + result.documents.push(block); + result.files.push(metadata); + }; const isBedrock = provider === Providers.BEDROCK; const isDocSupported = isDocumentSupportedProvider(provider); if (!isDocSupported && !isBedrock) { + recordRejections(result, files, 'unsupported'); return result; } - const processableFiles = filterProviderDocumentFiles(provider, files, model); + const { processable: processableFiles, unsupported } = filterProviderDocumentFiles( + provider, + files, + model, + ); + recordRejections(result, unsupported, 'unsupported'); if (!processableFiles.length) { return result; @@ -232,22 +306,29 @@ export async function encodeAndFormatDocuments( ), ); - for (const settledResult of results) { + for (let i = 0; i < results.length; i++) { + const settledResult = results[i]; + const sourceFile = processableFiles[i]; if (settledResult.status === 'rejected') { if (isAttachmentObjectNotFoundError(settledResult.reason)) { throw settledResult.reason; } console.error('Document processing failed:', settledResult.reason); + recordRejections(result, [sourceFile], 'integrity'); continue; } const processed = settledResult.value; - if (!processed) continue; + if (!processed) { + recordRejections(result, [sourceFile], 'integrity'); + continue; + } const { file, content, metadata } = processed; if (!content || !file) { - if (metadata) result.files.push(metadata); + if (metadata && !skipInvalid) result.files.push(metadata); + recordRejections(result, [sourceFile], 'integrity'); continue; } @@ -266,7 +347,12 @@ export async function encodeAndFormatDocuments( ); if (!validation.isValid) { - throw new Error(`Document validation failed: ${validation.error}`); + rejectInvalid( + file, + validation.reason ?? 'integrity', + `Document validation failed: ${validation.error}`, + ); + continue; } const sanitizedName = (file.filename || 'document') @@ -295,41 +381,41 @@ export async function encodeAndFormatDocuments( ); if (!validation.isValid) { - throw new Error(`PDF validation failed: ${validation.error}`); + rejectInvalid( + file, + validation.reason ?? 'integrity', + `PDF validation failed: ${validation.error}`, + ); + continue; } - const block = formatDocumentBlock( - provider, - mimeType, - content, - file.filename, - useResponsesApi, - model, + addBlock( + file, + formatDocumentBlock(provider, mimeType, content, file.filename, useResponsesApi, model), + metadata, ); - if (block) { - result.documents.push(block); - result.files.push(metadata); - } } else if (isDocSupported && !isBedrock) { const decodedByteCount = getBase64DecodedByteCount(content); - if (configuredFileSizeLimit && decodedByteCount > configuredFileSizeLimit) { - throw new Error( + const sizeLimit = getNativeDocumentSizeLimit({ + provider, + mimeType, + model, + configuredFileSizeLimit, + }); + if (sizeLimit !== undefined && decodedByteCount > sizeLimit) { + rejectInvalid( + file, + 'capacity', `File size (~${(decodedByteCount / 1024 / 1024).toFixed(1)}MB) exceeds the configured limit for ${provider}`, ); + continue; } - const block = formatDocumentBlock( - provider, - mimeType, - content, - file.filename, - useResponsesApi, - model, + addBlock( + file, + formatDocumentBlock(provider, mimeType, content, file.filename, useResponsesApi, model), + metadata, ); - if (block) { - result.documents.push(block); - result.files.push(metadata); - } } } diff --git a/packages/api/src/files/encode/utils.spec.ts b/packages/api/src/files/encode/utils.spec.ts index d4882d952dd..7ea74ece557 100644 --- a/packages/api/src/files/encode/utils.spec.ts +++ b/packages/api/src/files/encode/utils.spec.ts @@ -1,12 +1,16 @@ import { Readable } from 'node:stream'; import { Providers } from '@librechat/agents'; -import { audioMimeTypes } from 'librechat-data-provider'; +import { mbToBytes, audioMimeTypes, fileConfig as baseFileConfig } from 'librechat-data-provider'; +import type { TFileConfig } from 'librechat-data-provider'; +import type { FileSizeLimitParams } from './utils'; import type { ServerRequest } from '~/types'; import { - AttachmentObjectNotFoundError, getAudioFormat, getFileStream, + getConfiguredFileSizeLimit, + AttachmentObjectNotFoundError, isConfiguredProviderMediaType, + resolveConfiguredFileSizeLimit, } from './utils'; const file = { @@ -198,3 +202,66 @@ describe('getAudioFormat', () => { } }); }); + +describe('resolveConfiguredFileSizeLimit', () => { + const inheritedLimit = baseFileConfig.endpoints.default.fileSizeLimit; + + /** `getConfiguredFileSizeLimit` and its resolver agree for every fileConfig shape. */ + const parityCases: Array< + [string, TFileConfig | undefined, FileSizeLimitParams, number | undefined] + > = [ + ['no fileConfig', undefined, { provider: Providers.OPENAI }, undefined], + [ + 'an empty fileConfig, which inherits the 512 MB default', + {}, + { provider: Providers.ANTHROPIC }, + mbToBytes(512), + ], + [ + 'an explicit provider limit', + { endpoints: { [Providers.GOOGLE]: { fileSizeLimit: 25 } } }, + { provider: Providers.GOOGLE }, + mbToBytes(25), + ], + [ + 'an endpoint name shadowing the provider', + { + endpoints: { [Providers.OPENAI]: { fileSizeLimit: 15 }, MyGateway: { fileSizeLimit: 3 } }, + }, + { provider: Providers.OPENAI, endpoint: 'MyGateway' }, + mbToBytes(3), + ], + [ + 'an endpoint name with no entry, which does not fall back to the provider entry', + { endpoints: { [Providers.OPENAI]: { fileSizeLimit: 15 } } }, + { provider: Providers.OPENAI, endpoint: 'Unlisted' }, + inheritedLimit, + ], + [ + 'a configured default', + { endpoints: { default: { fileSizeLimit: 7 } } }, + { provider: Providers.BEDROCK }, + mbToBytes(7), + ], + [ + 'a configured default for a provider with a built-in entry', + { endpoints: { default: { fileSizeLimit: 7 } } }, + { provider: Providers.ANTHROPIC }, + inheritedLimit, + ], + [ + 'a configured 0', + { endpoints: { [Providers.OPENAI]: { fileSizeLimit: 0 } } }, + { provider: Providers.OPENAI }, + 0, + ], + ]; + + it.each(parityCases)( + 'resolves %s like getConfiguredFileSizeLimit', + (_label, config, params, expected) => { + expect(resolveConfiguredFileSizeLimit(config, params)).toBe(expected); + expect(getConfiguredFileSizeLimit(reqWith(config), params)).toBe(expected); + }, + ); +}); diff --git a/packages/api/src/files/encode/utils.ts b/packages/api/src/files/encode/utils.ts index 9c39ce2c13c..0e261731f6e 100644 --- a/packages/api/src/files/encode/utils.ts +++ b/packages/api/src/files/encode/utils.ts @@ -6,6 +6,7 @@ import { isExplicitMimeConfig, getEndpointFileConfig, } from 'librechat-data-provider'; +import type { TFileConfig } from 'librechat-data-provider'; import type { ServerRequest, StrategyFunctions, ProcessedFile } from '~/types'; import type { StoredFileRef } from '~/storage/path'; import { resolveDownloadPath } from '~/storage/path'; @@ -48,33 +49,47 @@ function isStorageNotFoundError(error: unknown): boolean { ); } +export interface FileSizeLimitParams { + provider: Providers; + endpoint?: string; +} + /** - * Extracts the configured file size limit for a specific provider from fileConfig - * @param req - The server request object containing config + * Resolves the configured file size limit for a provider or endpoint from a fileConfig. + * Any fileConfig, even an empty one, yields the merged endpoint limit, inherited default included. + * @param fileConfig - The app's fileConfig, if any * @param params - Object containing provider and optional endpoint * @param params.provider - The provider to get the limit for * @param params.endpoint - Optional endpoint name for lookup - * @returns The configured file size limit in bytes, or undefined if not configured + * @returns The configured file size limit in bytes, or undefined without a fileConfig */ -export const getConfiguredFileSizeLimit = ( - req: ServerRequest, - params: { - provider: Providers; - endpoint?: string; - }, +export const resolveConfiguredFileSizeLimit = ( + fileConfig: TFileConfig | undefined, + { provider, endpoint }: FileSizeLimitParams, ): number | undefined => { - if (!req.config?.fileConfig) { + if (!fileConfig) { return undefined; } - const { provider, endpoint } = params; - const fileConfig = mergeFileConfig(req.config.fileConfig); const endpointConfig = getEndpointFileConfig({ - fileConfig, + fileConfig: mergeFileConfig(fileConfig), endpoint: endpoint ?? provider, }); return endpointConfig?.fileSizeLimit; }; +/** + * Extracts the configured file size limit for a specific provider from fileConfig + * @param req - The server request object containing config + * @param params - Object containing provider and optional endpoint + * @param params.provider - The provider to get the limit for + * @param params.endpoint - Optional endpoint name for lookup + * @returns The configured file size limit in bytes, or undefined if not configured + */ +export const getConfiguredFileSizeLimit = ( + req: ServerRequest, + params: FileSizeLimitParams, +): number | undefined => resolveConfiguredFileSizeLimit(req.config?.fileConfig, params); + /** * Whether the admin explicitly allowed `mimeType` for this endpoint via * `fileConfig.endpoints..supportedMimeTypes`. The inherited default list does diff --git a/packages/api/src/files/filter.spec.ts b/packages/api/src/files/filter.spec.ts index f64a6017192..98dfb852374 100644 --- a/packages/api/src/files/filter.spec.ts +++ b/packages/api/src/files/filter.spec.ts @@ -1,6 +1,7 @@ import { Types } from 'mongoose'; import { Providers } from '@librechat/agents'; -import { EModelEndpoint, FileSources } from 'librechat-data-provider'; +import { EModelEndpoint, FileContext, FileSources } from 'librechat-data-provider'; +import type { TFileConfig, TurnDeliveryFile, TurnFileConsumers } from 'librechat-data-provider'; import type { IMongoFile } from '@librechat/data-schemas'; import type { AppConfig } from '@librechat/data-schemas'; import type { ServerRequest } from '~/types'; @@ -1445,3 +1446,152 @@ describe('filterFilesByEndpointConfig', () => { }); }); }); + +describe('filterFilesByEndpointRuntimeConfig under the automatic reading policy', () => { + const MB = 1024 * 1024; + + type ReadingRecord = TurnDeliveryFile & { + file_id: string; + filename: string; + type: string; + bytes: number; + }; + type EndpointConfigInput = NonNullable[string]; + + const record = (overrides: Partial = {}): ReadingRecord => ({ + file_id: 'eligible', + filename: 'report.pdf', + type: 'application/pdf', + bytes: 3 * MB, + source: FileSources.local, + context: FileContext.message_attachment, + llmDeliveryPath: 'provider', + metadata: { destinationChosen: false }, + ...overrides, + }); + + const automatic = (endpoint: EndpointConfigInput = {}): TFileConfig => ({ + endpoints: { + [EModelEndpoint.openAI]: { fileSizeLimit: 1, llmDeliveryPolicy: 'automatic', ...endpoint }, + }, + }); + + const runsCode: TurnFileConsumers = { executeCode: true, fileSearch: false }; + const searches: TurnFileConsumers = { executeCode: false, fileSearch: true }; + const noReader: TurnFileConsumers = { executeCode: false, fileSearch: false }; + + /** A file tool is loaded unless a test says otherwise: only then does the policy read at all. */ + const filter = ( + fileConfig: TFileConfig, + files: ReadingRecord[], + consumers: TurnFileConsumers | null | undefined = runsCode, + ): ReadingRecord[] => + filterFilesByEndpointRuntimeConfig({ config: {}, fileConfig } as AppConfig, { + files, + endpoint: EModelEndpoint.openAI, + consumers, + }); + + it('keeps an oversized eligible record, since its reading judges that size as capacity', () => { + const eligible = record(); + expect(filter(automatic(), [eligible])).toEqual([eligible]); + expect(filter(automatic(), [eligible], searches)).toEqual([eligible]); + }); + + it.each<[string, { consumers?: TurnFileConsumers | null }]>([ + ['no file tool is loaded', { consumers: noReader }], + ['the consumers are null', { consumers: null }], + ['the caller names no consumers', {}], + ])('drops the oversized eligible record as classic routing does when %s', (_case, consumers) => { + const classic = (files: ReadingRecord[]) => + filterFilesByEndpointRuntimeConfig({ config: {}, fileConfig: automatic() } as AppConfig, { + files, + endpoint: EModelEndpoint.openAI, + ...consumers, + }); + const withinLimit = record({ bytes: MB / 2 }); + expect(classic([record()])).toEqual([]); + expect(classic([withinLimit])).toEqual([withinLimit]); + }); + + it('keeps an eligible record whatever route it stored', () => { + const text = record({ file_id: 'text', llmDeliveryPath: 'text', text: 'extracted' }); + const none = record({ file_id: 'none', llmDeliveryPath: 'none' }); + expect(filter(automatic(), [text, none])).toEqual([text, none]); + }); + + it('applies a policy the endpoint inherits from the top-level file config', () => { + const eligible = record(); + const fileConfig: TFileConfig = { + llmDeliveryPolicy: 'automatic', + endpoints: { [EModelEndpoint.openAI]: { fileSizeLimit: 1 } }, + }; + expect(filter(fileConfig, [eligible])).toEqual([eligible]); + }); + + it.each<[string, { fileConfig?: TFileConfig; file?: Partial }]>([ + ['an image', { file: { type: 'image/png', filename: 'photo.png' } }], + ['a persistent agent file', { file: { context: FileContext.agents } }], + ['an explicit destination', { file: { metadata: { destinationChosen: true } } }], + ['an unmarked record', { file: { metadata: {} } }], + ['a legacy record', { file: { llmDeliveryPath: undefined } }], + ['a text-only record', { file: { source: FileSources.text, text: 'pasted' } }], + [ + 'a record on a classic endpoint', + { fileConfig: { endpoints: { [EModelEndpoint.openAI]: { fileSizeLimit: 1 } } } }, + ], + [ + 'a record on an endpoint set to classic', + { fileConfig: automatic({ llmDeliveryPolicy: 'classic' }) }, + ], + ['a record behind the legacy chooser', { fileConfig: automatic({ legacyFileUploadUX: true }) }], + [ + 'a type an endpoint route names', + { + fileConfig: automatic({ + defaultLLMDeliveryPath: { overrides: { 'application/pdf': 'provider' } }, + }), + }, + ], + [ + 'a type a global route names', + { fileConfig: { ...automatic(), defaultLLMDeliveryPath: { fallback: 'text' } } }, + ], + ])('still drops %s over the size limit', (_label, { fileConfig = automatic(), file = {} }) => { + const withinLimit = record({ ...file, bytes: MB / 2 }); + expect(filter(fileConfig, [record(file)])).toEqual([]); + expect(filter(fileConfig, [withinLimit])).toEqual([withinLimit]); + }); + + it('keeps the MIME allowlist, the disabled switch and the total size limit binding', () => { + expect(filter(automatic({ supportedMimeTypes: ['^text/plain$'] }), [record()])).toEqual([]); + expect(filter(automatic({ disabled: true }), [record()])).toEqual([]); + expect(filter(automatic({ totalSizeLimit: 2 }), [record()])).toEqual([]); + }); + + it('holds provider copies to the size limit when asked to bind them', () => { + const provider = record(); + const text = record({ file_id: 'text', llmDeliveryPath: 'text', text: 'extracted' }); + const bound = filterFilesByEndpointRuntimeConfig( + { config: {}, fileConfig: automatic() } as AppConfig, + { + files: [provider, text], + endpoint: EModelEndpoint.openAI, + bindProviderCopies: true, + consumers: runsCode, + }, + ); + expect(bound).toEqual([text]); + }); + + it('exempts only the eligible records in a mixed set', () => { + const eligible = record(); + const explicit = record({ file_id: 'explicit', metadata: { destinationChosen: true } }); + const small = record({ + file_id: 'small', + bytes: MB / 2, + metadata: { destinationChosen: true }, + }); + expect(filter(automatic(), [eligible, explicit, small])).toEqual([eligible, small]); + }); +}); diff --git a/packages/api/src/files/filter.ts b/packages/api/src/files/filter.ts index f6881ac1349..c85d0c2ef14 100644 --- a/packages/api/src/files/filter.ts +++ b/packages/api/src/files/filter.ts @@ -1,11 +1,18 @@ import { + fileConfig, FileSources, - getEndpointFileConfig, mergeFileConfig, - fileConfig, + getEndpointFileConfig, + hasFileToolConsumer, + isAutomaticReadingRecord, +} from 'librechat-data-provider'; +import type { + RegexLike, + TFile, + TurnDeliveryFile, + TurnFileConsumers, } from 'librechat-data-provider'; import type { AppConfig, IMongoFile } from '@librechat/data-schemas'; -import type { RegexLike, TFile } from 'librechat-data-provider'; import type { ServerRequest } from '~/types'; /** @@ -65,10 +72,13 @@ export function isLegacyFileUploadUX( return endpointFileConfig?.legacyFileUploadUX === true; } -type EndpointPolicyFile = Pick & { - source?: string; - metadata?: { routingMimeType?: string }; -}; +type EndpointPolicyFile = Pick & + Pick & { + metadata?: Pick< + NonNullable, + 'routingMimeType' | 'destinationChosen' | 'textDerivation' + > | null; + }; export function filterFilesByEndpointRuntimeConfig( appConfig: AppConfig | undefined, @@ -81,6 +91,13 @@ export function filterFilesByEndpointRuntimeConfig consumedBytes?: number; skipTotalSizeLimit?: boolean; preserveTextSources?: boolean; + /** Hold copies already routed to the provider to the size limit even when the automatic + * policy reads them: another agent's decision put their bytes on this endpoint's path. */ + bindProviderCopies?: boolean; + /** The turn's loaded file tools. The size limit is native capacity rather than admission + * only while one is loaded; otherwise the automatic policy routes nothing, so nothing + * else could take the file. */ + consumers?: TurnFileConsumers | null; }, ): T[] { const { @@ -90,6 +107,8 @@ export function filterFilesByEndpointRuntimeConfig consumedBytes = 0, skipTotalSizeLimit = false, preserveTextSources = false, + bindProviderCopies = false, + consumers, } = params; if (!files || files.length === 0) { @@ -116,11 +135,19 @@ export function filterFilesByEndpointRuntimeConfig /** Filter files based on individual file size and MIME type */ let filteredFiles = files; - /** Filter by individual file size limit */ + /** Filter by individual file size limit. For a record the automatic policy reads on a turn + * with a file tool, the limit is native-delivery capacity its reading decision already + * applies, not admission. */ if (fileSizeLimit !== undefined && fileSizeLimit > 0) { - filteredFiles = filteredFiles.filter((file) => { - return file.bytes <= fileSizeLimit; - }); + const readingConfig = { endpointConfig: endpointFileConfig, fileConfig: mergedFileConfig }; + const toolLoaded = hasFileToolConsumer(consumers); + const isCapacityOnly = (file: T): boolean => + toolLoaded && + !(bindProviderCopies && file.llmDeliveryPath === 'provider') && + isAutomaticReadingRecord(readingConfig, file); + filteredFiles = filteredFiles.filter( + (file) => file.bytes <= fileSizeLimit || isCapacityOnly(file), + ); } /** Filter by MIME type, against the type the upload was accepted as. Conversion rewrites diff --git a/packages/api/src/files/index.ts b/packages/api/src/files/index.ts index 66226ed4b4c..c66744f7d6a 100644 --- a/packages/api/src/files/index.ts +++ b/packages/api/src/files/index.ts @@ -14,6 +14,7 @@ export * from './ocr'; export * from './parse'; export * from './preflight'; export * from './provision'; +export * from './reading'; export * from './upload/routing'; export * from './upload/diagnostics'; export * from './upload/fallback'; diff --git a/packages/api/src/files/provision/callback.spec.ts b/packages/api/src/files/provision/callback.spec.ts index 28846bdb928..22479d03e70 100644 --- a/packages/api/src/files/provision/callback.spec.ts +++ b/packages/api/src/files/provision/callback.spec.ts @@ -1,14 +1,20 @@ import { Constants } from '@librechat/agents'; +import { logger } from '@librechat/data-schemas'; import { EToolResources, FileContext } from 'librechat-data-provider'; import type { TFile } from 'librechat-data-provider'; +import type { SearchPreparationState } from '../reading/turn'; import type { ProvisionState } from '~/agents/resources'; +import type { ReadingAgent } from '../reading/inventory'; import type { ProvisionToolContext } from './callback'; import type { CodeFileAgent } from '../code/queued'; import type { ServerRequest } from '~/types'; +import { collectInventoryEntries, prepareAgentFileContext } from '../reading/inventory'; +import { FileSearchPreparationError, createProvisionFilesCallback } from './callback'; +import { buildTurnReadingContext, getTurnReadingContext } from '../reading/turn'; import { mergeCodeFilesIntoContext } from '~/agents/codeFilesSession'; +import { resolveTurnDeliveryRouting } from '~/agents/files/delivery'; import { prepareQueuedCodeFileContext } from '../code/queued'; import { createSubagentCodeRouting } from '~/code/targets'; -import { createProvisionFilesCallback } from './callback'; jest.mock('@librechat/data-schemas', () => ({ logger: { error: jest.fn(), warn: jest.fn(), debug: jest.fn(), info: jest.fn() }, @@ -45,6 +51,49 @@ function state( }; } +function searchAgent( + files: TFile[], + id = 'agent-a', + scopedFileIds: string[] = [], +): CodeFileAgent & ReadingAgent { + const routing = resolveTurnDeliveryRouting({ + agent: { provider: 'openAI', endpoint: 'openAI' }, + config: { fileConfig: { endpoints: { openAI: { llmDeliveryPolicy: 'automatic' } } } }, + }); + const preparation = new Map(); + const provisionState = { ...state([], files, scopedFileIds), searchPreparation: preparation }; + const context = buildTurnReadingContext({ + routing, + provider: 'openAI', + fileTokenLimit: 1000, + configuredFileSizeLimit: 1024 * 1024, + countTokens: (text) => text.length, + }); + context?.setSearchEvidence({ + queued: files.map(({ file_id }) => file_id), + registered: [], + preparation, + }); + routing.reading = context; + return { + id, + deliveryRouting: routing, + provisionState, + fileConsumers: { executeCode: false, fileSearch: true }, + currentRequestAttachments: files, + }; +} + +const searchFile = (overrides: Partial = {}): TFile => + makeFile({ + filename: 'manual.pdf', + type: 'application/pdf', + bytes: 3_000_000, + llmDeliveryPath: 'none', + metadata: { destinationChosen: false }, + ...overrides, + }); + function buildHarness({ contexts, codeImpl, @@ -876,18 +925,24 @@ describe('createProvisionFilesCallback', () => { it('preserves cancellation raised during vector provisioning', async () => { const controller = new AbortController(); + const pending = searchFile(); + const agent = searchAgent([pending]); const vectorImpl = jest.fn(async () => { controller.abort(); controller.signal.throwIfAborted(); }); const { provisionFiles } = buildHarness({ - contexts: [['agent-a', { provisionState: state([], [makeFile()]) }]], + contexts: [[agent.id, agent]], vectorImpl, }); await expect( provisionFiles(['file_search'], 'agent-a', controller.signal), ).rejects.toMatchObject({ name: 'AbortError' }); + expect(agent.provisionState?.vectorDBFiles).toEqual([pending]); + expect(getTurnReadingContext(agent.deliveryRouting)?.searchState(pending.file_id)).toBe( + 'failed', + ); }); it('preserves a homogeneous Code API rate-limit failure from lazy provisioning', async () => { @@ -954,6 +1009,29 @@ describe('createProvisionFilesCallback', () => { }); }); + it('embeds a pending search attachment once, leaving nothing for the next batch', async () => { + const pending = makeFile({ + file_id: 'manual-pdf', + filename: 'manual.pdf', + filepath: '/uploads/manual.pdf', + type: 'application/pdf', + bytes: 3_000_000, + llmDeliveryPath: 'none', + metadata: { destinationChosen: false }, + }); + const { provisionFiles, provisionToVectorDB, agentToolContexts } = buildHarness({ + contexts: [['agent-a', { provisionState: state([], [pending]) }]], + }); + + await provisionFiles(['file_search'], 'agent-a'); + + expect(provisionToVectorDB).toHaveBeenCalledWith(expect.objectContaining({ file: pending })); + expect(pending.embedded).toBe(true); + expect(agentToolContexts.get('agent-a')?.provisionState?.vectorDBFiles).toEqual([]); + await provisionFiles(['file_search'], 'agent-a'); + expect(provisionToVectorDB).toHaveBeenCalledTimes(1); + }); + it('retries a failed upload on a later tool call instead of replaying the rejection', async () => { let attempts = 0; const codeImpl = jest.fn(async ({ file }: { file: TFile }) => { @@ -1053,7 +1131,7 @@ describe('createProvisionFilesCallback', () => { }); await expect(provisionFiles(['file_search'], 'agent-a')).rejects.toThrow( - /aborting tool execution rather than searching without them/, + FileSearchPreparationError, ); expect(agentToolContexts.get('agent-a')?.provisionState?.vectorDBFiles).toHaveLength(1); }); @@ -1068,11 +1146,150 @@ describe('createProvisionFilesCallback', () => { }); await expect(provisionFiles(['file_search'], 'agent-a')).rejects.toThrow( - /aborting tool execution rather than searching without them/, + FileSearchPreparationError, ); expect(agentToolContexts.get('agent-a')?.provisionState?.vectorDBFiles).toHaveLength(1); }); + it('retains failed search readiness until a declined embedding succeeds on retry', async () => { + const pending = searchFile({ text: 'complete cached text that was not sent' }); + const agent = searchAgent([pending]); + const context = getTurnReadingContext(agent.deliveryRouting); + let release: () => void = () => undefined; + const retry = new Promise((resolve) => { + release = resolve; + }); + const vectorImpl = jest + .fn() + .mockResolvedValueOnce({ embedded: false, fileUpdate: null }) + .mockImplementationOnce(async () => { + await retry; + return { embedded: true, fileUpdate: { file_id: pending.file_id, embedded: true } }; + }); + const { provisionFiles } = buildHarness({ contexts: [[agent.id, agent]], vectorImpl }); + const query = jest.fn(); + + await expect(provisionFiles(['file_search'], agent.id).then(query)).rejects.toMatchObject({ + code: 'file_search_preparation_failed', + }); + expect(query).not.toHaveBeenCalled(); + expect(agent.provisionState?.vectorDBFiles).toEqual([pending]); + expect(context?.searchState(pending.file_id)).toBe('failed'); + + const secondAttempt = provisionFiles(['file_search'], agent.id).then(query); + await new Promise((resolve) => setImmediate(resolve)); + expect(vectorImpl).toHaveBeenCalledTimes(2); + expect(context?.searchState(pending.file_id)).toBe('failed'); + expect(query).not.toHaveBeenCalled(); + release(); + await secondAttempt; + expect(query).toHaveBeenCalledTimes(1); + expect(agent.provisionState?.vectorDBFiles).toEqual([]); + expect(context?.searchState(pending.file_id)).toBe('ready'); + }); + + it('keeps partial indexing successes and retries only failed files', async () => { + const first = searchFile({ file_id: 'first', filename: 'first.pdf' }); + const second = searchFile({ file_id: 'second', filename: 'second.pdf' }); + const agent = searchAgent([first, second]); + let failed = false; + const vectorImpl = jest.fn(async ({ file }: { file: TFile }) => { + if (file.file_id === second.file_id && !failed) { + failed = true; + throw new Error('temporary indexing outage'); + } + return { embedded: true, fileUpdate: { file_id: file.file_id, embedded: true } }; + }); + const { provisionFiles } = buildHarness({ contexts: [[agent.id, agent]], vectorImpl }); + await expect(provisionFiles(['file_search'], agent.id)).rejects.toThrow( + FileSearchPreparationError, + ); + expect(agent.provisionState?.vectorDBFiles).toEqual([second]); + expect(getTurnReadingContext(agent.deliveryRouting)?.searchState(first.file_id)).toBe('ready'); + expect(getTurnReadingContext(agent.deliveryRouting)?.searchState(second.file_id)).toBe( + 'failed', + ); + + await provisionFiles(['file_search'], agent.id); + expect( + vectorImpl.mock.calls.filter(([args]) => args.file.file_id === first.file_id), + ).toHaveLength(1); + expect( + vectorImpl.mock.calls.filter(([args]) => args.file.file_id === second.file_id), + ).toHaveLength(2); + expect(agent.provisionState?.vectorDBFiles).toEqual([]); + expect(agent.tool_resources?.file_search?.files?.map((file) => file?.file_id)).toEqual([ + first.file_id, + second.file_id, + ]); + }); + + it('keeps failed readiness scoped to the namespace that failed', async () => { + const first = searchFile(); + const second = searchFile(); + const agentA = searchAgent([first], 'agent-a', [first.file_id]); + const agentB = searchAgent([second], 'agent-b', [second.file_id]); + const vectorImpl = jest.fn(async ({ file, entity_id }: { file: TFile; entity_id: string }) => { + if (entity_id === agentA.id) { + return { embedded: false, fileUpdate: null }; + } + return { embedded: true, fileUpdate: { file_id: file.file_id, embedded: true } }; + }); + const { provisionFiles, addEmbeddedEntity } = buildHarness({ + contexts: [ + [agentA.id, agentA], + [agentB.id, agentB], + ], + vectorImpl, + }); + await expect(provisionFiles(['file_search'], agentA.id)).rejects.toThrow( + FileSearchPreparationError, + ); + await provisionFiles(['file_search'], agentB.id); + + expect(vectorImpl).toHaveBeenCalledTimes(2); + expect(getTurnReadingContext(agentA.deliveryRouting)?.searchState(first.file_id)).toBe( + 'failed', + ); + expect(getTurnReadingContext(agentB.deliveryRouting)?.searchState(second.file_id)).toBe( + 'ready', + ); + expect(addEmbeddedEntity).toHaveBeenCalledWith({ + file_id: second.file_id, + entityId: agentB.id, + }); + expect(agentB.tool_resources?.file_search?.file_ids).toEqual([second.file_id]); + expect(agentA.provisionState?.vectorDBFiles).toEqual([first]); + }); + + it('sanitizes operational failures at the preparation boundary', async () => { + const pending = searchFile(); + const agent = searchAgent([pending]); + const secret = 'submitted-secret Bearer private-credential'; + const vectorImpl = jest.fn().mockRejectedValue( + Object.assign(new Error(secret), { + response: { status: 503, data: secret }, + config: { headers: { Authorization: secret } }, + }), + ); + const { provisionFiles } = buildHarness({ contexts: [[agent.id, agent]], vectorImpl }); + await expect(provisionFiles(['file_search'], agent.id)).rejects.toMatchObject({ + code: 'file_search_preparation_failed', + message: + 'File Search could not prepare the attached files. Try sending the message again.\n' + + '{"code":"file_search_preparation_failed"}', + }); + expect(logger.error).toHaveBeenLastCalledWith( + expect.stringContaining(`file ${pending.file_id}`), + { type: 'Error', status: 503 }, + ); + prepareAgentFileContext(agent, [agent], 'user-1', false); + expect(JSON.stringify(agent.dynamicToolContextMap)).toContain('could not prepare it'); + expect(JSON.stringify(agent.dynamicToolContextMap)).not.toContain(secret); + expect(JSON.stringify(collectInventoryEntries(agent))).not.toContain(secret); + expect(agent.provisionState?.searchPreparation?.get(pending.file_id)).toBe('failed'); + }); + it('ignores tool batches that need neither code nor search', async () => { const { provisionFiles, provisionToCodeEnv, provisionToVectorDB } = buildHarness({ contexts: [['agent-a', { provisionState: state([makeFile()], [makeFile()]) }]], diff --git a/packages/api/src/files/provision/callback.ts b/packages/api/src/files/provision/callback.ts index 7aee5546053..1278e45b973 100644 --- a/packages/api/src/files/provision/callback.ts +++ b/packages/api/src/files/provision/callback.ts @@ -7,6 +7,7 @@ import type { CodeEnvRefUpdate, ProvisionService } from './service'; import type { ProvisionToolContext } from '../code/queued'; import type { ServerRequest } from '~/types'; import { createCodeApiRateLimitBudget, isCodeApiRateLimitError } from '~/utils'; +import { getSafeErrorMetadata } from '~/utils/errors'; import { planCodeFileUploads } from '../code/queued'; import { isCodeFileToolName } from '~/agents/tools'; @@ -38,6 +39,22 @@ async function persistWithRetry( export type { ProvisionToolContext } from '../code/queued'; +const FILE_SEARCH_PREPARATION_CODE = 'file_search_preparation_failed'; + +/** A safe boundary error: indexing failed before any search query could run. */ +export class FileSearchPreparationError extends Error { + readonly code: typeof FILE_SEARCH_PREPARATION_CODE = FILE_SEARCH_PREPARATION_CODE; + + constructor() { + // Existing stream boundaries transmit error text; the client extracts this safe code to localize it. + super( + 'File Search could not prepare the attached files. Try sending the message again.\n' + + JSON.stringify({ code: FILE_SEARCH_PREPARATION_CODE }), + ); + this.name = 'FileSearchPreparationError'; + } +} + export interface ProvisionCallbackDeps { req: ServerRequest; agentToolContexts: Map; @@ -352,9 +369,16 @@ export function createProvisionFilesCallback({ if (needsSearch && provisionState.vectorDBFiles.length > 0) { const queuedVectorFiles = provisionState.vectorDBFiles; + provisionState.searchPreparation ??= new Map(); + const searchPreparation = provisionState.searchPreparation; + for (const file of queuedVectorFiles) { + if (!searchPreparation.has(file.file_id)) { + searchPreparation.set(file.file_id, 'queued'); + } + } const results = await Promise.allSettled( queuedVectorFiles.map(async (file) => { - const result = await shareProvisioning(shareKey('search', file), async () => { + await shareProvisioning(shareKey('search', file), async () => { const provisioned = await provisionToVectorDB({ req, file, @@ -362,9 +386,14 @@ export function createProvisionFilesCallback({ signal, }); signal?.throwIfAborted(); + /* A declined embedding must reject the shared work too, so a later tool + * call retries it rather than replaying a cached embedded:false result. */ + if (!provisioned.embedded) { + throw new FileSearchPreparationError(); + } /* The vectors are already stored, so a failed flag write costs a re-embed next * turn rather than this turn's results. Logged, not fatal. */ - if (provisioned.embedded && provisioned.fileUpdate) { + if (provisioned.fileUpdate) { const update = provisioned.fileUpdate; /* Vectors live under the entity that provisioned them, so the namespace is * recorded alongside the flag. Agents sharing a record, as a duplicate does @@ -379,39 +408,34 @@ export function createProvisionFilesCallback({ (error) => logger.error( `[provisionFiles] Failed to persist embedding state for file ${update.file_id}`, - error, + getSafeErrorMetadata(error), ), ); } return provisioned; }); - if (result.embedded) { - file.embedded = true; - const namespace = namespaceForFile(file); - if (namespace != null) { - const recorded = new Set(file.metadata?.embeddedEntities ?? []); - recorded.add(namespace); - file.metadata = { ...file.metadata, embeddedEntities: [...recorded] }; - } - addProvisionedFile( - file, - EToolResources.file_search, - entityIdForFile(file) !== undefined, - ); - return; + file.embedded = true; + const namespace = namespaceForFile(file); + if (namespace != null) { + const recorded = new Set(file.metadata?.embeddedEntities ?? []); + recorded.add(namespace); + file.metadata = { ...file.metadata, embeddedEntities: [...recorded] }; } - /* Resolving with embedded false is a normal outcome of the service, returned - * when the vector store declines the file, and it means the same thing as a - * throw: the vectors are not there. Treated as a failure so the file stays - * queued and search does not proceed without it. */ - throw new Error(`Vector store did not embed "${file.filename}" (${file.file_id})`); + addProvisionedFile(file, EToolResources.file_search, entityIdForFile(file) !== undefined); + searchPreparation.set(file.file_id, 'ready'); }), ); - signal?.throwIfAborted(); results.forEach((result, index) => { if (result.status === 'rejected') { - logger.error('[provisionFiles] Vector DB provisioning failed', result.reason); - failedVectorFiles.push(queuedVectorFiles[index]); + const file = queuedVectorFiles[index]; + if (!signal?.aborted) { + logger.error( + `[provisionFiles] Vector DB provisioning failed for file ${file.file_id}`, + getSafeErrorMetadata(result.reason), + ); + } + searchPreparation.set(file.file_id, 'failed'); + failedVectorFiles.push(file); } }); /* Preserved for a later retry, and fatal for this turn like a code failure. An @@ -419,6 +443,7 @@ export function createProvisionFilesCallback({ * narrows results, but a search that silently omits the file the user asked about * is a wrong answer, not a smaller one. */ provisionState.vectorDBFiles = failedVectorFiles; + signal?.throwIfAborted(); } /* Provisioning failed outright, so the sandbox or vector store does not have the @@ -436,9 +461,7 @@ export function createProvisionFilesCallback({ ); } if (failedVectorFiles.length > 0) { - throw new Error( - `Failed to provision ${failedVectorFiles.length} file(s) for search; aborting tool execution rather than searching without them`, - ); + throw new FileSearchPreparationError(); } if (needsCode) { diff --git a/packages/api/src/files/provision/service.spec.ts b/packages/api/src/files/provision/service.spec.ts index 253c97b8dd1..df92eb04d47 100644 --- a/packages/api/src/files/provision/service.spec.ts +++ b/packages/api/src/files/provision/service.spec.ts @@ -79,6 +79,50 @@ describe('createProvisionService', () => { }); }); + describe('openStoredFile', () => { + it('opens the stored original through the source strategy', async () => { + const stream = Readable.from('a,b'); + const getDownloadStream = jest.fn().mockResolvedValue(stream); + const getStrategyFunctions = jest.fn(() => ({ getDownloadStream })); + const { service } = buildService({ getStrategyFunctions }); + + await expect( + service.openStoredFile( + { file_id: 'f1', source: 's3', storageKey: 'uploads/u1/a.csv' }, + req, + ), + ).resolves.toBe(stream); + expect(getStrategyFunctions).toHaveBeenCalledWith('s3'); + expect(getDownloadStream).toHaveBeenCalledWith(req, 'uploads/u1/a.csv', { + signal: undefined, + }); + }); + + it('reads a record without a source from local storage', async () => { + const getStrategyFunctions = jest.fn(() => ({ + getDownloadStream: jest.fn().mockResolvedValue(Readable.from('a,b')), + })); + const { service } = buildService({ getStrategyFunctions }); + + await expect( + service.openStoredFile({ file_id: 'f1', filepath: '/uploads/u1/a.csv' }, req), + ).resolves.toBeInstanceOf(Readable); + expect(getStrategyFunctions).toHaveBeenCalledWith('local'); + }); + + it('opens nothing for a source without a storage download contract or a record without a path', async () => { + const { service, getDownloadStream } = buildService(); + + await expect( + service.openStoredFile({ file_id: 'f1', source: 'text', filepath: '/x/a.csv' }, req), + ).resolves.toBeNull(); + await expect( + service.openStoredFile({ file_id: 'f1', source: 'local' }, req), + ).resolves.toBeNull(); + expect(getDownloadStream).not.toHaveBeenCalled(); + }); + }); + describe('provisionToCodeEnv', () => { it('waits out Code API throttling and reopens the upload stream', async () => { const rateLimit = new AxiosError('Request failed', 'ERR_BAD_REQUEST'); @@ -182,6 +226,24 @@ describe('createProvisionService', () => { expect(uploadCodeEnvFile).not.toHaveBeenCalled(); }); + it('reads a record without a source from local storage, as the rest of the pipeline does', async () => { + const getDownloadStream = jest.fn().mockResolvedValue(Readable.from('a,b')); + const uploadCodeEnvFile = jest + .fn() + .mockResolvedValue({ storage_session_id: 's1', file_id: 'remote-1' }); + const getStrategyFunctions = jest.fn((source: string) => + source === 'execute_code' ? { handleFileUpload: uploadCodeEnvFile } : { getDownloadStream }, + ); + const { service } = buildService({ getStrategyFunctions }); + + await expect( + service.provisionToCodeEnv({ req, file: makeFile({ source: undefined }) }), + ).resolves.toMatchObject({ referenceSet: { codeEnvRef: { file_id: 'remote-1' } } }); + + expect(getStrategyFunctions).toHaveBeenCalledWith('local'); + expect(getDownloadStream).toHaveBeenCalledWith(req, '/x/data.csv', { signal: undefined }); + }); + it('prefers a durable storage key when opening the backing object', async () => { const { service, getDownloadStream } = buildService(); diff --git a/packages/api/src/files/provision/service.ts b/packages/api/src/files/provision/service.ts index e50ba1e8f77..5f9a900f9e6 100644 --- a/packages/api/src/files/provision/service.ts +++ b/packages/api/src/files/provision/service.ts @@ -9,6 +9,7 @@ import { FileSources, getCodeEnvRefs, mergeCodeEnvRef, + isOriginalBacked, resolveSandboxFilename, } from 'librechat-data-provider'; import type { CodeEnvRef, CodeEnvRefMap, TFile } from 'librechat-data-provider'; @@ -88,8 +89,26 @@ export interface CodeEnvReferenceSetResult { codeEnvRefs?: CodeEnvRefMap; } +/** What locates a record's retained original in storage. */ +export interface StoredOriginalRef { + file_id: string; + filename?: string; + source?: string | null; + filepath?: string; + storageKey?: string | null; +} + export interface ProvisionService { loadCodeApiKey: (userId: string) => Promise; + /** + * Opens a stream of the record's retained original, or null when its source has no storage + * download contract or the record names no stored object. A missing source reads as `local`. + */ + openStoredFile: ( + file: StoredOriginalRef, + req: ServerRequest, + signal?: AbortSignal, + ) => Promise; provisionToCodeEnv: (params: { req: ServerRequest; file: TFile; @@ -155,37 +174,34 @@ export function createProvisionService({ let axiosInstance; const getAxios = () => (axiosInstance ??= createAxiosInstance()); - /* Sources whose `getDownloadStream` takes `(req, filepath)` and returns a readable. - * Others diverge: `openai` takes `(file_id, client)` and `execute_code` takes - * `(fileIdentifier, identity, req)` returning an Axios response, so calling them - * through this contract fails. An allowlist keeps an unfamiliar source skipped - * rather than mis-invoked. */ - const STORAGE_STREAM_SOURCES = new Set([ - FileSources.local, - FileSources.s3, - FileSources.cloudfront, - FileSources.azure_blob, - FileSources.firebase, - ]); - - /** Resolves a storage download stream, or null when the source uses a different contract. */ + /** + * Resolves a storage download stream, or null when the source uses a different contract. + * Only original-backed sources take `(req, filepath)` and return a readable. Others diverge: + * `openai` takes `(file_id, client)` and `execute_code` takes `(fileIdentifier, identity, req)` + * returning an Axios response, so calling them through this contract fails. The allowlist + * keeps an unfamiliar source skipped rather than mis-invoked. + */ async function getStorageStream( - file: TFile, + file: StoredOriginalRef, req: ServerRequest, signal?: AbortSignal, ): Promise { - if (file.source == null || !STORAGE_STREAM_SOURCES.has(file.source)) { + if (!isOriginalBacked(file)) { logger.warn( `[provision] Cannot stream "${file.filename}" (${file.file_id}) from source "${file.source}": unsupported download contract`, ); return null; } - const { getDownloadStream } = getStrategyFunctions(file.source as string); + const downloadPath = resolveDownloadPath({ ...file, filepath: file.filepath ?? '' }); + if (!downloadPath) { + return null; + } + const { getDownloadStream } = getStrategyFunctions(file.source ?? FileSources.local); if (!getDownloadStream) { return null; } signal?.throwIfAborted(); - const stream = await getDownloadStream(req, resolveDownloadPath(file), { signal }); + const stream = await getDownloadStream(req, downloadPath, { signal }); if (signal?.aborted) { stream.destroy(signal.reason instanceof Error ? signal.reason : undefined); signal.throwIfAborted(); @@ -688,6 +704,7 @@ export function createProvisionService({ } return { loadCodeApiKey, + openStoredFile: getStorageStream, provisionToCodeEnv, provisionToVectorDB, checkCodeEnvFileAlive, diff --git a/packages/api/src/files/reading/derive.spec.ts b/packages/api/src/files/reading/derive.spec.ts new file mode 100644 index 00000000000..ed4a2a5fb08 --- /dev/null +++ b/packages/api/src/files/reading/derive.spec.ts @@ -0,0 +1,398 @@ +import fs from 'fs'; +import os from 'os'; +import path from 'path'; +import { Readable } from 'stream'; +import { FileSources } from 'librechat-data-provider'; +import type { FiltersConfig } from 'librechat-data-provider'; +import type { UploadFallbackTextExtractors } from '~/files/upload/fallback'; +import type { ProvisionService } from '~/files/provision/service'; +import type { TurnReadingFile } from './turn'; +import type { ServerRequest } from '~/types'; +import { DOCUMENT_PARSER_MAX_FILE_SIZE, parseDocument } from '~/files/documents/crud'; +import { AttachmentObjectNotFoundError } from '~/files/encode/utils'; +import { MAX_STORED_EXTRACTED_TEXT_BYTES } from '~/files/extract'; +import { ContentFilterError } from '~/middleware/contentFilter'; +import { UninspectableFileError } from '~/protection/files'; +import { createFileTextDeriver } from './derive'; +import { parseTextNative } from '~/files/text'; + +type OpenStoredFile = ProvisionService['openStoredFile']; + +const XLSX = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'; +const documentsDir = path.join(__dirname, '../documents'); + +let workDir: string; +let sequence = 0; + +beforeAll(() => { + workDir = fs.mkdtempSync(path.join(os.tmpdir(), 'reading-derive-')); +}); + +afterAll(() => { + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +const newRequest = (): ServerRequest => ({}) as ServerRequest; + +/** A deferred message attachment whose original lives at `sourcePath` in local storage. */ +function storedFile( + overrides: Partial & Pick, +): TurnReadingFile { + sequence += 1; + return { + file_id: `derive-spec-${process.pid}-${sequence}`, + source: FileSources.local, + filepath: `/uploads/u1/${overrides.filename}`, + bytes: 1024, + llmDeliveryPath: 'none', + metadata: { destinationChosen: false, textDerivation: { outcome: 'deferred' } }, + ...overrides, + }; +} + +/** Local storage: every open streams the file at `sourcePath`, observed. */ +const localStorage = (sourcePath: string) => + jest.fn, Parameters>(async () => + fs.createReadStream(sourcePath), + ); + +function writeSource(name: string, content: string | Buffer): string { + const filePath = path.join(workDir, name); + fs.writeFileSync(filePath, content); + return filePath; +} + +const tempCopies = (file: TurnReadingFile): string[] => + fs.readdirSync(os.tmpdir()).filter((name) => name.startsWith(`derive-${file.file_id}-`)); + +/** The real built-in extractors, observed. */ +function spiedExtractors() { + return { + parseDocument: jest.fn(parseDocument), + parseTextNative: jest.fn(parseTextNative), + } satisfies UploadFallbackTextExtractors; +} + +describe('createFileTextDeriver', () => { + it('derives a workbook from its stored original with the document parser', async () => { + const req = newRequest(); + const file = storedFile({ type: XLSX, filename: 'sample.xlsx' }); + const samplePath = path.join(documentsDir, 'sample.xlsx'); + const openStoredFile = localStorage(samplePath); + const extractors = spiedExtractors(); + + const result = await createFileTextDeriver({ req, openStoredFile, extractors })(file); + + expect(result).toEqual({ + status: 'derived', + text: 'Sheet One:\nData,on,first,sheet\nSecond Sheet:\nData,On\nSecond,Sheet\n', + textDerivation: { outcome: 'complete', extractor: 'document_parser', at: expect.any(Number) }, + }); + expect(openStoredFile).toHaveBeenCalledWith(file, req, undefined); + expect(extractors.parseDocument).toHaveBeenCalledWith({ + file: expect.objectContaining({ + originalname: 'sample.xlsx', + mimetype: XLSX, + size: fs.statSync(samplePath).size, + }), + }); + expect(extractors.parseTextNative).not.toHaveBeenCalled(); + expect(tempCopies(file)).toEqual([]); + }); + + it('derives a CSV with the native reader', async () => { + const content = 'region,total\nwest,4'; + const file = storedFile({ type: 'text/csv', filename: 'sales.csv', bytes: content.length }); + const extractors = spiedExtractors(); + + const result = await createFileTextDeriver({ + req: newRequest(), + openStoredFile: localStorage(writeSource('sales.csv', content)), + extractors, + })(file); + + expect(result).toEqual({ + status: 'derived', + text: content, + textDerivation: { outcome: 'complete', extractor: 'native_text', at: expect.any(Number) }, + }); + expect(extractors.parseTextNative).toHaveBeenCalledTimes(1); + expect(extractors.parseDocument).not.toHaveBeenCalled(); + }); + + it('fails too_large from the record alone, without opening the original', async () => { + const openStoredFile = localStorage(path.join(documentsDir, 'sample.xlsx')); + const derive = createFileTextDeriver({ req: newRequest(), openStoredFile }); + + await expect( + derive( + storedFile({ + type: XLSX, + filename: 'big.xlsx', + bytes: DOCUMENT_PARSER_MAX_FILE_SIZE + 1, + }), + ), + ).resolves.toEqual({ + status: 'failed', + textDerivation: { + outcome: 'failed', + extractor: 'document_parser', + reason: 'too_large', + at: expect.any(Number), + }, + persist: true, + }); + await expect( + derive( + storedFile({ + type: 'text/csv', + filename: 'big.csv', + bytes: MAX_STORED_EXTRACTED_TEXT_BYTES + 1, + }), + ), + ).resolves.toMatchObject({ status: 'failed', textDerivation: { reason: 'too_large' } }); + expect(openStoredFile).not.toHaveBeenCalled(); + }); + + const flagging: FiltersConfig = { + files: { + pii: { + fields: ['extracted_text'], + starterPatterns: [], + customPatterns: [{ id: 'private', label: 'private token', regex: 'PRIVATE-[A-Z]+' }], + }, + }, + }; + const blocking: FiltersConfig = { + files: { pii: { fields: ['extracted_text'], starterPatterns: [], uninspectable: 'block' } }, + }; + + it.each< + [string, FiltersConfig, string, typeof ContentFilterError | typeof UninspectableFileError] + >([ + ['flags', flagging, 'token,PRIVATE-SECRET', ContentFilterError], + ['cannot inspect', blocking, '', UninspectableFileError], + ])( + 'blocks text a content policy %s with its policy error', + async (_case, filters, content, error) => { + await expect( + createFileTextDeriver({ + req: newRequest(), + filters, + openStoredFile: localStorage(writeSource('blocked.csv', content)), + })(storedFile({ type: 'text/csv', filename: 'blocked.csv' })), + ).resolves.toEqual({ status: 'blocked', error: expect.any(error) }); + }, + ); + + it('fails original_missing, which persists, when storage no longer holds the original', async () => { + const missing = [ + localStorage(path.join(workDir, 'deleted.csv')), + jest.fn, Parameters>(async (file) => { + throw new AttachmentObjectNotFoundError(file.file_id); + }), + jest.fn, Parameters>(async () => { + throw Object.assign(new Error('The specified key does not exist.'), { name: 'NoSuchKey' }); + }), + ]; + for (const openStoredFile of missing) { + const file = storedFile({ type: 'text/csv', filename: 'deleted.csv' }); + await expect( + createFileTextDeriver({ req: newRequest(), openStoredFile })(file), + ).resolves.toEqual({ + status: 'failed', + textDerivation: { + outcome: 'failed', + extractor: 'native_text', + reason: 'original_missing', + at: expect.any(Number), + }, + persist: true, + }); + expect(tempCopies(file)).toEqual([]); + } + }); + + it('skips a transient storage error and a source with no download contract', async () => { + const throwing = (error: Error) => + jest.fn, Parameters>(async () => { + throw error; + }); + const unreachable = throwing(new Error('socket hang up')); + const missingBucket = throwing( + Object.assign(new Error('The specified bucket does not exist'), { + name: 'NoSuchBucket', + $metadata: { httpStatusCode: 404 }, + }), + ); + const bareNotFound = throwing(Object.assign(new Error('Not Found'), { status: 404 })); + const noContract = jest.fn, Parameters>( + async () => null, + ); + + for (const openStoredFile of [unreachable, missingBucket, bareNotFound, noContract]) { + await expect( + createFileTextDeriver({ req: newRequest(), openStoredFile })( + storedFile({ type: 'text/csv', filename: 'later.csv' }), + ), + ).resolves.toEqual({ status: 'skipped', reason: 'storage_unavailable' }); + } + }); + + it('skips, without keeping a marker, a system error the extractor hits reading the copy', async () => { + const extractors = { + parseDocument: jest.fn(parseDocument), + parseTextNative: jest.fn(async ({ path: tmpPath }: Express.Multer.File) => { + throw Object.assign(new Error('EBUSY: resource busy or locked'), { + code: 'EBUSY', + errno: -16, + path: tmpPath, + }); + }), + } satisfies UploadFallbackTextExtractors; + await expect( + createFileTextDeriver({ + req: newRequest(), + openStoredFile: localStorage(writeSource('busy.csv', 'a,b\n1,2\n')), + extractors, + })(storedFile({ type: 'text/csv', filename: 'busy.csv' })), + ).resolves.toEqual({ status: 'skipped', reason: 'storage_unavailable' }); + expect(extractors.parseTextNative).toHaveBeenCalledTimes(1); + }); + + it('keeps a parser failure the extractor raised without a system code', async () => { + const extractors = { + parseDocument: jest.fn(parseDocument), + parseTextNative: jest.fn(async () => { + throw Object.assign(new Error('Invalid structure'), { code: 'ERR_INVALID_ARG_VALUE' }); + }), + } satisfies UploadFallbackTextExtractors; + await expect( + createFileTextDeriver({ + req: newRequest(), + openStoredFile: localStorage(writeSource('broken.csv', 'a,b')), + extractors, + })(storedFile({ type: 'text/csv', filename: 'broken.csv' })), + ).resolves.toMatchObject({ + status: 'failed', + textDerivation: { outcome: 'failed', reason: 'parser' }, + persist: true, + }); + }); + + it('skips a natively readable type the text allowlist excludes without opening it', async () => { + const openStoredFile = localStorage(writeSource('notes.csv', 'a,b\n1,2\n')); + const textMimeTypes = [/^application\/json$/]; + + await expect( + createFileTextDeriver({ req: newRequest(), openStoredFile, textMimeTypes })( + storedFile({ type: 'text/csv', filename: 'notes.csv' }), + ), + ).resolves.toEqual({ status: 'skipped', reason: 'no_extractor' }); + expect(openStoredFile).not.toHaveBeenCalled(); + }); + + it('parses a document type whatever the text allowlist says, as upload extraction does', async () => { + const openStoredFile = localStorage(path.join(documentsDir, 'sample.xlsx')); + const extractors = spiedExtractors(); + + const result = await createFileTextDeriver({ + req: newRequest(), + openStoredFile, + extractors, + textMimeTypes: [/^application\/json$/], + })(storedFile({ type: XLSX, filename: 'sample.xlsx' })); + + expect(result).toMatchObject({ status: 'derived' }); + expect(extractors.parseDocument).toHaveBeenCalledTimes(1); + }); + + it('skips a type no built-in extractor reads without opening it', async () => { + const openStoredFile = localStorage(path.join(documentsDir, 'sample.xlsx')); + + await expect( + createFileTextDeriver({ req: newRequest(), openStoredFile })( + storedFile({ type: 'application/zip', filename: 'bundle.zip' }), + ), + ).resolves.toEqual({ status: 'skipped', reason: 'no_extractor' }); + expect(openStoredFile).not.toHaveBeenCalled(); + }); + + it('removes the temporary copy when the request is aborted mid-download', async () => { + const controller = new AbortController(); + const file = storedFile({ type: 'text/csv', filename: 'slow.csv' }); + const source = new Readable({ read() {} }); + source.push('region,total\n'); + let markOpened!: () => void; + const opened = new Promise((resolve) => { + markOpened = resolve; + }); + const openStoredFile = jest.fn, Parameters>( + async () => { + markOpened(); + return source; + }, + ); + + const pending = createFileTextDeriver({ req: newRequest(), openStoredFile })( + file, + controller.signal, + ); + await opened; + for (let attempt = 0; attempt < 50 && tempCopies(file).length === 0; attempt++) { + await new Promise((resolve) => setTimeout(resolve, 10)); + } + expect(tempCopies(file)).toHaveLength(1); + + controller.abort(); + + await expect(pending).resolves.toEqual({ status: 'skipped', reason: 'aborted' }); + expect(tempCopies(file)).toEqual([]); + expect(source.destroyed).toBe(true); + }); + + it('skips, without keeping anything, when the temporary copy cannot be written', async () => { + const file = storedFile({ type: 'text/csv', filename: 'nowhere.csv' }); + const tmpdir = jest.spyOn(os, 'tmpdir').mockReturnValue(path.join(workDir, 'missing-tmp')); + try { + await expect( + createFileTextDeriver({ + req: newRequest(), + openStoredFile: localStorage(writeSource('nowhere.csv', 'a,b')), + })(file), + ).resolves.toEqual({ status: 'skipped', reason: 'storage_unavailable' }); + } finally { + tmpdir.mockRestore(); + } + }); + + it('derives each file once per deriver, which the host builds once per request', async () => { + const file = storedFile({ type: 'text/csv', filename: 'once.csv' }); + const openStoredFile = localStorage(writeSource('once.csv', 'a,b')); + const derive = createFileTextDeriver({ req: newRequest(), openStoredFile }); + + const [a, b] = await Promise.all([derive(file), derive({ ...file })]); + + expect(a).toBe(b); + expect(a).toMatchObject({ status: 'derived', text: 'a,b' }); + expect(openStoredFile).toHaveBeenCalledTimes(1); + + await createFileTextDeriver({ req: newRequest(), openStoredFile })(file); + expect(openStoredFile).toHaveBeenCalledTimes(2); + }); + + it('forgets a derivation its caller aborted, so the next caller derives under its own signal', async () => { + const file = storedFile({ type: 'text/csv', filename: 'retry.csv' }); + const openStoredFile = localStorage(writeSource('retry.csv', 'a,b')); + const derive = createFileTextDeriver({ req: newRequest(), openStoredFile }); + + await expect(derive(file, AbortSignal.abort())).resolves.toEqual({ + status: 'skipped', + reason: 'aborted', + }); + await new Promise((resolve) => setImmediate(resolve)); + + await expect(derive(file)).resolves.toMatchObject({ status: 'derived', text: 'a,b' }); + expect(openStoredFile).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/api/src/files/reading/derive.ts b/packages/api/src/files/reading/derive.ts new file mode 100644 index 00000000000..ac7d440aa10 --- /dev/null +++ b/packages/api/src/files/reading/derive.ts @@ -0,0 +1,286 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { randomUUID } from 'node:crypto'; +import { pipeline } from 'node:stream/promises'; +import { logger } from '@librechat/data-schemas'; +import { fileConfig, selectBuiltInTextPlan } from 'librechat-data-provider'; +import type { + FiltersConfig, + BuiltInTextPlan, + TextDerivation, + RegexLike, +} from 'librechat-data-provider'; +import type { + DerivedText, + TurnReadingFile, + FileTextDeriver, + DerivationSkipReason, + DerivationPolicyError, +} from './turn'; +import type { UploadFallbackTextExtractors, BoundedText } from '~/files/upload/fallback'; +import type { ProvisionService } from '~/files/provision/service'; +import type { ServerRequest } from '~/types'; +import { DOCUMENT_PARSER_MAX_FILE_SIZE } from '~/files/documents/crud'; +import { isAttachmentObjectNotFoundError } from '~/files/encode/utils'; +import { MAX_STORED_EXTRACTED_TEXT_BYTES } from '~/files/extract'; +import { ContentFilterError } from '~/middleware/contentFilter'; +import { runGuardedEncode } from '~/files/encode/memoryGuard'; +import { extractBoundedText } from '~/files/upload/fallback'; +import { UninspectableFileError } from '~/protection/files'; +import { getSafeErrorMetadata } from '~/utils/errors'; + +export interface FileTextDeriverDeps { + req: ServerRequest; + /** Opens the record's retained original; the provision service's `openStoredFile`. */ + openStoredFile: ProvisionService['openStoredFile']; + filters?: FiltersConfig; + /** The built-in extractors; RAG and OCR never run at turn time. */ + extractors?: UploadFallbackTextExtractors; + /** + * The deployment's text allowlist (`fileConfig.text.supportedMimeTypes`). A natively readable + * type outside it is not parsed, as upload extraction refuses it; absent means unrestricted. + */ + textMimeTypes?: RegexLike[]; +} + +/** The largest file each built-in extractor accepts, checked before anything is downloaded. */ +const PLAN_MAX_BYTES: Readonly> = { + document_parser: DOCUMENT_PARSER_MAX_FILE_SIZE, + native_text: MAX_STORED_EXTRACTED_TEXT_BYTES, +}; + +/** Codes storage backends report for one missing object, as opposed to a missing bucket. */ +const OBJECT_NOT_FOUND_CODES: ReadonlySet = new Set([ + 'NoSuchKey', + 'BlobNotFound', + 'storage/object-not-found', +]); + +interface StorageErrorShape { + name?: string; + code?: string; + path?: string; +} + +/** + * Whether storage reports the original itself gone. A missing bucket or container, a status code + * alone, and any error on the temporary copy are not: those clear up, so a later turn retries. + * The same holds inside the extractor: a system error while it reads the temporary copy is not + * kept on the record, only a failure the parser names. + */ +function isOriginalMissing(error: unknown, tmpPath: string): boolean { + if (isAttachmentObjectNotFoundError(error)) { + return true; + } + if (typeof error !== 'object' || error == null) { + return false; + } + const storageError: StorageErrorShape = error; + if (storageError.code === 'ENOENT') { + return storageError.path != null && storageError.path !== tmpPath; + } + return ( + OBJECT_NOT_FOUND_CODES.has(storageError.name ?? '') || + OBJECT_NOT_FOUND_CODES.has(storageError.code ?? '') + ); +} + +const skipped = (reason: DerivationSkipReason): DerivedText => ({ status: 'skipped', reason }); + +const failed = ( + extractor: BuiltInTextPlan, + reason: NonNullable, +): DerivedText => ({ + status: 'failed', + textDerivation: { outcome: 'failed', extractor, reason, at: Date.now() }, + persist: true, +}); + +/** A policy refusal of the derived text, logged with the rule that refused it. */ +function blocked( + file: TurnReadingFile, + result: Exclude, +): DerivationPolicyError { + if (result.failure === 'policy') { + logger.warn( + `[readingText] file_id=${file.file_id} outcome=blocked rule=${result.finding.ruleId}`, + ); + return new ContentFilterError(result.finding); + } + logger.warn(`[readingText] file_id=${file.file_id} outcome=blocked reason=uninspectable`); + return result.error instanceof UninspectableFileError + ? result.error + : new UninspectableFileError('extracted_text'); +} + +function toDerivedText( + file: TurnReadingFile, + plan: BuiltInTextPlan, + result: BoundedText, +): DerivedText { + if ('text' in result) { + return { + status: 'derived', + text: result.text, + textDerivation: { outcome: 'complete', extractor: plan, at: Date.now() }, + }; + } + if (result.failure === 'policy' || result.failure === 'uninspectable') { + return { status: 'blocked', error: blocked(file, result) }; + } + if (result.failure === 'unavailable') { + logger.error( + `[readingText] file_id=${file.file_id} outcome=skipped reason=storage_unavailable`, + getSafeErrorMetadata(result.error), + ); + return skipped('storage_unavailable'); + } + return failed(plan, result.failure); +} + +function classifyReadError( + file: TurnReadingFile, + plan: BuiltInTextPlan, + error: unknown, + { signal, tmpPath }: { signal?: AbortSignal; tmpPath: string }, +): DerivedText { + if (signal?.aborted) { + return skipped('aborted'); + } + if (isOriginalMissing(error, tmpPath)) { + return failed(plan, 'original_missing'); + } + logger.error( + `[readingText] file_id=${file.file_id} outcome=skipped reason=storage_unavailable`, + getSafeErrorMetadata(error), + ); + return skipped('storage_unavailable'); +} + +function describeDerivation(result: DerivedText): string { + if (result.status === 'derived') { + return `outcome=complete extractor=${result.textDerivation.extractor}`; + } + if (result.status === 'failed') { + return `outcome=failed reason=${result.textDerivation.reason}`; + } + if (result.status === 'blocked') { + return 'outcome=blocked'; + } + return `outcome=skipped reason=${result.reason}`; +} + +/** An outcome a later caller should not inherit: a derivation its own caller's signal cut short. */ +const isAborted = (result: DerivedText): boolean => + result.status === 'skipped' && result.reason === 'aborted'; + +/** + * Builds a request's text deriver: reads a record's retained original back from storage and runs + * the one built-in extractor its type allows, under the same caps as upload extraction. Size caps + * are checked from the record before anything is downloaded; the download runs inside the encode + * memory guard, the parser checks the size actually downloaded, and the temporary copy is always + * removed. The deriver derives each file at most once, so the host builds one per request and + * hands it to every agent. A deterministic failure may be kept on the record and a policy refusal + * blocks the turn that needed the text. An unreachable store is not kept, and an aborted + * derivation is forgotten, so the next caller derives under its own signal. + */ +export function createFileTextDeriver({ + req, + openStoredFile, + filters, + extractors, + textMimeTypes, +}: FileTextDeriverDeps): FileTextDeriver { + const derivations = new Map>(); + + const readOriginal = async ( + file: TurnReadingFile, + plan: BuiltInTextPlan, + signal?: AbortSignal, + ): Promise => { + const tmpPath = path.join( + os.tmpdir(), + `derive-${file.file_id}-${randomUUID()}${path.extname(file.filename ?? '')}`, + ); + try { + signal?.throwIfAborted(); + const stream = await openStoredFile(file, req, signal); + if (stream == null) { + return skipped('storage_unavailable'); + } + await pipeline(stream, fs.createWriteStream(tmpPath), { signal }); + const { size } = await fs.promises.stat(tmpPath); + const upload = { + path: tmpPath, + originalname: file.filename ?? path.basename(tmpPath), + mimetype: file.type ?? '', + size, + } as Express.Multer.File; + const result = await extractBoundedText({ file: upload, filters, plan, extractors }); + if (signal?.aborted) { + return skipped('aborted'); + } + return toDerivedText(file, plan, result); + } catch (error) { + return classifyReadError(file, plan, error, { signal, tmpPath }); + } finally { + await fs.promises.rm(tmpPath, { force: true }).catch(() => undefined); + } + }; + + const deriveOnce = (file: TurnReadingFile, signal?: AbortSignal): Promise => { + const type = file.type ?? ''; + const plan = selectBuiltInTextPlan(type); + if (plan == null) { + return Promise.resolve(skipped('no_extractor')); + } + /* Upload extraction consults the allowlist only for types the native text reader takes; + * document types reach the parser regardless, so the same holds here. */ + if ( + plan === 'native_text' && + textMimeTypes != null && + !fileConfig.checkType(type, textMimeTypes) + ) { + return Promise.resolve(skipped('no_extractor')); + } + const bytes = file.bytes ?? 0; + if (bytes > PLAN_MAX_BYTES[plan]) { + return Promise.resolve(failed(plan, 'too_large')); + } + return runGuardedEncode(bytes, () => readOriginal(file, plan, signal)); + }; + + const derive = async (file: TurnReadingFile, signal?: AbortSignal): Promise => { + const startedAt = Date.now(); + const result = await deriveOnce(file, signal); + logger.debug( + `[readingText] file_id=${file.file_id} ${describeDerivation(result)} ms=${Date.now() - startedAt}`, + ); + return result; + }; + + const forget = (fileId: string, derivation: Promise): void => { + if (derivations.get(fileId) === derivation) { + derivations.delete(fileId); + } + }; + + return (file, signal) => { + const existing = derivations.get(file.file_id); + if (existing != null) { + return existing; + } + const derivation = derive(file, signal); + derivations.set(file.file_id, derivation); + derivation.then( + (result) => { + if (isAborted(result)) { + forget(file.file_id, derivation); + } + }, + () => forget(file.file_id, derivation), + ); + return derivation; + }; +} diff --git a/packages/api/src/files/reading/diagnostics.spec.ts b/packages/api/src/files/reading/diagnostics.spec.ts new file mode 100644 index 00000000000..6f5e0787d96 --- /dev/null +++ b/packages/api/src/files/reading/diagnostics.spec.ts @@ -0,0 +1,120 @@ +import { logger } from '@librechat/data-schemas'; +import { FileContext, FileSources } from 'librechat-data-provider'; +import type { TFileConfig, TurnFileConsumers } from 'librechat-data-provider'; +import type { FileTextDeriver, TurnReadingFile } from './turn'; +import { resolveTurnDeliveryRouting } from '~/agents/files/delivery'; +import { logTurnReading, logAllocation } from './diagnostics'; +import { buildTurnReadingContext } from './turn'; + +type EndpointFileConfigInput = NonNullable[string]; + +const searchOnly: TurnFileConsumers = { executeCode: false, fileSearch: true }; + +const attachment = ( + overrides: Partial & Pick, +): TurnReadingFile => ({ + type: 'application/pdf', + bytes: 1024, + filename: 'quarterly-report.pdf', + source: FileSources.local, + context: FileContext.message_attachment, + llmDeliveryPath: 'provider', + metadata: { destinationChosen: false }, + ...overrides, +}); + +const deriveText: FileTextDeriver = async () => ({ status: 'skipped', reason: 'policy' }); + +function agentFor(endpointConfig: EndpointFileConfigInput, withContext = true) { + const deliveryRouting = resolveTurnDeliveryRouting({ + agent: { provider: 'openAI', endpoint: 'openAI' }, + config: { fileConfig: { endpoints: { openAI: endpointConfig } } }, + }); + const context = withContext + ? buildTurnReadingContext({ + routing: deliveryRouting, + provider: 'openAI', + fileTokenLimit: 100_000, + configuredFileSizeLimit: 1024, + countTokens: (text) => text.length, + deriveText, + }) + : undefined; + if (context != null) { + deliveryRouting.reading = context; + } + return { + agent: { + id: 'agent_1', + deliveryRouting, + fileConsumers: searchOnly, + currentRequestAttachments: [ + attachment({ file_id: 'fits' }), + attachment({ file_id: 'large', bytes: 4096 }), + ], + }, + context, + }; +} + +describe('reading diagnostics', () => { + let info: jest.SpyInstance; + let debug: jest.SpyInstance; + + beforeEach(() => { + info = jest.spyOn(logger, 'info').mockImplementation(() => logger); + debug = jest.spyOn(logger, 'debug').mockImplementation(() => logger); + }); + + afterEach(() => { + info.mockRestore(); + debug.mockRestore(); + }); + + it('logs a summary and one line per file under the automatic policy, without filenames', () => { + const { agent, context } = agentFor({ llmDeliveryPolicy: 'automatic' }); + context?.recordDropped([attachment({ file_id: 'dropped' })]); + + logTurnReading(agent, 'init'); + + expect(info).toHaveBeenCalledTimes(1); + expect(info).toHaveBeenCalledWith( + '[fileReading] agent=agent_1 policy=automatic pass=init files=2 provider=1 text=0 search=1 code=0 unavailable=0 overflow=0 derived=0 rejected=0 dropped=1', + ); + expect(debug.mock.calls.map(([line]) => line)).toEqual([ + '[fileReading] file_id=fits category=document reader=provider path=provider classic=provider reason=native_supported skipped=none code=no_run_code', + '[fileReading] file_id=large category=document reader=search path=none classic=provider reason=native_capacity skipped=provider:native_capacity code=no_run_code', + ]); + const lines = [...info.mock.calls, ...debug.mock.calls].map(([line]) => String(line)); + expect(lines.some((line) => line.includes('quarterly-report'))).toBe(false); + }); + + it.each([ + ['without a reading context', { llmDeliveryPolicy: 'automatic' as const }, false], + ['under the classic policy with a derive-only context', {}, true], + ])('stays silent %s', (_label, endpointConfig, withContext) => { + const { agent } = agentFor(endpointConfig, withContext); + + logTurnReading(agent, 'final'); + + expect(info).not.toHaveBeenCalled(); + expect(debug).not.toHaveBeenCalled(); + }); + + it('stays silent for a turn with nothing to report', () => { + const { agent } = agentFor({ llmDeliveryPolicy: 'automatic' }); + + logTurnReading({ ...agent, currentRequestAttachments: [] }, 'run'); + + expect(info).not.toHaveBeenCalled(); + }); + + it('logs an allocation at debug', () => { + logAllocation({ agentId: 'agent_1', scope: 'history', candidates: 4, overflow: 1 }); + + expect(debug).toHaveBeenCalledWith( + '[allocation] agent=agent_1 scope=history candidates=4 overflow=1', + ); + expect(info).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/api/src/files/reading/diagnostics.ts b/packages/api/src/files/reading/diagnostics.ts new file mode 100644 index 00000000000..5818e18340c --- /dev/null +++ b/packages/api/src/files/reading/diagnostics.ts @@ -0,0 +1,102 @@ +import { logger } from '@librechat/data-schemas'; +import { decideFileReading } from 'librechat-data-provider'; +import type { + FileReading, + TurnDeliveryFile, + TurnFileConsumers, + TurnDeliveryRouting, +} from 'librechat-data-provider'; +import { getTurnReadingContext } from './turn'; + +/** Which reading the line describes: initialization's first or final decision, or a run's. */ +export type ReadingPass = 'init' | 'final' | 'run'; + +/** Whether an allocation covered the request alone or the request after its history. */ +export type AllocationScope = 'request' | 'history'; + +/** The fields of an initialized agent the turn's reading diagnostics read. */ +export interface TurnReadingAgent { + id?: string; + deliveryRouting?: TurnDeliveryRouting; + fileConsumers?: TurnFileConsumers; + requestAttachments?: readonly TurnDeliveryFile[]; + currentRequestAttachments?: readonly TurnDeliveryFile[]; +} + +const SUMMARY_READERS = ['provider', 'text', 'search', 'code', 'unavailable'] as const; + +type ReaderCounts = Record<(typeof SUMMARY_READERS)[number], number>; + +const formatSkipped = ({ skipped }: FileReading): string => + skipped.length === 0 + ? 'none' + : skipped.map(({ reader, reason }) => `${reader}:${reason}`).join(','); + +const formatFileReading = (fileId: string | undefined, reading: FileReading): string => + [ + `[fileReading] file_id=${fileId ?? 'unknown'}`, + `category=${reading.category}`, + `reader=${reading.reader}`, + `path=${reading.path ?? 'unset'}`, + `classic=${reading.classicPath ?? 'unset'}`, + `reason=${reading.reason}`, + `skipped=${formatSkipped(reading)}`, + `code=${reading.code}`, + ...(reading.needsText ? ['text=derive'] : []), + ].join(' '); + +/** + * Logs how the automatic policy reads the agent's current request attachments: one summary line + * at info and one line per file at debug. Lines carry file ids, codes and provider names only. + * Silent under the classic policy. + */ +export function logTurnReading(agent: TurnReadingAgent, pass: ReadingPass): void { + const routing = agent.deliveryRouting; + const context = getTurnReadingContext(routing); + if (routing == null || context == null || context.policy !== 'automatic') { + return; + } + const files = agent.currentRequestAttachments ?? agent.requestAttachments ?? []; + const stats = context.stats(); + if (files.length === 0 && stats.dropped === 0) { + return; + } + const counts: ReaderCounts = { provider: 0, text: 0, search: 0, code: 0, unavailable: 0 }; + const lines: string[] = []; + for (const file of files) { + const reading = decideFileReading({ routing, file, consumers: agent.fileConsumers }); + if (reading.reader !== 'unresolved') { + counts[reading.reader] += 1; + } + lines.push(formatFileReading(file.file_id, reading)); + } + logger.info( + [ + `[fileReading] agent=${agent.id ?? 'unknown'} policy=automatic pass=${pass}`, + `files=${files.length}`, + ...SUMMARY_READERS.map((reader) => `${reader}=${counts[reader]}`), + `overflow=${stats.overflow} derived=${stats.derived}`, + `rejected=${stats.rejected} dropped=${stats.dropped}`, + ].join(' '), + ); + for (const line of lines) { + logger.debug(line); + } +} + +/** Logs one first-fit allocation of direct model content. */ +export function logAllocation({ + agentId, + scope, + candidates, + overflow, +}: { + agentId?: string; + scope: AllocationScope; + candidates: number; + overflow: number; +}): void { + logger.debug( + `[allocation] agent=${agentId ?? 'unknown'} scope=${scope} candidates=${candidates} overflow=${overflow}`, + ); +} diff --git a/packages/api/src/files/reading/history.spec.ts b/packages/api/src/files/reading/history.spec.ts new file mode 100644 index 00000000000..1a5a23cdc7d --- /dev/null +++ b/packages/api/src/files/reading/history.spec.ts @@ -0,0 +1,557 @@ +import { FileContext, FileSources, decideFileReading } from 'librechat-data-provider'; +import type { TFileConfig, TurnFileConsumers, TurnDeliveryRouting } from 'librechat-data-provider'; +import type { AppConfig } from '@librechat/data-schemas'; +import type { TurnAttachmentsWithHistoryParams, HistoryAllocationFile } from './history'; +import type { FileTextDeriver, TurnReadingContext } from './turn'; +import { + isModelBoundAttachmentFile, + assertAgentAttachmentLimits, + collectHistoricalAttachmentIds, +} from '~/agents/attachments'; +import { + allocateTurnAttachmentsWithHistory, + allocateTurnAttachmentsWithRetainedContext, +} from './history'; +import { applyTurnDelivery, resolveTurnDeliveryRouting } from '~/agents/files/delivery'; +import { buildTurnReadingContext } from './turn'; + +type EndpointFileConfigInput = NonNullable[string]; +type AllocationParams = TurnAttachmentsWithHistoryParams; + +const MB = 1024 * 1024; +const PDF = 'application/pdf'; +const XLSX = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'; +const ENDPOINT = 'openAI'; + +const noReader: TurnFileConsumers = { executeCode: false, fileSearch: false }; +const searchOnly: TurnFileConsumers = { executeCode: false, fileSearch: true }; + +const attachment = ( + overrides: Partial & Pick, +): HistoryAllocationFile => ({ + type: PDF, + bytes: 3 * MB, + source: FileSources.local, + context: FileContext.message_attachment, + llmDeliveryPath: 'provider', + metadata: { destinationChosen: false }, + ...overrides, +}); + +const workbook = (file_id: string, text: string): HistoryAllocationFile => + attachment({ file_id, type: XLSX, bytes: 1024, llmDeliveryPath: 'text', text }); + +interface Turn { + agent: AllocationParams['agent']; + req: NonNullable; + routing: TurnDeliveryRouting; + context: TurnReadingContext; +} + +function setup({ + endpointConfig = { llmDeliveryPolicy: 'automatic' }, + fileConfig = {}, + consumers = searchOnly, + deriveText, +}: { + endpointConfig?: EndpointFileConfigInput; + fileConfig?: Omit; + consumers?: TurnFileConsumers; + deriveText?: FileTextDeriver; +} = {}): Turn { + const appConfig: AppConfig = { + config: {}, + fileStrategy: FileSources.local, + imageOutputType: 'png', + fileConfig: { ...fileConfig, endpoints: { [ENDPOINT]: endpointConfig } }, + }; + const routing = resolveTurnDeliveryRouting({ + agent: { provider: ENDPOINT, endpoint: ENDPOINT }, + config: appConfig, + }); + const context = buildTurnReadingContext({ + routing, + provider: ENDPOINT, + model: 'gpt-4o', + fileTokenLimit: 100_000, + configuredFileSizeLimit: undefined, + countTokens: (text) => text.length, + deriveText, + }); + if (context == null) { + throw new Error('expected a reading context'); + } + routing.reading = context; + return { + agent: { id: 'agent_1', deliveryRouting: routing, fileConsumers: consumers }, + req: { config: appConfig }, + routing, + context, + }; +} + +const allocate = ( + { agent, req }: Turn, + historical: readonly HistoryAllocationFile[], + current: HistoryAllocationFile[], +): Promise => + allocateTurnAttachmentsWithHistory({ agent, req, historical, current, endpoint: ENDPOINT }); + +const pathsById = ( + files: readonly HistoryAllocationFile[], +): Record => + Object.fromEntries(files.map((file) => [file.file_id, file.llmDeliveryPath])); + +/** The check AgentClient's `assertTurnAttachmentLimits` runs on the shared set. */ +const assertTurnLimits = ( + { req }: Turn, + historical: readonly HistoryAllocationFile[], + current: readonly HistoryAllocationFile[], +) => + assertAgentAttachmentLimits({ + attachments: [...historical, ...current.filter(isModelBoundAttachmentFile)], + req, + endpoint: ENDPOINT, + countRepeatedExtractedText: true, + enforceAttachmentCount: false, + useGlobalContextSizeLimit: true, + }); + +/** The check the topology assertion runs on the same set for the primary agent. */ +const assertAgentLimits = ( + { req }: Turn, + historical: readonly HistoryAllocationFile[], + current: readonly HistoryAllocationFile[], +) => + assertAgentAttachmentLimits({ + attachments: [...historical, ...current.filter(isModelBoundAttachmentFile)], + req, + endpoint: ENDPOINT, + countRepeatedExtractedText: true, + historicalFileIds: collectHistoricalAttachmentIds(historical, current), + }); + +describe('allocateTurnAttachmentsWithHistory', () => { + describe('outside the automatic policy', () => { + interface ClassicTurn { + agent: AllocationParams['agent']; + req?: AllocationParams['req']; + context?: TurnReadingContext; + } + + it.each<[string, () => ClassicTurn]>([ + [ + 'the routing carries no reading context', + () => { + const routing = resolveTurnDeliveryRouting({ + agent: { provider: ENDPOINT, endpoint: ENDPOINT }, + config: { + fileConfig: { + fileContextSizeLimit: 1, + endpoints: { [ENDPOINT]: { llmDeliveryPolicy: 'automatic' } }, + }, + }, + }); + return { agent: { deliveryRouting: routing, fileConsumers: searchOnly } }; + }, + ], + [ + 'the policy is classic', + () => { + const turn = setup({ + endpointConfig: {}, + fileConfig: { fileContextSizeLimit: 1 }, + deriveText: async () => ({ status: 'skipped', reason: 'no_extractor' }), + }); + expect(turn.context.policy).toBe('classic'); + return turn; + }, + ], + ])( + 'returns the current files themselves when %s, asking its context nothing', + async (_case, arrange) => { + const { agent, req, context } = arrange(); + const methods = [ + 'judge', + 'derive', + 'addOverflow', + 'markTextFailed', + 'flush', + 'knownTokenCount', + ] as const; + const spies = context == null ? [] : methods.map((method) => jest.spyOn(context, method)); + const current = [attachment({ file_id: 'a' }), attachment({ file_id: 'b' })]; + + const allocated = await allocateTurnAttachmentsWithHistory({ + agent, + req, + historical: [attachment({ file_id: 'history', bytes: 6 * MB })], + current, + endpoint: ENDPOINT, + }); + + expect(allocated).toBe(current); + for (const spy of spies) { + expect(spy).not.toHaveBeenCalled(); + } + }, + ); + }); + + describe('under the automatic policy', () => { + const history = attachment({ file_id: 'history', bytes: 6 * MB }); + const current = (): HistoryAllocationFile[] => [ + attachment({ file_id: 'a' }), + attachment({ file_id: 'b' }), + ]; + + it('charges history first, so a current PDF that no longer fits moves to File Search', async () => { + const turn = setup({ fileConfig: { fileContextSizeLimit: 10 } }); + const { routing, context } = turn; + const historyBefore = structuredClone(history); + const files = current(); + expect(() => assertTurnLimits(turn, [history], files)).toThrow( + expect.objectContaining({ limitType: 'bytes' }), + ); + + const allocated = await allocate(turn, [history], files); + + expect(pathsById(allocated)).toEqual({ a: 'provider', b: 'none' }); + expect( + decideFileReading({ routing, file: allocated[1], consumers: searchOnly }), + ).toMatchObject({ reader: 'search', reason: 'aggregate_overflow' }); + expect(allocated[0]).toBe(files[0]); + expect(context.stats().overflow).toBe(1); + expect(() => assertTurnLimits(turn, [history], allocated)).not.toThrow(); + expect(() => assertAgentLimits(turn, [history], allocated)).not.toThrow(); + expect(history).toEqual(historyBefore); + expect(context.judge(history).overflow).toBeUndefined(); + expect(decideFileReading({ routing, file: history, consumers: searchOnly })).toMatchObject({ + reader: 'provider', + }); + }); + + it('admits the same request in full without the history in front of it', async () => { + const turn = setup({ fileConfig: { fileContextSizeLimit: 10 } }); + const files = current(); + + const allocated = await allocate(turn, [], files); + + expect(allocated).toBe(files); + expect(turn.context.stats().overflow).toBe(0); + }); + + it('leaves the overflow unavailable rather than failing the turn when File Search cannot reach it', async () => { + /* File Search queued and registered nothing, as a failed provisioning leaves it, and no + * code tool is loaded, so no reader is left for the overflow. */ + const turn = setup({ fileConfig: { fileContextSizeLimit: 10 } }); + turn.context.setSearchEvidence({ queued: [], registered: [] }); + + const allocated = await allocate(turn, [history], current()); + + expect(pathsById(allocated)).toEqual({ a: 'provider', b: 'none' }); + expect( + decideFileReading({ routing: turn.routing, file: allocated[1], consumers: searchOnly }), + ).toMatchObject({ reader: 'unavailable', reason: 'aggregate_overflow' }); + expect(() => assertTurnLimits(turn, [history], allocated)).not.toThrow(); + }); + + it('keeps the current files on their classic route when no file tool is loaded, so the limit check still fails the turn', async () => { + const turn = setup({ fileConfig: { fileContextSizeLimit: 10 }, consumers: noReader }); + const files = current(); + + const allocated = await allocate(turn, [history], files); + + expect(pathsById(allocated)).toEqual({ a: 'provider', b: 'provider' }); + expect(turn.context.stats().overflow).toBe(0); + expect( + decideFileReading({ routing: turn.routing, file: allocated[1], consumers: noReader }), + ).toMatchObject({ reader: 'provider', reason: 'no_file_tools', automatic: false }); + expect(() => assertTurnLimits(turn, [history], allocated)).toThrow( + expect.objectContaining({ limitType: 'bytes' }), + ); + }); + + it('moves a text reading that no longer fits the extracted-text budget', async () => { + const turn = setup({ fileConfig: { fileContextCharLimit: 100 } }); + const replayed = workbook('replayed', 'r'.repeat(70)); + const sheet = workbook('sheet', 's'.repeat(40)); + expect( + decideFileReading({ routing: turn.routing, file: sheet, consumers: searchOnly }), + ).toMatchObject({ reader: 'text' }); + + const allocated = await allocate(turn, [replayed], [sheet]); + + expect(pathsById(allocated)).toEqual({ sheet: 'none' }); + expect( + decideFileReading({ routing: turn.routing, file: allocated[0], consumers: searchOnly }), + ).toMatchObject({ reader: 'search', reason: 'aggregate_overflow' }); + expect(() => assertTurnLimits(turn, [replayed], allocated)).not.toThrow(); + }); + + it.each([ + ['replayed', (file: HistoryAllocationFile) => ({ historical: [file], files: current() })], + [ + 'submitted', + (file: HistoryAllocationFile) => ({ historical: [], files: [file, ...current()] }), + ], + ])('does not charge a %s file this endpoint refuses', async (_label, arrange) => { + const turn = setup({ + endpointConfig: { + llmDeliveryPolicy: 'automatic', + supportedMimeTypes: ['^application/pdf$'], + }, + fileConfig: { fileContextSizeLimit: 10 }, + }); + const screenshot = attachment({ file_id: 'screenshot', type: 'image/png', bytes: 6 * MB }); + const { historical, files } = arrange(screenshot); + + const allocated = await allocate(turn, historical, files); + + expect(allocated).toBe(files); + expect(turn.context.stats().overflow).toBe(0); + }); + + it('does not charge replayed history that is no longer model-bound', async () => { + const turn = setup({ fileConfig: { fileContextSizeLimit: 10 } }); + const searched = attachment({ file_id: 'searched', bytes: 6 * MB, llmDeliveryPath: 'none' }); + const files = current(); + + expect(await allocate(turn, [searched], files)).toBe(files); + }); + + describe('count allowance', () => { + const countLimited = () => + setup({ endpointConfig: { llmDeliveryPolicy: 'automatic', fileLimit: 2 } }); + const small = (file_id: string): HistoryAllocationFile => + attachment({ file_id, bytes: 1024 }); + + it('spends no count allowance on replayed history', async () => { + const turn = countLimited(); + const historical = ['h1', 'h2', 'h3'].map(small); + const files = ['a', 'b', 'c'].map(small); + + const allocated = await allocate(turn, historical, files); + + expect(pathsById(allocated)).toEqual({ a: 'provider', b: 'provider', c: 'none' }); + expect(() => assertAgentLimits(turn, historical, allocated)).not.toThrow(); + expect(() => assertAgentLimits(turn, historical, files)).toThrow( + expect.objectContaining({ limitType: 'count' }), + ); + }); + + it('counts a replayed file submitted again once, as the current submission', async () => { + const turn = countLimited(); + const historical = ['h1', 'a'].map(small); + const files = ['a', 'b', 'c'].map(small); + + const allocated = await allocate(turn, historical, files); + + expect(pathsById(allocated)).toEqual({ a: 'provider', b: 'provider', c: 'none' }); + expect(() => assertAgentLimits(turn, historical, allocated)).not.toThrow(); + }); + + it('commits a resubmitted file with its history rather than moving it', async () => { + const turn = countLimited(); + const historical = ['h1', 'c'].map(small); + const files = ['a', 'b', 'c'].map(small); + + const allocated = await allocate(turn, historical, files); + + expect(pathsById(allocated)).toEqual({ a: 'provider', b: 'none', c: 'provider' }); + expect(turn.context.stats().overflow).toBe(1); + expect(() => assertAgentLimits(turn, historical, allocated)).not.toThrow(); + expect( + decideFileReading({ routing: turn.routing, file: historical[1], consumers: searchOnly }) + .reader, + ).toBe('provider'); + }); + }); + + it('hands the moved files to the caller, and nothing when none moved', async () => { + const turn = setup({ fileConfig: { fileContextSizeLimit: 4 } }); + const onAllocated = jest.fn(); + const files = current(); + + const allocated = await allocateTurnAttachmentsWithHistory({ + ...turn, + historical: [], + current: files, + endpoint: ENDPOINT, + onAllocated, + }); + const fitting = [attachment({ file_id: 'small', bytes: MB })]; + const unchanged = await allocateTurnAttachmentsWithHistory({ + ...setup(), + historical: [], + current: fitting, + endpoint: ENDPOINT, + onAllocated, + }); + + expect(onAllocated.mock.calls).toEqual([[allocated]]); + expect(pathsById(allocated)).toEqual({ a: 'provider', b: 'none' }); + expect(unchanged).toBe(fitting); + }); + + it('holds the shared set to a secondary agent endpoint with a tighter count', async () => { + const appConfig: AppConfig = { + config: {}, + fileStrategy: FileSources.local, + imageOutputType: 'png', + fileConfig: { + endpoints: { + [ENDPOINT]: { llmDeliveryPolicy: 'automatic', fileLimit: 10 }, + anthropic: { fileLimit: 1 }, + }, + }, + }; + const turn = setup(); + const req = { config: appConfig }; + const endpointsByAgentId = new Map([ + ['agent_1', { endpoint: ENDPOINT }], + ['agent_2', { endpoint: 'anthropic' }], + ]); + const files = ['a', 'b'].map((file_id) => attachment({ file_id, bytes: 1024 })); + /** The per-agent check the topology assertion runs for the secondary agent. */ + const assertSecondary = (shared: readonly HistoryAllocationFile[]) => + assertAgentAttachmentLimits({ + attachments: shared.filter(isModelBoundAttachmentFile), + req, + endpoint: 'anthropic', + countRepeatedExtractedText: true, + }); + + const allocated = await allocateTurnAttachmentsWithHistory({ + agent: turn.agent, + req, + historical: [], + current: files, + endpoint: ENDPOINT, + endpointsByAgentId, + }); + + expect(pathsById(allocated)).toEqual({ a: 'provider', b: 'none' }); + expect(() => assertSecondary(allocated)).not.toThrow(); + expect(() => assertSecondary(files)).toThrow(expect.objectContaining({ limitType: 'count' })); + }); + + describe('against retained file contexts', () => { + const retainedContext = (text: string): HistoryAllocationFile => ({ + file_id: 'retained-file-context:msg-1', + source: FileSources.text, + type: 'text/plain', + text, + bytes: Buffer.byteLength(text), + }); + + it('moves a text reading that no longer fits beside the retained text', async () => { + const turn = setup({ fileConfig: { fileContextCharLimit: 1000 } }); + const retained = [retainedContext('r'.repeat(500))]; + const files = [workbook('sheet', 's'.repeat(600))]; + + const allocated = await allocateTurnAttachmentsWithRetainedContext({ + ...turn, + resendFiles: false, + retained, + current: files, + endpoint: ENDPOINT, + }); + + expect(pathsById(allocated)).toEqual({ sheet: 'none' }); + expect(() => assertTurnLimits(turn, retained, allocated)).not.toThrow(); + expect(() => assertTurnLimits(turn, retained, files)).toThrow( + expect.objectContaining({ limitType: 'extracted_text' }), + ); + }); + + it('leaves the request alone when the conversation resends files', async () => { + const turn = setup({ fileConfig: { fileContextCharLimit: 1000 } }); + const files = [workbook('sheet', 's'.repeat(600))]; + + await expect( + allocateTurnAttachmentsWithRetainedContext({ + ...turn, + resendFiles: true, + retained: [retainedContext('r'.repeat(500))], + current: files, + endpoint: ENDPOINT, + }), + ).resolves.toBe(files); + expect(turn.context.stats().overflow).toBe(0); + }); + }); + + it('depends on the current order alone, not on the order of the history', async () => { + const historical = ['h1', 'h2', 'h3'].map((file_id) => + attachment({ file_id, bytes: 2 * MB }), + ); + const run = async ( + replay: readonly HistoryAllocationFile[], + files: HistoryAllocationFile[], + ) => { + const turn = setup({ fileConfig: { fileContextSizeLimit: 10 } }); + return pathsById(await allocate(turn, replay, files)); + }; + const [a, b] = current(); + + const expected = { a: 'provider', b: 'none' }; + expect(await run(historical, [a, b])).toEqual(expected); + expect(await run([...historical].reverse(), [a, b])).toEqual(expected); + expect(await run([historical[1], historical[2], historical[0]], [a, b])).toEqual(expected); + expect(await run(historical, [b, a])).toEqual({ a: 'none', b: 'provider' }); + }); + + describe('limits', () => { + it('holds the shared set to the global limits the turn check enforces', async () => { + const turn = setup({ + endpointConfig: { llmDeliveryPolicy: 'automatic', fileLimit: 4 }, + fileConfig: { fileContextSizeLimit: 10, fileContextCharLimit: 500 }, + }); + const exact = 10 * MB - 7 * MB; + const historical = [attachment({ file_id: 'history', bytes: 7 * MB })]; + const files = [ + attachment({ file_id: 'exact', bytes: exact }), + attachment({ file_id: 'byte', bytes: 1 }), + ]; + + const allocated = await allocate(turn, historical, files); + + expect(pathsById(allocated)).toEqual({ exact: 'provider', byte: 'none' }); + expect(assertTurnLimits(turn, historical, allocated).totalKnownBytes).toBe(10 * MB); + expect(() => assertTurnLimits(turn, historical, files)).toThrow( + expect.objectContaining({ limitType: 'bytes', limit: 10 * MB }), + ); + }); + + it('holds the shared set to a tighter aggregate the agent endpoint configures', async () => { + const turn = setup({ + endpointConfig: { llmDeliveryPolicy: 'automatic', totalSizeLimit: 8 }, + fileConfig: { fileContextSizeLimit: 10 }, + }); + const historical = [attachment({ file_id: 'history', bytes: 4 * MB })]; + const files = [attachment({ file_id: 'a' }), attachment({ file_id: 'b', bytes: 2 * MB })]; + + const allocated = await allocate(turn, historical, files); + + expect(pathsById(allocated)).toEqual({ a: 'provider', b: 'none' }); + expect(() => assertAgentLimits(turn, historical, allocated)).not.toThrow(); + expect(() => assertAgentLimits(turn, historical, files)).toThrow( + expect.objectContaining({ limitType: 'bytes', limit: 8 * MB }), + ); + }); + }); + + it('returns copies decided by the turn routing for files it did not move', async () => { + const turn = setup({ fileConfig: { fileContextSizeLimit: 10 } }); + const stale = attachment({ file_id: 'stale', bytes: MB, llmDeliveryPath: 'text' }); + + const allocated = await allocate(turn, [], [stale]); + + expect(allocated).toEqual( + applyTurnDelivery([stale], { routing: turn.routing, consumers: searchOnly }), + ); + expect(allocated[0].llmDeliveryPath).toBe('provider'); + }); + }); +}); diff --git a/packages/api/src/files/reading/history.ts b/packages/api/src/files/reading/history.ts new file mode 100644 index 00000000000..d3900ed676a --- /dev/null +++ b/packages/api/src/files/reading/history.ts @@ -0,0 +1,237 @@ +import { EModelEndpoint } from 'librechat-data-provider'; +import type { + TFile, + DirectContentEntry, + DirectContentLimits, + TurnFileConsumers, +} from 'librechat-data-provider'; +import type { AgentAttachmentEndpointsByAgentId } from '~/agents/attachments'; +import type { TurnReadingAgent } from './diagnostics'; +import type { TurnReadingFile } from './turn'; +import type { ServerRequest } from '~/types'; +import { + collectFileIds, + listAgentEntries, + isModelBoundAttachmentFile, + measureModelBoundAttachment, + resolveAgentAttachmentLimits, + collectHistoricalAttachmentIds, +} from '~/agents/attachments'; +import { filterFilesByEndpointRuntimeConfig } from '~/files/filter'; +import { getTurnReadingContext } from './turn'; +import { settleTurnFiles } from './settle'; + +/** A turn copy of an attachment the endpoint's file policy can judge, as AgentClient holds it. */ +export type HistoryAllocationFile = TurnReadingFile & Pick; + +/** The endpoint whose attachment policy and limits hold the turn's shared attachments. */ +interface AttachmentScope { + req?: Pick; + endpoint: string; + endpointType?: string | null; + consumers?: TurnFileConsumers | null; +} + +export interface TurnAttachmentsWithHistoryParams { + agent: Pick; + /** The replayed files the conversation's history sends again, as this turn decided them. */ + historical: readonly T[]; + /** The request's attachments in submission order; earlier files are admitted first. */ + current: T[]; + req?: Pick; + /** The endpoint the turn's attachment checks run against; the agents endpoint when absent. */ + endpoint?: string | null; + endpointType?: string | null; + /** + * Every agent the topology check holds the shared set to, with its endpoint, so the + * allocation uses the strictest limits any of those checks applies. + */ + endpointsByAgentId?: AgentAttachmentEndpointsByAgentId; + /** Receives the re-decided request attachments, only when the allocation moved one. */ + onAllocated?: (files: T[]) => void; + signal?: AbortSignal; +} + +export interface RetainedContextAllocationParams + extends Omit, 'historical'> { + /** The conversation's setting; with files resent, history replay already allocated the turn. */ + resendFiles?: boolean | null; + /** The extracted text earlier messages keep in place of their files. */ + retained: readonly T[]; +} + +/** The tighter of two limits, where a missing or zero limit leaves the dimension unlimited. */ +const tighter = (first?: number, second?: number): number | undefined => { + if (!first) { + return second; + } + if (!second) { + return first; + } + return Math.min(first, second); +}; + +const tightenLimits = ( + first: DirectContentLimits, + second: DirectContentLimits, +): DirectContentLimits => ({ + count: tighter(first.count, second.count), + bytes: tighter(first.bytes, second.bytes), + textChars: tighter(first.textChars, second.textChars), +}); + +/** + * The limits every check on the shared set enforces together: the global context limits of + * `assertTurnAttachmentLimits`, and each reachable agent's own count allowance and endpoint + * aggregate, which the topology check applies to the same set. Agent-scoped context injections + * are counted after this allocation, so they can still exceed these limits. + */ +function resolveSharedLimits( + { req, endpoint, endpointType }: AttachmentScope, + endpointsByAgentId?: AgentAttachmentEndpointsByAgentId, +): DirectContentLimits { + const turn = resolveAgentAttachmentLimits({ + req, + endpoint, + endpointType, + enforceAttachmentCount: false, + useGlobalContextSizeLimit: true, + }); + return [{ endpoint, endpointType }, ...listAgentEntries(endpointsByAgentId)] + .map((agentEndpoint) => + resolveAgentAttachmentLimits({ + req, + endpoint: agentEndpoint.endpoint, + endpointType: agentEndpoint.endpointType, + }), + ) + .reduce(tightenLimits, turn); +} + +/** The files the endpoint's runtime policy keeps, as AgentClient selects model-bound files. */ +const selectCompatible = ( + files: T[], + { req, endpoint, endpointType, consumers }: AttachmentScope, +): T[] => + filterFilesByEndpointRuntimeConfig(req?.config, { + files, + endpoint, + endpointType, + skipTotalSizeLimit: true, + preserveTextSources: true, + consumers, + }); + +const uncharged = (file: HistoryAllocationFile): DirectContentEntry => ({ + fileId: file.file_id, + counts: false, + bytes: 0, + textChars: 0, +}); + +/** + * Charges replayed content as the shared-set checks charge `[...history, ...current]`: every copy + * spends the context budgets, and a file counts once, at its first copy, unless history alone + * replays it. + */ +function chargeReplayed( + measured: ReadonlyArray<{ file: HistoryAllocationFile; entry: DirectContentEntry }>, + historyOnlyIds: ReadonlySet, +): DirectContentEntry[] { + const firstCopy = new Map(); + measured.forEach(({ file }, index) => { + if (!firstCopy.has(file.file_id)) { + firstCopy.set(file.file_id, index); + } + }); + return measured.map(({ file, entry }, index) => ({ + ...entry, + counts: + entry.counts && firstCopy.get(file.file_id) === index && !historyOnlyIds.has(file.file_id), + })); +} + +/** + * Allocates the request's direct content after the turn's replayed history, so the history + * checks AgentClient runs next cannot refuse what the automatic policy admitted. The compatible + * model-bound history is charged first, then every current file the policy cannot move; the + * files the policy reads natively or as text are admitted in submission order while they fit. + * A file that does not fit is recorded as overflow and decided again, so it moves to File Search + * or Run Code, or is left unavailable, instead of failing the turn. A current file history + * replays as well is committed with it rather than allocated, since history sends it anyway. + * Returns `current` itself unless the agent's turn reads under the automatic policy and a file + * moved. + */ +export async function allocateTurnAttachmentsWithHistory({ + agent, + historical, + current, + req, + endpoint, + endpointType, + endpointsByAgentId, + onAllocated, + signal, +}: TurnAttachmentsWithHistoryParams): Promise { + const routing = agent.deliveryRouting; + if (getTurnReadingContext(routing)?.policy !== 'automatic' || current.length === 0) { + return current; + } + const scope: AttachmentScope = { + req, + endpoint: endpoint ?? EModelEndpoint.agents, + endpointType, + consumers: agent.fileConsumers, + }; + const committedHistory = selectCompatible(historical.filter(isModelBoundAttachmentFile), scope); + const replayedIds = collectFileIds(committedHistory); + const compatibleFileIds = collectFileIds(selectCompatible(current, scope)); + const measureCurrent = (file: T): DirectContentEntry => + compatibleFileIds.has(file.file_id) ? measureModelBoundAttachment(file) : uncharged(file); + const replayed = chargeReplayed( + [ + ...committedHistory.map((file) => ({ file, entry: measureModelBoundAttachment(file) })), + ...current + .filter((file) => replayedIds.has(file.file_id)) + .map((file) => ({ file, entry: measureCurrent(file) })), + ], + collectHistoricalAttachmentIds(committedHistory, current), + ); + const allocated = await settleTurnFiles({ + routing, + consumers: agent.fileConsumers, + files: current, + allocation: { + requestFileIds: current + .filter((file) => !replayedIds.has(file.file_id)) + .map((file) => file.file_id), + limits: resolveSharedLimits(scope, endpointsByAgentId), + measure: measureCurrent, + committedEntries: replayed, + agentId: agent.id, + scope: 'history', + }, + flush: true, + signal, + }); + if (allocated !== current) { + onAllocated?.(allocated); + } + return allocated; +} + +/** + * Allocates the request against the extracted text earlier messages retain when the conversation + * does not resend files, which the turn check charges in their place. With files resent, history + * replay ran {@link allocateTurnAttachmentsWithHistory} already, so this returns `current`. + */ +export function allocateTurnAttachmentsWithRetainedContext({ + resendFiles, + retained, + ...params +}: RetainedContextAllocationParams): Promise { + if (resendFiles !== false) { + return Promise.resolve(params.current); + } + return allocateTurnAttachmentsWithHistory({ ...params, historical: retained }); +} diff --git a/packages/api/src/files/reading/index.ts b/packages/api/src/files/reading/index.ts new file mode 100644 index 00000000000..0b66db69496 --- /dev/null +++ b/packages/api/src/files/reading/index.ts @@ -0,0 +1,7 @@ +export * from './turn'; +export * from './settle'; +export * from './diagnostics'; +export * from './upload'; +export * from './derive'; +export * from './inventory'; +export * from './native'; diff --git a/packages/api/src/files/reading/inventory.spec.ts b/packages/api/src/files/reading/inventory.spec.ts new file mode 100644 index 00000000000..e76f2e8b734 --- /dev/null +++ b/packages/api/src/files/reading/inventory.spec.ts @@ -0,0 +1,709 @@ +import { Tools, FileContext, FileSources } from 'librechat-data-provider'; +import type { + TFile, + TFileConfig, + FiltersConfig, + TurnFileConsumers, + TurnDeliveryRouting, +} from 'librechat-data-provider'; +import type { BuildTurnReadingContextParams, TurnReadingContext } from './turn'; +import type { CodeExecutionContext } from '~/agents/execution'; +import type { CodeFileAgent } from '~/files/code/queued'; +import type { ReadingAgent } from './inventory'; +import { buildTurnReadingContext, getTurnReadingContext, recordNativeRejections } from './turn'; +import { prepareAgentFileContext, renderLeftOutFiles } from './inventory'; +import { resolveTurnDeliveryRouting } from '~/agents/files/delivery'; +import { ContentFilterError } from '~/middleware/contentFilter'; + +type EndpointFileConfigInput = NonNullable[string]; + +const MB = 1024 * 1024; +const PDF = 'application/pdf'; +const DOCX = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'; +const XLSX = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'; +const PARQUET = 'application/vnd.apache.parquet'; +const SENTINEL = 'SENTINEL-7F3A'; +const NO_FILES_NOTE = `- Note: Semantic search is available through the ${Tools.file_search} tool but no files are currently loaded. Request the user to upload documents to search through.`; + +const AUTOMATIC: EndpointFileConfigInput = { llmDeliveryPolicy: 'automatic' }; +const RUNS_CODE: TurnFileConsumers = { executeCode: true, fileSearch: false }; +const SEARCHES: TurnFileConsumers = { executeCode: false, fileSearch: true }; +const NO_READER: TurnFileConsumers = { executeCode: false, fileSearch: false }; + +const attachment = (overrides: Partial & Pick): TFile => ({ + user: 'user-1', + object: 'file', + filepath: `/uploads/user-1/${overrides.filename}`, + type: XLSX, + bytes: 2048, + embedded: false, + usage: 0, + source: FileSources.local, + context: FileContext.message_attachment, + llmDeliveryPath: 'none', + metadata: { destinationChosen: false }, + ...overrides, +}); + +interface AgentSetup { + provider?: string; + files?: TFile[]; + consumers?: TurnFileConsumers; + endpointConfig?: EndpointFileConfigInput; + /** Absent leaves the agent without a provision state, as a failed provisioning does. */ + codeEnvFiles?: TFile[]; + vectorDBFiles?: TFile[]; + reading?: Partial; + primedSearchFileIds?: string[]; + dynamicToolContextMap?: Record; + codeExecutionContext?: CodeExecutionContext; +} + +type TestAgent = CodeFileAgent & ReadingAgent & { deliveryRouting: TurnDeliveryRouting }; + +/** A file tool is loaded unless a test says otherwise: the inventory is written only then. */ +function agentWith({ + provider = 'openAI', + files = [], + consumers = SEARCHES, + endpointConfig = AUTOMATIC, + codeEnvFiles, + vectorDBFiles = [], + reading, + primedSearchFileIds, + dynamicToolContextMap, + codeExecutionContext, +}: AgentSetup = {}): TestAgent { + const routing = resolveTurnDeliveryRouting({ + agent: { provider, endpoint: provider }, + config: { fileConfig: { endpoints: { [provider]: endpointConfig } } }, + }); + const context = buildTurnReadingContext({ + routing, + provider, + model: 'gpt-4o', + fileTokenLimit: 100_000, + configuredFileSizeLimit: undefined, + countTokens: (text) => text.length, + ...reading, + }); + if (context != null) { + routing.reading = context; + } + const provisioned = codeEnvFiles != null || vectorDBFiles.length > 0; + return { + id: 'agent-1', + deliveryRouting: routing, + fileConsumers: consumers, + currentRequestAttachments: files, + primedSearchFileIds, + dynamicToolContextMap, + codeExecutionContext, + provisionState: provisioned + ? { + codeEnvFiles: codeEnvFiles ?? [], + vectorDBFiles, + aliveFileIds: new Set(), + agentScopedFileIds: new Set(), + } + : undefined, + }; +} + +function readingContextOf(agent: TestAgent): TurnReadingContext { + const context = getTurnReadingContext(agent.deliveryRouting); + if (context == null) { + throw new Error('expected a turn reading context'); + } + return context; +} + +function inventoryOf(agent: TestAgent): string { + const inventory = agent.dynamicToolContextMap?.file_inventory; + if (typeof inventory !== 'string') { + throw new Error('expected a rendered inventory'); + } + return inventory; +} + +const lineFor = (agent: TestAgent, filename: string): string | undefined => + inventoryOf(agent) + .split('\n') + .find((line) => line.includes(JSON.stringify(filename))); + +const prepare = (agent: TestAgent, filters?: FiltersConfig): void => + prepareAgentFileContext(agent, [agent], 'user-1', false, { filters }); + +describe('prepareAgentFileContext', () => { + describe('one line per reading', () => { + it('lists a queued spreadsheet at the path the planner advertised in the same call', () => { + const workbook = attachment({ file_id: 'q3', filename: 'Q3.xlsx' }); + const agent = agentWith({ + files: [workbook], + consumers: RUNS_CODE, + codeEnvFiles: [workbook], + }); + + prepare(agent); + + const destination = agent.provisionState?.codeEnvDestinations?.get('q3'); + expect(destination).toBe('Q3.xlsx'); + expect(agent.dynamicToolContextMap?.queued_code_files).toContain(`/mnt/data/${destination}`); + expect(inventoryOf(agent).split('\n')).toEqual([ + '- Attached files and how you can read them on this turn (file contents are not listed here):', + `\t- "Q3.xlsx" (spreadsheet, file_id q3): read it with Run Code at /mnt/data/${destination}. It is copied when code first runs.`, + ]); + }); + + it('restates the programmatic data directory for an attached workspace', () => { + const workbook = attachment({ file_id: 'q3', filename: 'Q3.xlsx' }); + const agent = agentWith({ + files: [workbook], + consumers: RUNS_CODE, + codeEnvFiles: [workbook], + codeExecutionContext: { + baseUrl: 'https://code.example', + codeSessionKey: 'attached', + executionProfile: 'stateful', + statefulSessions: true, + environmentType: 'attached', + codeWorkspace: { + environmentId: 'personal-machine', + workspaceId: 'project', + operations: ['read_file'], + }, + }, + }); + + prepare(agent); + + const destination = agent.provisionState?.codeEnvDestinations?.get('q3'); + const path = `$LIBRECHAT_CODE_DATA_DIR/${destination}`; + expect(agent.dynamicToolContextMap?.queued_code_files).toContain(path); + expect(lineFor(agent, 'Q3.xlsx')).toBe( + `\t- "Q3.xlsx" (spreadsheet, file_id q3): scripts run by the programmatic Bash tool can read it at ${path}. It is copied when such a script first runs.`, + ); + expect(inventoryOf(agent)).not.toContain('/mnt/data/'); + expect(inventoryOf(agent)).not.toContain('Run Code'); + }); + + it('points a spreadsheet already in the code environment to the code tool file list', () => { + const workbook = attachment({ + file_id: 'q2', + filename: 'Q2.xlsx', + metadata: { + destinationChosen: false, + codeEnvRefs: { + default: { kind: 'user', id: 'user-1', storage_session_id: 'store', file_id: 'remote' }, + }, + }, + }); + const agent = agentWith({ files: [workbook], consumers: RUNS_CODE, codeEnvFiles: [] }); + + prepare(agent); + + expect(lineFor(agent, 'Q2.xlsx')).toBe( + '\t- "Q2.xlsx" (spreadsheet): read it with Run Code; it is already in the code environment (see the code tool\'s file list).', + ); + }); + + it('says a spreadsheet Run Code could not be prepared for has not been read', () => { + const workbook = attachment({ file_id: 'q1', filename: 'Q1.xlsx' }); + const agent = agentWith({ files: [workbook], consumers: RUNS_CODE }); + + prepare(agent); + + expect(agent.dynamicToolContextMap?.queued_code_files).toBeUndefined(); + expect(lineFor(agent, 'Q1.xlsx')).toBe( + '\t- "Q1.xlsx" (spreadsheet): Run Code is available, but this file could not be prepared on this turn. Do not claim to have read it.', + ); + }); + + it('notes where Run Code can also open a PDF sent with the message', () => { + const pdf = attachment({ + file_id: 'brief', + filename: 'brief.pdf', + type: PDF, + llmDeliveryPath: 'provider', + }); + const agent = agentWith({ files: [pdf], consumers: RUNS_CODE, codeEnvFiles: [pdf] }); + + prepare(agent); + + expect(lineFor(agent, 'brief.pdf')).toBe( + '\t- "brief.pdf" (PDF): sent with this message. Run Code can also open it at /mnt/data/brief.pdf.', + ); + }); + + it('says complete extracted text is included without listing it', () => { + const docx = attachment({ + file_id: 'notes', + filename: 'notes.docx', + type: DOCX, + text: `Board notes ${SENTINEL}`, + llmDeliveryPath: 'text', + }); + const agent = agentWith({ files: [docx] }); + + prepare(agent); + + expect(lineFor(agent, 'notes.docx')).toBe( + '\t- "notes.docx" (document): this model cannot read this type directly. Its complete extracted text is included in this message.', + ); + expect(inventoryOf(agent)).not.toContain(SENTINEL); + }); + + it('leaves a PDF too large to send to File Search, pending its first search', () => { + const pdf = attachment({ + file_id: 'manual', + filename: 'manual.pdf', + type: PDF, + bytes: 5 * MB, + llmDeliveryPath: 'provider', + }); + const agent = agentWith({ + files: [pdf], + consumers: SEARCHES, + vectorDBFiles: [pdf], + reading: { configuredFileSizeLimit: MB }, + }); + + prepare(agent); + + expect(lineFor(agent, 'manual.pdf')).toBe( + `\t- "manual.pdf" (PDF): too large to send directly. Search it with ${Tools.file_search}; it is indexed when you first search. Results are excerpts, not the whole document.`, + ); + expect(inventoryOf(agent).match(/manual\.pdf/g)).toHaveLength(1); + }); + + it('describes failed search preparation and its retry without claiming cached text was sent', () => { + const pdf = attachment({ + file_id: 'manual', + filename: 'manual.pdf', + type: PDF, + bytes: 5 * MB, + text: SENTINEL, + llmDeliveryPath: 'none', + }); + const agent = agentWith({ + files: [pdf], + consumers: SEARCHES, + vectorDBFiles: [pdf], + reading: { configuredFileSizeLimit: MB }, + }); + const preparation = new Map([[pdf.file_id, 'failed']]); + readingContextOf(agent).setSearchEvidence({ + queued: [pdf.file_id], + registered: [], + preparation, + }); + + prepare(agent); + expect(lineFor(agent, pdf.filename)).toContain('File Search could not prepare it'); + expect(lineFor(agent, pdf.filename)).toContain('Retry file_search to try indexing again'); + expect(lineFor(agent, pdf.filename)).toContain('do not claim to have read it'); + expect(inventoryOf(agent)).not.toContain(SENTINEL); + expect(inventoryOf(agent)).not.toContain('text is included'); + + preparation.set(pdf.file_id, 'ready'); + prepare(agent); + expect(lineFor(agent, pdf.filename)).toContain('Search it with file_search.'); + expect(lineFor(agent, pdf.filename)).not.toContain('could not prepare'); + expect(lineFor(agent, pdf.filename)).not.toContain('indexed when'); + }); + + it('sends text too long to include to File Search', () => { + const log = attachment({ + file_id: 'huge', + filename: 'huge.log', + type: 'text/plain', + text: `${SENTINEL} ${'x'.repeat(64)}`, + llmDeliveryPath: 'text', + }); + const agent = agentWith({ + files: [log], + consumers: SEARCHES, + reading: { fileTokenLimit: 10 }, + }); + + prepare(agent); + + expect(lineFor(agent, 'huge.log')).toBe( + `\t- "huge.log" (text): too long to include. Search it with ${Tools.file_search}; it is indexed when you first search. Results are excerpts, not the whole document.`, + ); + expect(inventoryOf(agent)).not.toContain(SENTINEL); + }); + + it('says a file no reader can take cannot be read on this turn', () => { + const parquet = attachment({ file_id: 'data', filename: 'data.parquet', type: PARQUET }); + const agent = agentWith({ files: [parquet], consumers: SEARCHES }); + + prepare(agent); + + expect(lineFor(agent, 'data.parquet')).toBe( + '\t- "data.parquet" (spreadsheet): cannot be read on this turn because Run Code is not available. Say so if asked; do not claim to have read it.', + ); + }); + + it('says only an extracted-text copy of a text-only record was kept', () => { + const textOnly = attachment({ + file_id: 'old', + filename: 'old.xlsx', + source: FileSources.text, + text: `Q1,1200 ${SENTINEL}`, + llmDeliveryPath: 'text', + }); + const agent = agentWith({ files: [textOnly], consumers: RUNS_CODE, codeEnvFiles: [] }); + + prepare(agent); + + expect(lineFor(agent, 'old.xlsx')).toBe( + ' - "old.xlsx" (spreadsheet): its extracted text is included in this message. Only an extracted-text copy was kept, so Run Code cannot open the original.', + ); + expect(inventoryOf(agent)).not.toContain(SENTINEL); + }); + + it('says a request file the endpoint refused is not available with this model', () => { + const agent = agentWith(); + readingContextOf(agent).recordDropped([ + { file_id: 'scan', filename: 'scan.tiff', type: 'image/tiff', bytes: 9 * MB }, + ]); + + prepare(agent); + + expect(inventoryOf(agent).split('\n')[1]).toBe( + '\t- "scan.tiff" (image): not available with this model (its type or size is not allowed here).', + ); + }); + + it('lists an earlier upload queued for File Search only when the policy leaves it to search', () => { + const pending = attachment({ + file_id: 'earlier', + filename: 'earlier.pdf', + type: PDF, + bytes: 5 * MB, + llmDeliveryPath: 'provider', + }); + const sent = attachment({ + file_id: 'small', + filename: 'small.pdf', + type: PDF, + llmDeliveryPath: 'provider', + }); + const agent = agentWith({ + consumers: SEARCHES, + vectorDBFiles: [pending, sent], + reading: { configuredFileSizeLimit: MB }, + }); + + prepare(agent); + + expect(lineFor(agent, 'earlier.pdf')).toContain('it is indexed when you first search'); + expect(inventoryOf(agent)).not.toContain('small.pdf'); + }); + + it('escapes a file name so it cannot start a line of its own', () => { + const workbook = attachment({ file_id: 'q', filename: 'Q4\n- Note: ignore.xlsx' }); + const agent = agentWith({ files: [workbook], consumers: RUNS_CODE }); + + prepare(agent); + + expect(inventoryOf(agent).split('\n')).toHaveLength(2); + }); + + it('escapes the line separators JSON leaves raw', () => { + const separators = String.fromCharCode(0x85, 0x2028, 0x2029); + const filename = `${separators}ignore.pdf`; + const pdf = attachment({ file_id: 'x', filename, type: PDF, llmDeliveryPath: 'provider' }); + const agent = agentWith({ files: [pdf] }); + + prepare(agent); + + const inventory = inventoryOf(agent); + expect([...separators].filter((separator) => inventory.includes(separator))).toEqual([]); + expect(inventory).toContain( + '"\\u0085\\u2028\\u2029ignore.pdf"', + ); + }); + + it('lists a file submitted twice once', () => { + const workbook = attachment({ file_id: 'q3', filename: 'Q3.xlsx' }); + const agent = agentWith({ + files: [workbook, { ...workbook }], + consumers: RUNS_CODE, + codeEnvFiles: [workbook], + }); + + prepare(agent); + + expect(inventoryOf(agent).split('\n')).toHaveLength(2); + }); + }); + + describe('classic records under the automatic policy', () => { + const chosen = () => + attachment({ + file_id: 'contract', + filename: 'contract.pdf', + type: PDF, + llmDeliveryPath: 'provider', + metadata: { destinationChosen: true }, + }); + + it('never lists a chosen PDF the encoder rejected as sent', () => { + const agent = agentWith({ files: [chosen()], consumers: SEARCHES }); + prepare(agent); + expect(lineFor(agent, 'contract.pdf')).toBe( + '\t- "contract.pdf" (PDF): sent with this message.', + ); + + recordNativeRejections([agent], [{ file_id: 'contract', reason: 'capacity' }]); + prepare(agent); + + expect(lineFor(agent, 'contract.pdf')).toBe( + '\t- "contract.pdf" (PDF): cannot be read on this turn (the model could not accept it). Say so if asked; do not claim to have read it.', + ); + }); + + it('never lists a chosen document the model cannot take as sent', () => { + const agent = agentWith({ + files: [{ ...chosen(), filename: 'contract.docx', type: DOCX }], + consumers: SEARCHES, + provider: 'anthropic', + }); + + prepare(agent); + + expect(lineFor(agent, 'contract.docx')).toBe( + '\t- "contract.docx" (document): cannot be read on this turn (this model cannot read this type directly). Say so if asked; do not claim to have read it.', + ); + }); + }); + + describe('for an agent that does not receive the request', () => { + const childPrepare = (agent: TestAgent): void => + prepareAgentFileContext(agent, [agent], 'user-1', false, { receivesRequest: false }); + + it('never says a request file was sent or included, and keeps the Run Code paths', () => { + const pdf = attachment({ + file_id: 'brief', + filename: 'brief.pdf', + type: PDF, + llmDeliveryPath: 'provider', + }); + const docx = attachment({ + file_id: 'notes', + filename: 'notes.docx', + type: DOCX, + text: 'Board notes', + llmDeliveryPath: 'text', + }); + const workbook = attachment({ file_id: 'q3', filename: 'Q3.xlsx' }); + const agent = agentWith({ + files: [pdf, docx, workbook], + consumers: RUNS_CODE, + codeEnvFiles: [pdf, workbook], + }); + + childPrepare(agent); + + const inventory = inventoryOf(agent); + expect(inventory).not.toMatch(/sent with this message|included in this message/); + expect(lineFor(agent, 'brief.pdf')).toBe( + '\t- "brief.pdf" (PDF): not sent with your task. Do not claim to have read it unless it is shared with you. Run Code can also open it at /mnt/data/brief.pdf.', + ); + expect(lineFor(agent, 'notes.docx')).toBe( + '\t- "notes.docx" (document): not sent with your task. Do not claim to have read it unless it is shared with you.', + ); + expect(lineFor(agent, 'Q3.xlsx')).toContain('read it with Run Code at /mnt/data/Q3.xlsx.'); + }); + }); + + describe('renderLeftOutFiles', () => { + it('names a document the encoder rejected as read by File Search, in the inventory and the left-out list', () => { + const pdf = attachment({ + file_id: 'brief', + filename: 'brief.pdf', + type: PDF, + llmDeliveryPath: 'provider', + }); + const agent = agentWith({ files: [pdf], consumers: SEARCHES, vectorDBFiles: [pdf] }); + prepare(agent); + expect(lineFor(agent, 'brief.pdf')).toBe('\t- "brief.pdf" (PDF): sent with this message.'); + + recordNativeRejections([agent], [{ file_id: 'brief', reason: 'integrity' }]); + prepare(agent); + + const leftOut = `\t- "brief.pdf" (PDF): the model could not accept it. Search it with ${Tools.file_search}; it is indexed when you first search. Results are excerpts, not the whole document.`; + expect(lineFor(agent, 'brief.pdf')).toBe(leftOut); + expect(renderLeftOutFiles(agent, [pdf, pdf]).split('\n')).toEqual([ + '- Shared files left out of this message (file contents are not listed here):', + leftOut, + ]); + expect(renderLeftOutFiles(agent, [])).toBe(''); + }); + }); + + const classicAgent = (endpointConfig: EndpointFileConfigInput): TestAgent => { + const pdf = attachment({ + file_id: 'manual', + filename: 'manual.pdf', + type: PDF, + bytes: 5 * MB, + llmDeliveryPath: 'provider', + }); + const workbook = attachment({ file_id: 'q3', filename: 'Q3.xlsx' }); + return agentWith({ + files: [pdf, workbook], + consumers: { executeCode: true, fileSearch: true }, + endpointConfig, + codeEnvFiles: [workbook], + vectorDBFiles: [pdf], + primedSearchFileIds: [], + dynamicToolContextMap: { [Tools.file_search]: NO_FILES_NOTE, file_inventory: 'stale' }, + }); + }; + const untooledAgent = (): TestAgent => { + const pdf = attachment({ + file_id: 'manual', + filename: 'manual.pdf', + type: PDF, + bytes: 5 * MB, + llmDeliveryPath: 'provider', + }); + const workbook = attachment({ file_id: 'q3', filename: 'Q3.xlsx' }); + const agent = agentWith({ + files: [pdf, workbook], + consumers: NO_READER, + reading: { configuredFileSizeLimit: MB }, + primedSearchFileIds: [], + dynamicToolContextMap: { [Tools.file_search]: NO_FILES_NOTE, file_inventory: 'stale' }, + }); + readingContextOf(agent).recordDropped([ + { file_id: 'scan', filename: 'scan.tiff', type: 'image/tiff', bytes: 9 * MB }, + ]); + return agent; + }; + const queuedCodeOnly = { + [Tools.file_search]: NO_FILES_NOTE, + queued_code_files: expect.stringContaining('/mnt/data/Q3.xlsx'), + }; + + it.each<[string, () => TestAgent, Record]>([ + ['under classic routing with no policy set', () => classicAgent({}), queuedCodeOnly], + ['under classic routing', () => classicAgent({ llmDeliveryPolicy: 'classic' }), queuedCodeOnly], + ['when no file tool is loaded', untooledAgent, { [Tools.file_search]: NO_FILES_NOTE }], + ])('writes no inventory and leaves the File Search note alone %s', (_case, arrange, expected) => { + const agent = arrange(); + + prepare(agent); + + expect(agent.dynamicToolContextMap).toEqual(expected); + }); + + describe('the File Search note', () => { + const oversized = () => + attachment({ + file_id: 'manual', + filename: 'manual.pdf', + type: PDF, + bytes: 5 * MB, + llmDeliveryPath: 'provider', + }); + const agentFor = (file: TFile, primedSearchFileIds?: string[]) => + agentWith({ + files: [file], + consumers: SEARCHES, + vectorDBFiles: [file], + reading: { configuredFileSizeLimit: MB }, + primedSearchFileIds, + dynamicToolContextMap: { [Tools.file_search]: NO_FILES_NOTE }, + }); + + it('is replaced by the inventory when nothing was primed and a file awaits indexing', () => { + const agent = agentFor(oversized(), []); + + prepare(agent); + + expect(agent.dynamicToolContextMap?.[Tools.file_search]).toBeUndefined(); + expect(lineFor(agent, 'manual.pdf')).toContain('it is indexed when you first search'); + }); + + it.each([ + ['the loader primed files', ['registered'], oversized()], + ['the primed files are unknown', undefined, oversized()], + [ + 'the file is already indexed', + [], + { ...oversized(), embedded: true, metadata: { destinationChosen: false } }, + ], + ])('stays when %s', (_case, primedSearchFileIds, file) => { + const agent = agentFor(file, primedSearchFileIds); + + prepare(agent); + + expect(agent.dynamicToolContextMap?.[Tools.file_search]).toBe(NO_FILES_NOTE); + }); + + it('stays when no file is left to File Search', () => { + const workbook = attachment({ file_id: 'q3', filename: 'Q3.xlsx' }); + const agent = agentWith({ + files: [workbook], + consumers: { executeCode: true, fileSearch: true }, + codeEnvFiles: [workbook], + primedSearchFileIds: [], + dynamicToolContextMap: { [Tools.file_search]: NO_FILES_NOTE }, + }); + + prepare(agent); + + expect(agent.dynamicToolContextMap?.[Tools.file_search]).toBe(NO_FILES_NOTE); + expect(lineFor(agent, 'Q3.xlsx')).toContain('read it with Run Code'); + }); + }); + + describe('content policy', () => { + const filters: FiltersConfig = { + files: { + pii: { + fields: ['content'], + starterPatterns: [], + customPatterns: [{ id: 'private', label: 'private value', regex: 'PRIVATE-[A-Z]+' }], + }, + }, + }; + + it('inspects the rendered inventory before writing it', () => { + const workbook = attachment({ file_id: 'plan', filename: 'PRIVATE-PLAN.xlsx' }); + const agent = agentWith({ files: [workbook], consumers: RUNS_CODE }); + + expect(() => prepare(agent, filters)).toThrow(ContentFilterError); + expect(agent.dynamicToolContextMap?.file_inventory).toBeUndefined(); + }); + + it('passes an inventory whose file text alone matches, since contents are never listed', () => { + const workbook = attachment({ + file_id: 'plan', + filename: 'plan.xlsx', + source: FileSources.text, + text: 'PRIVATE-VALUE', + llmDeliveryPath: 'text', + }); + const agent = agentWith({ files: [workbook], consumers: RUNS_CODE }); + + prepare(agent, filters); + + expect(inventoryOf(agent)).not.toContain('PRIVATE-VALUE'); + }); + + it('skips the check without filters and under classic routing', () => { + const workbook = attachment({ file_id: 'plan', filename: 'PRIVATE-PLAN.xlsx' }); + const automatic = agentWith({ files: [workbook], consumers: RUNS_CODE }); + const classic = agentWith({ files: [workbook], consumers: RUNS_CODE, endpointConfig: {} }); + + expect(() => prepare(automatic)).not.toThrow(); + expect(() => prepare(classic, filters)).not.toThrow(); + expect(inventoryOf(automatic)).toContain('PRIVATE-PLAN.xlsx'); + }); + }); +}); diff --git a/packages/api/src/files/reading/inventory.ts b/packages/api/src/files/reading/inventory.ts new file mode 100644 index 00000000000..9b10317478e --- /dev/null +++ b/packages/api/src/files/reading/inventory.ts @@ -0,0 +1,467 @@ +import { + getRoutingMimeType, + hasFileToolConsumer, + Tools, + EToolResources, + isTextOnlyRecord, + decideFileReading, + categorizeForReading, + getCodeEnvRefForProfile, + resolveLLMDeliveryPolicy, + hasToolResourceProvisioning, +} from 'librechat-data-provider'; +import type { + FileReading, + FiltersConfig, + ReadingReason, + TurnDeliveryRouting, +} from 'librechat-data-provider'; +import type { CodeFileAgent, ProvisionToolContext } from '~/files/code/queued'; +import type { SearchPreparationState, TurnReadingFile } from './turn'; +import type { CodeFileLocation } from '~/files/code/priming'; +import { CODE_FILE_CONTEXT_ROOTS, getCodeFileLocation } from '~/files/code/priming'; +import { assertModelBoundContent } from '~/middleware/modelBoundContent'; +import { prepareQueuedCodeFileContext } from '~/files/code/queued'; +import { getCodeExecutionRouteKey } from '~/agents/execution'; +import { getTurnReadingContext } from './turn'; + +/** The fields of an initialized agent the file inventory reads. */ +export interface ReadingAgent + extends Pick< + CodeFileAgent, + 'id' | 'fileConsumers' | 'provisionState' | 'codeExecutionContext' | 'dynamicToolContextMap' + > { + deliveryRouting?: TurnDeliveryRouting; + currentRequestAttachments?: readonly TurnReadingFile[]; + /** Files the File Search advert listed on this turn; unknown when absent. */ + primedSearchFileIds?: readonly string[]; +} + +/** A file the File Search advert listed, as the tool loader primed it. */ +export interface PrimedSearchFile { + file_id: string; + filename: string; + fromAgent: boolean; +} + +/** The fields one file's reading and code location are decided from. */ +type DecidingAgent = Pick< + ReadingAgent, + 'deliveryRouting' | 'fileConsumers' | 'provisionState' | 'codeExecutionContext' +>; + +/** Where Run Code can open a file on this turn. */ +export type CodeFileState = + | { state: 'queued'; path: string; location: CodeFileLocation } + | { state: 'live' } + | { state: 'not_prepared' }; + +type QueuedCodeFile = Extract; + +/** Why a classic provider reading was not delivered, from the turn's encoder evidence. */ +type WithheldReason = Extract; + +/** One inventory line's inputs: a file with its reading, or a file the endpoint refused. */ +export type InventoryEntry = + | { + kind: 'read'; + file: TurnReadingFile; + reading: FileReading; + code?: CodeFileState; + search?: SearchPreparationState; + withheld?: WithheldReason; + } + | { kind: 'dropped'; file: TurnReadingFile }; + +type ReadEntry = Extract; + +export interface AgentFileContextOptions { + /** The content policy the rendered inventory must pass before it reaches the model. */ + filters?: FiltersConfig; + /** + * Whether the agent's messages carry the request and its attachments. A subagent child + * receives only its task, so nothing sent with the request is described as sent to it. + */ + receivesRequest?: boolean; +} + +/** Whose messages the rendered lines describe. */ +export interface InventoryView { + receivesRequest: boolean; +} + +const REQUEST_VIEW: InventoryView = { receivesRequest: true }; + +const INVENTORY_KEY = 'file_inventory'; + +const INVENTORY_HEADER = + '- Attached files and how you can read them on this turn (file contents are not listed here):'; + +const LEFT_OUT_HEADER = + '- Shared files left out of this message (file contents are not listed here):'; + +const REASON_PHRASES: Readonly>> = { + native_capacity: 'too large to send directly', + native_unsupported: 'this model cannot read this type directly', + native_rejected: 'the model could not accept it', + text_exceeds: 'too long to include', + aggregate_overflow: 'did not fit with the other attachments', +}; + +const NOT_READ = 'Say so if asked; do not claim to have read it.'; +const NOT_IN_TASK = + 'not sent with your task. Do not claim to have read it unless it is shared with you.'; +const TEXT_ONLY = 'only an extracted-text copy was kept, so Run Code cannot open the original.'; +const SEARCH_EXCERPTS = 'Results are excerpts, not the whole document.'; +const CODE_FILE_LIST = "(see the code tool's file list)"; + +const READABLE_CLASSIC_READERS: ReadonlySet = new Set([ + 'provider', + 'text', + 'search', + 'code', +]); + +/** Readers whose content travels inside the request's own message. */ +const MESSAGE_READERS: ReadonlySet = new Set(['provider', 'text']); + +/** Line separators JSON leaves raw, which some renderers still break a line on. */ +const RAW_LINE_BREAKS = new RegExp(`[${String.fromCharCode(0x85, 0x2028, 0x2029)}]`, 'g'); + +const NOT_PREPARED: CodeFileState = { state: 'not_prepared' }; + +const capitalize = (sentence: string): string => + sentence.charAt(0).toUpperCase() + sentence.slice(1); + +const codeRouteKey = (agent: DecidingAgent): string => + agent.codeExecutionContext == null + ? 'default' + : getCodeExecutionRouteKey(agent.codeExecutionContext); + +/** + * Queued paths come only from the destinations the planner just wrote, so the inventory and the + * lazy upload name the same path (#16578). + */ +function locateCodeFile(agent: DecidingAgent, file: TurnReadingFile): CodeFileState { + const destination = agent.provisionState?.codeEnvDestinations?.get(file.file_id); + if (destination != null) { + const location = getCodeFileLocation(agent.codeExecutionContext); + return { + state: 'queued', + path: `${CODE_FILE_CONTEXT_ROOTS[location]}/${destination}`, + location, + }; + } + if (getCodeEnvRefForProfile(file.metadata, codeRouteKey(agent)) != null) { + return { state: 'live' }; + } + return NOT_PREPARED; +} + +/** + * A classic provider reading names no other reader, so the turn's encoder evidence is what says + * whether the file went out: a recorded rejection, or a type the encoder never emits. + */ +function findWithheldReason( + agent: DecidingAgent, + file: TurnReadingFile, + reading: FileReading, +): WithheldReason | undefined { + if (reading.automatic || reading.reader !== 'provider') { + return undefined; + } + const evidence = getTurnReadingContext(agent.deliveryRouting)?.judge(file); + if (evidence?.rejected != null) { + return evidence.rejected === 'unsupported' ? 'native_unsupported' : 'native_rejected'; + } + return evidence?.native === 'unsupported' ? 'native_unsupported' : undefined; +} + +function readEntry(agent: DecidingAgent, file: TurnReadingFile): ReadEntry { + const reading = decideFileReading({ + routing: agent.deliveryRouting, + file, + consumers: agent.fileConsumers, + }); + const withheld = findWithheldReason(agent, file, reading); + const search = + reading.reader === 'search' + ? getTurnReadingContext(agent.deliveryRouting)?.searchState(file.file_id) + : undefined; + return { + kind: 'read', + file, + reading, + ...(reading.code === 'eligible' && { code: locateCodeFile(agent, file) }), + ...(search != null && { search }), + ...(withheld != null && { withheld }), + }; +} + +/** Classic routes are listed only where they name a reader; the automatic walk always does. */ +const isListed = ({ file, reading }: ReadEntry): boolean => + reading.automatic || isTextOnlyRecord(file) || READABLE_CLASSIC_READERS.has(reading.reader); + +const isAutomaticSearch = ({ reading }: ReadEntry): boolean => + reading.automatic && reading.reader === 'search'; + +/** The files not in `seen`, once each and in order. */ +function selectUnseen( + files: readonly TurnReadingFile[], + seen: ReadonlySet, +): TurnReadingFile[] { + const selected = new Map(); + for (const file of files) { + if (!seen.has(file.file_id) && !selected.has(file.file_id)) { + selected.set(file.file_id, file); + } + } + return [...selected.values()]; +} + +/** + * Decides each file the inventory lists once, from the turn's current evidence: the request's + * attachments, files queued for File Search that the automatic policy leaves to it, and request + * attachments the endpoint refused. + */ +export function collectInventoryEntries(agent: ReadingAgent): InventoryEntry[] { + const currentFiles = selectUnseen(agent.currentRequestAttachments ?? [], new Set()); + const seen = new Set(currentFiles.map(({ file_id }) => file_id)); + const pendingSearch = selectUnseen(agent.provisionState?.vectorDBFiles ?? [], seen) + .map((file) => readEntry(agent, file)) + .filter(isAutomaticSearch); + pendingSearch.forEach(({ file }) => seen.add(file.file_id)); + const dropped = selectUnseen(getTurnReadingContext(agent.deliveryRouting)?.dropped() ?? [], seen); + return [ + ...currentFiles.map((file) => readEntry(agent, file)).filter(isListed), + ...pendingSearch, + ...dropped.map((file): InventoryEntry => ({ kind: 'dropped', file })), + ]; +} + +const isPendingSearch = (entry: InventoryEntry): boolean => + entry.kind === 'read' && + entry.reading.reader === 'search' && + !hasToolResourceProvisioning(entry.file, EToolResources.file_search); + +/** + * With nothing primed, the File Search advert can only be its "no files are currently loaded" + * note, which a file awaiting indexing contradicts; the inventory lists that file instead. + */ +function reconcileSearchAdvert(agent: ReadingAgent, entries: readonly InventoryEntry[]): void { + if (agent.primedSearchFileIds?.length !== 0 || agent.dynamicToolContextMap == null) { + return; + } + if (entries.some(isPendingSearch)) { + delete agent.dynamicToolContextMap[Tools.file_search]; + } +} + +function describeType( + file: TurnReadingFile, + category: FileReading['category'] = categorizeForReading(getRoutingMimeType(file)), +): string { + const mimeType = getRoutingMimeType(file); + if (category === 'tabular') { + return 'spreadsheet'; + } + if (category === 'media') { + return mimeType.split('/', 1)[0].toLowerCase(); + } + if (mimeType === 'application/pdf') { + return 'PDF'; + } + return mimeType.startsWith('text/') ? 'text' : 'document'; +} + +/** Primed files on an attached workspace reach only scripts run by programmatic Bash. */ +const describeQueuedReader = (code: QueuedCodeFile, also = ''): string => + code.location === 'programmatic' + ? `scripts run by the programmatic Bash tool can${also} read it at ${code.path}` + : `Run Code can${also} open it at ${code.path}`; + +function describeCodeReader(code: CodeFileState): string { + if (code.state === 'queued' && code.location === 'programmatic') { + return `${describeQueuedReader(code)}. It is copied when such a script first runs.`; + } + if (code.state === 'queued') { + return `read it with Run Code at ${code.path}. It is copied when code first runs.`; + } + if (code.state === 'live') { + return `read it with Run Code; it is already in the code environment ${CODE_FILE_LIST}.`; + } + return 'Run Code is available, but this file could not be prepared on this turn. Do not claim to have read it.'; +} + +function describeAlsoInCode(code?: CodeFileState): string | undefined { + if (code?.state === 'queued') { + return `${capitalize(describeQueuedReader(code, ' also'))}.`; + } + return code?.state === 'live' ? `Run Code can also open it ${CODE_FILE_LIST}.` : undefined; +} + +const withAlsoInCode = (lead: string, code?: CodeFileState): string => { + const alsoInCode = describeAlsoInCode(code); + return alsoInCode == null ? lead : `${lead} ${alsoInCode}`; +}; + +function describeSearch(file: TurnReadingFile, state?: SearchPreparationState): string { + if (state === 'failed') { + return `File Search could not prepare it on this turn. Retry ${Tools.file_search} to try indexing again. ${NOT_READ}`; + } + const indexed = + state === 'ready' || hasToolResourceProvisioning(file, EToolResources.file_search); + const pending = indexed ? '' : '; it is indexed when you first search'; + return `search it with ${Tools.file_search}${pending}. ${SEARCH_EXCERPTS}`; +} + +function describeUnavailable(reading: FileReading): string { + if (reading.reason === 'code_unavailable') { + return `cannot be read on this turn because Run Code is not available. ${NOT_READ}`; + } + const phrase = REASON_PHRASES[reading.reason]; + return `cannot be read on this turn${phrase == null ? '' : ` (${phrase})`}. ${NOT_READ}`; +} + +function describeReader( + file: TurnReadingFile, + reading: FileReading, + code: CodeFileState = NOT_PREPARED, + search?: SearchPreparationState, +): string { + switch (reading.reader) { + case 'provider': + return 'sent with this message.'; + case 'text': + return reading.automatic + ? 'its complete extracted text is included in this message.' + : 'its extracted text is included in this message.'; + case 'search': + return describeSearch(file, search); + case 'code': + return describeCodeReader(code); + default: + return describeUnavailable(reading); + } +} + +function describeRead( + { file, reading, code, search, withheld }: ReadEntry, + { receivesRequest }: InventoryView, +): string { + if (!receivesRequest && MESSAGE_READERS.has(reading.reader)) { + return withAlsoInCode(NOT_IN_TASK, code); + } + if (withheld != null) { + return withAlsoInCode( + `cannot be read on this turn (${REASON_PHRASES[withheld]}). ${NOT_READ}`, + code, + ); + } + if (isTextOnlyRecord(file)) { + return reading.reader === 'text' + ? `its extracted text is included in this message. ${capitalize(TEXT_ONLY)}` + : TEXT_ONLY; + } + const status = describeReader(file, reading, code, search); + if (reading.reader === 'search' && search === 'failed') { + return withAlsoInCode(status, code); + } + if (reading.reader === 'unavailable' || reading.reader === 'unresolved') { + return status; + } + const phrase = REASON_PHRASES[reading.reason]; + const lead = phrase == null ? status : `${phrase}. ${capitalize(status)}`; + return reading.reader === 'code' ? lead : withAlsoInCode(lead, code); +} + +const escapeLineBreak = (separator: string): string => + `\\u${separator.charCodeAt(0).toString(16).padStart(4, '0')}`; + +/** A file name as one quoted token: JSON escapes plus the line separators JSON leaves raw. */ +const quoteFilename = (filename?: string | null): string => + JSON.stringify(filename ?? 'attachment').replace(RAW_LINE_BREAKS, escapeLineBreak); + +function renderEntry(entry: InventoryEntry, view: InventoryView): string { + const { file } = entry; + const name = quoteFilename(file.filename); + if (entry.kind === 'dropped') { + return `${name} (${describeType(file)}): not available with this model (its type or size is not allowed here).`; + } + const { reading, code } = entry; + const type = describeType(file, reading.category); + const queuedForCode = reading.reader === 'code' && code?.state === 'queued'; + const label = queuedForCode ? `${type}, file_id ${file.file_id}` : type; + return `${name} (${label}): ${describeRead(entry, view)}`; +} + +const renderLines = ( + header: string, + entries: readonly InventoryEntry[], + view: InventoryView, +): string => + entries.length === 0 + ? '' + : [header, ...entries.map((entry) => `\t- ${renderEntry(entry, view)}`)].join('\n'); + +/** + * The model's view of how each attachment is read on this turn. Pending preparation is worded + * apart from delivery, and no line carries file contents. Empty when nothing is listed. + */ +export function renderFileInventory( + entries: readonly InventoryEntry[], + view: InventoryView = REQUEST_VIEW, +): string { + return renderLines(INVENTORY_HEADER, entries, view); +} + +/** + * Names the shared files a message left out once the turn's evidence for them is recorded, and + * how each can still be read, so none disappears unnoticed. Empty without such files. + */ +export function renderLeftOutFiles( + agent: DecidingAgent, + files: readonly TurnReadingFile[], +): string { + return renderLines( + LEFT_OUT_HEADER, + selectUnseen(files, new Set()).map((file) => readEntry(agent, file)), + REQUEST_VIEW, + ); +} + +/** + * Plans the agent's queued code files, then lists how the automatic policy reads each attachment + * as `file_inventory`. The planner runs first in the same call, so the inventory restates its + * destinations and a re-plan re-renders both. Classic routing, and a turn that loads no file + * tool, write no inventory and leave the File Search advert alone. + */ +export function prepareAgentFileContext( + agent: CodeFileAgent & ReadingAgent, + contexts: Iterable, + userId?: string, + useAdvertisedNames = false, + { filters, receivesRequest = true }: AgentFileContextOptions = {}, +): void { + prepareQueuedCodeFileContext(agent, contexts, userId, useAdvertisedNames); + if (agent.dynamicToolContextMap) { + delete agent.dynamicToolContextMap[INVENTORY_KEY]; + } + if ( + resolveLLMDeliveryPolicy(agent.deliveryRouting?.endpointConfig) !== 'automatic' || + !hasFileToolConsumer(agent.fileConsumers) + ) { + return; + } + const entries = collectInventoryEntries(agent); + reconcileSearchAdvert(agent, entries); + const inventory = renderFileInventory(entries, { receivesRequest }); + if (!inventory) { + return; + } + if (filters) { + assertModelBoundContent({ filters, files: [{ content: inventory }] }); + } + agent.dynamicToolContextMap ??= {}; + agent.dynamicToolContextMap[INVENTORY_KEY] = inventory; +} diff --git a/packages/api/src/files/reading/native.spec.ts b/packages/api/src/files/reading/native.spec.ts new file mode 100644 index 00000000000..829f9fe8c2b --- /dev/null +++ b/packages/api/src/files/reading/native.spec.ts @@ -0,0 +1,678 @@ +import mongoose from 'mongoose'; +import { Readable } from 'stream'; +import { Providers } from '@librechat/agents'; +import { createModels } from '@librechat/data-schemas'; +import { megabyte, FileContext, FileSources, decideFileReading } from 'librechat-data-provider'; +import type { TFile, FiltersConfig, TurnFileConsumers } from 'librechat-data-provider'; +import type { IMongoFile } from '@librechat/data-schemas'; +import type { DocumentBlock, ServerRequest, StrategyFunctions } from '~/types'; +import { + getTurnTextOptions, + encodeNativeDocuments, + recordNativeRejections, + buildTurnReadingContext, +} from './turn'; +import { assertAgentAttachmentLimits, isModelBoundAttachmentFile } from '~/agents/attachments'; +import { resolveTurnDeliveryRouting, applyTurnDelivery } from '~/agents/files/delivery'; +import { admitNativeFallbackAttachments, prepareMessageAttachments } from './native'; +import { encodeAndFormatDocuments } from '~/files/encode/document'; +import { ContentFilterError } from '~/middleware/contentFilter'; +import { parseDocument } from '~/files/documents/crud'; +import { extractFileContext } from '~/files/context'; +import { createFileTextDeriver } from './derive'; +import { parseTextNative } from '~/files/text'; +import { prepareTurnFiles } from './settle'; +import Tokenizer from '~/utils/tokenizer'; + +/* Bridge Jest's CommonJS loader to the real PDF.js ESM implementation, including its worker. */ +jest.mock('pdfjs-dist/legacy/build/pdf.mjs', () => { + const { createRequire } = process.getBuiltinModule('node:module'); + return createRequire(__filename)('pdfjs-dist/legacy/build/pdf.mjs'); +}); + +const FileModel = createModels(mongoose).File; +const noTools: TurnFileConsumers = { executeCode: false, fileSearch: false }; +const search: TurnFileConsumers = { executeCode: false, fileSearch: true }; +const code: TurnFileConsumers = { executeCode: true, fileSearch: false }; +let countTokens: (text: string) => number; + +beforeAll(async () => { + countTokens = await Tokenizer.createExactTokenCounter('o200k_base'); +}); + +interface StoredPdf { + file: TFile; + original: Buffer; +} + +interface Message { + documents: DocumentBlock[]; + fileContext?: string; +} + +/** An uncompressed PDF with a correct xref table, text on its first page and blank later pages. */ +function pdfBytes(title: string, pages = 101): Buffer { + const content = `BT /F1 12 Tf 20 100 Td (${title}) Tj ET`; + const objects = [ + '<< /Type /Catalog /Pages 2 0 R >>', + `<< /Type /Pages /Kids [${Array.from({ length: pages }, (_, i) => `${i + 6} 0 R`).join(' ')}] /Count ${pages} >>`, + '<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>', + `<< /Length ${content.length} >>\nstream\n${content}\nendstream`, + '<< /Length 0 >>\nstream\n\nendstream', + ...Array.from( + { length: pages }, + (_, i) => + `<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Resources << /Font << /F1 3 0 R >> >> /Contents ${i === 0 ? 4 : 5} 0 R >>`, + ), + ]; + let document = '%PDF-1.4\n'; + const offsets = [0]; + for (const [index, object] of objects.entries()) { + offsets.push(Buffer.byteLength(document)); + document += `${index + 1} 0 obj\n${object}\nendobj\n`; + } + const xref = Buffer.byteLength(document); + document += `xref\n0 ${objects.length + 1}\n0000000000 65535 f \n`; + document += offsets + .slice(1) + .map((offset) => `${String(offset).padStart(10, '0')} 00000 n \n`) + .join(''); + document += `trailer\n<< /Size ${objects.length + 1} /Root 1 0 R >>\nstartxref\n${xref}\n%%EOF\n`; + return Buffer.from(document); +} + +function storedPdf( + id: string, + { text, pages = 101 }: { text?: string; pages?: number } = {}, +): StoredPdf { + const original = pdfBytes(id, pages); + return { + original, + file: { + file_id: id, + filename: `${id}.pdf`, + filepath: `/files/${id}.pdf`, + user: 'user', + object: 'file', + usage: 0, + embedded: false, + type: 'application/pdf', + bytes: original.length, + source: FileSources.local, + context: FileContext.message_attachment, + llmDeliveryPath: 'provider', + metadata: { destinationChosen: false, textDerivation: { outcome: 'deferred' } }, + ...(text != null && { text }), + }, + }; +} + +/** + * File Search is loaded unless a test says otherwise, but has queued and registered none of the + * files, as a failed provisioning leaves it: the walk passes search by, no code tool is loaded, + * and a PDF the provider rejects falls back to its text. `queuedForSearch` makes the files + * reachable, so the walk ends at File Search instead. Without any file tool the automatic + * policy reads nothing and a provider rejection fails the turn as classic routing does. + */ +function setup({ + current = [], + historical = [], + consumers = search, + queuedForSearch = false, + fileTokenLimit = 10_000, + fileContextCharLimit = 100_000, + fileLimit, + fileContextSizeLimit, + endpointTotalSizeLimit, + abortController, + filters, +}: { + current?: StoredPdf[]; + historical?: StoredPdf[]; + consumers?: TurnFileConsumers; + queuedForSearch?: boolean; + fileTokenLimit?: number; + fileContextCharLimit?: number; + fileLimit?: number; + fileContextSizeLimit?: number; + endpointTotalSizeLimit?: number; + abortController?: AbortController; + filters?: FiltersConfig; +} = {}) { + const fileConfig: NonNullable['fileConfig'] = { + fileContextCharLimit, + fileContextSizeLimit, + endpoints: { + anthropic: { + llmDeliveryPolicy: 'automatic', + fileLimit, + totalSizeLimit: endpointTotalSizeLimit, + }, + }, + }; + const req = { + body: { fileTokenLimit }, + config: { + filters, + fileConfig, + }, + } as ServerRequest; + const originals = new Map( + [...historical, ...current].map(({ file, original }) => [file.filepath, original]), + ); + const openStoredFile = jest.fn(async (file: { filepath?: string }) => { + const original = originals.get(file.filepath ?? ''); + return original == null ? null : Readable.from([original]); + }); + const storage: StrategyFunctions = { + getDownloadStream: jest.fn(async (_req, filepath) => { + const original = originals.get(filepath); + if (original == null) throw Object.assign(new Error('Missing original'), { code: 'ENOENT' }); + return Readable.from([original]); + }), + }; + const extractors = { parseDocument: jest.fn(parseDocument), parseTextNative }; + const deriveText = jest.fn(createFileTextDeriver({ req, openStoredFile, filters, extractors })); + const persistDerivation = jest.fn(async () => true); + const routing = resolveTurnDeliveryRouting({ + agent: { provider: Providers.ANTHROPIC }, + config: req.config, + }); + const context = buildTurnReadingContext({ + routing, + provider: Providers.ANTHROPIC, + fileTokenLimit, + configuredFileSizeLimit: undefined, + countTokens, + deriveText, + persistDerivation, + }); + if (context == null) throw new Error('Expected automatic reading context'); + routing.reading = context; + context.setSearchEvidence({ + queued: queuedForSearch ? [...historical, ...current].map(({ file }) => file.file_id) : [], + registered: [], + }); + const primary = { + id: 'primary', + endpoint: Providers.ANTHROPIC, + deliveryRouting: routing, + fileConsumers: consumers, + currentRequestAttachments: current.map(({ file }) => file), + requestAttachments: current.map(({ file }) => file), + attachments: current.map(({ file }) => file), + }; + const extractText = jest.fn(extractFileContext); + const addFileContextToMessage = jest.fn(async (message: Message, files: TFile[]) => { + const textFiles = applyTurnDelivery(files, { routing, consumers }).filter( + (file) => file.llmDeliveryPath === 'text', + ); + if (textFiles.length > 0) { + message.fileContext = await extractText({ + attachments: textFiles, + req, + tokenCountFn: countTokens, + ...getTurnTextOptions(routing), + }); + } + }); + const admitPreparedAttachments = (files: TFile[]): Promise => + admitNativeFallbackAttachments(client, files); + const client = { + options: { + req, + agent: primary, + abortController, + attachments: Promise.resolve(current.map(({ file }) => file)), + }, + authorizedHistoricalFiles: new Map(historical.map(({ file }) => [file.file_id, file])), + authorizedHistoricalReplayFiles: new Map(historical.map(({ file }) => [file.file_id, file])), + message_file_map: {} as Record, + modelBoundCurrentFiles: current.map(({ file }) => file), + modelBoundHistoricalSteerFiles: historical.map(({ file }) => file), + turnSharedAttachmentFiles: [...historical, ...current].map(({ file }) => file), + turnScopedAttachmentsByAgentId: new Map(), + turnAttachmentEndpointsByAgentId: new Map([ + ['primary', { endpoint: Providers.ANTHROPIC }], + ]), + getConversationAgents: () => [primary], + assertTurnAttachmentLimits: (shared: TFile[], scoped: TFile[]) => { + assertAgentAttachmentLimits({ + attachments: [...shared, ...scoped], + req, + endpoint: Providers.ANTHROPIC, + historicalFileIds: new Set(historical.map(({ file }) => file.file_id)), + countRepeatedExtractedText: true, + enforceAttachmentCount: false, + useGlobalContextSizeLimit: true, + }); + }, + processAttachments: jest.fn(async (message: Message, files: TFile[]) => { + const processed = applyTurnDelivery(files, { routing, consumers }); + const result = await encodeNativeDocuments( + processed.filter((file) => file.llmDeliveryPath === 'provider'), + primary, + (nativeFiles, onValidationFailure) => + encodeAndFormatDocuments( + req, + nativeFiles.map((file) => FileModel.hydrate(file).toObject()), + { provider: Providers.ANTHROPIC, onValidationFailure }, + () => storage, + ), + ); + recordNativeRejections([primary], result.rejected); + message.documents.push(...result.documents); + return processed; + }), + prepareTurnAttachments: jest.fn((files: TFile[]) => + prepareTurnFiles({ routing, consumers, files }), + ), + addFileContextToMessage, + admitPreparedAttachments, + }; + const prepare = async (files: TFile[]) => { + const message: Message = { documents: [] }; + const prepared = await prepareMessageAttachments({ client, message, files }); + return { message, prepared }; + }; + return { + client, + context, + primary, + prepare, + storage, + deriveText, + persistDerivation, + extractText, + extractors, + }; +} + +describe('native attachment fallback', () => { + it.each([ + ['File Search', search, true, 'search', 'none'], + ['Run Code', code, false, 'code', 'none'], + ['File Search it cannot reach', search, false, 'text', 'text'], + ] as const)( + 'recovers a real 101-page PDF initially and in a fresh history turn with %s', + async (_name, consumers, queuedForSearch, reader, path) => { + const fixture = storedPdf('CACHED-SENTINEL', { text: 'Complete cached attachment text' }); + for (const historical of [false, true]) { + const harness = setup({ + [historical ? 'historical' : 'current']: [fixture], + consumers, + queuedForSearch, + }); + expect(harness.context.judge(fixture.file).native).toBe('fits'); + + const { message, prepared } = await harness.prepare([fixture.file]); + + expect(harness.context.judge(fixture.file).rejected).toBe('capacity'); + expect(prepared?.[0].llmDeliveryPath).toBe(path); + expect( + decideFileReading({ + routing: harness.primary.deliveryRouting, + file: prepared![0], + consumers, + }).reader, + ).toBe(reader); + expect(message.documents).toEqual([]); + expect(harness.storage.getDownloadStream).toHaveBeenCalledTimes(1); + expect(harness.deriveText).not.toHaveBeenCalled(); + if (reader === 'text') { + expect(message.fileContext?.split(fixture.file.text!).length).toBe(2); + expect(message.fileContext).not.toContain('Truncated'); + if (historical) { + expect(harness.client.authorizedHistoricalFiles.get(fixture.file.file_id)).toBe( + prepared![0], + ); + expect(harness.client.modelBoundHistoricalSteerFiles[0]).toBe(prepared![0]); + } else { + expect((await harness.client.options.attachments)[0]).toBe(prepared![0]); + expect(harness.primary.currentRequestAttachments[0]).toBe(prepared![0]); + } + } else { + expect(message.fileContext).toBeUndefined(); + expect(harness.extractText).not.toHaveBeenCalled(); + } + } + expect(fixture.file.llmDeliveryPath).toBe('provider'); + }, + ); + + it.each([ + [ + 'a 101-page PDF', + () => storedPdf('NO-TOOLS', { text: 'Complete cached attachment text' }), + /100-page limit/, + ], + [ + 'a corrupt PDF', + () => { + const fixture = storedPdf('NO-TOOLS-INTEGRITY', { text: 'Cached text' }); + fixture.original = Buffer.from('not a PDF'); + fixture.file.bytes = fixture.original.length; + return fixture; + }, + /missing PDF header/, + ], + ])( + 'fails the turn on %s as classic routing does when no file tool is loaded', + async (_name, fixtureOf, error) => { + const fixture = fixtureOf(); + for (const historical of [false, true]) { + const harness = setup({ + [historical ? 'historical' : 'current']: [fixture], + consumers: noTools, + }); + + await expect(harness.prepare([fixture.file])).rejects.toThrow(error); + + expect(harness.context.judge(fixture.file).rejected).toBeUndefined(); + expect(harness.client.prepareTurnAttachments).not.toHaveBeenCalled(); + expect(harness.extractText).not.toHaveBeenCalled(); + expect(harness.deriveText).not.toHaveBeenCalled(); + } + expect(fixture.file.llmDeliveryPath).toBe('provider'); + }, + ); + + it('keeps a fitting native PDF as a real provider block without duplicating its cached text', async () => { + const fixture = storedPdf('NATIVE-SENTINEL', { pages: 1, text: 'Do not duplicate this text' }); + const harness = setup({ current: [fixture] }); + + const { message } = await harness.prepare([fixture.file]); + + expect(message.documents).toEqual([ + expect.objectContaining({ + type: 'document', + source: { + type: 'base64', + media_type: 'application/pdf', + data: fixture.original.toString('base64'), + }, + }), + ]); + expect(message.fileContext).toBeUndefined(); + expect(harness.extractText).not.toHaveBeenCalled(); + expect(harness.context.judge(fixture.file).rejected).toBeUndefined(); + }); + + it('derives complete PDF text once, then sends the same body from its restored cache', async () => { + const fixture = storedPdf('DERIVED-SENTINEL'); + const harness = setup({ current: [fixture] }); + + const first = await harness.prepare([fixture.file]); + + expect(first.prepared?.[0]).toMatchObject({ + llmDeliveryPath: 'text', + text: `DERIVED-SENTINEL${'\n'.repeat(101)}`, + }); + expect(first.message.documents).toEqual([]); + expect(first.message.fileContext?.split('DERIVED-SENTINEL').length).toBe(3); // filename and text + expect(harness.deriveText).toHaveBeenCalledTimes(1); + expect(harness.persistDerivation).toHaveBeenCalledTimes(1); + expect(fixture.file.text).toBeUndefined(); + const restored = { ...fixture, file: first.prepared![0] }; + const history = setup({ historical: [restored] }); + + const next = await history.prepare([restored.file]); + + expect(next.message.fileContext).toBe(first.message.fileContext); + expect(next.message.documents).toEqual([]); + expect(history.deriveText).not.toHaveBeenCalled(); + }); + + it('leaves excessive cached text unavailable instead of sending a truncated fallback', async () => { + const fixture = storedPdf('TOKEN-LIMIT', { + text: 'A long sentence with many distinct tokens. '.repeat(50), + }); + const harness = setup({ current: [fixture], fileTokenLimit: 5 }); + + const { message, prepared } = await harness.prepare([fixture.file]); + + expect(prepared?.[0].llmDeliveryPath).toBe('none'); + expect(message.documents).toEqual([]); + expect(message.fileContext).toBeUndefined(); + expect(harness.extractText).not.toHaveBeenCalled(); + expect(harness.deriveText).not.toHaveBeenCalled(); + }); + + it.each([ + [search, true], + [search, false], + [code, false], + ])( + 'never substitutes text for an integrity rejection (%j, queued for search: %s)', + async (consumers, queuedForSearch) => { + const fixture = storedPdf('INTEGRITY', { text: 'Cached text must not bypass corruption' }); + fixture.original = Buffer.from('not a PDF'); + fixture.file.bytes = fixture.original.length; + const harness = setup({ current: [fixture], consumers, queuedForSearch }); + + const { message, prepared } = await harness.prepare([fixture.file]); + + expect(harness.context.judge(fixture.file).rejected).toBe('integrity'); + expect(prepared?.[0].llmDeliveryPath).toBe('none'); + expect(message.documents).toEqual([]); + expect(message.fileContext).toBeUndefined(); + expect(harness.extractText).not.toHaveBeenCalled(); + expect(harness.deriveText).not.toHaveBeenCalled(); + }, + ); + + it('allocates newly derived fallback text against other admitted files before text injection', async () => { + const candidate = storedPdf('AGGREGATE-FALLBACK'); + const committed = storedPdf('COMMITTED', { pages: 1, text: 'Existing context' }); + committed.file.llmDeliveryPath = 'text'; + committed.file.metadata = { destinationChosen: true }; + const harness = setup({ current: [candidate, committed], fileContextCharLimit: 120 }); + + const { message, prepared } = await harness.prepare([candidate.file]); + + expect(harness.deriveText).toHaveBeenCalledTimes(1); + expect(prepared?.[0].llmDeliveryPath).toBe('none'); + expect(harness.context.judge(candidate.file).overflow).toBe(true); + expect(message.documents).toEqual([]); + expect(message.fileContext).toBeUndefined(); + expect(harness.extractText).not.toHaveBeenCalled(); + expect((await harness.client.options.attachments)[1]).toBe(committed.file); + }); + + it('serializes concurrent historical fallback allocation against one shared text budget', async () => { + const first = storedPdf('FIRST'); + const second = storedPdf('SECOND'); + const harness = setup({ historical: [first, second], fileContextCharLimit: 150 }); + + const outcomes = await Promise.all([ + harness.prepare([first.file]), + harness.prepare([second.file]), + ]); + + expect(outcomes.map(({ prepared }) => prepared?.[0].llmDeliveryPath).sort()).toEqual([ + 'none', + 'text', + ]); + expect(outcomes.filter(({ message }) => message.fileContext != null)).toHaveLength(1); + expect(harness.extractText).toHaveBeenCalledTimes(1); + expect( + harness.client.turnSharedAttachmentFiles.filter(isModelBoundAttachmentFile), + ).toHaveLength(1); + }); + + it.each([ + ['a current attachment beside one scoped agent', false, 1, ['primary'], 'none'], + ['a historical attachment beside one scoped agent', true, 1, ['primary'], 'text'], + ['a current attachment beside two scoped agents', false, 2, ['primary', 'parallel'], 'text'], + ] as const)( + "charges each agent's own scoped count allowance, exempting history, for %s", + async (_name, historical, fileLimit, agentIds, path) => { + const fixture = storedPdf('SHARED', { text: 'Fits the text budget' }); + const harness = setup({ [historical ? 'historical' : 'current']: [fixture], fileLimit }); + for (const id of agentIds) { + const scoped = storedPdf(`${id}-SCOPED`, { pages: 1, text: 'Scoped text' }).file; + scoped.llmDeliveryPath = 'text'; + harness.client.turnScopedAttachmentsByAgentId.set(id, [scoped]); + harness.client.turnAttachmentEndpointsByAgentId.set(id, { endpoint: Providers.ANTHROPIC }); + } + + const { message, prepared } = await harness.prepare([fixture.file]); + + expect(prepared?.[0].llmDeliveryPath).toBe(path); + if (path === 'text') { + expect(message.fileContext).toContain(fixture.file.text); + } else { + expect(message.fileContext).toBeUndefined(); + } + }, + ); + + it.each([ + ['equal endpoint budgets', false, 'text'], + ['a tighter parallel endpoint', true, 'none'], + ] as const)( + 'reserves scoped bytes separately for %s and the combined global budget', + async (_name, tighterParallel, path) => { + const fixture = storedPdf('SHARED-BYTES', { text: 'Complete fallback text' }); + const unit = fixture.file.bytes; + const harness = setup({ + current: [fixture], + fileContextSizeLimit: (3 * unit) / megabyte, + endpointTotalSizeLimit: (2 * unit) / megabyte, + }); + harness.client.options.req.config!.fileConfig!.endpoints!.parallel = { + totalSizeLimit: ((tighterParallel ? 1.5 : 2) * unit) / megabyte, + }; + for (const id of ['primary', 'parallel']) { + const scoped = storedPdf(`${id}-SCOPED`, { pages: 1, text: 'Scoped text' }).file; + scoped.bytes = unit; + scoped.llmDeliveryPath = 'text'; + harness.client.turnScopedAttachmentsByAgentId.set(id, [scoped]); + harness.client.turnAttachmentEndpointsByAgentId.set(id, { + endpoint: id === 'primary' ? Providers.ANTHROPIC : 'parallel', + }); + } + + const { message, prepared } = await harness.prepare([fixture.file]); + + expect(prepared?.[0].llmDeliveryPath).toBe(path); + expect(message.fileContext != null).toBe(path === 'text'); + expect(harness.context.judge(fixture.file).overflow).toBe(tighterParallel ? true : undefined); + }, + ); + + it('charges the full text of a current attachment replayed in history twice', async () => { + const fixture = storedPdf('REPEATED', { text: 'r'.repeat(90) }); + const harness = setup({ current: [fixture], historical: [fixture], fileContextCharLimit: 100 }); + + const { message, prepared } = await harness.prepare([fixture.file]); + + expect(prepared?.[0].llmDeliveryPath).toBe('none'); + expect(harness.context.judge(fixture.file).overflow).toBe(true); + expect(message.fileContext).toBeUndefined(); + expect(harness.extractText).not.toHaveBeenCalled(); + }); + + it('allows a later admission after the same host rejected sensitive fallback', async () => { + const blocked = storedPdf('BLOCKED', { text: 'PRIVATE-SENTINEL' }); + const safe = storedPdf('SAFE', { text: 'Safe complete text' }); + const filters: FiltersConfig = { + files: { + pii: { + fields: ['extracted_text'], + starterPatterns: [], + customPatterns: [{ id: 'private', label: 'private token', regex: 'PRIVATE-[A-Z]+' }], + }, + }, + }; + const harness = setup({ current: [blocked, safe], filters }); + harness.context.recordRejections( + [blocked, safe].map(({ file }) => ({ file_id: file.file_id, reason: 'capacity' })), + ); + const blockedFiles = await harness.client.prepareTurnAttachments([blocked.file]); + const safeFiles = await harness.client.prepareTurnAttachments([safe.file]); + + const rejected = admitNativeFallbackAttachments(harness.client, blockedFiles); + const recovered = admitNativeFallbackAttachments(harness.client, safeFiles); + + await expect(rejected).rejects.toBeInstanceOf(ContentFilterError); + await expect(recovered).resolves.toEqual(safeFiles); + expect((await harness.client.options.attachments)[1]).toBe(safeFiles[0]); + expect((await harness.client.options.attachments)[0]).toBe(blocked.file); + }); + + it('waits for admission when another message already recorded the native rejection', async () => { + const fixture = storedPdf('KNOWN-REJECTION', { text: 'Complete cached fallback' }); + const harness = setup({ current: [fixture] }); + harness.context.recordRejections([{ file_id: fixture.file.file_id, reason: 'capacity' }]); + const entered = Promise.withResolvers(); + const allowed = Promise.withResolvers(); + const admit = harness.client.admitPreparedAttachments; + harness.client.admitPreparedAttachments = async (files) => { + entered.resolve(); + await allowed.promise; + return admit(files); + }; + + const pending = harness.prepare([fixture.file]); + await entered.promise; + + expect(harness.client.addFileContextToMessage).not.toHaveBeenCalled(); + expect(harness.extractText).not.toHaveBeenCalled(); + allowed.resolve(); + const { message, prepared } = await pending; + expect(prepared?.[0].llmDeliveryPath).toBe('text'); + expect(message.fileContext).toContain(fixture.file.text); + }); + + it('stops after real validation if the request aborts before cached fallback injection', async () => { + const fixture = storedPdf('ABORTED', { text: 'Complete text never injected' }); + const controller = new AbortController(); + const aborted = new Error('Canceled request'); + const harness = setup({ current: [fixture], abortController: controller }); + const process = harness.client.processAttachments.getMockImplementation()!; + harness.client.processAttachments.mockImplementation(async (message, files) => { + const processed = await process(message, files); + controller.abort(aborted); + return processed; + }); + + await expect(harness.prepare([fixture.file])).rejects.toBe(aborted); + + expect(harness.context.judge(fixture.file).rejected).toBe('capacity'); + expect(harness.client.prepareTurnAttachments).not.toHaveBeenCalled(); + expect(harness.client.addFileContextToMessage).not.toHaveBeenCalled(); + expect(harness.extractText).not.toHaveBeenCalled(); + }); + + it.each([ + ['cached', { text: 'PRIVATE-SENTINEL' }, false], + ['derived', {}, false], + ['already rejected cached', { text: 'PRIVATE-SENTINEL' }, true], + ] as const)( + 'blocks sensitive %s fallback before injection or persistence', + async (_kind, stored, rejectedBefore) => { + const filters: FiltersConfig = { + files: { + pii: { + fields: ['extracted_text'], + starterPatterns: [], + customPatterns: [{ id: 'private', label: 'private token', regex: 'PRIVATE-[A-Z]+' }], + }, + }, + }; + const fixture = storedPdf('PRIVATE-SENTINEL', stored); + const harness = setup({ current: [fixture], filters }); + if (rejectedBefore) { + harness.context.recordRejections([{ file_id: fixture.file.file_id, reason: 'capacity' }]); + } + + await expect(harness.prepare([fixture.file])).rejects.toBeInstanceOf(ContentFilterError); + + expect(harness.client.addFileContextToMessage).not.toHaveBeenCalled(); + expect(harness.extractText).not.toHaveBeenCalled(); + expect(harness.persistDerivation).not.toHaveBeenCalled(); + expect(fixture.file.llmDeliveryPath).toBe('provider'); + }, + ); +}); diff --git a/packages/api/src/files/reading/native.ts b/packages/api/src/files/reading/native.ts new file mode 100644 index 00000000000..d2572f93fd9 --- /dev/null +++ b/packages/api/src/files/reading/native.ts @@ -0,0 +1,396 @@ +import { EModelEndpoint, resolveTurnLLMDeliveryPath } from 'librechat-data-provider'; +import type { + DirectContentEntry, + DirectContentLimits, + TurnFileConsumers, +} from 'librechat-data-provider'; +import type { AgentAttachmentEndpointsByAgentId } from '~/agents/attachments'; +import type { HistoryAllocationFile } from './history'; +import type { NativeDeliveryAgent } from './turn'; +import type { ServerRequest } from '~/types'; +import { + getAgentEntry, + listAgentIds, + isModelBoundAttachmentFile, + measureModelBoundAttachment, + resolveAgentAttachmentLimits, + collectHistoricalAttachmentIds, +} from '~/agents/attachments'; +import { getNativeValidationPolicy, getTurnReadingContext } from './turn'; +import { assertModelBoundContent } from '~/middleware/modelBoundContent'; +import { settleTurnFiles } from './settle'; + +interface MessageAttachmentClient { + options?: { agent?: NativeDeliveryAgent | null; abortController?: AbortController }; + processAttachments( + message: M, + files: T[], + consumers?: TurnFileConsumers, + ): Promise; + prepareTurnAttachments(files: T[], consumers?: TurnFileConsumers): Promise; + addFileContextToMessage(message: M, files: T[], consumers?: TurnFileConsumers): Promise; + admitPreparedAttachments?(files: T[], consumers?: TurnFileConsumers): Promise; +} + +/** + * Waits for native validation before extracting text when an automatic document may fall back. + * Other attachments keep their independent extraction and encoding in parallel. The settled + * copies are admitted before their text reaches the message and returned for history/projection. + */ +export async function prepareMessageAttachments({ + client, + message, + files, + consumers, + contextFiles = files, +}: { + client: MessageAttachmentClient; + message: M; + files: T[]; + consumers?: TurnFileConsumers; + contextFiles?: T[]; +}): Promise { + const agent = client.options?.agent; + const signal = client.options?.abortController?.signal; + const modeOf = getNativeValidationPolicy(agent); + const context = getTurnReadingContext(agent?.deliveryRouting); + const mayReject = files.some( + (file) => + modeOf(file) === 'skip' && + (context?.judge(file).rejected != null || + resolveTurnLLMDeliveryPath( + agent?.deliveryRouting ?? undefined, + file, + consumers ?? agent?.fileConsumers, + ) === 'provider'), + ); + if (!mayReject) { + const [, processed] = await Promise.all([ + consumers == null + ? client.addFileContextToMessage(message, contextFiles) + : client.addFileContextToMessage(message, contextFiles, consumers), + consumers == null + ? client.processAttachments(message, files) + : client.processAttachments(message, files, consumers), + ]); + return mergeAttachmentSurvivors(files, processed); + } + + signal?.throwIfAborted(); + const processed = await client.processAttachments(message, files, consumers); + signal?.throwIfAborted(); + let prepared = await client.prepareTurnAttachments( + mergeAttachmentSurvivors(files, processed) ?? files, + consumers, + ); + if (prepared.some((file) => context?.judge(file).rejected != null)) { + prepared = (await client.admitPreparedAttachments?.(prepared, consumers)) ?? prepared; + } + signal?.throwIfAborted(); + await client.addFileContextToMessage(message, prepared, consumers); + return prepared; +} + +/** Encoder metadata may be sparse; retain the authorized original record on surviving copies. */ +function mergeAttachmentSurvivors( + files: T[], + processed: T[] | undefined, +): T[] | undefined { + if (processed == null) { + return processed; + } + const originals = new Map(files.map((file) => [file.file_id, file])); + return processed.map((file) => { + const original = originals.get(file.file_id); + return original == null || original === file + ? file + : { ...original, ...file, metadata: file.metadata ?? original.metadata }; + }); +} + +/** The host's already loaded attachment state and limits, shared by current/history/steering. */ +interface NativeFallbackHost { + options: { + req?: ServerRequest; + endpoint?: string | null; + endpointType?: string | null; + agent?: NativeDeliveryAgent & { endpoint?: string | null }; + attachments?: T[] | Promise; + abortController?: AbortController; + }; + authorizedHistoricalFiles?: Map; + authorizedHistoricalReplayFiles?: Map; + message_file_map?: Record; + modelBoundCurrentFiles?: T[]; + modelBoundHistoricalSteerFiles?: T[]; + turnSharedAttachmentFiles?: T[]; + turnAggregateOnlyAttachmentFiles?: T[]; + turnScopedAttachmentsByAgentId?: Map; + turnAttachmentEndpointsByAgentId?: AgentAttachmentEndpointsByAgentId; + turnHistoricalAttachmentIds?: ReadonlySet; + admittedSteerAttachments?: Map; + attachmentMemoryContext?: { attachments?: T[] }; + assertTurnAttachmentLimits?(shared: T[], scoped: T[]): void; + getConversationAgents?(): Iterable< + | (NativeDeliveryAgent & { + currentRequestAttachments?: T[]; + requestAttachments?: T[]; + attachments?: T[]; + }) + | null + | undefined + >; +} + +const admissions = new WeakMap>(); + +/** Keeps the already admitted historical replay topology available before message encoding. */ +export function retainNativeAttachmentTopology( + host: NativeFallbackHost, + shared: T[], + scoped: Map, + endpoints: AgentAttachmentEndpointsByAgentId, +): void { + host.turnSharedAttachmentFiles = shared; + host.turnScopedAttachmentsByAgentId = scoped; + host.turnAttachmentEndpointsByAgentId = endpoints; +} + +/** + * Shared candidates spend the same footprint in every agent's payload. Reserve each endpoint's + * own scoped content and the global combined context separately, then use the smallest remaining + * allowance per dimension. Keeping that scope's positive limit and charged entries also handles + * an exactly exhausted dimension without treating a remaining zero as unlimited. + */ +function reserveFallbackCapacity( + host: NativeFallbackHost, + committed: T[], + scoped: Map, + historicalFileIds: ReadonlySet, +): { limits: DirectContentLimits; committedEntries: DirectContentEntry[] } { + const req = host.options.req; + const endpoint = host.options.agent?.endpoint ?? host.options.endpoint; + const endpointType = host.options.endpointType; + const measureCommitted = (files: T[]): DirectContentEntry[] => { + const seen = new Set(); + return files.map((file) => { + const measured = measureModelBoundAttachment(file); + const counts = + measured.counts && !historicalFileIds.has(file.file_id) && !seen.has(file.file_id); + seen.add(file.file_id); + return { ...measured, counts }; + }); + }; + const budgets: Array<{ limits: DirectContentLimits; entries: DirectContentEntry[] }> = [ + { + limits: resolveAgentAttachmentLimits({ + req, + endpoint, + endpointType, + enforceAttachmentCount: false, + useGlobalContextSizeLimit: true, + }), + entries: measureCommitted([ + ...committed, + ...(host.turnAggregateOnlyAttachmentFiles ?? []), + ...[...scoped.values()].flat(), + ]), + }, + ]; + const endpoints = host.turnAttachmentEndpointsByAgentId; + const agentIds = new Set([...scoped.keys(), ...listAgentIds(endpoints)]); + const appendEndpointBudget = (agentId?: string): void => { + const selectedEndpoint = agentId == null ? undefined : getAgentEntry(endpoints, agentId); + budgets.push({ + limits: resolveAgentAttachmentLimits({ + req, + endpoint: selectedEndpoint?.endpoint ?? endpoint ?? EModelEndpoint.agents, + endpointType: selectedEndpoint?.endpointType ?? endpointType, + }), + entries: measureCommitted([ + ...committed, + ...(agentId == null ? [] : (scoped.get(agentId) ?? [])), + ]), + }); + }; + if (agentIds.size === 0) { + appendEndpointBudget(); + } else { + for (const agentId of agentIds) appendEndpointBudget(agentId); + } + const limits: DirectContentLimits = {}; + const committedEntries: DirectContentEntry[] = []; + for (const dimension of ['count', 'bytes', 'textChars'] as const) { + let remaining = Infinity; + let selected: (typeof budgets)[number] | undefined; + for (const budget of budgets) { + const limit = budget.limits[dimension]; + if (!limit) continue; + const spent = budget.entries.reduce( + (sum, entry) => sum + (dimension === 'count' ? Number(entry.counts) : entry[dimension]), + 0, + ); + if (limit - spent < remaining) { + remaining = limit - spent; + selected = budget; + } + } + if (selected == null) continue; + limits[dimension] = selected.limits[dimension]; + committedEntries.push( + ...selected.entries.map((entry) => ({ + fileId: entry.fileId, + counts: dimension === 'count' && entry.counts, + bytes: dimension === 'bytes' ? entry.bytes : 0, + textChars: dimension === 'textChars' ? entry.textChars : 0, + })), + ); + } + return { limits, committedEntries }; +} + +/** + * Admits text selected after native rejection against the host's complete loaded shared/scoped + * set. Original types/bytes passed endpoint admission before encoding; this rechecks the added + * text and the aggregate before replacing the turn copies used by subsequent projection. + */ +async function admitFallback( + host: NativeFallbackHost, + files: T[], + consumers: TurnFileConsumers | undefined, +): Promise { + const signal = host.options.abortController?.signal; + signal?.throwIfAborted(); + const replacements = new Map(files.map((file) => [file.file_id, file])); + const replace = (file: T): T => replacements.get(file.file_id) ?? file; + const current = (await host.options.attachments) ?? []; + const historical = [...(host.authorizedHistoricalFiles?.values() ?? [])]; + const shared = (host.turnSharedAttachmentFiles ?? [...historical, ...current]).map(replace); + const sharedIds = new Set(shared.map((file) => file.file_id)); + for (const file of files) { + if (!sharedIds.has(file.file_id)) { + shared.push(file); + sharedIds.add(file.file_id); + } + } + const historicalFileIds = + host.turnHistoricalAttachmentIds ?? collectHistoricalAttachmentIds(historical, current); + const req = host.options.req; + const scoped = host.turnScopedAttachmentsByAgentId ?? new Map(); + const candidates = files.filter((file) => file.llmDeliveryPath === 'text'); + const candidateIds = new Set(candidates.map((file) => file.file_id)); + const copies = new Map(); + for (const file of shared) { + copies.set(file.file_id, (copies.get(file.file_id) ?? 0) + 1); + } + const { limits, committedEntries } = reserveFallbackCapacity( + host, + shared.filter((file) => !candidateIds.has(file.file_id)), + scoped, + historicalFileIds, + ); + const prepared = await settleTurnFiles({ + routing: host.options.agent?.deliveryRouting ?? undefined, + consumers: consumers ?? host.options.agent?.fileConsumers, + files, + allocation: { + requestFileIds: candidates.map((file) => file.file_id), + limits, + measure: (file) => { + const measured = measureModelBoundAttachment(file); + const multiplicity = copies.get(file.file_id) ?? 1; + return { + ...measured, + bytes: measured.bytes * multiplicity, + textChars: measured.textChars * multiplicity, + counts: measured.counts && !historicalFileIds.has(file.file_id), + }; + }, + committedEntries, + scope: 'history', + }, + flush: true, + signal, + }); + signal?.throwIfAborted(); + for (const file of prepared) { + replacements.set(file.file_id, file); + } + const admittedShared = shared.map(replace).filter(isModelBoundAttachmentFile); + host.assertTurnAttachmentLimits?.( + [...admittedShared, ...(host.turnAggregateOnlyAttachmentFiles ?? [])], + [...scoped.values()].flat(), + ); + assertModelBoundContent({ + filters: req?.config?.filters, + files: prepared.filter(isModelBoundAttachmentFile).map((file) => ({ + ...file, + source: file.source ?? undefined, + })), + }); + + const replaceMap = (map?: Map): void => { + for (const [fileId, file] of map ?? []) { + map?.set(fileId, replace(file)); + } + }; + replaceMap(host.authorizedHistoricalFiles); + replaceMap(host.authorizedHistoricalReplayFiles); + const messageFiles = host.message_file_map; + if (messageFiles != null) { + for (const [messageId, attachments] of Object.entries(messageFiles)) { + messageFiles[messageId] = attachments.map(replace).filter(isModelBoundAttachmentFile); + } + } + host.modelBoundCurrentFiles = host.modelBoundCurrentFiles?.map(replace); + host.modelBoundHistoricalSteerFiles = host.modelBoundHistoricalSteerFiles?.map(replace); + for (const [steerId, reserved] of host.admittedSteerAttachments ?? []) { + host.admittedSteerAttachments?.set(steerId, reserved.map(replace)); + } + if (host.attachmentMemoryContext?.attachments != null) { + const memoryFiles = host.attachmentMemoryContext.attachments; + for (let index = 0; index < memoryFiles.length; index++) { + memoryFiles[index] = replace(memoryFiles[index]); + } + } + if (host.turnSharedAttachmentFiles != null) { + const sharedFiles = host.turnSharedAttachmentFiles; + sharedFiles.length = shared.length; + for (let index = 0; index < shared.length; index++) { + sharedFiles[index] = replace(shared[index]); + } + } + if (host.options.attachments != null) { + host.options.attachments = current.map(replace); + } + const primary = getTurnReadingContext(host.options.agent?.deliveryRouting); + const overflowIds = prepared + .filter((file) => primary?.judge(file).overflow) + .map((file) => file.file_id); + for (const agent of host.getConversationAgents?.() ?? []) { + if (agent == null) { + continue; + } + agent.currentRequestAttachments = agent.currentRequestAttachments?.map(replace); + agent.requestAttachments = agent.requestAttachments?.map(replace); + agent.attachments = agent.attachments?.map(replace); + getTurnReadingContext(agent.deliveryRouting)?.addOverflow(overflowIds); + } + return prepared; +} + +/** Serializes allocation for parallel historical messages that spend one turn's text budget. */ +export function admitNativeFallbackAttachments( + host: NativeFallbackHost, + files: T[], + consumers?: TurnFileConsumers, +): Promise { + const previous = admissions.get(host) ?? Promise.resolve(); + const admitted = previous.then(() => admitFallback(host, files, consumers)); + const settled = admitted.then( + () => undefined, + () => undefined, + ); + admissions.set(host, settled); + return admitted; +} diff --git a/packages/api/src/files/reading/settle.spec.ts b/packages/api/src/files/reading/settle.spec.ts new file mode 100644 index 00000000000..6911376966e --- /dev/null +++ b/packages/api/src/files/reading/settle.spec.ts @@ -0,0 +1,513 @@ +import { FileContext, FileSources, decideFileReading } from 'librechat-data-provider'; +import type { TurnFileConsumers, TFileConfig, TurnDeliveryRouting } from 'librechat-data-provider'; +import type { FileTextDeriver, TurnReadingContext, TurnReadingFile, DerivedText } from './turn'; +import type { DirectContentAllocation } from './settle'; +import { applyTurnDelivery, resolveTurnDeliveryRouting } from '~/agents/files/delivery'; +import { measureAttachment, measureModelBoundAttachment } from '~/agents/attachments'; +import { settleTurnFiles, prepareTurnFiles } from './settle'; +import { buildTurnReadingContext } from './turn'; + +type EndpointFileConfigInput = NonNullable[string]; + +/** The module object settle reads through, so a spy on it sees every decision settle asks for. */ +const dataProvider = + jest.requireActual('librechat-data-provider'); + +const MB = 1024 * 1024; +const PDF = 'application/pdf'; +const XLSX = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'; + +const noReader: TurnFileConsumers = { executeCode: false, fileSearch: false }; +const searchOnly: TurnFileConsumers = { executeCode: false, fileSearch: true }; + +const derivedText: DerivedText = { + status: 'derived', + text: 'quarter,total\nQ1,42', + textDerivation: { outcome: 'complete', extractor: 'document_parser', at: 1 }, +}; + +const attachment = ( + overrides: Partial & Pick, +): TurnReadingFile => ({ + type: PDF, + bytes: 4 * MB, + source: FileSources.local, + context: FileContext.message_attachment, + llmDeliveryPath: 'provider', + metadata: { destinationChosen: false }, + ...overrides, +}); + +const deferredWorkbook = attachment({ + file_id: 'xlsx', + type: XLSX, + bytes: 64 * 1024, + llmDeliveryPath: 'none', + metadata: { destinationChosen: false, textDerivation: { outcome: 'deferred' } }, +}); + +function setup({ + endpointConfig = { llmDeliveryPolicy: 'automatic' }, + deriveText, +}: { + endpointConfig?: EndpointFileConfigInput; + deriveText?: FileTextDeriver; +} = {}): { routing: TurnDeliveryRouting; context: TurnReadingContext } { + const routing = resolveTurnDeliveryRouting({ + agent: { provider: 'openAI', endpoint: 'openAI' }, + config: { fileConfig: { endpoints: { openAI: endpointConfig } } }, + }); + const context = buildTurnReadingContext({ + routing, + provider: 'openAI', + model: 'gpt-4o', + fileTokenLimit: 100_000, + configuredFileSizeLimit: undefined, + countTokens: (text) => text.length, + deriveText, + }); + if (context == null) { + throw new Error('expected a reading context'); + } + routing.reading = context; + return { routing, context }; +} + +const byteBudget = ( + requestFileIds: readonly string[], + bytes: number, + committedExtra?: readonly TurnReadingFile[], +): DirectContentAllocation => ({ + requestFileIds, + limits: { bytes }, + measure: measureModelBoundAttachment, + committedExtra, +}); + +const pathsById = (files: readonly TurnReadingFile[]): Record => + Object.fromEntries(files.map((file) => [file.file_id, file.llmDeliveryPath])); + +describe('settleTurnFiles', () => { + describe('text derivation', () => { + it.each<[string, TurnFileConsumers, string, boolean]>([ + ['File Search', searchOnly, 'code_unavailable', true], + /* Without Run Code or File Search the policy decides nothing, but a deferred workbook's + * classic text route still has no text, so settle derives it as a no-tools turn reads it. */ + ['no file tool', noReader, 'no_file_tools', false], + ])( + 'derives the text a reading needs with %s loaded, then decides again on the derived copy', + async (_name, consumers, reason, automatic) => { + const deriveText = jest.fn, Parameters>( + async () => derivedText, + ); + const { routing } = setup({ deriveText }); + const files = [deferredWorkbook]; + + expect(decideFileReading({ routing, file: deferredWorkbook, consumers })).toMatchObject({ + reader: 'text', + reason, + needsText: true, + automatic, + }); + + const [settled] = await settleTurnFiles({ routing, consumers, files }); + + expect(settled).toMatchObject({ + file_id: 'xlsx', + llmDeliveryPath: 'text', + text: derivedText.text, + metadata: { destinationChosen: false, textDerivation: derivedText.textDerivation }, + }); + expect(decideFileReading({ routing, file: settled, consumers })).toMatchObject({ + reader: 'text', + reason, + needsText: false, + automatic, + }); + expect(deferredWorkbook.text).toBeUndefined(); + expect(deferredWorkbook.metadata?.textDerivation).toEqual({ outcome: 'deferred' }); + + await settleTurnFiles({ routing, consumers, files }); + expect(deriveText).toHaveBeenCalledTimes(1); + expect(deriveText).toHaveBeenCalledWith( + expect.objectContaining({ file_id: 'xlsx', llmDeliveryPath: 'text' }), + undefined, + ); + }, + ); + + const parserFailure: DerivedText = { + status: 'failed', + textDerivation: { outcome: 'failed', reason: 'parser' }, + persist: true, + }; + const movedToSearch = { + reader: 'search', + needsText: false, + skipped: expect.arrayContaining([{ reader: 'text', reason: 'text_unavailable' }]), + }; + + it.each<[string, TurnFileConsumers, DerivedText, Record]>([ + ['a failed derivation, with File Search loaded', searchOnly, parserFailure, movedToSearch], + [ + 'a failed derivation, with no file tool loaded', + noReader, + parserFailure, + { reader: 'unavailable', reason: 'no_file_tools', needsText: false, automatic: false }, + ], + [ + 'a skipped derivation, with File Search loaded', + searchOnly, + { status: 'skipped', reason: 'storage_unavailable' }, + movedToSearch, + ], + ])( + 'treats %s as unavailable text for the rest of the request', + async (_name, consumers, outcome, decision) => { + const { routing, context } = setup({ deriveText: async () => outcome }); + + const [settled] = await settleTurnFiles({ routing, consumers, files: [deferredWorkbook] }); + + expect(settled.llmDeliveryPath).toBe('none'); + expect(settled.text).toBeUndefined(); + expect(context.judge(deferredWorkbook).textFailed).toBe(true); + expect(decideFileReading({ routing, file: settled, consumers })).toMatchObject(decision); + }, + ); + + it('never decides or derives an unmarked record in a classic derive-only context', async () => { + const { routing, context } = setup({ + endpointConfig: {}, + deriveText: async () => derivedText, + }); + const deriveSpy = jest.spyOn(context, 'derive'); + const decideSpy = jest.spyOn(dataProvider, 'decideFileReading'); + const unmarked = [ + attachment({ file_id: 'pdf' }), + attachment({ file_id: 'plain-xlsx', type: XLSX, llmDeliveryPath: 'none' }), + attachment({ file_id: 'legacy-xlsx', type: XLSX, llmDeliveryPath: undefined }), + ]; + try { + const settled = await settleTurnFiles({ + routing, + consumers: noReader, + files: [...unmarked, deferredWorkbook], + }); + + expect(deriveSpy).toHaveBeenCalledTimes(1); + expect(deriveSpy).toHaveBeenCalledWith( + expect.objectContaining({ file_id: 'xlsx' }), + undefined, + ); + expect(decideSpy.mock.calls.map(([input]) => input.file.file_id)).toEqual(['xlsx']); + expect(settled.slice(0, unmarked.length)).toEqual( + applyTurnDelivery(unmarked, { routing, consumers: noReader }), + ); + expect(settled[unmarked.length]).toMatchObject({ file_id: 'xlsx', text: derivedText.text }); + } finally { + decideSpy.mockRestore(); + } + }); + + it('stops before deriving once the request is aborted', async () => { + const deriveText = jest.fn, Parameters>( + async () => derivedText, + ); + const { routing } = setup({ deriveText }); + const controller = new AbortController(); + controller.abort(); + + await expect( + settleTurnFiles({ + routing, + consumers: noReader, + files: [deferredWorkbook], + signal: controller.signal, + }), + ).rejects.toThrow(); + expect(deriveText).not.toHaveBeenCalled(); + }); + }); + + describe('allocation', () => { + const pdfs = ['a', 'b', 'c'].map((file_id) => attachment({ file_id })); + + it('admits direct content first-fit in request order and moves the overflow to File Search', async () => { + const { routing, context } = setup(); + + const settled = await settleTurnFiles({ + routing, + consumers: searchOnly, + files: pdfs, + allocation: byteBudget(['a', 'b', 'c'], 10 * MB), + }); + + expect(pathsById(settled)).toEqual({ a: 'provider', b: 'provider', c: 'none' }); + expect(context.judge(pdfs[2]).overflow).toBe(true); + expect(decideFileReading({ routing, file: settled[2], consumers: searchOnly })).toMatchObject( + { reader: 'search', reason: 'aggregate_overflow' }, + ); + }); + + it('leaves the overflow unavailable when File Search is loaded but cannot reach it', async () => { + /* File Search queued and registered nothing, as a failed provisioning leaves it, and no + * code tool is loaded, so the walk past the overflow finds no reader. */ + const { routing, context } = setup(); + context.setSearchEvidence({ queued: [], registered: [] }); + + const settled = await settleTurnFiles({ + routing, + consumers: searchOnly, + files: pdfs, + allocation: byteBudget(['a', 'b', 'c'], 10 * MB), + }); + + expect(settled[2].llmDeliveryPath).toBe('none'); + expect(decideFileReading({ routing, file: settled[2], consumers: searchOnly })).toMatchObject( + { reader: 'unavailable', reason: 'aggregate_overflow' }, + ); + }); + + it('allocates nothing when no file tool is loaded, keeping every file on its classic route', async () => { + const { routing, context } = setup(); + + const settled = await settleTurnFiles({ + routing, + consumers: noReader, + files: pdfs, + allocation: byteBudget(['a', 'b', 'c'], 10 * MB), + }); + + expect(pathsById(settled)).toEqual({ a: 'provider', b: 'provider', c: 'provider' }); + expect(context.stats().overflow).toBe(0); + expect(decideFileReading({ routing, file: settled[2], consumers: noReader })).toMatchObject({ + reader: 'provider', + reason: 'no_file_tools', + automatic: false, + }); + }); + + it('depends on the request order alone, not on the order of the files', async () => { + const settle = async (files: TurnReadingFile[], requestFileIds: string[]) => { + const { routing } = setup(); + const settled = await settleTurnFiles({ + routing, + consumers: searchOnly, + files, + allocation: byteBudget(requestFileIds, 10 * MB), + }); + return pathsById(settled); + }; + const [a, b, c] = pdfs; + + const expected = { a: 'provider', b: 'provider', c: 'none' }; + expect(await settle([a, b, c], ['a', 'b', 'c'])).toEqual(expected); + expect(await settle([c, a, b], ['a', 'b', 'c'])).toEqual(expected); + expect(await settle([b, c, a], ['a', 'a', 'b', 'c'])).toEqual(expected); + expect(await settle([a, b, c], ['c', 'a', 'b'])).toEqual({ + a: 'provider', + b: 'none', + c: 'provider', + }); + }); + + it('charges committed content before candidates, whatever order it arrives in', async () => { + const chosen = attachment({ + file_id: 'chosen', + bytes: 3 * MB, + metadata: { destinationChosen: true }, + }); + const history = attachment({ file_id: 'history', bytes: MB }); + const settle = async (requestFileIds: string[], extra: TurnReadingFile[]) => { + const { routing } = setup(); + const settled = await settleTurnFiles({ + routing, + consumers: searchOnly, + files: [...pdfs, chosen], + allocation: byteBudget(requestFileIds, 12 * MB, extra), + }); + return pathsById(settled); + }; + + const expected = { a: 'provider', b: 'provider', c: 'none', chosen: 'provider' }; + expect(await settle(['chosen', 'a', 'b', 'c'], [history])).toEqual(expected); + expect(await settle(['a', 'b', 'chosen', 'c'], [history])).toEqual(expected); + expect(await settle(['a', 'b', 'c', 'chosen'], [history])).toEqual(expected); + }); + + it('charges committed entries the caller measured beside the committed files', async () => { + const history = attachment({ file_id: 'history', bytes: MB }); + const settle = async ( + allocation: Pick< + DirectContentAllocation, + 'committedExtra' | 'committedEntries' + >, + ) => { + const { routing } = setup(); + const settled = await settleTurnFiles({ + routing, + consumers: searchOnly, + files: pdfs, + allocation: { + requestFileIds: ['a', 'b', 'c'], + limits: { count: 3 }, + measure: measureModelBoundAttachment, + ...allocation, + }, + }); + return pathsById(settled); + }; + const historicalFileIds = new Set(['history']); + + expect(await settle({ committedExtra: [history] })).toEqual({ + a: 'provider', + b: 'provider', + c: 'none', + }); + expect( + await settle({ committedEntries: [measureAttachment(history, { historicalFileIds })] }), + ).toEqual({ a: 'provider', b: 'provider', c: 'provider' }); + }); + + it('does not charge files left to tools', async () => { + const { routing } = setup(); + const toolFile = attachment({ + file_id: 'tool', + bytes: 50 * MB, + llmDeliveryPath: 'none', + metadata: { destinationChosen: true }, + }); + + const settled = await settleTurnFiles({ + routing, + consumers: searchOnly, + files: [toolFile, ...pdfs], + allocation: byteBudget(['tool', 'a', 'b', 'c'], 12 * MB), + }); + + expect(pathsById(settled)).toEqual({ + tool: 'none', + a: 'provider', + b: 'provider', + c: 'provider', + }); + }); + + it('charges a text-source request file as admission does, whatever its route', async () => { + const { routing } = setup(); + const pasted = attachment({ + file_id: 'pasted', + type: 'text/plain', + bytes: 3 * MB, + source: FileSources.text, + text: 'pasted notes', + llmDeliveryPath: 'none', + metadata: { destinationChosen: true }, + }); + + const settled = await settleTurnFiles({ + routing, + consumers: searchOnly, + files: [pasted, ...pdfs], + allocation: byteBudget(['pasted', 'a', 'b', 'c'], 12 * MB), + }); + + expect(pathsById(settled)).toEqual({ + pasted: 'none', + a: 'provider', + b: 'provider', + c: 'none', + }); + }); + + it('returns the same array when nothing changes', async () => { + const { routing } = setup(); + + const settled = await settleTurnFiles({ + routing, + consumers: searchOnly, + files: pdfs, + allocation: byteBudget(['a', 'b', 'c'], 100 * MB), + }); + + expect(settled).toBe(pdfs); + }); + + it('never allocates under the classic policy', async () => { + const { routing, context } = setup({ + endpointConfig: {}, + deriveText: async () => derivedText, + }); + const measureSpy = jest.fn(measureModelBoundAttachment); + + const settled = await settleTurnFiles({ + routing, + consumers: searchOnly, + files: pdfs, + allocation: { requestFileIds: ['a', 'b', 'c'], limits: { bytes: MB }, measure: measureSpy }, + }); + + expect(settled).toBe(pdfs); + expect(measureSpy).not.toHaveBeenCalled(); + expect(context.stats().overflow).toBe(0); + }); + }); + + it('is exactly applyTurnDelivery when the routing carries no reading context', async () => { + const routing = resolveTurnDeliveryRouting({ + agent: { provider: 'openAI', endpoint: 'openAI' }, + config: { fileConfig: { endpoints: { openAI: { llmDeliveryPolicy: 'automatic' } } } }, + }); + const files = [deferredWorkbook, ...['a', 'b'].map((file_id) => attachment({ file_id }))]; + + const settled = await settleTurnFiles({ + routing, + consumers: noReader, + files, + allocation: byteBudget(['a', 'b'], MB), + }); + + expect(settled).toEqual(applyTurnDelivery(files, { routing, consumers: noReader })); + expect(pathsById(settled)).toEqual({ xlsx: 'none', a: 'provider', b: 'provider' }); + expect(await settleTurnFiles({ consumers: noReader, files })).toBe(files); + }); + + describe('flush', () => { + it('defers flushing to the caller by default', async () => { + const { routing, context } = setup({ deriveText: async () => derivedText }); + const flushSpy = jest.spyOn(context, 'flush'); + + await settleTurnFiles({ routing, consumers: noReader, files: [deferredWorkbook] }); + expect(flushSpy).not.toHaveBeenCalled(); + + await settleTurnFiles({ + routing, + consumers: noReader, + files: [deferredWorkbook], + flush: true, + }); + expect(flushSpy).toHaveBeenCalledTimes(1); + }); + + it('flushes before returning when files are prepared outside initialization', async () => { + const { routing, context } = setup({ deriveText: async () => derivedText }); + const flushSpy = jest.spyOn(context, 'flush'); + + const [prepared] = await prepareTurnFiles({ + routing, + consumers: noReader, + files: [deferredWorkbook], + }); + expect(prepared.text).toBe(derivedText.text); + expect(flushSpy).toHaveBeenCalledTimes(1); + + await prepareTurnFiles({ + routing, + consumers: noReader, + files: [deferredWorkbook], + flush: false, + }); + expect(flushSpy).toHaveBeenCalledTimes(1); + }); + }); +}); diff --git a/packages/api/src/files/reading/settle.ts b/packages/api/src/files/reading/settle.ts new file mode 100644 index 00000000000..b050c2df06c --- /dev/null +++ b/packages/api/src/files/reading/settle.ts @@ -0,0 +1,188 @@ +import { decideFileReading, allocateDirectContent } from 'librechat-data-provider'; +import type { + DirectContentEntry, + TurnFileConsumers, + DirectContentLimits, + TurnDeliveryRouting, +} from 'librechat-data-provider'; +import type { TurnReadingFile, TurnReader } from './turn'; +import type { AllocationScope } from './diagnostics'; +import { + withDerivedText, + needsDerivedText, + deriveRequestedText, + getTurnReadingContext, +} from './turn'; +import { applyTurnDelivery } from '~/agents/files/delivery'; +import { logAllocation } from './diagnostics'; + +/** First-fit allocation of the request's direct model content against the turn's limits. */ +export interface DirectContentAllocation { + /** The request's attachments in submission order; earlier files are admitted first. */ + requestFileIds: readonly string[]; + limits: DirectContentLimits; + /** + * The direct content one file puts on the model path, as the matching admission check charges + * it. A file that is not model-bound measures as zero. + */ + measure: (file: T) => DirectContentEntry; + /** Content already charged against the same limits, such as replayed history. */ + committedExtra?: readonly T[]; + /** + * Content already charged that the caller measured itself, such as replayed history, which + * spends the context budgets but counts as the checks on it count repeated copies. + */ + committedEntries?: readonly DirectContentEntry[]; + agentId?: string; + scope?: AllocationScope; +} + +export interface SettleTurnFilesParams { + routing?: TurnDeliveryRouting; + consumers?: TurnFileConsumers; + files: T[]; + allocation?: DirectContentAllocation; + /** + * Save the derivations the context queued before returning. Derived text was already + * content-inspected at derivation, and a policy refusal throws instead of saving; this decides + * only whether the save waits for the caller's later checks, as initialization's does. + */ + flush?: boolean; + signal?: AbortSignal; +} + +interface SettleStep extends TurnReader { + signal?: AbortSignal; +} + +const route = (files: T[], { routing, consumers }: SettleStep): T[] => + applyTurnDelivery(files, { routing, consumers }); + +/** + * Derives text for every copy whose reading needs it. Failures are recorded on the context, so the + * next decision passes text over for the rest of the request. Returns null when no copy needed text. + * A derive-only classic context decides only the records the automatic policy marked. + */ +async function deriveNeededText( + files: T[], + step: SettleStep, +): Promise { + const { context, signal } = step; + const needed = files.filter((file) => needsDerivedText(file, step)); + if (needed.length === 0) { + return null; + } + signal?.throwIfAborted(); + const texts = await deriveRequestedText( + needed.map((file) => ({ file, contexts: [context] as const })), + signal, + ); + if (texts.size === 0) { + return files; + } + return files.map((file) => { + const derived = texts.get(file.file_id); + return derived == null ? file : withDerivedText(file, derived); + }); +} + +const isDirectReading = (file: TurnReadingFile, { routing, consumers }: SettleStep): boolean => { + const reading = decideFileReading({ routing, file, consumers }); + return reading.automatic && (reading.reader === 'provider' || reading.reader === 'text'); +}; + +/** The request's files in submission order, once each, among those still on the turn. */ +function selectRequestFiles( + files: readonly T[], + requestFileIds: readonly string[], +): T[] { + const byId = new Map(); + for (const file of files) { + if (!byId.has(file.file_id)) { + byId.set(file.file_id, file); + } + } + const selected: T[] = []; + for (const fileId of new Set(requestFileIds)) { + const file = byId.get(fileId); + if (file != null) { + selected.push(file); + } + } + return selected; +} + +/** + * Charges what the automatic policy cannot move first, measured exactly as admission measures it, + * then admits each request file it reads directly in submission order while it fits. A file that does not fit is recorded as overflow, + * so the next decision moves it to another reader or leaves it unavailable. + */ +function allocate( + files: T[], + allocation: DirectContentAllocation, + step: SettleStep, +): T[] { + const { measure, limits, committedExtra = [], committedEntries = [] } = allocation; + const candidates: DirectContentEntry[] = []; + const committed: DirectContentEntry[] = [...committedEntries, ...committedExtra.map(measure)]; + for (const file of selectRequestFiles(files, allocation.requestFileIds)) { + (isDirectReading(file, step) ? candidates : committed).push(measure(file)); + } + const overflow = allocateDirectContent(committed, candidates, limits); + logAllocation({ + agentId: allocation.agentId, + scope: allocation.scope ?? 'request', + candidates: candidates.length, + overflow: overflow.size, + }); + if (overflow.size === 0) { + return files; + } + step.context.addOverflow(overflow); + return route(files, step); +} + +/** + * Settles how a turn reads its files: applies the turn route, derives text the reading needs and + * decides again, then, with an allocation under the automatic policy, moves direct content that + * does not fit the turn's limits to another reader. Returns the input array when nothing changed. + * Without a reading context on the routing this is exactly {@link applyTurnDelivery}. + */ +export async function settleTurnFiles({ + routing, + consumers, + files, + allocation, + flush = false, + signal, +}: SettleTurnFilesParams): Promise { + const resolved = applyTurnDelivery(files, { routing, consumers }); + const context = getTurnReadingContext(routing); + if (routing == null || context == null) { + return resolved; + } + const step: SettleStep = { routing, consumers, context, signal }; + const derived = await deriveNeededText(resolved, step); + const decided = derived == null ? resolved : route(derived, step); + const settled = + allocation != null && context.policy === 'automatic' + ? allocate(decided, allocation, step) + : decided; + if (flush) { + await context.flush(); + } + return settled; +} + +/** + * {@link settleTurnFiles} without allocation, saving its derivations before returning: for readers + * outside initialization, such as history replay, steering and the run-file encoder. Those saves + * happen before the caller's endpoint filter and model-bound admission; the text they keep passed + * the `extracted_text` inspection when it was derived, and a refusal throws without saving. + */ +export function prepareTurnFiles({ + flush = true, + ...params +}: Omit, 'allocation'>): Promise { + return settleTurnFiles({ ...params, flush }); +} diff --git a/packages/api/src/files/reading/turn.spec.ts b/packages/api/src/files/reading/turn.spec.ts new file mode 100644 index 00000000000..36ad6df12a9 --- /dev/null +++ b/packages/api/src/files/reading/turn.spec.ts @@ -0,0 +1,1024 @@ +import { logger } from '@librechat/data-schemas'; +import { FileContext, FileSources, decideFileReading } from 'librechat-data-provider'; +import type { TFileConfig, TurnFileConsumers, TurnDeliveryRouting } from 'librechat-data-provider'; +import type { + DerivedText, + FileTextDeriver, + TurnReadingFile, + EncodedDocuments, + NativeDeliveryAgent, + TextDerivationPersister, + TextDerivationSave, + BuildTurnReadingContextParams, +} from './turn'; +import type { NativeValidationMode } from '~/types'; +import { + needsDerivedText, + getTurnTextOptions, + deriveRequestedText, + getTurnReadingContext, + encodeNativeDocuments, + recordNativeRejections, + buildTurnReadingContext, + getNativeValidationPolicy, + createDerivationPersister, +} from './turn'; +import { resolveTurnDeliveryRouting } from '~/agents/files/delivery'; +import { UninspectableFileError } from '~/protection/files'; + +type EndpointFileConfigInput = NonNullable[string]; + +const MB = 1024 * 1024; +const PDF = 'application/pdf'; +const DOCX = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'; +const PPTX = 'application/vnd.openxmlformats-officedocument.presentationml.presentation'; +const XLSX = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'; + +const routingFor = ( + endpoint: string, + endpointConfig: EndpointFileConfigInput = { llmDeliveryPolicy: 'automatic' }, +): TurnDeliveryRouting => + resolveTurnDeliveryRouting({ + agent: { provider: endpoint, endpoint }, + config: { fileConfig: { endpoints: { [endpoint]: endpointConfig } } }, + }); + +const attachment = ( + overrides: Partial & Pick, +): TurnReadingFile => ({ + type: PDF, + bytes: MB, + source: FileSources.local, + context: FileContext.message_attachment, + llmDeliveryPath: 'provider', + metadata: { destinationChosen: false }, + ...overrides, +}); + +const countByLength = (text: string): number => text.length; + +function contextFor(overrides: Partial = {}) { + const params: BuildTurnReadingContextParams = { + routing: routingFor('openAI'), + provider: 'openAI', + model: 'gpt-4o', + fileTokenLimit: 100_000, + configuredFileSizeLimit: undefined, + countTokens: countByLength, + ...overrides, + }; + const context = buildTurnReadingContext(params); + if (context == null) { + throw new Error('expected a reading context'); + } + return context; +} + +const nativeVerdict = ( + provider: string, + file: TurnReadingFile, + overrides: Partial = {}, +) => contextFor({ routing: routingFor(provider), provider, ...overrides }).judge(file).native; + +describe('buildTurnReadingContext', () => { + it('builds nothing under the classic policy without a text deriver', () => { + const params = { + provider: 'openAI', + fileTokenLimit: 100_000, + configuredFileSizeLimit: undefined, + countTokens: countByLength, + }; + expect(buildTurnReadingContext({ ...params, routing: routingFor('openAI', {}) })).toBe( + undefined, + ); + expect( + buildTurnReadingContext({ + ...params, + routing: routingFor('openAI', { llmDeliveryPolicy: 'classic' }), + }), + ).toBe(undefined); + expect( + buildTurnReadingContext({ + ...params, + routing: routingFor('openAI', { llmDeliveryPolicy: 'automatic', legacyFileUploadUX: true }), + }), + ).toBe(undefined); + }); + + it('builds a derive-only context under the classic policy when a deriver is wired', () => { + const deriveText: FileTextDeriver = jest.fn(); + const context = contextFor({ routing: routingFor('openAI', {}), deriveText }); + const file = attachment({ file_id: 'pdf', text: 'x'.repeat(10), bytes: 500 * MB }); + + expect(context.policy).toBe('classic'); + expect(context.canDerive).toBe(true); + expect(context.judge(file)).toEqual({}); + + context.addOverflow(['pdf']); + context.recordRejections([{ file_id: 'pdf', reason: 'capacity' }]); + context.markTextFailed('pdf'); + expect(context.judge(file)).toEqual({ textFailed: true }); + }); + + it('cannot derive under the automatic policy without a deriver', () => { + const context = contextFor(); + expect(context.policy).toBe('automatic'); + expect(context.canDerive).toBe(false); + }); + + it('is found again through the routing it is attached to, and through a copy of it', () => { + const routing = routingFor('openAI'); + const context = contextFor({ routing }); + expect(getTurnReadingContext(routing)).toBe(undefined); + + routing.reading = context; + expect(getTurnReadingContext(routing)).toBe(context); + expect(getTurnReadingContext({ ...routing })).toBe(context); + expect(getTurnReadingContext({ reading: { judge: () => ({}), canDerive: false } })).toBe( + undefined, + ); + expect(getTurnReadingContext(undefined)).toBe(undefined); + }); +}); + +describe('getTurnTextOptions', () => { + it('supplies nothing without a reading context', () => { + expect(getTurnTextOptions(undefined)).toEqual({}); + expect(getTurnTextOptions(routingFor('openAI'))).toEqual({}); + }); + + it('reuses judged counts and marks truncation under the automatic policy', () => { + const routing = routingFor('openAI'); + const context = contextFor({ routing, fileTokenLimit: 10, countTokens: () => 4 }); + routing.reading = context; + const file = attachment({ file_id: 'notes', text: 'x'.repeat(40), llmDeliveryPath: 'text' }); + expect(context.judge(file).text).toBe('fits'); + + const options = getTurnTextOptions(routing); + + expect(options.markTruncation).toBe(true); + expect(options.knownTokenCount?.(file)).toBe(4); + }); + + it('reuses counts without marking truncation in a derive-only classic context', () => { + const routing = routingFor('openAI', {}); + routing.reading = contextFor({ routing, deriveText: jest.fn() }); + + expect(getTurnTextOptions(routing)).toMatchObject({ markTruncation: false }); + }); +}); + +describe('native validation', () => { + const searchOnly: TurnFileConsumers = { executeCode: false, fileSearch: true }; + const rejectedPdf = attachment({ file_id: 'rejected-pdf' }); + const chosenPdf = attachment({ + file_id: 'chosen-pdf', + metadata: { destinationChosen: true }, + }); + const unmarkedPdf = attachment({ file_id: 'unmarked-pdf', metadata: {} }); + + const automaticAgent = (overrides: Partial = {}): NativeDeliveryAgent => { + const routing = routingFor('openAI'); + routing.reading = contextFor({ routing }); + return { + id: 'agent-1', + deliveryRouting: routing, + fileConsumers: searchOnly, + currentRequestAttachments: [rejectedPdf, chosenPdf, unmarkedPdf], + ...overrides, + }; + }; + + describe('getNativeValidationPolicy', () => { + it('leaves out only a file the automatic policy reads', () => { + const modeOf = getNativeValidationPolicy(automaticAgent()); + + expect(modeOf(rejectedPdf)).toBe('skip'); + expect(modeOf(chosenPdf)).toBe('throw'); + expect(modeOf(unmarkedPdf)).toBe('throw'); + expect(modeOf(attachment({ file_id: 'agent-file', context: FileContext.agents }))).toBe( + 'throw', + ); + }); + + it('keeps a type the operator routed to a fixed path failing as classic', () => { + const routing = routingFor('openAI', { + llmDeliveryPolicy: 'automatic', + defaultLLMDeliveryPath: { overrides: { [PDF]: 'provider' } }, + }); + routing.reading = contextFor({ routing }); + + expect( + getNativeValidationPolicy(automaticAgent({ deliveryRouting: routing }))(rejectedPdf), + ).toBe('throw'); + }); + + it('fails every file outside an automatic reading, without known consumers or without a file tool', () => { + const derivingClassic = routingFor('openAI', {}); + derivingClassic.reading = contextFor({ + routing: derivingClassic, + deriveText: async () => ({ status: 'skipped', reason: 'no_extractor' }), + }); + const agents = [ + automaticAgent({ deliveryRouting: derivingClassic }), + automaticAgent({ deliveryRouting: routingFor('openAI') }), + automaticAgent({ fileConsumers: undefined }), + automaticAgent({ fileConsumers: { executeCode: false, fileSearch: false } }), + undefined, + ]; + + expect(agents.map((agent) => getNativeValidationPolicy(agent)(rejectedPdf))).toEqual([ + 'throw', + 'throw', + 'throw', + 'throw', + 'throw', + ]); + expect( + getNativeValidationPolicy( + automaticAgent({ fileConsumers: { executeCode: true, fileSearch: false } }), + )(rejectedPdf), + ).toBe('skip'); + }); + + it('allows an automatic historical attachment to use the same fallback as a request file', () => { + const replayed = attachment({ file_id: 'replayed-pdf' }); + const agent = automaticAgent({ currentRequestAttachments: [] }); + const modeOf = getNativeValidationPolicy(agent); + + expect(modeOf(rejectedPdf)).toBe('skip'); + expect(modeOf(replayed)).toBe('skip'); + expect(modeOf({ ...replayed, metadata: { destinationChosen: true } })).toBe('throw'); + }); + }); + + describe('encodeNativeDocuments', () => { + type Encoded = EncodedDocuments; + const isInvalid = ({ file_id }: TurnReadingFile): boolean => + file_id.startsWith('rejected') || file_id.startsWith('chosen'); + /** Encodes every file but the rejected and chosen PDFs, which fail validation. */ + const encoder = () => + jest.fn(async (files: TurnReadingFile[], mode: NativeValidationMode): Promise => { + const invalid = files.filter(isInvalid); + if (mode === 'throw' && invalid.length > 0) { + throw new Error('PDF validation failed'); + } + const valid = files.filter((file) => !isInvalid(file)); + return { + documents: valid.map(({ file_id }) => `block:${file_id}`), + files: valid.map(({ file_id }) => file_id), + ...(mode === 'skip' && { + rejected: invalid.map(({ file_id }) => ({ file_id, reason: 'integrity' as const })), + }), + }; + }); + const callsOf = (encode: ReturnType) => + encode.mock.calls.map(([files, mode]) => [files.map(({ file_id }) => file_id), mode]); + + it('encodes a batch that shares one mode in a single call', async () => { + const encode = encoder(); + const fits = attachment({ file_id: 'fits-pdf' }); + + const result = await encodeNativeDocuments([rejectedPdf, fits], automaticAgent(), encode); + + expect(callsOf(encode)).toEqual([[['rejected-pdf', 'fits-pdf'], 'skip']]); + expect(result).toEqual({ + documents: ['block:fits-pdf'], + files: ['fits-pdf'], + rejected: [{ file_id: 'rejected-pdf', reason: 'integrity' }], + }); + }); + + it('still fails the turn on a classic record that fails beside an automatic one', async () => { + const encode = encoder(); + + await expect( + encodeNativeDocuments([rejectedPdf, chosenPdf], automaticAgent(), encode), + ).rejects.toThrow('PDF validation failed'); + expect(callsOf(encode)).toEqual([ + [['rejected-pdf'], 'skip'], + [['chosen-pdf'], 'throw'], + ]); + }); + + it('merges a split encode and lists only the files it left out', async () => { + const encode = encoder(); + + const result = await encodeNativeDocuments( + [rejectedPdf, unmarkedPdf], + automaticAgent(), + encode, + ); + + expect(result).toEqual({ + documents: ['block:unmarked-pdf'], + files: ['unmarked-pdf'], + rejected: [{ file_id: 'rejected-pdf', reason: 'integrity' }], + }); + }); + + it('encodes in throw mode under classic routing', async () => { + const encode = encoder(); + + await expect( + encodeNativeDocuments([rejectedPdf], { deliveryRouting: routingFor('openAI', {}) }, encode), + ).rejects.toThrow('PDF validation failed'); + expect(callsOf(encode)).toEqual([[['rejected-pdf'], 'throw']]); + }); + }); + + describe('recordNativeRejections', () => { + it('records the rejections on each agent carrying the payload, once per reading', () => { + const primary = automaticAgent(); + const handoff = automaticAgent({ id: 'agent-2' }); + const sharing = { ...primary, id: 'agent-3' }; + const levelSpy = jest.spyOn(logger, 'isDebugEnabled').mockReturnValue(true); + const debugSpy = jest.spyOn(logger, 'debug').mockImplementation(() => logger); + + recordNativeRejections( + [primary, handoff, sharing, undefined], + [{ file_id: rejectedPdf.file_id, reason: 'capacity' }], + ); + + expect( + [primary, handoff].map((agent) => { + const context = getTurnReadingContext(agent.deliveryRouting); + return [context?.judge(rejectedPdf).rejected, context?.stats().rejected]; + }), + ).toEqual([ + ['capacity', 1], + ['capacity', 1], + ]); + expect(debugSpy.mock.calls.map(([line]) => line)).toEqual([ + '[nativeDelivery] agent=agent-1 file_id=rejected-pdf provider=openAI rejected=capacity mode=skip next=search', + '[nativeDelivery] agent=agent-2 file_id=rejected-pdf provider=openAI rejected=capacity mode=skip next=search', + ]); + debugSpy.mockRestore(); + levelSpy.mockRestore(); + }); + + it('ignores rejections without a reading context or without rejections', () => { + const classic = automaticAgent({ deliveryRouting: routingFor('openAI') }); + expect(() => + recordNativeRejections([classic], [{ file_id: rejectedPdf.file_id, reason: 'integrity' }]), + ).not.toThrow(); + const agent = automaticAgent(); + + recordNativeRejections([agent], undefined); + recordNativeRejections([agent], []); + + expect(getTurnReadingContext(agent.deliveryRouting)?.stats().rejected).toBe(0); + }); + }); +}); + +describe('text verdicts', () => { + it('settles text within the limit by its byte length without counting tokens', () => { + const countTokens = jest.fn(countByLength); + const context = contextFor({ fileTokenLimit: 10, countTokens }); + + expect(context.judge(attachment({ file_id: 'short', text: 'tiny' })).text).toBe('fits'); + expect(countTokens).not.toHaveBeenCalled(); + expect(context.knownTokenCount(attachment({ file_id: 'short', text: 'tiny' }))).toBe(undefined); + }); + + it('counts tokens once when the byte length exceeds the limit', () => { + const countTokens = jest.fn((text: string) => Math.ceil(text.length / 4)); + const context = contextFor({ fileTokenLimit: 10, countTokens }); + const file = attachment({ file_id: 'long', text: 'x'.repeat(40) }); + + expect(context.judge(file).text).toBe('fits'); + expect(context.judge({ ...file }).text).toBe('fits'); + expect(countTokens).toHaveBeenCalledTimes(1); + expect(context.knownTokenCount(file)).toBe(10); + }); + + it('exceeds when the complete text has more tokens than the limit', () => { + const context = contextFor({ fileTokenLimit: 10 }); + const file = attachment({ file_id: 'long', text: 'x'.repeat(11) }); + + expect(context.judge(file).text).toBe('exceeds'); + expect(context.knownTokenCount(file)).toBe(11); + }); + + it.each([0, undefined])('exceeds when the token limit is %p', (fileTokenLimit) => { + const countTokens = jest.fn(countByLength); + const context = contextFor({ fileTokenLimit, countTokens }); + + expect(context.judge(attachment({ file_id: 'text', text: 'a' })).text).toBe('exceeds'); + expect(countTokens).not.toHaveBeenCalled(); + }); + + it('judges the text again once its length changes', () => { + const countTokens = jest.fn(countByLength); + const context = contextFor({ fileTokenLimit: 10, countTokens }); + + expect(context.judge(attachment({ file_id: 'grows', text: 'x'.repeat(12) })).text).toBe( + 'exceeds', + ); + expect(context.judge(attachment({ file_id: 'grows', text: 'x'.repeat(11) })).text).toBe( + 'exceeds', + ); + expect(countTokens).toHaveBeenCalledTimes(2); + }); + + it('never counts a stored text the reading walk gives to Run Code', () => { + const countTokens = jest.fn(countByLength); + const routing = routingFor('openAI'); + const context = contextFor({ routing, fileTokenLimit: 100_000, countTokens }); + routing.reading = context; + const csv = attachment({ + file_id: 'large-csv', + type: 'text/csv', + llmDeliveryPath: 'text', + text: 'x'.repeat(700_000), + }); + + const reading = decideFileReading({ + routing, + file: csv, + consumers: { executeCode: true, fileSearch: false }, + }); + + expect(reading.reader).toBe('code'); + expect(countTokens).not.toHaveBeenCalled(); + expect(context.knownTokenCount(csv)).toBe(undefined); + + expect(context.judge(csv).text).toBe('exceeds'); + expect(countTokens).toHaveBeenCalledTimes(1); + expect(context.knownTokenCount(csv)).toBe(700_000); + }); + + it('judges no text verdict for a record without text', () => { + const context = contextFor(); + expect(context.judge(attachment({ file_id: 'empty', text: '' }))).not.toHaveProperty('text'); + expect(context.judge(attachment({ file_id: 'none' }))).not.toHaveProperty('text'); + }); +}); + +describe('native verdicts', () => { + it.each<{ + why: string; + provider: string; + model?: string; + type: string; + bytes: number; + configuredFileSizeLimit?: number; + verdict: 'fits' | 'capacity' | 'unsupported'; + }>([ + { + why: 'a PDF on Azure OpenAI, where the encoder emits nothing', + provider: 'azureOpenAI', + type: PDF, + bytes: MB, + verdict: 'unsupported', + }, + { + why: 'a Word document on Anthropic', + provider: 'anthropic', + type: DOCX, + bytes: MB, + verdict: 'unsupported', + }, + { why: 'a PDF on Anthropic', provider: 'anthropic', type: PDF, bytes: MB, verdict: 'fits' }, + { + why: 'a Word document for a Claude model behind an OpenAI-compatible gateway', + provider: 'openAI', + model: 'claude-sonnet-4-5', + type: DOCX, + bytes: MB, + verdict: 'unsupported', + }, + { + why: 'a Word document for an OpenAI model', + provider: 'openAI', + model: 'gpt-4o', + type: DOCX, + bytes: MB, + verdict: 'fits', + }, + { + why: 'a 20 MB PDF for Claude 4 on Bedrock, exempt from the 4.5 MB document limit', + provider: 'bedrock', + model: 'us.anthropic.claude-sonnet-4-20250514-v1:0', + type: PDF, + bytes: 20 * MB, + verdict: 'fits', + }, + { + why: 'a 20 MB PDF for Claude 3.5 on Bedrock', + provider: 'bedrock', + model: 'anthropic.claude-3-5-sonnet-20240620-v1:0', + type: PDF, + bytes: 20 * MB, + verdict: 'capacity', + }, + { + why: 'a presentation on Bedrock', + provider: 'bedrock', + type: PPTX, + bytes: MB, + verdict: 'unsupported', + }, + { + why: 'a PDF above the configured limit', + provider: 'openAI', + type: PDF, + bytes: 2 * MB, + configuredFileSizeLimit: MB, + verdict: 'capacity', + }, + { + why: 'a Word document above the configured limit', + provider: 'openAI', + type: DOCX, + bytes: 2 * MB, + configuredFileSizeLimit: MB, + verdict: 'capacity', + }, + { + why: 'a PDF at the configured limit', + provider: 'openAI', + type: PDF, + bytes: MB, + configuredFileSizeLimit: MB, + verdict: 'fits', + }, + { + why: 'a PDF above the provider limit when none is configured', + provider: 'openAI', + type: PDF, + bytes: 11 * MB, + verdict: 'capacity', + }, + { + why: 'a PDF at the provider limit when none is configured', + provider: 'openAI', + type: PDF, + bytes: 10 * MB, + verdict: 'fits', + }, + ])( + 'judges $why as $verdict', + ({ provider, model, type, bytes, configuredFileSizeLimit, verdict }) => { + const overrides = { + ...(model != null && { model }), + ...(configuredFileSizeLimit != null && { configuredFileSizeLimit }), + }; + expect(nativeVerdict(provider, attachment({ file_id: 'file', type, bytes }), overrides)).toBe( + verdict, + ); + }, + ); + + it('does not judge images, audio or video', () => { + expect( + contextFor().judge(attachment({ file_id: 'image', type: 'image/png' })), + ).not.toHaveProperty('native'); + }); + + it('memoizes the verdict per file', () => { + const context = contextFor(); + expect(context.judge(attachment({ file_id: 'pdf', bytes: MB })).native).toBe('fits'); + expect(context.judge(attachment({ file_id: 'pdf', bytes: 50 * MB })).native).toBe('fits'); + expect(context.judge(attachment({ file_id: 'other', bytes: 50 * MB })).native).toBe('capacity'); + }); +}); + +describe('request evidence', () => { + it('reports overflow, encode-time rejections and derivation failures per file', () => { + const context = contextFor(); + const file = attachment({ file_id: 'pdf' }); + expect(context.judge(file)).toEqual({ native: 'fits' }); + + context.addOverflow(new Set(['pdf'])); + context.recordRejections([ + { file_id: 'pdf', reason: 'integrity' }, + { file_id: 'pdf', reason: 'capacity' }, + ]); + context.markTextFailed('pdf'); + + expect(context.judge(file)).toEqual({ + native: 'fits', + overflow: true, + rejected: 'integrity', + textFailed: true, + }); + expect(context.judge(attachment({ file_id: 'other' }))).toEqual({ native: 'fits' }); + }); + + it('judges File Search reachability only once search evidence is set', () => { + const context = contextFor(); + const queued = attachment({ file_id: 'queued' }); + const registered = attachment({ file_id: 'registered' }); + const missing = attachment({ file_id: 'missing' }); + const unsearchable = attachment({ file_id: 'zip', type: 'application/zip' }); + const chosen = attachment({ file_id: 'chosen', metadata: { destinationChosen: true } }); + + expect(context.judge(missing)).not.toHaveProperty('search'); + + context.setSearchEvidence({ queued: ['queued'], registered: ['registered'] }); + + expect(context.judge(queued).search).toBe('reachable'); + expect(context.judge(registered).search).toBe('reachable'); + expect(context.judge(missing).search).toBe('unreachable'); + expect(context.judge(unsearchable).search).toBe('reachable'); + expect(context.judge(chosen).search).toBe('reachable'); + }); + + it('reads shared preparation outcomes without rerouting already-sent attachments', () => { + const context = contextFor(); + const queued = attachment({ file_id: 'queued' }); + const preparation = new Map(); + context.setSearchEvidence({ queued: ['queued'], registered: ['indexed'], preparation }); + + expect(context.searchState('queued')).toBe('queued'); + expect(context.searchState('indexed')).toBe('ready'); + expect(context.searchState('missing')).toBeUndefined(); + + preparation.set('queued', 'failed'); + expect(context.searchState('queued')).toBe('failed'); + expect(context.judge(queued).search).toBe('reachable'); + + preparation.set('queued', 'ready'); + expect(context.searchState('queued')).toBe('ready'); + expect(context.judge(queued).search).toBe('reachable'); + expect(contextFor().searchState('queued')).toBeUndefined(); + }); + + it('keeps each dropped request file once', () => { + const context = contextFor(); + const first = attachment({ file_id: 'a', bytes: 1 }); + context.recordDropped([first, attachment({ file_id: 'b' })]); + context.recordDropped([attachment({ file_id: 'a', bytes: 2 })]); + + expect(context.dropped().map((file) => file.file_id)).toEqual(['a', 'b']); + expect(context.dropped()[0]).toBe(first); + expect(context.stats()).toEqual({ + overflow: 0, + rejected: 0, + textFailed: 0, + derived: 0, + dropped: 2, + }); + }); +}); + +describe('derive', () => { + const xlsx = attachment({ + file_id: 'xlsx', + type: XLSX, + llmDeliveryPath: 'none', + metadata: { destinationChosen: false, textDerivation: { outcome: 'deferred' } }, + }); + + it('derives each file once per request with the request signal', async () => { + const controller = new AbortController(); + const deriveText = jest.fn, Parameters>( + async () => ({ + status: 'derived', + text: 'a,b', + textDerivation: { outcome: 'complete', extractor: 'document_parser' }, + }), + ); + const context = contextFor({ deriveText, signal: controller.signal }); + + const [first, second] = await Promise.all([context.derive(xlsx), context.derive({ ...xlsx })]); + + expect(first).toBe(second); + expect(first).toMatchObject({ status: 'derived', text: 'a,b' }); + expect(deriveText).toHaveBeenCalledTimes(1); + expect(deriveText).toHaveBeenCalledWith(xlsx, controller.signal); + expect(context.stats().derived).toBe(1); + }); + + it('skips without a deriver', async () => { + await expect(contextFor().derive(xlsx)).resolves.toEqual({ + status: 'skipped', + reason: 'no_extractor', + }); + }); + + it('turns a deriver error into a skipped derivation and logs only safe metadata', async () => { + const errorSpy = jest.spyOn(logger, 'error').mockImplementation(() => logger); + try { + const context = contextFor({ + deriveText: async () => { + throw new Error('secret path /data/report.xlsx'); + }, + }); + + await expect(context.derive(xlsx)).resolves.toEqual({ + status: 'skipped', + reason: 'storage_unavailable', + }); + expect(errorSpy).toHaveBeenCalledWith( + '[readingText] file_id=xlsx outcome=skipped reason=derivation_error', + { type: 'Error' }, + ); + expect(context.stats().derived).toBe(0); + } finally { + errorSpy.mockRestore(); + } + }); + + describe('persistence', () => { + const derived: DerivedText = { + status: 'derived', + text: 'quarter,total\nQ1,42', + textDerivation: { outcome: 'complete', extractor: 'document_parser', at: 1 }, + }; + const persister = () => + jest.fn, Parameters>( + async () => true, + ); + + it.each([ + ['automatic', { llmDeliveryPolicy: 'automatic' as const }], + ['classic', {}], + ])( + 'saves text derived for a marked record once, under the %s policy', + async (_policy, endpointConfig) => { + const persistDerivation = persister(); + const context = contextFor({ + routing: routingFor('openAI', endpointConfig), + deriveText: async () => derived, + persistDerivation, + }); + + await Promise.all([context.derive(xlsx), context.derive({ ...xlsx })]); + await context.flush(); + + expect(persistDerivation).toHaveBeenCalledTimes(1); + expect(persistDerivation).toHaveBeenCalledWith({ + file_id: 'xlsx', + text: derived.text, + textDerivation: derived.textDerivation, + }); + }, + ); + + it('saves a deterministic failure as a marker alone, and nothing it may not keep', async () => { + const outcomes: Record = { + parser: { + status: 'failed', + textDerivation: { outcome: 'failed', reason: 'parser' }, + persist: true, + }, + transient: { + status: 'failed', + textDerivation: { outcome: 'failed', reason: 'extractor_unavailable' }, + persist: false, + }, + skipped: { status: 'skipped', reason: 'storage_unavailable' }, + }; + const persistDerivation = persister(); + const context = contextFor({ + deriveText: async (file) => outcomes[file.file_id], + persistDerivation, + }); + + await Promise.all( + Object.keys(outcomes).map((file_id) => context.derive({ ...xlsx, file_id })), + ); + await context.flush(); + + expect(persistDerivation).toHaveBeenCalledTimes(1); + const [[update]] = persistDerivation.mock.calls; + expect(update).toEqual({ + file_id: 'parser', + textDerivation: { outcome: 'failed', reason: 'parser' }, + }); + expect(update).not.toHaveProperty('text'); + }); + + it('caches text derived for an unmarked original and never for a settled record', async () => { + const persistDerivation = persister(); + const context = contextFor({ deriveText: async () => derived, persistDerivation }); + const unmarked = attachment({ file_id: 'pdf' }); + const settled = { + ...xlsx, + file_id: 'settled', + metadata: { textDerivation: derived.textDerivation }, + }; + + await Promise.all([context.derive(unmarked), context.derive(settled)]); + await context.flush(); + + expect(persistDerivation).toHaveBeenCalledTimes(1); + expect(persistDerivation).toHaveBeenCalledWith({ + file_id: 'pdf', + text: derived.text, + textDerivation: derived.textDerivation, + }); + }); + + it('keeps a failed marker only on a record the automatic policy deferred', async () => { + const persistDerivation = persister(); + const failure: DerivedText = { + status: 'failed', + textDerivation: { outcome: 'failed', reason: 'parser' }, + persist: true, + }; + const context = contextFor({ deriveText: async () => failure, persistDerivation }); + + await context.derive(attachment({ file_id: 'pdf' })); + await context.flush(); + + expect(persistDerivation).not.toHaveBeenCalled(); + }); + + it('forgets an aborted derivation so a later caller derives again', async () => { + const deriveText = jest + .fn(async (): Promise => derived) + .mockResolvedValueOnce({ status: 'skipped', reason: 'aborted' }); + const context = contextFor({ deriveText }); + + await expect(context.derive(xlsx)).resolves.toEqual({ status: 'skipped', reason: 'aborted' }); + await expect(context.derive(xlsx)).resolves.toBe(derived); + expect(deriveText).toHaveBeenCalledTimes(2); + }); + + it('writes nothing until the flush, so a caller saves only files that passed its checks', async () => { + const persistDerivation = persister(); + const context = contextFor({ deriveText: async () => derived, persistDerivation }); + + await context.derive(xlsx); + await new Promise((resolve) => setImmediate(resolve)); + expect(persistDerivation).not.toHaveBeenCalled(); + + await context.flush(); + expect(persistDerivation).toHaveBeenCalledTimes(1); + await context.flush(); + expect(persistDerivation).toHaveBeenCalledTimes(1); + }); + + it('settles the flush only after pending derivations and their queued writes', async () => { + let finishDerive: (() => void) | undefined; + let finishWrite: ((saved: boolean) => void) | undefined; + const context = contextFor({ + deriveText: () => + new Promise((resolve) => { + finishDerive = () => resolve(derived); + }), + persistDerivation: () => + new Promise((resolve) => { + finishWrite = resolve; + }), + }); + void context.derive(xlsx); + + let flushed = false; + const flushing = context.flush().then(() => { + flushed = true; + }); + await new Promise((resolve) => setImmediate(resolve)); + expect(finishDerive).toBeDefined(); + expect(finishWrite).toBeUndefined(); + expect(flushed).toBe(false); + + finishDerive?.(); + await new Promise((resolve) => setImmediate(resolve)); + expect(finishWrite).toBeDefined(); + expect(flushed).toBe(false); + + finishWrite?.(true); + await flushing; + expect(flushed).toBe(true); + await expect(context.flush()).resolves.toBeUndefined(); + }); + + it('logs a failed write with safe metadata and still settles the flush', async () => { + const errorSpy = jest.spyOn(logger, 'error').mockImplementation(() => logger); + try { + const context = contextFor({ + deriveText: async () => derived, + persistDerivation: async () => { + throw new Error('mongodb://user:secret@db/files'); + }, + }); + + await expect(context.derive(xlsx)).resolves.toBe(derived); + await expect(context.flush()).resolves.toBeUndefined(); + expect(errorSpy).toHaveBeenCalledWith( + '[readingText] file_id=xlsx outcome=complete persisted=failed', + { type: 'Error' }, + ); + } finally { + errorSpy.mockRestore(); + } + }); + }); + + describe('createDerivationPersister', () => { + const update = { + file_id: 'xlsx', + text: 'quarter,total', + textDerivation: { outcome: 'complete' as const, extractor: 'document_parser' as const }, + }; + const scope = { user: 'user-1', tenantId: 'tenant-1' }; + + it('writes a file once for every agent context sharing it', async () => { + const save = jest.fn, Parameters>( + async () => true, + ); + const persistDerivation = createDerivationPersister(save, scope); + const deriveText: FileTextDeriver = async () => ({ + status: 'derived', + text: update.text, + textDerivation: update.textDerivation, + }); + const primary = contextFor({ deriveText, persistDerivation }); + const handoff = contextFor({ deriveText, persistDerivation }); + + await Promise.all([primary.derive(xlsx), handoff.derive({ ...xlsx })]); + await Promise.all([primary.flush(), handoff.flush()]); + + expect(save).toHaveBeenCalledTimes(1); + expect(save).toHaveBeenCalledWith(update, scope); + }); + + it('forgets a failed write, so a later context tries again', async () => { + const save = jest + .fn, Parameters>() + .mockRejectedValueOnce(new Error('write conflict')) + .mockResolvedValueOnce(true); + const persistDerivation = createDerivationPersister(save, scope); + + await expect(persistDerivation(update)).rejects.toThrow('write conflict'); + await expect(persistDerivation(update)).resolves.toBe(true); + await expect(persistDerivation(update)).resolves.toBe(true); + expect(save).toHaveBeenCalledTimes(2); + }); + }); +}); + +describe('deriveRequestedText', () => { + const workbook = attachment({ + file_id: 'xlsx', + type: XLSX, + llmDeliveryPath: 'none', + metadata: { destinationChosen: false, textDerivation: { outcome: 'deferred' } }, + }); + + it('derives once through the first context that asked and returns the text by file id', async () => { + const firstDeriver = jest.fn, Parameters>( + async () => ({ + status: 'derived', + text: 'a,b', + textDerivation: { outcome: 'complete', extractor: 'document_parser' }, + }), + ); + const secondDeriver = jest.fn, Parameters>(); + const first = contextFor({ deriveText: firstDeriver }); + const second = contextFor({ deriveText: secondDeriver }); + + const texts = await deriveRequestedText([{ file: workbook, contexts: [first, second] }]); + + expect(texts.get('xlsx')).toMatchObject({ status: 'derived', text: 'a,b' }); + expect(firstDeriver).toHaveBeenCalledTimes(1); + expect(secondDeriver).not.toHaveBeenCalled(); + expect(second.judge(workbook).textFailed).toBeUndefined(); + }); + + it('throws the policy error of derived text a content policy refuses', async () => { + const error = new UninspectableFileError('extracted_text'); + const context = contextFor({ deriveText: async () => ({ status: 'blocked', error }) }); + + await expect(deriveRequestedText([{ file: workbook, contexts: [context] }])).rejects.toBe( + error, + ); + }); + + it('marks a file left without text failed on every context that asked', async () => { + const first = contextFor({ + deriveText: async () => ({ status: 'skipped', reason: 'storage_unavailable' }), + }); + const second = contextFor({ routing: routingFor('openAI', {}), deriveText: jest.fn() }); + + const texts = await deriveRequestedText([{ file: workbook, contexts: [first, second] }]); + + expect(texts.size).toBe(0); + expect(first.judge(workbook).textFailed).toBe(true); + expect(second.judge(workbook).textFailed).toBe(true); + }); +}); + +describe('needsDerivedText', () => { + it('decides only marked records for a derive-only classic context', () => { + const routing = routingFor('openAI', {}); + const context = contextFor({ routing, deriveText: jest.fn() }); + routing.reading = context; + const consumers = { executeCode: false, fileSearch: false }; + const marked = attachment({ + file_id: 'marked', + type: XLSX, + llmDeliveryPath: 'none', + metadata: { destinationChosen: false, textDerivation: { outcome: 'deferred' } }, + }); + const unmarked = { ...marked, file_id: 'unmarked', metadata: { destinationChosen: false } }; + + expect(needsDerivedText(marked, { routing, consumers, context })).toBe(true); + expect(needsDerivedText(unmarked, { routing, consumers, context })).toBe(false); + }); +}); diff --git a/packages/api/src/files/reading/turn.ts b/packages/api/src/files/reading/turn.ts new file mode 100644 index 00000000000..3de4275d573 --- /dev/null +++ b/packages/api/src/files/reading/turn.ts @@ -0,0 +1,767 @@ +import { Providers } from '@librechat/agents'; +import { logger } from '@librechat/data-schemas'; +import { + getRoutingMimeType, + hasFileToolConsumer, + EToolResources, + decideFileReading, + isBedrockDocumentType, + categorizeForReading, + canToolResourceConsume, + isAnthropicDocumentType, + resolveLLMDeliveryPolicy, + isAutomaticReadingRecord, + isDocumentSupportedProvider, +} from 'librechat-data-provider'; +import type { + TFile, + TextDerivation, + ReadingEvidence, + TurnDeliveryFile, + TurnFileConsumers, + TurnReadingInputs, + TLLMDeliveryPolicy, + TurnDeliveryRouting, +} from 'librechat-data-provider'; +import type { FileTextDerivationUpdate } from '@librechat/data-schemas'; +import type { DocumentRejection, NativeValidationMode } from '~/types'; +import type { ContentFilterError } from '~/middleware/contentFilter'; +import type { UninspectableFileError } from '~/protection/files'; +import { usesAnthropicDocumentCapabilities } from '~/files/encode/document'; +import { getNativeDocumentSizeLimit } from '~/files/validation'; +import { getSafeErrorMetadata } from '~/utils/errors'; + +/** Where a record's retained original is stored, for the deriver that reads it back. */ +type StoredOriginal = Partial>; + +/** A turn copy of an attachment record, identified so request-scoped evidence can name it. */ +export type TurnReadingFile = TurnDeliveryFile & StoredOriginal & { file_id: string }; + +/** Why no text was derived on this request; none of these is persisted, so a later turn retries. */ +export type DerivationSkipReason = 'policy' | 'storage_unavailable' | 'aborted' | 'no_extractor'; + +/** The content-policy error derived text raised: a finding, or text the policy cannot inspect. */ +export type DerivationPolicyError = ContentFilterError | UninspectableFileError; + +/** + * The outcome of deriving a file's text from its retained original. A `failed` outcome is + * deterministic, and `persist` says whether the record may keep it as a failed marker. A + * `blocked` outcome is a content-policy refusal of the derived text, which fails the turn that + * needed it exactly as the same text stored at upload would. + */ +export type DerivedText = + | { status: 'derived'; text: string; textDerivation: TextDerivation } + | { status: 'failed'; textDerivation: TextDerivation; persist: boolean } + | { status: 'blocked'; error: DerivationPolicyError } + | { status: 'skipped'; reason: DerivationSkipReason }; + +export type FileTextDeriver = (file: TurnReadingFile, signal?: AbortSignal) => Promise; + +/** A derivation that changed the copy's text, as {@link withDerivedText} applies it. */ +export type DerivedTextSuccess = Extract; + +/** What a derivation writes back onto a record: see {@link selectDerivationUpdate}. */ +export type TextDerivationUpdate = FileTextDerivationUpdate; + +/** Saves a derivation onto its record and resolves whether a record changed. */ +export type TextDerivationPersister = (update: TextDerivationUpdate) => Promise; + +/** The owner a derivation is saved under, as `saveFileTextDerivation` scopes its write. */ +export interface DerivationOwnerScope { + user: string; + tenantId?: string | null; +} + +/** Saves a derivation under an owner scope: the data-schemas `saveFileTextDerivation`. */ +export type TextDerivationSave = ( + update: TextDerivationUpdate, + scope: DerivationOwnerScope, +) => Promise; + +/** + * A request's derivation persister: saves under the request's file owner and writes each file at + * most once, so the reading contexts of several agents that share one deriver issue one write per + * derived file. A failed write is forgotten, so a later context may try again. The host builds + * one per request, next to the deriver, and hands it to every agent. + */ +export function createDerivationPersister( + save: TextDerivationSave, + scope: DerivationOwnerScope, +): TextDerivationPersister { + const writes = new Map>(); + return (update) => { + const existing = writes.get(update.file_id); + if (existing != null) { + return existing; + } + const write = save(update, scope); + writes.set(update.file_id, write); + write.catch(() => { + if (writes.get(update.file_id) === write) { + writes.delete(update.file_id); + } + }); + return write; + }; +} + +/** Preparation of one file in this turn's authorized File Search namespace. */ +export type SearchPreparationState = 'queued' | 'ready' | 'failed'; + +/** Which files File Search will receive on this turn, known once tools have loaded. */ +export interface SearchEvidence { + /** Files queued for embedding on this request. */ + queued: readonly string[]; + /** Files registered under the turn's `file_search` tool resource. */ + registered: readonly string[]; + /** Shared with lazy provisioning; a failed attempt stays failed until a retry succeeds. */ + preparation?: ReadonlyMap; +} + +/** Request-scoped counts for the turn's reading diagnostics. */ +export interface TurnReadingStats { + overflow: number; + rejected: number; + textFailed: number; + derived: number; + dropped: number; +} + +/** + * One agent's reading evidence for one request. Attached as `deliveryRouting.reading`, so every + * reader holding the routing sees the verdicts and request-scoped evidence the others recorded. + */ +export interface TurnReadingContext extends TurnReadingInputs { + readonly policy: TLLMDeliveryPolicy; + /** The provider the document encoder receives for this agent. */ + readonly provider: string; + /** The token count of the file's current text, when judging it required a count. */ + knownTokenCount(file: TurnDeliveryFile): number | undefined; + addOverflow(fileIds: Iterable): void; + recordRejections(rejections: ReadonlyArray): void; + markTextFailed(fileId: string): void; + setSearchEvidence(evidence: SearchEvidence): void; + /** Whether an eligible search file is indexed, queued, or failed on this turn. */ + searchState(fileId: string): SearchPreparationState | undefined; + /** Request attachments the endpoint's runtime policy removed before delivery. */ + recordDropped(files: Iterable): void; + dropped(): readonly TurnReadingFile[]; + /** Derives the file's text once per request; repeated calls share the first derivation. */ + derive(file: TurnReadingFile, signal?: AbortSignal): Promise; + /** + * Starts the record writes the request's derivations queued, once each derivation settles, and + * waits for them. Never rejects. Nothing is written before a flush, but a flush is not what + * makes the text safe to keep: derived text is content-inspected when it is derived + * (`extractBoundedText` applies the `extracted_text` policy), text a policy refuses is never + * written, and its refusal throws from {@link deriveRequestedText}. Initialization flushes only + * after endpoint admission and the full-set inspection. History replay, steering and the + * run-file encoder flush right after deriving, before their own endpoint filter and model-bound + * admission run; the scoped handoff candidates do too, but derive only for files each + * receiver's endpoint accepts. + */ + flush(): Promise; + stats(): TurnReadingStats; +} + +export interface BuildTurnReadingContextParams { + routing: TurnDeliveryRouting; + /** The provider the document encoder receives for this agent. */ + provider: string; + model?: string; + /** The request's `fileTokenLimit`; a missing or zero limit delivers no text. */ + fileTokenLimit: number | undefined; + /** The endpoint `fileSizeLimit` the document encoder applies, in bytes. */ + configuredFileSizeLimit: number | undefined; + /** A synchronous token count, ready for use. */ + countTokens: (text: string) => number; + deriveText?: FileTextDeriver; + /** + * Saves derived text, or a deterministic failure, onto the record it came from. The caller + * binds the request's file owner scope. A record marked `deferred`, or an unmarked original + * without text, takes derived text; a failed marker is kept only by a deferred record. + */ + persistDerivation?: TextDerivationPersister; + signal?: AbortSignal; +} + +type NativeVerdict = NonNullable; + +interface TextVerdict { + fit: NonNullable; + tokens?: number; +} + +const TURN_READING = Symbol('turnReading'); + +type BrandedTurnReadingContext = TurnReadingContext & { readonly [TURN_READING]: true }; + +const isTurnReadingContext = ( + reading: TurnReadingInputs | undefined, +): reading is BrandedTurnReadingContext => reading != null && TURN_READING in reading; + +/** The reading context {@link buildTurnReadingContext} built for this routing, if any. */ +export function getTurnReadingContext( + routing?: Pick | null, +): TurnReadingContext | undefined { + const reading = routing?.reading; + return isTurnReadingContext(reading) ? reading : undefined; +} + +/** The `extractFileContext` options a turn's reading supplies for its text injection. */ +export interface TurnTextOptions { + knownTokenCount?: (file: TurnDeliveryFile) => number | undefined; + markTruncation?: boolean; +} + +/** + * Reuses the token counts the reading judged text fit with, and under the automatic policy makes + * any truncation visible: the policy never sends partial text on purpose, so a truncated file + * means a reader judged it without the turn's evidence. Empty without a reading context. + */ +export function getTurnTextOptions( + routing?: Pick | null, +): TurnTextOptions { + const context = getTurnReadingContext(routing); + if (context == null) { + return {}; + } + return { + knownTokenCount: context.knownTokenCount, + markTruncation: context.policy === 'automatic', + }; +} + +/** The agent fields that decide how the document encoder treats the files it cannot send. */ +export interface NativeDeliveryAgent { + id?: string; + deliveryRouting?: TurnDeliveryRouting | null; + fileConsumers?: TurnFileConsumers; + currentRequestAttachments?: readonly TurnDeliveryFile[]; +} + +/** What a document encoder returns, as a split encode merges it. */ +export interface EncodedDocuments { + documents: D[]; + files?: M[]; + rejected?: readonly DocumentRejection[]; +} + +/** + * How the document encoder treats each file it cannot send. Only a file the automatic policy + * reads has another reader to move to, so only it is left out instead of failing the turn; + * explicit, configured, unmarked and legacy records keep failing exactly as classic. + */ +export function getNativeValidationPolicy( + agent: NativeDeliveryAgent | null | undefined, +): (file: TurnDeliveryFile) => NativeValidationMode { + const routing = agent?.deliveryRouting ?? undefined; + if ( + getTurnReadingContext(routing)?.policy !== 'automatic' || + !hasFileToolConsumer(agent?.fileConsumers) + ) { + return () => 'throw'; + } + return (file) => (isAutomaticReadingRecord(routing, file) ? 'skip' : 'throw'); +} + +const mergeEncoded = ( + skipped: EncodedDocuments, + thrown: EncodedDocuments, +): EncodedDocuments => ({ + documents: [...skipped.documents, ...thrown.documents], + files: [...(skipped.files ?? []), ...(thrown.files ?? [])], + rejected: [...(skipped.rejected ?? []), ...(thrown.rejected ?? [])], +}); + +/** + * Encodes documents with each file's own validation mode: files the automatic policy may leave + * out are encoded in skip mode, the rest in throw mode, so a classic record still fails the turn. + * A batch whose files share one mode is encoded in a single call. + */ +export async function encodeNativeDocuments( + files: F[], + agent: NativeDeliveryAgent | null | undefined, + encode: ( + files: F[], + onValidationFailure: NativeValidationMode, + ) => Promise>, +): Promise> { + const validationMode = getNativeValidationPolicy(agent); + const skippable = files.filter((file) => validationMode(file) === 'skip'); + if (skippable.length === 0) { + return encode(files, 'throw'); + } + if (skippable.length === files.length) { + return encode(files, 'skip'); + } + const skippableSet = new Set(skippable); + const [skipped, thrown] = await Promise.all([ + encode(skippable, 'skip'), + encode( + files.filter((file) => !skippableSet.has(file)), + 'throw', + ), + ]); + return mergeEncoded(skipped, thrown); +} + +/** The reader each rejected request file moves to, by file id, for the delivery log. */ +function describeNextReaders( + agent: NativeDeliveryAgent, + rejected: readonly DocumentRejection[], +): Map { + const rejectedIds = new Set(rejected.map(({ file_id }) => file_id)); + const next = new Map(); + for (const file of agent.currentRequestAttachments ?? []) { + if (file.file_id == null || !rejectedIds.has(file.file_id) || next.has(file.file_id)) { + continue; + } + const reading = decideFileReading({ + routing: agent.deliveryRouting ?? undefined, + file, + consumers: agent.fileConsumers, + }); + next.set(file.file_id, reading.reader); + } + return next; +} + +/** + * Records the documents the encoder left out on every agent whose messages carry the encoded + * payload, so each agent's later decisions and inventory move them on. Each reading context is + * recorded once, even when agents share it. + */ +export function recordNativeRejections( + agents: Iterable, + rejected?: readonly DocumentRejection[], +): void { + if (rejected == null || rejected.length === 0) { + return; + } + const recorded = new Set(); + for (const agent of agents) { + const context = getTurnReadingContext(agent?.deliveryRouting); + if (agent == null || context == null || recorded.has(context)) { + continue; + } + recorded.add(context); + context.recordRejections(rejected); + if (!logger.isDebugEnabled()) { + continue; + } + const next = describeNextReaders(agent, rejected); + for (const { file_id, reason } of rejected) { + logger.debug( + `[nativeDelivery] agent=${agent.id ?? 'unknown'} file_id=${file_id} provider=${context.provider} rejected=${reason} mode=skip next=${next.get(file_id) ?? 'unknown'}`, + ); + } + } +} + +const hasText = (text: TurnDeliveryFile['text']): text is string => + typeof text === 'string' && text.length > 0; + +/** Whether the document encoder emits any block for this type on this provider and model. */ +function encoderEmits(provider: string, mimeType: string, model?: string): boolean { + if (provider === Providers.BEDROCK) { + return isBedrockDocumentType(mimeType); + } + if (!isDocumentSupportedProvider(provider)) { + return false; + } + return ( + !usesAnthropicDocumentCapabilities(provider as Providers, model) || + isAnthropicDocumentType(mimeType) + ); +} + +/** + * Text fits when the complete text is within the token limit. Every token spans at least one + * byte, so a byte length within the limit settles it without counting. + */ +function judgeTextFit( + text: string, + limit: number | undefined, + countTokens: (text: string) => number, +): TextVerdict { + if (!limit) { + return { fit: 'exceeds' }; + } + if (Buffer.byteLength(text) <= limit) { + return { fit: 'fits' }; + } + const tokens = countTokens(text); + return { fit: tokens <= limit ? 'fits' : 'exceeds', tokens }; +} + +/** + * The write a derivation makes on its record: the text with its marker on success, the marker + * alone for a deterministic failure the record may keep. Derived text is kept by the record that + * deferred it, and cached on an unmarked original that stored none, so a document the provider + * could not take is parsed once rather than on every turn that replays it. A failed marker is + * kept only where the record was deferred; a settled or failed record is never written. + */ +function selectDerivationUpdate( + file: TurnReadingFile, + result: DerivedText, +): TextDerivationUpdate | undefined { + const outcome = file.metadata?.textDerivation?.outcome; + const hasText = typeof file.text === 'string' && file.text.length > 0; + if (result.status === 'derived' && (outcome === 'deferred' || (outcome == null && !hasText))) { + return { file_id: file.file_id, text: result.text, textDerivation: result.textDerivation }; + } + if (result.status === 'failed' && result.persist && outcome === 'deferred') { + return { file_id: file.file_id, textDerivation: result.textDerivation }; + } + return undefined; +} + +function createTurnReadingContext( + policy: TLLMDeliveryPolicy, + { + routing, + provider, + model, + fileTokenLimit, + configuredFileSizeLimit, + countTokens, + deriveText, + persistDerivation, + signal: requestSignal, + }: BuildTurnReadingContextParams, +): BrandedTurnReadingContext { + const automatic = policy === 'automatic'; + const nativeVerdicts = new Map(); + const textVerdicts = new Map(); + const overflow = new Set(); + const rejected = new Map(); + const textFailed = new Set(); + const droppedFiles = new Map(); + const derivations = new Map>(); + const queuedWrites: Array<{ file: TurnReadingFile; derivation: Promise }> = []; + let searchFileIds: ReadonlySet | undefined; + let registeredSearchFileIds: ReadonlySet | undefined; + let searchPreparation: ReadonlyMap | undefined; + let derived = 0; + + const judgeNativeUncached = (file: TurnDeliveryFile): NativeVerdict | null => { + const routingMimeType = getRoutingMimeType(file); + if (categorizeForReading(routingMimeType) === 'media') { + return null; + } + const mimeType = file.type ?? ''; + if (!encoderEmits(provider, mimeType, model)) { + return 'unsupported'; + } + const limit = getNativeDocumentSizeLimit({ + provider, + mimeType, + model, + configuredFileSizeLimit, + }); + return limit !== undefined && (file.bytes ?? 0) > limit ? 'capacity' : 'fits'; + }; + + const judgeNative = (file: TurnDeliveryFile): NativeVerdict | null => { + if (file.file_id == null) { + return judgeNativeUncached(file); + } + const cached = nativeVerdicts.get(file.file_id); + if (cached !== undefined) { + return cached; + } + const verdict = judgeNativeUncached(file); + nativeVerdicts.set(file.file_id, verdict); + return verdict; + }; + + const textKey = (file: TurnDeliveryFile, text: string): string | undefined => + file.file_id == null ? undefined : `${file.file_id}:${text.length}`; + + const judgeText = (file: TurnDeliveryFile): TextVerdict | undefined => { + if (!hasText(file.text)) { + return undefined; + } + const key = textKey(file, file.text); + const cached = key == null ? undefined : textVerdicts.get(key); + if (cached != null) { + return cached; + } + const verdict = judgeTextFit(file.text, fileTokenLimit, countTokens); + if (key != null) { + textVerdicts.set(key, verdict); + } + return verdict; + }; + + const judgeSearch = (file: TurnDeliveryFile): ReadingEvidence['search'] => { + if (searchFileIds == null) { + return undefined; + } + const unreachable = + (file.file_id == null || !searchFileIds.has(file.file_id)) && + canToolResourceConsume(EToolResources.file_search, file.type ?? '') && + isAutomaticReadingRecord(routing, file); + return unreachable ? 'unreachable' : 'reachable'; + }; + + const requestEvidence = (fileId: string | undefined): ReadingEvidence => ({ + ...(fileId != null && textFailed.has(fileId) && { textFailed: true as const }), + }); + + /** + * Exposes the text verdict as an enumerable getter judged on first read, so a reading walk that + * settles on Run Code, the provider or File Search never counts tokens in text it will not + * deliver. The verdict is memoized here and, by file and text length, for knownTokenCount. + */ + const withLazyText = (evidence: ReadingEvidence, file: TurnDeliveryFile): ReadingEvidence => { + let verdict: TextVerdict | undefined; + Object.defineProperty(evidence, 'text', { + enumerable: true, + configurable: true, + get: () => { + verdict ??= judgeText(file); + return verdict?.fit; + }, + }); + return evidence; + }; + + const judge = (file: TurnDeliveryFile): ReadingEvidence => { + if (!automatic) { + return requestEvidence(file.file_id); + } + const fileId = file.file_id; + const native = judgeNative(file); + const rejection = fileId == null ? undefined : rejected.get(fileId); + const search = judgeSearch(file); + const evidence: ReadingEvidence = { + ...(native != null && { native }), + ...(rejection != null && { rejected: rejection }), + ...(fileId != null && overflow.has(fileId) && { overflow: true as const }), + ...requestEvidence(fileId), + ...(search != null && { search }), + }; + return hasText(file.text) ? withLazyText(evidence, file) : evidence; + }; + + const knownTokenCount = (file: TurnDeliveryFile): number | undefined => { + if (!hasText(file.text)) { + return undefined; + } + const key = textKey(file, file.text); + return key == null ? undefined : textVerdicts.get(key)?.tokens; + }; + + const runDerivation = (file: TurnReadingFile, signal?: AbortSignal): Promise => { + if (deriveText == null) { + return Promise.resolve({ status: 'skipped', reason: 'no_extractor' }); + } + return deriveText(file, signal).then( + (result): DerivedText => { + if (result.status === 'derived') { + derived += 1; + } + return result; + }, + (error: unknown): DerivedText => { + logger.error( + `[readingText] file_id=${file.file_id} outcome=skipped reason=derivation_error`, + getSafeErrorMetadata(error), + ); + return { status: 'skipped', reason: signal?.aborted ? 'aborted' : 'storage_unavailable' }; + }, + ); + }; + + /** Saves the derivation onto a marked record; a failed write is logged, never thrown. */ + const persist = async (file: TurnReadingFile, result: DerivedText): Promise => { + const update = selectDerivationUpdate(file, result); + if (update == null || persistDerivation == null) { + return; + } + const line = `[readingText] file_id=${update.file_id} outcome=${update.textDerivation.outcome}`; + try { + const saved = await persistDerivation(update); + logger.debug(`${line} persisted=${saved}`); + } catch (error) { + logger.error(`${line} persisted=failed`, getSafeErrorMetadata(error)); + } + }; + + const derive = (file: TurnReadingFile, signal?: AbortSignal): Promise => { + const existing = derivations.get(file.file_id); + if (existing != null) { + return existing; + } + const derivation = runDerivation(file, signal ?? requestSignal); + derivations.set(file.file_id, derivation); + queuedWrites.push({ file, derivation }); + /* An aborted derivation answers only its own caller, like the deriver's memo. */ + const forget = () => { + if (derivations.get(file.file_id) === derivation) { + derivations.delete(file.file_id); + } + }; + derivation.then((result) => { + if (result.status === 'skipped' && result.reason === 'aborted') { + forget(); + } + }, forget); + return derivation; + }; + + const flush = async (): Promise => { + const writes = queuedWrites.splice(0); + await Promise.allSettled( + writes.map(({ file, derivation }) => derivation.then((result) => persist(file, result))), + ); + }; + + return { + [TURN_READING]: true, + policy, + provider, + canDerive: deriveText != null, + judge, + knownTokenCount, + addOverflow: (fileIds) => { + for (const fileId of fileIds) { + overflow.add(fileId); + } + }, + recordRejections: (rejections) => { + for (const { file_id, reason } of rejections) { + if (!rejected.has(file_id)) { + rejected.set(file_id, reason); + } + } + }, + markTextFailed: (fileId) => { + textFailed.add(fileId); + }, + setSearchEvidence: ({ queued, registered, preparation }) => { + searchFileIds = new Set([...queued, ...registered]); + registeredSearchFileIds = new Set(registered); + searchPreparation = preparation; + }, + searchState: (fileId) => { + const preparation = searchPreparation?.get(fileId); + if (preparation != null) { + return preparation; + } + if (registeredSearchFileIds?.has(fileId)) { + return 'ready'; + } + if (searchFileIds?.has(fileId)) { + return 'queued'; + } + return undefined; + }, + recordDropped: (files) => { + for (const file of files) { + if (!droppedFiles.has(file.file_id)) { + droppedFiles.set(file.file_id, file); + } + } + }, + dropped: () => [...droppedFiles.values()], + derive, + flush, + stats: () => ({ + overflow: overflow.size, + rejected: rejected.size, + textFailed: textFailed.size, + derived, + dropped: droppedFiles.size, + }), + }; +} + +/** + * Builds the reading evidence for one agent's turn, or nothing where the turn needs none: under + * the classic policy without a text deriver every route stays classic. Under classic with a + * deriver the context only derives text, for records the automatic policy marked, and its judge + * reports nothing but this request's derivation failures. + */ +export function buildTurnReadingContext( + params: BuildTurnReadingContextParams, +): TurnReadingContext | undefined { + const policy = resolveLLMDeliveryPolicy(params.routing.endpointConfig); + if (policy !== 'automatic' && params.deriveText == null) { + return undefined; + } + return createTurnReadingContext(policy, params); +} + +/** One agent's turn routing, the tools judged as its readers, and its reading context. */ +export interface TurnReader { + routing: TurnDeliveryRouting; + consumers?: TurnFileConsumers; + context: TurnReadingContext; +} + +/** + * Whether the file's reading needs text derived on this turn. A derive-only classic context + * serves only records the automatic policy marked, so an unmarked record costs it no decision. + */ +export function needsDerivedText( + file: TurnReadingFile, + { routing, consumers, context }: TurnReader, +): boolean { + if (context.policy !== 'automatic' && file.metadata?.textDerivation == null) { + return false; + } + return decideFileReading({ routing, file, consumers }).needsText; +} + +/** The copy carrying derived text and the marker saying how it was obtained. */ +export const withDerivedText = ( + file: T, + derived: DerivedTextSuccess, +): T => ({ + ...file, + text: derived.text, + metadata: { ...file.metadata, textDerivation: derived.textDerivation }, +}); + +/** A file whose reading needs text, with every reading context whose decision asked for it. */ +export interface TextRequest { + file: T; + /** The first context derives; the rest share its outcome. */ + contexts: readonly [TurnReadingContext, ...TurnReadingContext[]]; +} + +/** + * Derives each requested file's text once, through the first context that asked for it, and + * returns the successes by file id. A file left without text is marked failed on every context + * that asked, so their next decisions pass text over for the rest of the request. Derived text a + * content policy refuses throws its policy error: the turn needed that text, and the same text + * stored at upload would have been refused there. + */ +export async function deriveRequestedText( + requests: readonly TextRequest[], + signal?: AbortSignal, +): Promise> { + const outcomes = await Promise.all( + requests.map(async ({ file, contexts }) => ({ + file, + contexts, + result: await contexts[0].derive(file, signal), + })), + ); + const texts = new Map(); + for (const { file, contexts, result } of outcomes) { + if (result.status === 'derived') { + texts.set(file.file_id, result); + continue; + } + if (result.status === 'blocked') { + throw result.error; + } + for (const context of contexts) { + context.markTextFailed(file.file_id); + } + } + return texts; +} diff --git a/packages/api/src/files/reading/upload.spec.ts b/packages/api/src/files/reading/upload.spec.ts new file mode 100644 index 00000000000..8ce42300552 --- /dev/null +++ b/packages/api/src/files/reading/upload.spec.ts @@ -0,0 +1,513 @@ +import fs from 'fs'; +import os from 'os'; +import path from 'path'; +import { logger } from '@librechat/data-schemas'; +import type { EndpointFileConfig, FiltersConfig, TextDerivation } from 'librechat-data-provider'; +import type { ResolveUploadReadingInput, ResolvedUploadReading } from './upload'; +import { + logUploadReading, + acquireUploadText, + orderToolsForReading, + resolveUploadReading, + getUploadReadingMetadata, + ExtractorUnavailableError, + resolveUploadCodePossible, +} from './upload'; +import { UninspectableFileError, ContentFilterInputTooLargeError } from '~/protection/files'; +import { extractFileContent } from '~/protection/adapters/submissions'; +import { UnsupportedProviderAudioError } from '~/files/upload/errors'; +import { MAX_STORED_EXTRACTED_TEXT_BYTES } from '~/files/extract'; +import { ContentFilterError } from '~/middleware/contentFilter'; +import { ZipBombError } from '~/files/documents/zipSafety'; +import { parseDocument } from '~/files/documents/crud'; +import { inspectContent } from '~/protection/runtime'; + +const XLSX = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'; +const DOCX = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'; +const PPTX = 'application/vnd.openxmlformats-officedocument.presentationml.presentation'; +const PDF = 'application/pdf'; +const CSV = 'text/csv'; + +const automatic: EndpointFileConfig = { llmDeliveryPolicy: 'automatic' }; +const extractedTextBlock: FiltersConfig = { + files: { pii: { fields: ['extracted_text'], starterPatterns: [], uninspectable: 'block' } }, +}; +const contentAndTextBlock: FiltersConfig = { + files: { + pii: { fields: ['content', 'extracted_text'], starterPatterns: [], uninspectable: 'block' }, + }, +}; +const privateTokenFilters: FiltersConfig = { + files: { + pii: { + fields: ['extracted_text'], + starterPatterns: [], + customPatterns: [{ id: 'private', label: 'private token', regex: 'PRIVATE-[A-Z]+' }], + }, + }, +}; + +function decide( + overrides: Partial & Pick, + codePossible = true, +) { + const resolveCodePossible = jest.fn(async () => codePossible); + const reading = resolveUploadReading({ + endpoint: 'openAI', + endpointConfig: automatic, + isMessageAttachment: true, + resolveCodePossible, + ...overrides, + }); + return { reading, resolveCodePossible }; +} + +describe('resolveUploadReading', () => { + it('leaves a workbook to Run Code when code is possible, deferring its text', async () => { + const { reading, resolveCodePossible } = decide({ mimeType: XLSX }); + + await expect(reading).resolves.toEqual({ + path: 'none', + policy: 'automatic', + category: 'tabular', + reason: 'code_preferred', + codePreferred: true, + needsCodeAvailability: false, + keepOriginalOnExtractionFailure: false, + deferredMarker: true, + mimeType: XLSX, + }); + expect(resolveCodePossible).toHaveBeenCalledTimes(1); + }); + + it('extracts a workbook at upload when code is not possible, keeping it on failure', async () => { + const { reading, resolveCodePossible } = decide({ mimeType: XLSX }, false); + + await expect(reading).resolves.toMatchObject({ + path: 'text', + reason: 'code_unavailable', + codePreferred: false, + keepOriginalOnExtractionFailure: true, + deferredMarker: false, + }); + expect(resolveCodePossible).toHaveBeenCalledTimes(1); + }); + + it('never asks about code for documents, which keep their original on failure', async () => { + for (const mimeType of [PDF, DOCX, 'application/zip']) { + const { reading, resolveCodePossible } = decide({ mimeType }); + await expect(reading).resolves.toMatchObject({ + reason: 'automatic_default', + category: 'document', + codePreferred: false, + keepOriginalOnExtractionFailure: true, + deferredMarker: false, + }); + expect(resolveCodePossible).not.toHaveBeenCalled(); + } + }); + + it('extracts at upload, and rejects on failure, where extraction is the inspection', async () => { + for (const mimeType of [XLSX, DOCX]) { + const { reading, resolveCodePossible } = decide({ mimeType, filters: extractedTextBlock }); + await expect(reading).resolves.toMatchObject({ + path: 'text', + codePreferred: false, + keepOriginalOnExtractionFailure: false, + deferredMarker: false, + }); + expect(resolveCodePossible).not.toHaveBeenCalled(); + } + await expect( + decide({ mimeType: XLSX, filters: extractedTextBlock }).reading, + ).resolves.toMatchObject({ reason: 'inspection_requires_text' }); + }); + + it('still defers where a blocked content field, not extraction, is the inspection', async () => { + await expect( + decide({ mimeType: XLSX, filters: contentAndTextBlock }).reading, + ).resolves.toMatchObject({ reason: 'code_preferred', codePreferred: true }); + await expect( + decide({ mimeType: XLSX, filters: privateTokenFilters }).reading, + ).resolves.toMatchObject({ reason: 'code_preferred', codePreferred: true }); + }); + + it('keeps a classic row classic without asking about code', async () => { + const { reading, resolveCodePossible } = decide({ mimeType: XLSX, endpointConfig: undefined }); + + await expect(reading).resolves.toMatchObject({ + reason: 'classic_policy', + codePreferred: false, + keepOriginalOnExtractionFailure: false, + deferredMarker: false, + }); + expect(resolveCodePossible).not.toHaveBeenCalled(); + }); +}); + +describe('resolveUploadCodePossible', () => { + function resolvers({ capability = true, grant }: { capability?: boolean; grant?: boolean }) { + return { + checkCodeCapability: jest.fn(async () => capability), + getRunCodeGrant: jest.fn(async () => grant), + }; + } + + it.each([ + [['file_search', 'execute_code'], true], + [['file_search'], false], + [[], false], + ])( + 'reads saved agent tools %p as %p, with no capability or role read', + async (tools, expected) => { + const saved = resolvers({}); + await expect(resolveUploadCodePossible({ agentTools: tools, ...saved })).resolves.toBe( + expected, + ); + expect(saved.checkCodeCapability).not.toHaveBeenCalled(); + expect(saved.getRunCodeGrant).not.toHaveBeenCalled(); + }, + ); + + it.each([ + [true, undefined, true], + [true, true, true], + [true, false, false], + [false, undefined, false], + [false, true, false], + ])( + 'reads an ephemeral agent from capability %p and grant %p as %p', + async (capability, grant, expected) => { + const ephemeral = resolvers({ capability, grant }); + await expect(resolveUploadCodePossible(ephemeral)).resolves.toBe(expected); + expect(ephemeral.checkCodeCapability).toHaveBeenCalledTimes(1); + }, + ); +}); + +describe('orderToolsForReading', () => { + const tools = ['file_search', 'execute_code', 'web_search']; + + it('files a code-preferred upload under Run Code first', () => { + expect(orderToolsForReading({ codePreferred: true }, tools)).toEqual([ + 'execute_code', + 'file_search', + 'web_search', + ]); + }); + + it('leaves the order alone otherwise', () => { + expect(orderToolsForReading({ codePreferred: false }, tools)).toBe(tools); + const withoutCode = ['file_search']; + expect(orderToolsForReading({ codePreferred: true }, withoutCode)).toBe(withoutCode); + expect(orderToolsForReading({ codePreferred: true }, undefined)).toBeUndefined(); + }); +}); + +describe('acquireUploadText', () => { + let workDir: string; + + beforeAll(() => { + workDir = fs.mkdtempSync(path.join(os.tmpdir(), 'upload-reading-')); + }); + + afterAll(() => { + fs.rmSync(workDir, { recursive: true, force: true }); + }); + + const keep = (mimeType: string): ResolvedUploadReading => ({ + path: 'text', + policy: 'automatic', + category: 'document', + reason: 'automatic_default', + codePreferred: false, + needsCodeAvailability: false, + keepOriginalOnExtractionFailure: true, + deferredMarker: false, + mimeType, + }); + const classic = (mimeType: string): ResolvedUploadReading => ({ + ...keep(mimeType), + policy: 'classic', + reason: 'classic_policy', + keepOriginalOnExtractionFailure: false, + }); + + /** The real document parser over a file written to disk. */ + function parse(name: string, mimetype: string, content: string | Buffer) { + const filePath = path.join(workDir, name); + fs.writeFileSync(filePath, content); + const file = { originalname: name, path: filePath, mimetype } as Express.Multer.File; + return async () => ({ text: (await parseDocument({ file })).text }); + } + + const failedWith = (reason: TextDerivation['reason'], extractor = 'document_parser') => ({ + textDerivation: { outcome: 'failed', extractor, reason, at: expect.any(Number) }, + path: 'none', + }); + const deferredWith = (textDerivation: Omit) => ({ + textDerivation: { outcome: 'deferred', ...textDerivation, at: expect.any(Number) }, + path: 'none', + }); + const fileId = 'file-1'; + + it('passes text and errors through unchanged where failure rejects the upload', async () => { + const acquired = { text: 'a'.repeat(MAX_STORED_EXTRACTED_TEXT_BYTES + 1) }; + await expect( + acquireUploadText({ fileId, reading: classic('text/csv'), acquire: async () => acquired }), + ).resolves.toBe(acquired); + await expect( + acquireUploadText({ + fileId, + reading: classic(DOCX), + acquire: parse('broken.docx', DOCX, 'no zip'), + }), + ).rejects.toThrow(); + const empty = { text: '' }; + await expect( + acquireUploadText({ fileId, reading: classic('text/csv'), acquire: async () => empty }), + ).resolves.toBe(empty); + }); + + it('returns acquired text within the caps unchanged', async () => { + const acquired = { text: 'region,total', isTranscript: false }; + await expect( + acquireUploadText({ fileId, reading: keep('text/csv'), acquire: async () => acquired }), + ).resolves.toBe(acquired); + }); + + it('keeps the original, deferred for a later turn, when extraction fails unrecognized', async () => { + const warn = jest.spyOn(logger, 'warn'); + try { + await expect( + acquireUploadText({ + fileId, + reading: keep(DOCX), + acquire: parse('broken.docx', DOCX, 'no zip'), + }), + ).resolves.toEqual(deferredWith({ reason: 'parser' })); + await expect( + acquireUploadText({ + fileId, + reading: keep(PDF), + acquire: () => Promise.reject(new Error('connect ECONNREFUSED 10.0.0.5:8000')), + }), + ).resolves.toEqual(deferredWith({ reason: 'parser' })); + expect(warn).toHaveBeenCalledWith( + '[uploadReading] file_id=file-1 extraction=deferred failure=parser', + expect.objectContaining({ type: expect.any(String) }), + ); + expect(JSON.stringify(warn.mock.calls)).not.toContain('10.0.0.5'); + await expect( + acquireUploadText({ + fileId, + reading: keep(PPTX), + acquire: () => Promise.reject(new TypeError('cannot read properties of undefined')), + }), + ).resolves.toEqual(failedWith('parser', 'configured')); + } finally { + warn.mockRestore(); + } + }); + + it('keeps the original, marked failed, when a built-in extractor names its failure', async () => { + await expect( + acquireUploadText({ + fileId, + reading: keep(DOCX), + acquire: parse( + 'empty.docx', + DOCX, + fs.readFileSync(path.join(__dirname, '../documents/empty.docx')), + ), + }), + ).resolves.toEqual(failedWith('empty')); + await expect( + acquireUploadText({ + fileId, + reading: keep(XLSX), + acquire: () => Promise.reject(new ZipBombError('sheet: entry exceeds the cap')), + }), + ).resolves.toEqual(failedWith('expansion_limit')); + }); + + it('treats the stored-text cap and empty text as soft failures', async () => { + await expect( + acquireUploadText({ + fileId, + reading: keep('text/csv'), + acquire: async () => ({ text: 'a'.repeat(MAX_STORED_EXTRACTED_TEXT_BYTES + 1) }), + }), + ).resolves.toEqual(failedWith('too_large', 'native_text')); + await expect( + acquireUploadText({ + fileId, + reading: keep('text/csv'), + acquire: async () => ({ text: ' \n' }), + }), + ).resolves.toEqual(failedWith('empty', 'native_text')); + }); + + it('defers when only a configured step was unavailable and a built-in extractor exists', async () => { + await expect( + acquireUploadText({ + fileId, + reading: keep(PDF), + acquire: () => + Promise.reject( + new ExtractorUnavailableError( + 'OCR capability is not enabled for Agents', + 'extractor_unavailable', + ), + ), + }), + ).resolves.toEqual(deferredWith({ extractor: 'configured', reason: 'extractor_unavailable' })); + await expect( + acquireUploadText({ + fileId, + reading: keep(PPTX), + acquire: () => + Promise.reject( + new ExtractorUnavailableError( + `File type ${PPTX} is not supported for text parsing.`, + 'no_extractor', + ), + ), + }), + ).resolves.toEqual(failedWith('no_extractor', 'configured')); + /* A type the text allowlist excludes stays failed even where a built-in extractor exists. */ + await expect( + acquireUploadText({ + fileId, + reading: keep(CSV), + acquire: () => + Promise.reject( + new ExtractorUnavailableError( + `File type ${CSV} is not supported for text parsing.`, + 'no_extractor', + ), + ), + }), + ).resolves.toEqual(failedWith('no_extractor', 'configured')); + }); + + it('rejects an unavailable configured step unchanged where failure rejects the upload', async () => { + const error = new ExtractorUnavailableError( + 'OCR capability is not enabled for Agents', + 'extractor_unavailable', + ); + await expect( + acquireUploadText({ fileId, reading: classic(PDF), acquire: () => Promise.reject(error) }), + ).rejects.toBe(error); + expect(error).toBeInstanceOf(Error); + expect(error.message).toBe('OCR capability is not enabled for Agents'); + }); + + it('fails closed on text a blocking policy cannot inspect before it counts as empty', async () => { + for (const text of ['', ' \n']) { + await expect( + acquireUploadText({ + fileId, + reading: keep(DOCX), + filters: contentAndTextBlock, + acquire: async () => ({ text }), + }), + ).rejects.toBeInstanceOf(UninspectableFileError); + } + await expect( + acquireUploadText({ + fileId, + reading: keep(DOCX), + filters: privateTokenFilters, + acquire: async () => ({ text: ' ' }), + }), + ).resolves.toEqual(failedWith('empty')); + }); + + it('still rejects inspection, content policy, audio and abort errors', async () => { + const finding = inspectContent(extractFileContent({ extractedText: 'PRIVATE-SECRET' }), { + filters: privateTokenFilters, + }); + if (finding == null) { + throw new Error('expected the private token to match'); + } + const hardErrors = [ + new UninspectableFileError('extracted_text'), + new ContentFilterError(finding), + new ContentFilterInputTooLargeError('extracted_text'), + new UnsupportedProviderAudioError(), + AbortSignal.abort().reason, + ]; + for (const error of hardErrors) { + await expect( + acquireUploadText({ fileId, reading: keep(DOCX), acquire: () => Promise.reject(error) }), + ).rejects.toBe(error); + } + }); +}); + +describe('getUploadReadingMetadata', () => { + const marker: TextDerivation = { outcome: 'failed', reason: 'parser', at: 1 }; + + it('writes nothing under the classic policy', () => { + expect(getUploadReadingMetadata({ policy: 'classic', deferredMarker: false })).toEqual({}); + expect(getUploadReadingMetadata({ policy: 'classic', deferredMarker: true }, marker)).toEqual( + {}, + ); + }); + + it('writes the acquisition marker, or a deferred marker for an upload left to Run Code', () => { + expect( + getUploadReadingMetadata({ policy: 'automatic', deferredMarker: false }, marker), + ).toEqual({ textDerivation: marker }); + expect(getUploadReadingMetadata({ policy: 'automatic', deferredMarker: true })).toEqual({ + textDerivation: { outcome: 'deferred', at: expect.any(Number) }, + }); + expect(getUploadReadingMetadata({ policy: 'automatic', deferredMarker: false })).toEqual({}); + }); +}); + +describe('logUploadReading', () => { + let debug: jest.SpyInstance; + let info: jest.SpyInstance; + + beforeEach(() => { + debug = jest.spyOn(logger, 'debug'); + info = jest.spyOn(logger, 'info'); + }); + + afterEach(() => { + debug.mockRestore(); + info.mockRestore(); + }); + + it('logs a deferred upload at debug with codes only', async () => { + const reading = await decide({ mimeType: XLSX }).reading; + logUploadReading('file-1', reading); + + expect(debug).toHaveBeenCalledWith( + '[uploadReading] file_id=file-1 policy=automatic category=tabular path=none reason=code_preferred extraction=deferred', + ); + }); + + it('logs a kept original at info', async () => { + const reading = await decide({ mimeType: DOCX }).reading; + logUploadReading( + 'file-2', + { ...reading, path: 'none' }, + { outcome: 'failed', reason: 'parser' }, + ); + + expect(info).toHaveBeenCalledWith( + '[uploadReading] file_id=file-2 policy=automatic category=document path=none reason=automatic_default extraction=failed failure=parser kept=original', + ); + }); + + it('stays silent under the classic policy', async () => { + const reading = await decide({ mimeType: XLSX, endpointConfig: undefined }).reading; + logUploadReading('file-3', reading); + + expect(debug).not.toHaveBeenCalledWith(expect.stringContaining('[uploadReading]')); + expect(info).not.toHaveBeenCalledWith(expect.stringContaining('[uploadReading]')); + }); +}); diff --git a/packages/api/src/files/reading/upload.ts b/packages/api/src/files/reading/upload.ts new file mode 100644 index 00000000000..8f32e6d68f8 --- /dev/null +++ b/packages/api/src/files/reading/upload.ts @@ -0,0 +1,293 @@ +import { logger } from '@librechat/data-schemas'; +import { + EToolResources, + decideUploadReading, + selectBuiltInTextPlan, +} from 'librechat-data-provider'; +import type { + FiltersConfig, + UploadReading, + TextDerivation, + UploadReadingInput, +} from 'librechat-data-provider'; +import { + hasActiveFilePolicy, + UninspectableFileError, + ContentFilterInputTooLargeError, + getBlockedUninspectableFileField, +} from '~/protection/files'; +import { assertExtractedTextInspectable, MAX_STORED_EXTRACTED_TEXT_BYTES } from '~/files/extract'; +import { UnsupportedProviderAudioError } from '~/files/upload/errors'; +import { isAbortError, getSafeErrorMetadata } from '~/utils/errors'; +import { matchExtractionFailure } from '~/files/upload/fallback'; +import { ContentFilterError } from '~/middleware/contentFilter'; + +/** An upload's reading decision, with the type it was decided for. */ +export interface ResolvedUploadReading extends UploadReading { + mimeType: string; +} + +export type ResolveUploadReadingInput = Omit< + UploadReadingInput, + 'extractionRequiredForInspection' | 'codePossible' +> & { + filters?: FiltersConfig; + /** Asked only when the decision depends on whether Run Code can read the file. */ + resolveCodePossible: () => Promise; +}; + +/** + * Decides how an upload is read before anything is extracted. Code preference is off where the + * upload preflight defers an extracted-text fail-close to the context route, because there the + * extraction is the inspection; Run Code availability is resolved only when the decision needs it. + */ +export async function resolveUploadReading({ + filters, + resolveCodePossible, + ...input +}: ResolveUploadReadingInput): Promise { + const extractionRequiredForInspection = + hasActiveFilePolicy(filters) && + getBlockedUninspectableFileField(filters, ['content', 'extracted_text']) === 'extracted_text'; + const routing: UploadReadingInput = { ...input, extractionRequiredForInspection }; + const first = decideUploadReading(routing); + if (!first.needsCodeAvailability) { + return { ...first, mimeType: input.mimeType }; + } + const codePossible = await resolveCodePossible(); + return { ...decideUploadReading({ ...routing, codePossible }), mimeType: input.mimeType }; +} + +export interface UploadCodePossibleParams { + /** + * The saved agent's tools, already narrowed by capability and role grants: the same list the + * upload is filed against. Undefined when no agent record backs the upload. + */ + agentTools?: string[]; + checkCodeCapability: () => Promise; + getRunCodeGrant: () => Promise; +} + +/** + * Whether Run Code can read the upload: a saved agent whose narrowed tools include it, or, with no + * saved agent, the capability enabled and the role's grant not denied. + */ +export async function resolveUploadCodePossible({ + agentTools, + checkCodeCapability, + getRunCodeGrant, +}: UploadCodePossibleParams): Promise { + if (agentTools != null) { + return agentTools.includes(EToolResources.execute_code); + } + const [enabled, grant] = await Promise.all([checkCodeCapability(), getRunCodeGrant()]); + return enabled && grant !== false; +} + +/** + * Files a deferred upload under the reader that will use it: `execute_code` first, since upload + * filing takes the first compatible tool. + */ +export function orderToolsForReading( + reading: Pick, + tools?: string[], +): string[] | undefined { + if (!reading.codePreferred || tools?.includes(EToolResources.execute_code) !== true) { + return tools; + } + return [ + EToolResources.execute_code, + ...tools.filter((tool) => tool !== EToolResources.execute_code), + ]; +} + +/** Text the context route acquired, before it is inspected and stored. */ +export interface AcquiredUploadText { + text: string; + isTranscript?: boolean; +} + +/** The upload keeps its original as plain storage without text, and records why. */ +export interface UploadTextMarker { + textDerivation: TextDerivation; + /** The route the kept original takes: no reader gets text from it at upload. */ + path: 'none'; +} + +/** Why a configured extraction step could not run for an upload. */ +export type ExtractorUnavailableReason = Extract< + TextDerivation['reason'], + 'no_extractor' | 'extractor_unavailable' +>; + +/** + * A configured extraction step the upload needed is off or absent, such as OCR without its + * capability, or a type the text route does not parse. It rejects the upload where failure + * rejects it; where the original is kept, a built-in extractor may derive the text later. + */ +export class ExtractorUnavailableError extends Error { + constructor( + message: string, + readonly reason: ExtractorUnavailableReason, + ) { + super(message); + this.name = 'ExtractorUnavailableError'; + Object.setPrototypeOf(this, ExtractorUnavailableError.prototype); + } +} + +const isHardUploadFailure = (error: unknown): boolean => + error instanceof UninspectableFileError || + error instanceof ContentFilterError || + error instanceof ContentFilterInputTooLargeError || + error instanceof UnsupportedProviderAudioError || + isAbortError(error); + +/** + * The marker an extraction failure leaves. A failure a built-in step names (a parser limit, empty + * text, the zip guard) is `failed`, as is a type the deployment's text allowlist excludes: a + * later turn must not parse what the administrator kept out of text. An unavailable configured + * step, or an error nothing recognizes and that may be a passing outage, is `deferred` where a + * built-in extractor can derive the text on a later turn, and `failed` where none can. + */ +function markExtractionFailure(mimeType: string, error: unknown): TextDerivation { + const at = Date.now(); + const plan = selectBuiltInTextPlan(mimeType); + if (error instanceof ExtractorUnavailableError) { + return plan == null || error.reason === 'no_extractor' + ? { outcome: 'failed', extractor: 'configured', reason: error.reason, at } + : { outcome: 'deferred', extractor: 'configured', reason: 'extractor_unavailable', at }; + } + const recognized = matchExtractionFailure(error); + if (recognized != null) { + return { outcome: 'failed', extractor: plan ?? 'configured', reason: recognized, at }; + } + return plan == null + ? { outcome: 'failed', extractor: 'configured', reason: 'parser', at } + : { outcome: 'deferred', reason: 'parser', at }; +} + +function checkAcquiredText({ text }: AcquiredUploadText): TextDerivation['reason'] | undefined { + if (text.trim().length === 0) { + return 'empty'; + } + return Buffer.byteLength(text, 'utf8') > MAX_STORED_EXTRACTED_TEXT_BYTES + ? 'too_large' + : undefined; +} + +/** Logs the error behind a kept original as safe metadata, never the filename or text. */ +function logExtractionError(fileId: string, marker: TextDerivation, error: unknown): void { + logger.warn( + `[uploadReading] file_id=${fileId} extraction=${marker.outcome} failure=${marker.reason}`, + getSafeErrorMetadata(error), + ); +} + +/** + * Runs the context route's text acquisition. Where the reading keeps the original on failure, an + * extraction failure, empty text or text past the storage cap returns a marker and the `none` + * route instead of throwing, and logs any error behind it (see {@link markExtractionFailure}). + * Content policy, inspection, audio and abort errors still reject, and text a blocking policy + * cannot inspect fails closed before it can count as empty. Everywhere else the acquisition's + * result or error passes through unchanged. + */ +export async function acquireUploadText({ + reading, + acquire, + fileId, + filters, +}: { + reading: Pick; + acquire: () => Promise; + fileId: string; + filters?: FiltersConfig; +}): Promise { + if (!reading.keepOriginalOnExtractionFailure) { + return acquire(); + } + let acquired: T; + try { + acquired = await acquire(); + } catch (error) { + if (isHardUploadFailure(error)) { + throw error; + } + const textDerivation = markExtractionFailure(reading.mimeType, error); + logExtractionError(fileId, textDerivation, error); + return { textDerivation, path: 'none' }; + } + if (acquired.isTranscript !== true) { + assertExtractedTextInspectable({ filters, text: acquired.text }); + } + const reason = checkAcquiredText(acquired); + if (reason == null) { + return acquired; + } + const textDerivation: TextDerivation = { + outcome: 'failed', + extractor: selectBuiltInTextPlan(reading.mimeType) ?? 'configured', + reason, + at: Date.now(), + }; + return { textDerivation, path: 'none' }; +} + +/** + * The `textDerivation` an upload's record stores: the marker its text acquisition returned, or a + * deferred marker for an upload left to Run Code that a built-in extractor can read. Nothing under + * the classic policy. + */ +export function getUploadReadingMetadata( + reading: Pick, + marker?: TextDerivation, +): { textDerivation?: TextDerivation } { + if (reading.policy !== 'automatic') { + return {}; + } + if (marker != null) { + return { textDerivation: marker }; + } + return reading.deferredMarker ? { textDerivation: { outcome: 'deferred', at: Date.now() } } : {}; +} + +function describeExtraction(reading: UploadReading, marker?: TextDerivation): string { + if (marker != null) { + return [ + `extraction=${marker.outcome}`, + ...(marker.reason != null ? [`failure=${marker.reason}`] : []), + 'kept=original', + ].join(' '); + } + if (reading.codePreferred) { + return `extraction=${reading.deferredMarker ? 'deferred' : 'none'}`; + } + return `extraction=${reading.path === 'text' ? 'eager' : 'none'}`; +} + +/** + * Logs how the automatic policy read one upload. Carries the file id and codes only, never the + * filename or text. Silent under the classic policy. + */ +export function logUploadReading( + fileId: string, + reading: UploadReading, + marker?: TextDerivation, +): void { + if (reading.policy !== 'automatic') { + return; + } + const line = [ + `[uploadReading] file_id=${fileId}`, + `policy=${reading.policy}`, + `category=${reading.category}`, + `path=${reading.path}`, + `reason=${reading.reason}`, + describeExtraction(reading, marker), + ].join(' '); + if (marker?.outcome === 'failed') { + logger.info(line); + return; + } + logger.debug(line); +} diff --git a/packages/api/src/files/upload/fallback.spec.ts b/packages/api/src/files/upload/fallback.spec.ts index 1b65139d1fe..2be2e71a6c6 100644 --- a/packages/api/src/files/upload/fallback.spec.ts +++ b/packages/api/src/files/upload/fallback.spec.ts @@ -1,18 +1,25 @@ import fs from 'fs'; import os from 'os'; import path from 'path'; -import type { FiltersConfig } from 'librechat-data-provider'; +import { logger } from '@librechat/data-schemas'; +import { excelFileTypes, fullMimeTypesList, selectBuiltInTextPlan } from 'librechat-data-provider'; +import type { FiltersConfig, TextDerivation } from 'librechat-data-provider'; import type { UploadFallbackTextExtractors } from './fallback'; import { + extractBoundedText, UPLOAD_FALLBACK_TEXT_PLANS, getUploadFallbackTextPlan, + matchExtractionFailure, resolveUploadFallbackText, + classifyExtractionFailure, } from './fallback'; +import { DOCUMENT_PARSER_MAX_FILE_SIZE, parseDocument } from '~/files/documents/crud'; import { MAX_STORED_EXTRACTED_TEXT_BYTES } from '~/files/extract'; -import { parseDocument } from '~/files/documents/crud'; +import { ZipBombError } from '~/files/documents/zipSafety'; import { parseTextNative } from '~/files/text'; const XLSX_MIME = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'; +const DOCX_MIME = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'; const csvRoute = { deliveryPath: 'none' as const, destinationChosen: false, @@ -59,6 +66,49 @@ describe('getUploadFallbackTextPlan', () => { it('runs nothing for a file kept on an agent, which no turn delivers as text', () => { expect(getUploadFallbackTextPlan({ ...csvRoute, isMessageAttachment: false })).toBeNull(); }); + + it('runs nothing for an upload left to Run Code, whose text a turn derives later', () => { + const codePreferred = { reading: { codePreferred: true } }; + expect(getUploadFallbackTextPlan({ ...csvRoute, ...codePreferred })).toBeNull(); + expect( + getUploadFallbackTextPlan({ ...csvRoute, mimeType: XLSX_MIME, ...codePreferred }), + ).toBeNull(); + expect(getUploadFallbackTextPlan({ ...csvRoute, reading: { codePreferred: false } })).toBe( + UPLOAD_FALLBACK_TEXT_PLANS.nativeText, + ); + }); + + it('runs nothing for an upload whose text acquisition kept the original with a marker', () => { + const failed: TextDerivation = { outcome: 'failed', reason: 'parser', at: 1 }; + const deferred: TextDerivation = { outcome: 'deferred', reason: 'extractor_unavailable' }; + for (const textDerivation of [failed, deferred]) { + expect( + getUploadFallbackTextPlan({ + ...csvRoute, + reading: { codePreferred: false }, + textDerivation, + }), + ).toBeNull(); + } + }); + + it('picks the extractor turn-time derivation picks, for every supported type', () => { + const mimeTypes = new Set([ + ...fullMimeTypesList, + ...excelFileTypes, + 'text/csv', + 'text/tab-separated-values', + ]); + const plans = new Set(); + for (const mimeType of mimeTypes) { + const plan = getUploadFallbackTextPlan({ ...csvRoute, mimeType }); + expect({ mimeType, plan }).toEqual({ mimeType, plan: selectBuiltInTextPlan(mimeType) }); + if (plan != null) { + plans.add(plan); + } + } + expect([...plans].sort()).toEqual(Object.values(UPLOAD_FALLBACK_TEXT_PLANS).sort()); + }); }); describe('resolveUploadFallbackText', () => { @@ -146,6 +196,46 @@ describe('resolveUploadFallbackText', () => { expect(extractors.parseTextNative).not.toHaveBeenCalled(); }); + it('extracts nothing for a deferred upload', async () => { + const extractors = spiedExtractors(); + + await expect( + resolveUploadFallbackText({ + ...route, + reading: { codePreferred: true }, + file: csvUpload('deferred.csv', 'region,total'), + fileId: 'file-1', + extractors, + }), + ).resolves.toBeUndefined(); + expect(extractors.parseTextNative).not.toHaveBeenCalled(); + }); + + it('warns as before for each kind of failure, and stays silent for empty text', async () => { + const warn = jest.spyOn(logger, 'warn'); + const oversized: UploadFallbackTextExtractors = { + parseDocument, + parseTextNative: async () => ({ text: 'a'.repeat(MAX_STORED_EXTRACTED_TEXT_BYTES + 1) }), + }; + try { + await resolveUploadFallbackText({ ...route, file: csvUpload('gone.csv'), fileId: 'f1' }); + await resolveUploadFallbackText({ + ...route, + file: csvUpload('big.csv', 'a'), + fileId: 'f2', + extractors: oversized, + }); + await resolveUploadFallbackText({ ...route, file: csvUpload('none.csv', ' '), fileId: 'f3' }); + + expect(warn.mock.calls.map(([message]) => message)).toEqual([ + '[resolveUploadFallbackText] No fallback text for "gone.csv": extraction failed', + '[resolveUploadFallbackText] No fallback text for "big.csv": extracted text exceeds the storage limit', + ]); + } finally { + warn.mockRestore(); + } + }); + it('keeps the upload when extraction fails', async () => { await expect( resolveUploadFallbackText({ ...route, file: csvUpload('missing.csv'), fileId: 'file-1' }), @@ -208,3 +298,166 @@ describe('resolveUploadFallbackText', () => { ).resolves.toBeUndefined(); }); }); + +describe('extractBoundedText', () => { + const documentsDir = path.join(__dirname, '../documents'); + let workDir: string; + + beforeAll(() => { + workDir = fs.mkdtempSync(path.join(os.tmpdir(), 'bounded-text-')); + }); + + afterAll(() => { + fs.rmSync(workDir, { recursive: true, force: true }); + }); + + function upload(name: string, mimetype: string, content: string): Express.Multer.File { + const filePath = path.join(workDir, name); + fs.writeFileSync(filePath, content); + return { + originalname: name, + path: filePath, + mimetype, + size: Buffer.byteLength(content), + } as Express.Multer.File; + } + + const fixture = (name: string, mimetype: string): Express.Multer.File => + ({ originalname: name, path: path.join(documentsDir, name), mimetype }) as Express.Multer.File; + + it('returns text within every cap', async () => { + await expect( + extractBoundedText({ + file: fixture('sample.xlsx', XLSX_MIME), + plan: UPLOAD_FALLBACK_TEXT_PLANS.documentParser, + }), + ).resolves.toEqual({ + text: 'Sheet One:\nData,on,first,sheet\nSecond Sheet:\nData,On\nSecond,Sheet\n', + }); + }); + + it('fails empty for blank text and for a document the parser finds no text in', async () => { + await expect( + extractBoundedText({ + file: upload('blank.csv', 'text/csv', ' \n '), + plan: UPLOAD_FALLBACK_TEXT_PLANS.nativeText, + }), + ).resolves.toEqual({ failure: 'empty' }); + await expect( + extractBoundedText({ + file: fixture('empty.docx', DOCX_MIME), + plan: UPLOAD_FALLBACK_TEXT_PLANS.documentParser, + }), + ).resolves.toMatchObject({ failure: 'empty', error: expect.any(Error) }); + }); + + it('fails parser for a document the parser cannot read', async () => { + await expect( + extractBoundedText({ + file: upload('broken.docx', DOCX_MIME, 'not a zip archive'), + plan: UPLOAD_FALLBACK_TEXT_PLANS.documentParser, + }), + ).resolves.toMatchObject({ failure: 'parser', error: expect.any(Error) }); + }); + + it('fails too_large past the parser input cap and past the stored-text cap', async () => { + await expect( + extractBoundedText({ + file: { ...fixture('sample.docx', DOCX_MIME), size: DOCUMENT_PARSER_MAX_FILE_SIZE + 1 }, + plan: UPLOAD_FALLBACK_TEXT_PLANS.documentParser, + }), + ).resolves.toMatchObject({ failure: 'too_large' }); + await expect( + extractBoundedText({ + file: upload('huge.csv', 'text/csv', 'a'), + plan: UPLOAD_FALLBACK_TEXT_PLANS.nativeText, + extractors: { + parseDocument, + parseTextNative: async () => ({ text: 'a'.repeat(MAX_STORED_EXTRACTED_TEXT_BYTES + 1) }), + }, + }), + ).resolves.toEqual({ failure: 'too_large' }); + }); + + it('fails expansion_limit when the zip guard refuses the archive', async () => { + const bomb = new ZipBombError('sheet: entry exceeds the 25MB per-entry decompressed cap'); + + await expect( + extractBoundedText({ + file: upload('bomb.xlsx', XLSX_MIME, 'x'), + plan: UPLOAD_FALLBACK_TEXT_PLANS.documentParser, + extractors: { parseDocument: () => Promise.reject(bomb), parseTextNative }, + }), + ).resolves.toEqual({ failure: 'expansion_limit', error: bomb }); + }); + + it('refuses text a content policy flags, and text a blocking policy cannot inspect', async () => { + const flagging: FiltersConfig = { + files: { + pii: { + fields: ['extracted_text'], + starterPatterns: [], + customPatterns: [{ id: 'private', label: 'private token', regex: 'PRIVATE-[A-Z]+' }], + }, + }, + }; + const blocking: FiltersConfig = { + files: { pii: { fields: ['extracted_text'], starterPatterns: [], uninspectable: 'block' } }, + }; + + await expect( + extractBoundedText({ + file: upload('flagged.csv', 'text/csv', 'token,PRIVATE-SECRET'), + filters: flagging, + plan: UPLOAD_FALLBACK_TEXT_PLANS.nativeText, + }), + ).resolves.toEqual({ + failure: 'policy', + finding: expect.objectContaining({ ruleId: expect.any(String), field: 'extracted_text' }), + }); + await expect( + extractBoundedText({ + file: upload('blank-blocked.csv', 'text/csv', ''), + filters: blocking, + plan: UPLOAD_FALLBACK_TEXT_PLANS.nativeText, + }), + ).resolves.toMatchObject({ failure: 'uninspectable' }); + }); +}); + +describe('classifyExtractionFailure', () => { + it.each([ + [new ZipBombError('archive: total decompressed size exceeds the 100MB cap'), 'expansion_limit'], + [new Error('ODT content.xml exceeds the 50MB decompressed limit'), 'expansion_limit'], + [new Error('File "a.pdf" exceeds the 15MB document parser limit (16MB).'), 'too_large'], + [new Error('No text found in document'), 'empty'], + [new Error('Unable to extract text from "a.pdf". It may be image-based.'), 'empty'], + [new Error('Invalid PDF structure'), 'parser'], + [Object.assign(new Error('Invalid PDF structure'), { code: 'ERR_OUT_OF_RANGE' }), 'parser'], + ['not an error', 'parser'], + [Object.assign(new Error('no space left on device'), { code: 'ENOSPC' }), 'unavailable'], + [Object.assign(new Error('i/o error'), { code: 'EIO' }), 'unavailable'], + [Object.assign(new Error('resource busy or locked'), { code: 'EBUSY' }), 'unavailable'], + [Object.assign(new Error('too many open files'), { code: 'EMFILE' }), 'unavailable'], + [Object.assign(new Error('try again'), { code: 'EAGAIN' }), 'unavailable'], + [Object.assign(new Error('read failed'), { errno: -5, syscall: 'read' }), 'unavailable'], + [ + Object.assign(new Error('incorrect header check'), { errno: -3, code: 'Z_DATA_ERROR' }), + 'parser', + ], + [ + Object.assign(new ZipBombError('archive: total decompressed size exceeds the 100MB cap'), { + code: 'EIO', + }), + 'expansion_limit', + ], + ])('classifies %p as %s', (error, reason) => { + expect(classifyExtractionFailure(error)).toBe(reason); + }); + + it('recognizes only the failures it can name, leaving the rest unclassified', () => { + expect(matchExtractionFailure(new Error('No text found in document'))).toBe('empty'); + expect(matchExtractionFailure(new Error('Invalid PDF structure'))).toBeUndefined(); + expect(matchExtractionFailure('not an error')).toBeUndefined(); + }); +}); diff --git a/packages/api/src/files/upload/fallback.ts b/packages/api/src/files/upload/fallback.ts index 96b8bce20ac..b08a4e61b05 100644 --- a/packages/api/src/files/upload/fallback.ts +++ b/packages/api/src/files/upload/fallback.ts @@ -1,17 +1,22 @@ import { logger } from '@librechat/data-schemas'; -import { isNativelyReadableText, documentParserMimeTypes } from 'librechat-data-provider'; +import { selectBuiltInTextPlan } from 'librechat-data-provider'; import type { FiltersConfig, + UploadReading, + TextDerivation, + BuiltInTextPlan, EndpointFileConfig, TDefaultLLMDeliveryPath, } from 'librechat-data-provider'; +import type { ProtectionFinding } from '~/protection/types'; import { extractInspectableFileText, getFileExtractionLogDetails, MAX_STORED_EXTRACTED_TEXT_BYTES, } from '~/files/extract'; +import { hasActiveFileFieldPolicy, UninspectableFileError } from '~/protection/files'; import { extractFileContent } from '~/protection/adapters/submissions'; -import { hasActiveFileFieldPolicy } from '~/protection/files'; +import { ZipBombError } from '~/files/documents/zipSafety'; import { parseDocument } from '~/files/documents/crud'; import { inspectContent } from '~/protection/runtime'; import { parseTextNative } from '~/files/text'; @@ -19,10 +24,9 @@ import { parseTextNative } from '~/files/text'; export const UPLOAD_FALLBACK_TEXT_PLANS = { documentParser: 'document_parser', nativeText: 'native_text', -} as const; +} as const satisfies Record; -export type UploadFallbackTextPlan = - (typeof UPLOAD_FALLBACK_TEXT_PLANS)[keyof typeof UPLOAD_FALLBACK_TEXT_PLANS]; +export type UploadFallbackTextPlan = BuiltInTextPlan; export interface UploadFallbackTextRoute { /** The route the upload resolved to. */ @@ -34,6 +38,10 @@ export interface UploadFallbackTextRoute { mimeType: string; /** The upload endpoint's file config, which opts in through `textFallbackWithoutTools`. */ endpointConfig?: Pick; + /** The upload's reading decision; an upload left to Run Code reads no text now. */ + reading?: Pick; + /** The marker the upload's text acquisition returned, when it kept the original without text. */ + textDerivation?: TextDerivation; } interface ExtractedText { @@ -46,8 +54,92 @@ export interface UploadFallbackTextExtractors { parseTextNative: (file: Express.Multer.File) => Promise; } +/** Why a built-in extractor produced no text a file can keep. */ +export type ExtractionFailureReason = Extract< + NonNullable, + 'empty' | 'parser' | 'too_large' | 'expansion_limit' +>; + +/** + * Why bounded extraction kept no text: the extractor failed, a file policy refused its text, or + * the system could not serve the read (`unavailable`), which says nothing about the file. + */ +export type BoundedTextFailure = + | ExtractionFailureReason + | 'policy' + | 'uninspectable' + | 'unavailable'; + +/** + * Text within every cap, or why there is none: `error` is set when extraction threw, and + * `finding` is the content policy's finding when it refused the text. + */ +export type BoundedText = + | { text: string } + | { failure: 'policy'; finding: ProtectionFinding } + | { failure: Exclude; error?: unknown }; + const builtInExtractors: UploadFallbackTextExtractors = { parseDocument, parseTextNative }; +/** Fragments of the built-in extractors' plain error messages, mapped to stable reasons. */ +const EXTRACTION_FAILURE_MESSAGES: ReadonlyArray = [ + ['No text found in document', 'empty'], + ['Unable to extract text from', 'empty'], + ['MB document parser limit', 'too_large'], + ['MB storage limit', 'too_large'], + ['MB decompressed limit', 'expansion_limit'], +]; + +/** The stable reason a built-in extractor failed with, or undefined for an unrecognized error. */ +export function matchExtractionFailure(error: unknown): ExtractionFailureReason | undefined { + if (error instanceof ZipBombError) { + return 'expansion_limit'; + } + const message = error instanceof Error ? error.message : ''; + return EXTRACTION_FAILURE_MESSAGES.find(([fragment]) => message.includes(fragment))?.[1]; +} + +interface SystemErrorShape { + code?: string; + errno?: number; + syscall?: string; +} + +/** POSIX error names (`ENOSPC`, `EIO`, `EBUSY`, `EMFILE`, `EAGAIN`...), never Node's `ERR_*` codes. */ +const SYSTEM_ERROR_CODE = /^E[A-Z0-9]+$/; + +/** + * Whether the error is the operating system refusing a read, such as a full disk, a busy or + * locked file, or descriptor exhaustion, rather than anything the file's content caused. These + * clear up, so a later read may succeed. An `errno` alone is not enough: zlib reports a corrupt + * archive with one (`Z_DATA_ERROR`), so it counts only with the system call that failed. + */ +export function isSystemError(error: unknown): boolean { + if (typeof error !== 'object' || error == null) { + return false; + } + const { code, errno, syscall }: SystemErrorShape = error; + return ( + (typeof code === 'string' && SYSTEM_ERROR_CODE.test(code)) || + (typeof errno === 'number' && typeof syscall === 'string') + ); +} + +/** + * The stable reason a built-in extractor failed with. A failure the extractor names keeps its + * reason, a system error is `unavailable`, and anything else unrecognized is a parser error. + */ +export function classifyExtractionFailure(error: unknown): ExtractionFailureReason | 'unavailable' { + return matchExtractionFailure(error) ?? (isSystemError(error) ? 'unavailable' : 'parser'); +} + +/** Whether the upload's text is left to a turn, which derives it from the stored original. */ +const isUploadTextDeferred = ({ + reading, + textDerivation, +}: Pick): boolean => + reading?.codePreferred === true || textDerivation != null; + /** * Which built-in extractor stores text for an upload left to tools, or `null` when none runs. * @@ -57,7 +149,7 @@ const builtInExtractors: UploadFallbackTextExtractors = { parseDocument, parseTe * and files kept on an agent's tool resources never reach a prompt. A file filed * under a tool that reads it still gets text, because a later turn may run without that tool: a * handoff agent, or the same agent after its tools or grants change. Only built-in extractors - * run, so a file meant for a tool never costs a RAG or OCR call. + * run, so a file meant for a tool never costs a RAG or OCR call. A deferred upload runs none. */ export function getUploadFallbackTextPlan( route: UploadFallbackTextRoute, @@ -66,16 +158,65 @@ export function getUploadFallbackTextPlan( route.endpointConfig?.textFallbackWithoutTools !== true || route.deliveryPath !== 'none' || route.destinationChosen || - !route.isMessageAttachment + !route.isMessageAttachment || + isUploadTextDeferred(route) ) { return null; } - if (documentParserMimeTypes.some((pattern) => pattern.test(route.mimeType))) { - return UPLOAD_FALLBACK_TEXT_PLANS.documentParser; + return selectBuiltInTextPlan(route.mimeType); +} + +/** + * Reads a file with one built-in extractor and keeps its text only within every cap: non-empty, + * within the stored-text limit, and clear of any `extracted_text` content policy. Extraction runs + * under the policy's fail-close, so text a blocking policy cannot inspect is refused. Never + * throws: upload fallback and turn-time derivation share these caps and each decides what a + * failure means for its file. + */ +export async function extractBoundedText({ + file, + filters, + plan, + extractors = builtInExtractors, +}: { + file: Express.Multer.File; + filters?: FiltersConfig; + plan: UploadFallbackTextPlan; + extractors?: UploadFallbackTextExtractors; +}): Promise { + try { + const result = await extractInspectableFileText({ + filters, + extract: () => + plan === UPLOAD_FALLBACK_TEXT_PLANS.documentParser + ? extractors.parseDocument({ file }) + : extractors.parseTextNative(file), + }); + const text = result?.text; + if (typeof text !== 'string' || text.trim().length === 0) { + return { failure: 'empty' }; + } + if (Buffer.byteLength(text, 'utf8') > MAX_STORED_EXTRACTED_TEXT_BYTES) { + return { failure: 'too_large' }; + } + if (filters == null || !hasActiveFileFieldPolicy(filters, ['extracted_text'])) { + return { text }; + } + const finding = inspectContent(extractFileContent({ extractedText: text }), { filters }); + return finding == null ? { text } : { failure: 'policy', finding }; + } catch (error) { + const failure = + error instanceof UninspectableFileError ? 'uninspectable' : classifyExtractionFailure(error); + return { failure, error }; } - return isNativelyReadableText(route.mimeType) ? UPLOAD_FALLBACK_TEXT_PLANS.nativeText : null; } +/** The warning a failure without a thrown error logs; an empty result logs nothing. */ +const FALLBACK_SKIP_REASONS: Partial> = { + too_large: 'extracted text exceeds the storage limit', + policy: 'extracted text matched a content policy', +}; + /** * Text a turn without a reading tool can fall back to for this upload, when one applies. * @@ -114,30 +255,13 @@ export async function resolveUploadFallbackText({ ); return undefined; }; - try { - const result = await extractInspectableFileText({ - filters, - extract: () => - plan === UPLOAD_FALLBACK_TEXT_PLANS.documentParser - ? extractors.parseDocument({ file }) - : extractors.parseTextNative(file), - }); - const text = result?.text; - if (typeof text !== 'string' || text.trim().length === 0) { - return undefined; - } - if (Buffer.byteLength(text, 'utf8') > MAX_STORED_EXTRACTED_TEXT_BYTES) { - return skip('extracted text exceeds the storage limit'); - } - if ( - filters != null && - hasActiveFileFieldPolicy(filters, ['extracted_text']) && - inspectContent(extractFileContent({ extractedText: text }), { filters }) != null - ) { - return skip('extracted text matched a content policy'); - } - return text; - } catch (error) { - return skip('extraction failed', error); + const result = await extractBoundedText({ file, filters, plan, extractors }); + if ('text' in result) { + return result.text; + } + if ('error' in result) { + return skip('extraction failed', result.error); } + const reason = FALLBACK_SKIP_REASONS[result.failure]; + return reason == null ? undefined : skip(reason); } diff --git a/packages/api/src/files/validation.spec.ts b/packages/api/src/files/validation.spec.ts index 97c8ee56979..c07a682ec23 100644 --- a/packages/api/src/files/validation.spec.ts +++ b/packages/api/src/files/validation.spec.ts @@ -1,6 +1,14 @@ import { Providers } from '@librechat/agents'; import { mbToBytes } from 'librechat-data-provider'; -import { validatePdf, validateBedrockDocument, validateVideo, validateAudio } from './validation'; +import type { NativeDocumentSizeLimitParams } from './validation'; +import { + validatePdf, + validateAudio, + validateImage, + validateVideo, + validateBedrockDocument, + getNativeDocumentSizeLimit, +} from './validation'; describe('PDF Validation with fileConfig.endpoints.*.fileSizeLimit', () => { /** Helper to create a PDF buffer with valid header */ @@ -882,3 +890,357 @@ describe('PDF Validation with fileConfig.endpoints.*.fileSizeLimit', () => { }); }); }); + +describe('getNativeDocumentSizeLimit', () => { + const pdf = 'application/pdf'; + const docx = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'; + const claude4 = 'anthropic.claude-sonnet-4-20250514-v1:0'; + const nova = 'amazon.nova-pro-v1:0'; + + it.each<[string, NativeDocumentSizeLimitParams, number | undefined]>([ + [ + 'Anthropic PDF, unconfigured', + { provider: Providers.ANTHROPIC, mimeType: pdf }, + mbToBytes(32), + ], + [ + 'Anthropic PDF, configured lower', + { provider: Providers.ANTHROPIC, mimeType: pdf, configuredFileSizeLimit: mbToBytes(5) }, + mbToBytes(5), + ], + [ + 'Anthropic PDF, configured higher', + { provider: Providers.ANTHROPIC, mimeType: pdf, configuredFileSizeLimit: mbToBytes(50) }, + mbToBytes(50), + ], + [ + 'Anthropic PDF, configured 0', + { provider: Providers.ANTHROPIC, mimeType: pdf, configuredFileSizeLimit: 0 }, + 0, + ], + ['OpenAI PDF, unconfigured', { provider: Providers.OPENAI, mimeType: pdf }, mbToBytes(10)], + ['Azure PDF, unconfigured', { provider: Providers.AZURE, mimeType: pdf }, mbToBytes(10)], + [ + 'OpenRouter PDF, unconfigured', + { provider: Providers.OPENROUTER, mimeType: pdf }, + mbToBytes(10), + ], + [ + 'OpenAI PDF, configured 0', + { provider: Providers.OPENAI, mimeType: pdf, configuredFileSizeLimit: 0 }, + 0, + ], + ['Google PDF, unconfigured', { provider: Providers.GOOGLE, mimeType: pdf }, mbToBytes(20)], + ['VertexAI PDF, unconfigured', { provider: Providers.VERTEXAI, mimeType: pdf }, mbToBytes(20)], + [ + 'Google PDF, configured', + { provider: Providers.GOOGLE, mimeType: pdf, configuredFileSizeLimit: mbToBytes(25) }, + mbToBytes(25), + ], + [ + 'PDF on a provider without a PDF limit', + { provider: 'unsupported', mimeType: pdf }, + undefined, + ], + [ + 'PDF on a provider without a PDF limit, configured', + { provider: 'unsupported', mimeType: pdf, configuredFileSizeLimit: mbToBytes(5) }, + undefined, + ], + ['Bedrock PDF, default model', { provider: Providers.BEDROCK, mimeType: pdf }, mbToBytes(4.5)], + [ + 'Bedrock PDF, Claude 4', + { provider: Providers.BEDROCK, mimeType: pdf, model: claude4 }, + mbToBytes(32), + ], + [ + 'Bedrock PDF, Nova', + { provider: Providers.BEDROCK, mimeType: pdf, model: nova }, + mbToBytes(32), + ], + [ + 'Bedrock DOCX, Nova', + { provider: Providers.BEDROCK, mimeType: docx, model: nova }, + mbToBytes(32), + ], + [ + 'Bedrock DOCX, Claude 4', + { provider: Providers.BEDROCK, mimeType: docx, model: claude4 }, + mbToBytes(4.5), + ], + [ + 'Bedrock CSV, Nova', + { provider: Providers.BEDROCK, mimeType: 'text/csv', model: nova }, + mbToBytes(4.5), + ], + [ + 'Bedrock PDF, Claude 4, configured lower than the exemption', + { + provider: Providers.BEDROCK, + mimeType: pdf, + model: claude4, + configuredFileSizeLimit: mbToBytes(10), + }, + mbToBytes(10), + ], + [ + 'Bedrock CSV, configured 0', + { provider: Providers.BEDROCK, mimeType: 'text/csv', configuredFileSizeLimit: 0 }, + 0, + ], + [ + 'generic text, unconfigured', + { provider: Providers.ANTHROPIC, mimeType: 'text/plain' }, + undefined, + ], + [ + 'generic text, configured', + { + provider: Providers.ANTHROPIC, + mimeType: 'text/plain', + configuredFileSizeLimit: mbToBytes(1), + }, + mbToBytes(1), + ], + [ + 'generic text, configured 0', + { provider: Providers.OPENAI, mimeType: 'text/plain', configuredFileSizeLimit: 0 }, + undefined, + ], + [ + 'generic spreadsheet on Google, configured', + { + provider: Providers.GOOGLE, + mimeType: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + configuredFileSizeLimit: mbToBytes(25), + }, + mbToBytes(25), + ], + ])('%s', (_label, params, expected) => { + expect(getNativeDocumentSizeLimit(params)).toBe(expected); + }); +}); + +describe('validator parity with getNativeDocumentSizeLimit', () => { + const pdfBuffer = Buffer.alloc(1024); + pdfBuffer.write('%PDF-1.4\n', 0); + + const pdfCases: Array<{ + name: string; + provider: Providers; + model?: string; + configured?: number; + limit: number; + }> = [ + { name: 'Anthropic, unconfigured', provider: Providers.ANTHROPIC, limit: mbToBytes(32) }, + { + name: 'Anthropic, configured', + provider: Providers.ANTHROPIC, + configured: mbToBytes(3), + limit: mbToBytes(3), + }, + { name: 'OpenAI, unconfigured', provider: Providers.OPENAI, limit: mbToBytes(10) }, + { + name: 'OpenAI, configured', + provider: Providers.OPENAI, + configured: mbToBytes(15), + limit: mbToBytes(15), + }, + { name: 'Google, unconfigured', provider: Providers.GOOGLE, limit: mbToBytes(20) }, + { + name: 'VertexAI, configured', + provider: Providers.VERTEXAI, + configured: mbToBytes(8), + limit: mbToBytes(8), + }, + { name: 'Bedrock, default model', provider: Providers.BEDROCK, limit: mbToBytes(4.5) }, + { + name: 'Bedrock, Claude 4', + provider: Providers.BEDROCK, + model: 'anthropic.claude-sonnet-4-20250514-v1:0', + limit: mbToBytes(32), + }, + { + name: 'Bedrock, Nova, configured', + provider: Providers.BEDROCK, + model: 'amazon.nova-pro-v1:0', + configured: mbToBytes(8), + limit: mbToBytes(8), + }, + ]; + + describe.each(pdfCases)('validatePdf on $name', ({ provider, model, configured, limit }) => { + it('accepts a PDF exactly at the limit', async () => { + const result = await validatePdf(pdfBuffer, limit, provider, configured, model); + expect(result).toEqual({ isValid: true }); + }); + + it('rejects a PDF one byte above the limit as a capacity failure', async () => { + const result = await validatePdf(pdfBuffer, limit + 1, provider, configured, model); + expect(result).toMatchObject({ isValid: false, reason: 'capacity' }); + }); + }); + + const bedrockCases: Array<{ name: string; mimeType: string; model?: string; limit: number }> = [ + { name: 'CSV', mimeType: 'text/csv', limit: mbToBytes(4.5) }, + { + name: 'Nova DOCX', + mimeType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + model: 'amazon.nova-pro-v1:0', + limit: mbToBytes(32), + }, + { + name: 'Claude 4 DOCX', + mimeType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + model: 'anthropic.claude-sonnet-4-20250514-v1:0', + limit: mbToBytes(4.5), + }, + ]; + + describe.each(bedrockCases)('validateBedrockDocument on $name', ({ mimeType, model, limit }) => { + it('accepts a document exactly at the limit', async () => { + const result = await validateBedrockDocument(limit, mimeType, undefined, undefined, model); + expect(result).toEqual({ isValid: true }); + }); + + it('rejects a document one byte above the limit as a capacity failure', async () => { + const result = await validateBedrockDocument( + limit + 1, + mimeType, + undefined, + undefined, + model, + ); + expect(result).toMatchObject({ isValid: false, reason: 'capacity' }); + }); + }); + + it.each([ + Providers.ANTHROPIC, + Providers.OPENAI, + Providers.GOOGLE, + Providers.VERTEXAI, + Providers.BEDROCK, + ])('rejects a 1-byte PDF on %s when the configured limit is 0', async (provider) => { + const result = await validatePdf(pdfBuffer, 1, provider, 0); + expect(result).toMatchObject({ isValid: false, reason: 'capacity' }); + }); + + it('accepts any PDF size on a provider without a PDF limit, configured or not', async () => { + const provider = 'unsupported' as Providers; + await expect(validatePdf(pdfBuffer, mbToBytes(100), provider)).resolves.toEqual({ + isValid: true, + }); + await expect(validatePdf(pdfBuffer, mbToBytes(100), provider, 0)).resolves.toEqual({ + isValid: true, + }); + }); +}); + +describe('validation failure reasons', () => { + const validPdf = (): Buffer => { + const buffer = Buffer.alloc(1024); + buffer.write('%PDF-1.4\n', 0); + return buffer; + }; + + it.each([ + [Providers.ANTHROPIC, 35, 'PDF file size (35MB) exceeds the 32MB limit'], + [Providers.OPENAI, 12, 'PDF file size (12MB) exceeds the 10MB limit'], + [Providers.GOOGLE, 25, 'PDF file size (25MB) exceeds the 20MB limit'], + ])( + 'reports an oversized %s PDF as capacity with the unchanged message', + async (provider, sizeMB, error) => { + const result = await validatePdf(validPdf(), mbToBytes(sizeMB), provider); + expect(result).toEqual({ isValid: false, reason: 'capacity', error }); + }, + ); + + it('reports an Anthropic page estimate over 100 as capacity', async () => { + const pages = '/Type /Page\n'.repeat(101); + const result = await validatePdf(Buffer.from(`%PDF-1.4\n${pages}`), 1024, Providers.ANTHROPIC); + expect(result).toEqual({ + isValid: false, + reason: 'capacity', + error: "PDF has approximately 101 pages, exceeding Anthropic's 100-page limit", + }); + }); + + const tooSmall = Buffer.alloc(3); + const headerless = Buffer.alloc(1024); + headerless.write('INVALID', 0); + + it.each([ + [ + 'a too-small Anthropic PDF', + () => validatePdf(tooSmall, tooSmall.length, Providers.ANTHROPIC), + 'Invalid PDF file: too small or corrupted', + ], + [ + 'a missing Anthropic PDF header', + () => validatePdf(headerless, headerless.length, Providers.ANTHROPIC), + 'Invalid PDF file: missing PDF header', + ], + [ + 'a too-small Bedrock PDF', + () => validateBedrockDocument(tooSmall.length, 'application/pdf', tooSmall), + 'Invalid PDF file: too small or corrupted', + ], + [ + 'a missing Bedrock PDF header', + () => validateBedrockDocument(headerless.length, 'application/pdf', headerless), + 'Invalid PDF file: missing PDF header', + ], + ])('reports %s as integrity with the unchanged message', async (_label, validate, error) => { + await expect(validate()).resolves.toEqual({ isValid: false, reason: 'integrity', error }); + }); + + it('reports an encrypted Anthropic PDF as integrity', async () => { + const buffer = validPdf(); + buffer.write('/Encrypt ', 100); + const result = await validatePdf(buffer, buffer.length, Providers.ANTHROPIC); + expect(result).toEqual({ + isValid: false, + reason: 'integrity', + error: 'PDF is password-protected or encrypted. Anthropic requires unencrypted PDFs.', + }); + }); + + it('reports an oversized Bedrock document as capacity with the unchanged message', async () => { + const result = await validateBedrockDocument(mbToBytes(5), 'text/csv'); + expect(result).toEqual({ + isValid: false, + reason: 'capacity', + error: 'File size (5.0MB) exceeds the 4.5MB limit for Bedrock', + }); + }); + + it('reports oversized media as capacity and truncated media as integrity', async () => { + const media = Buffer.alloc(1024); + const tiny = Buffer.alloc(5); + await expect(validateVideo(media, mbToBytes(25), Providers.GOOGLE)).resolves.toMatchObject({ + isValid: false, + reason: 'capacity', + }); + await expect(validateAudio(media, mbToBytes(25), Providers.GOOGLE)).resolves.toMatchObject({ + isValid: false, + reason: 'capacity', + }); + await expect(validateImage(media, mbToBytes(6), Providers.ANTHROPIC)).resolves.toMatchObject({ + isValid: false, + reason: 'capacity', + }); + await expect(validateVideo(tiny, tiny.length, Providers.GOOGLE)).resolves.toMatchObject({ + isValid: false, + reason: 'integrity', + }); + await expect(validateAudio(tiny, tiny.length, Providers.GOOGLE)).resolves.toMatchObject({ + isValid: false, + reason: 'integrity', + }); + await expect(validateImage(tiny, tiny.length, Providers.OPENAI)).resolves.toMatchObject({ + isValid: false, + reason: 'integrity', + }); + }); +}); diff --git a/packages/api/src/files/validation.ts b/packages/api/src/files/validation.ts index 0bde847b876..26ab3facd86 100644 --- a/packages/api/src/files/validation.ts +++ b/packages/api/src/files/validation.ts @@ -1,31 +1,32 @@ import { Providers } from '@librechat/agents'; import { mbToBytes, isOpenAILikeProvider } from 'librechat-data-provider'; +import type { ValidationFailureReason } from '~/types'; export interface ValidationResult { isValid: boolean; error?: string; + reason?: ValidationFailureReason; } -export interface PDFValidationResult { - isValid: boolean; - error?: string; -} +export type PDFValidationResult = ValidationResult; -export interface VideoValidationResult { - isValid: boolean; - error?: string; -} +export type VideoValidationResult = ValidationResult; -export interface AudioValidationResult { - isValid: boolean; - error?: string; -} +export type AudioValidationResult = ValidationResult; -export interface ImageValidationResult { - isValid: boolean; - error?: string; +export type ImageValidationResult = ValidationResult; + +export interface NativeDocumentSizeLimitParams { + provider: Providers | string; + mimeType: string; + model?: string; + /** Configured endpoint `fileSizeLimit` in bytes */ + configuredFileSizeLimit?: number; } +const pdfMimeType = 'application/pdf'; +const docxMimeType = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'; + export async function validatePdf( pdfBuffer: Buffer, fileSize: number, @@ -40,19 +41,38 @@ export async function validatePdf( if (provider === Providers.BEDROCK) { return validateBedrockDocument( fileSize, - 'application/pdf', + pdfMimeType, pdfBuffer, configuredFileSizeLimit, model, ); } - if (isOpenAILikeProvider(provider)) { - return validateOpenAIPdf(fileSize, configuredFileSizeLimit); - } + return validatePdfSize(fileSize, provider, configuredFileSizeLimit); +} - if (provider === Providers.GOOGLE || provider === Providers.VERTEXAI) { - return validateGooglePdf(fileSize, configuredFileSizeLimit); +/** + * Validates a PDF's size against the limit the provider's native document path enforces. + * Providers without a PDF limit (anything but Anthropic, OpenAI-like and Google/Vertex) pass. + */ +function validatePdfSize( + fileSize: number, + provider: Providers, + configuredFileSizeLimit?: number, +): PDFValidationResult { + const effectiveLimit = getNativeDocumentSizeLimit({ + provider, + mimeType: pdfMimeType, + configuredFileSizeLimit, + }); + + if (effectiveLimit !== undefined && fileSize > effectiveLimit) { + const limitMB = Math.round(effectiveLimit / (1024 * 1024)); + return { + isValid: false, + reason: 'capacity', + error: `PDF file size (${Math.round(fileSize / (1024 * 1024))}MB) exceeds the ${limitMB}MB limit`, + }; } return { isValid: true }; @@ -71,20 +91,15 @@ async function validateAnthropicPdf( configuredFileSizeLimit?: number, ): Promise { try { - const providerLimit = mbToBytes(32); - const effectiveLimit = configuredFileSizeLimit ?? providerLimit; - - if (fileSize > effectiveLimit) { - const limitMB = Math.round(effectiveLimit / (1024 * 1024)); - return { - isValid: false, - error: `PDF file size (${Math.round(fileSize / (1024 * 1024))}MB) exceeds the ${limitMB}MB limit`, - }; + const sizeValidation = validatePdfSize(fileSize, Providers.ANTHROPIC, configuredFileSizeLimit); + if (!sizeValidation.isValid) { + return sizeValidation; } if (!pdfBuffer || pdfBuffer.length < 5) { return { isValid: false, + reason: 'integrity', error: 'Invalid PDF file: too small or corrupted', }; } @@ -93,6 +108,7 @@ async function validateAnthropicPdf( if (!pdfHeader.startsWith('%PDF-')) { return { isValid: false, + reason: 'integrity', error: 'Invalid PDF file: missing PDF header', }; } @@ -105,6 +121,7 @@ async function validateAnthropicPdf( ) { return { isValid: false, + reason: 'integrity', error: 'PDF is password-protected or encrypted. Anthropic requires unencrypted PDFs.', }; } @@ -115,6 +132,7 @@ async function validateAnthropicPdf( if (estimatedPages > 100) { return { isValid: false, + reason: 'capacity', error: `PDF has approximately ${estimatedPages} pages, exceeding Anthropic's 100-page limit`, }; } @@ -124,6 +142,7 @@ async function validateAnthropicPdf( console.error('PDF validation error:', error); return { isValid: false, + reason: 'integrity', error: 'Failed to validate PDF file', }; } @@ -160,9 +179,6 @@ const isBedrockClaude4Plus = (model?: string): boolean => const isBedrockNova = (model?: string): boolean => model != null && /(?:^|\.)amazon\.nova-/.test(model); -const pdfMimeType = 'application/pdf'; -const docxMimeType = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'; - /** * Returns true when the given model + MIME type combination is exempt from * Bedrock's default 4.5 MB per-document limit. @@ -181,6 +197,51 @@ const isExemptFromBedrockDocLimit = (model?: string, mimeType?: string): boolean return false; }; +/** Provider fallback for a PDF sent as a native document, or undefined when it has none. */ +const getPdfProviderLimit = (provider: Providers | string): number | undefined => { + if (provider === Providers.ANTHROPIC) { + return mbToBytes(32); + } + if (isOpenAILikeProvider(provider)) { + return mbToBytes(10); + } + if (provider === Providers.GOOGLE || provider === Providers.VERTEXAI) { + return mbToBytes(20); + } + return undefined; +}; + +/** + * The per-file byte limit the document encoder enforces before sending a file natively, + * resolved from metadata alone. A configured limit replaces the provider fallback rather + * than capping it, so a configured 0 rejects every Bedrock document and every PDF with a + * provider limit. Outside those, only a truthy configured limit applies. + * @returns The limit in bytes, or undefined when no size limit applies + */ +export function getNativeDocumentSizeLimit({ + provider, + mimeType, + model, + configuredFileSizeLimit, +}: NativeDocumentSizeLimitParams): number | undefined { + if (provider === Providers.BEDROCK) { + const providerLimit = isExemptFromBedrockDocLimit(model, mimeType) + ? mbToBytes(32) + : mbToBytes(4.5); + return configuredFileSizeLimit ?? providerLimit; + } + + if (mimeType !== pdfMimeType) { + return configuredFileSizeLimit || undefined; + } + + const providerLimit = getPdfProviderLimit(provider); + if (providerLimit === undefined) { + return undefined; + } + return configuredFileSizeLimit ?? providerLimit; +} + /** * Validates a document against Bedrock size limits. The default limit is 4.5 MB, * but Claude 4+ (PDF) and Nova (PDF/DOCX) models are exempt per AWS docs. @@ -200,15 +261,18 @@ export async function validateBedrockDocument( model?: string, ): Promise { try { - const exempt = isExemptFromBedrockDocLimit(model, mimeType); - /** Default 4.5 MB; exempt models (Claude 4+ PDF, Nova PDF/DOCX) default to 32 MB when unconfigured */ - const providerLimit = exempt ? mbToBytes(32) : mbToBytes(4.5); - const effectiveLimit = configuredFileSizeLimit ?? providerLimit; + const effectiveLimit = getNativeDocumentSizeLimit({ + provider: Providers.BEDROCK, + mimeType, + model, + configuredFileSizeLimit, + }); - if (fileSize > effectiveLimit) { + if (effectiveLimit !== undefined && fileSize > effectiveLimit) { const limitMB = (effectiveLimit / (1024 * 1024)).toFixed(1); return { isValid: false, + reason: 'capacity', error: `File size (${(fileSize / (1024 * 1024)).toFixed(1)}MB) exceeds the ${limitMB}MB limit for Bedrock`, }; } @@ -217,6 +281,7 @@ export async function validateBedrockDocument( if (fileBuffer.length < 5) { return { isValid: false, + reason: 'integrity', error: 'Invalid PDF file: too small or corrupted', }; } @@ -225,6 +290,7 @@ export async function validateBedrockDocument( if (!pdfHeader.startsWith('%PDF-')) { return { isValid: false, + reason: 'integrity', error: 'Invalid PDF file: missing PDF header', }; } @@ -235,59 +301,12 @@ export async function validateBedrockDocument( console.error('Bedrock document validation error:', error); return { isValid: false, + reason: 'integrity', error: 'Failed to validate document file', }; } } -/** - * Validates if a PDF meets OpenAI's requirements - * @param fileSize - The file size in bytes - * @param configuredFileSizeLimit - Optional configured file size limit from fileConfig (in bytes) - * @returns Promise that resolves to validation result - */ -async function validateOpenAIPdf( - fileSize: number, - configuredFileSizeLimit?: number, -): Promise { - const providerLimit = mbToBytes(10); - const effectiveLimit = configuredFileSizeLimit ?? providerLimit; - - if (fileSize > effectiveLimit) { - const limitMB = Math.round(effectiveLimit / (1024 * 1024)); - return { - isValid: false, - error: `PDF file size (${Math.round(fileSize / (1024 * 1024))}MB) exceeds the ${limitMB}MB limit`, - }; - } - - return { isValid: true }; -} - -/** - * Validates if a PDF meets Google's requirements - * @param fileSize - The file size in bytes - * @param configuredFileSizeLimit - Optional configured file size limit from fileConfig (in bytes) - * @returns Promise that resolves to validation result - */ -async function validateGooglePdf( - fileSize: number, - configuredFileSizeLimit?: number, -): Promise { - const providerLimit = mbToBytes(20); - const effectiveLimit = configuredFileSizeLimit ?? providerLimit; - - if (fileSize > effectiveLimit) { - const limitMB = Math.round(effectiveLimit / (1024 * 1024)); - return { - isValid: false, - error: `PDF file size (${Math.round(fileSize / (1024 * 1024))}MB) exceeds the ${limitMB}MB limit`, - }; - } - - return { isValid: true }; -} - /** * Validates video files for different providers * @param videoBuffer - The video file as a buffer @@ -310,6 +329,7 @@ export async function validateVideo( const limitMB = Math.round(effectiveLimit / (1024 * 1024)); return { isValid: false, + reason: 'capacity', error: `Video file size (${Math.round(fileSize / (1024 * 1024))}MB) exceeds the ${limitMB}MB limit`, }; } @@ -318,6 +338,7 @@ export async function validateVideo( if (!videoBuffer || videoBuffer.length < 10) { return { isValid: false, + reason: 'integrity', error: 'Invalid video file: too small or corrupted', }; } @@ -347,6 +368,7 @@ export async function validateAudio( const limitMB = Math.round(effectiveLimit / (1024 * 1024)); return { isValid: false, + reason: 'capacity', error: `Audio file size (${Math.round(fileSize / (1024 * 1024))}MB) exceeds the ${limitMB}MB limit`, }; } @@ -355,6 +377,7 @@ export async function validateAudio( if (!audioBuffer || audioBuffer.length < 10) { return { isValid: false, + reason: 'integrity', error: 'Invalid audio file: too small or corrupted', }; } @@ -384,6 +407,7 @@ export async function validateImage( const limitMB = Math.round(effectiveLimit / (1024 * 1024)); return { isValid: false, + reason: 'capacity', error: `Image file size (${Math.round(fileSize / (1024 * 1024))}MB) exceeds the ${limitMB}MB limit`, }; } @@ -397,6 +421,7 @@ export async function validateImage( const limitMB = Math.round(effectiveLimit / (1024 * 1024)); return { isValid: false, + reason: 'capacity', error: `Image file size (${Math.round(fileSize / (1024 * 1024))}MB) exceeds the ${limitMB}MB limit`, }; } @@ -405,6 +430,7 @@ export async function validateImage( if (!imageBuffer || imageBuffer.length < 10) { return { isValid: false, + reason: 'integrity', error: 'Invalid image file: too small or corrupted', }; } diff --git a/packages/api/src/index.ts b/packages/api/src/index.ts index 54ba5cd1852..7bdb47a8b09 100644 --- a/packages/api/src/index.ts +++ b/packages/api/src/index.ts @@ -101,6 +101,7 @@ export * from './plugins'; export * from './endpoints'; /* Files */ export * from './files'; +export * from './files/reading/history'; /* Images */ export * from './images'; /* Storage */ diff --git a/packages/api/src/types/files.ts b/packages/api/src/types/files.ts index 83c3bfc1230..7490befcc65 100644 --- a/packages/api/src/types/files.ts +++ b/packages/api/src/types/files.ts @@ -1,4 +1,4 @@ -import type { BedrockDocumentFormat } from 'librechat-data-provider'; +import type { BedrockDocumentFormat, ReadingEvidence } from 'librechat-data-provider'; import type { IMongoFile } from '@librechat/data-schemas'; import type { Readable } from 'stream'; import type { DownloadURLParams } from '~/storage/types'; @@ -124,6 +124,21 @@ export type DocumentBlock = | OpenAIInputFileBlock | BedrockDocumentBlock; +/** + * `capacity`: larger than the provider accepts. + * `integrity`: the stored bytes could not be read or are not a usable file. + */ +export type ValidationFailureReason = Exclude; + +/** Whether the document encoder fails the turn on an invalid file or leaves the file out. */ +export type NativeValidationMode = 'throw' | 'skip'; + +/** A document the encoder left out in skip mode instead of failing the turn, and why. */ +export interface DocumentRejection { + file_id: string; + reason: NonNullable; +} + export interface DocumentResult { documents: DocumentBlock[]; files: Array<{ @@ -134,6 +149,8 @@ export interface DocumentResult { filename: string; type: string; }>; + /** Present only when encoding with `onValidationFailure: 'skip'` */ + rejected?: DocumentRejection[]; } /** Google audio block format */ diff --git a/packages/api/src/utils/message.ts b/packages/api/src/utils/message.ts index 968aad51130..bf3006b3095 100644 --- a/packages/api/src/utils/message.ts +++ b/packages/api/src/utils/message.ts @@ -1,8 +1,8 @@ import { Constants } from 'librechat-data-provider'; -import type { TFile, TMessage } from 'librechat-data-provider'; +import type { TMessage } from 'librechat-data-provider'; /** Minimal shape for request file entries (from `req.body.files`) */ -type RequestFile = { file_id?: string }; +export type RequestFile = { file_id?: string }; type GetMessagesByParentId = ( filter: { user: string; messageId: string; conversationId?: string }, @@ -12,6 +12,9 @@ type GetMessagesByParentId = ( /** Fields to strip from files before client transmission */ const FILE_STRIP_FIELDS = ['text', '_id', '__v'] as const; +/** A file as sent to the client: without its extracted text or database ids. */ +export type TransmittedFile = Omit; + /** Fields to strip from messages before client transmission. * Both are prompt-building inputs: `fileContext` is text extracted from * attachments and `image_urls` holds base64 image inputs. Terminal events are @@ -35,9 +38,7 @@ const MESSAGE_STRIP_FIELDS = ['fileContext', 'image_urls'] as const; * } * } */ -export function sanitizeFileForTransmit>( - file: T, -): Omit { +export function sanitizeFileForTransmit(file: T): TransmittedFile { const sanitized = { ...file }; for (const field of FILE_STRIP_FIELDS) { delete sanitized[field as keyof typeof sanitized]; @@ -46,10 +47,10 @@ export function sanitizeFileForTransmit>( } /** Filters attachments to those whose `file_id` appears in `requestFiles`, then sanitizes each. */ -export function buildMessageFiles>( +export function buildMessageFiles( requestFiles: RequestFile[], attachments: T[], -): Omit[] { +): TransmittedFile[] { const requestFileIds = new Set(); for (const f of requestFiles) { if (f.file_id) { @@ -57,7 +58,7 @@ export function buildMessageFiles>( } } - const files: Omit[] = []; + const files: TransmittedFile[] = []; for (const attachment of attachments) { if (attachment.file_id != null && requestFileIds.has(attachment.file_id)) { files.push(sanitizeFileForTransmit(attachment)); diff --git a/packages/api/src/utils/text.spec.ts b/packages/api/src/utils/text.spec.ts index 5a0c21a018c..2a8d625775b 100644 --- a/packages/api/src/utils/text.spec.ts +++ b/packages/api/src/utils/text.spec.ts @@ -405,6 +405,51 @@ describe('processTextWithTokenLimit', () => { }); }); + describe('with a known token count', () => { + it('uses the known count instead of counting text under the limit', async () => { + const tokenCountFn = jest.fn((text: string): number => Math.ceil(text.length / 4)); + const text = createTextOfLength(40); + + const result = await processTextWithTokenLimit({ + text, + tokenLimit: 100, + tokenCountFn, + knownTokenCount: 10, + }); + + expect(result).toEqual({ text, tokenCount: 10, wasTruncated: false }); + expect(tokenCountFn).not.toHaveBeenCalled(); + }); + + it('truncates from the known count without counting the full text again', async () => { + const tokenCountFn = jest.fn((text: string): number => Math.ceil(text.length / 4)); + const text = createTextOfLength(4000); + const counted = await processTextWithTokenLimit({ text, tokenLimit: 100, tokenCountFn }); + tokenCountFn.mockClear(); + + const result = await processTextWithTokenLimit({ + text, + tokenLimit: 100, + tokenCountFn, + knownTokenCount: 1000, + }); + + expect(result).toEqual(counted); + expect(result.wasTruncated).toBe(true); + expect(tokenCountFn).not.toHaveBeenCalledWith(text); + }); + + it('counts the text when no known count is supplied', async () => { + const tokenCountFn = jest.fn((text: string): number => Math.ceil(text.length / 4)); + const text = createTextOfLength(40); + + await processTextWithTokenLimit({ text, tokenLimit: 100, tokenCountFn }); + + expect(tokenCountFn).toHaveBeenCalledTimes(1); + expect(tokenCountFn).toHaveBeenCalledWith(text); + }); + }); + describe('edge cases', () => { it('should handle empty text', async () => { const { tokenCountFn } = createMockTokenCounter(); diff --git a/packages/api/src/utils/text.ts b/packages/api/src/utils/text.ts index 2d3977465ad..2c590d8107f 100644 --- a/packages/api/src/utils/text.ts +++ b/packages/api/src/utils/text.ts @@ -32,6 +32,8 @@ const TRUNCATION_SAFETY_BUFFER = 0.98; * @param text - The text content to process * @param tokenLimit - The maximum number of tokens allowed * @param tokenCountFn - Function to count tokens (can be sync or async) + * @param knownTokenCount - The text's token count when the caller already measured it; used as + * the original count instead of counting the text again * @returns Promise resolving to object with processed text, token count, and truncation status * * @remarks @@ -44,13 +46,15 @@ export async function processTextWithTokenLimit({ tokenLimit, tokenCountFn, preserve = 'start', + knownTokenCount, }: { text: string; tokenLimit: number; tokenCountFn: TokenCountFn; preserve?: 'start' | 'end'; + knownTokenCount?: number; }): Promise<{ text: string; tokenCount: number; wasTruncated: boolean }> { - const originalTokenCount = await tokenCountFn(text); + const originalTokenCount = knownTokenCount ?? (await tokenCountFn(text)); if (originalTokenCount <= tokenLimit) { return { diff --git a/packages/data-provider/src/file-config.spec.ts b/packages/data-provider/src/file-config.spec.ts index 3acee7e7d82..a6480f80e06 100644 --- a/packages/data-provider/src/file-config.spec.ts +++ b/packages/data-provider/src/file-config.spec.ts @@ -10,6 +10,7 @@ import { getConfiguredMimeAccept, getDocumentFileExtension, bedrockDocumentMimeTypes, + resolveLLMDeliveryPolicy, isAnthropicDocumentType, isPermissiveMimeConfig, isExplicitMimeConfig, @@ -2077,6 +2078,137 @@ describe('textFallbackWithoutTools config merging', () => { }); }); +describe('llmDeliveryPolicy config merging', () => { + type DynamicConfig = Parameters[0]; + const endpointConfigFor = (dynamic: DynamicConfig, endpoint: string) => + getEndpointFileConfig({ fileConfig: mergeFileConfig(dynamic), endpoint }); + const policyFor = (dynamic: DynamicConfig, endpoint: string) => + resolveLLMDeliveryPolicy(endpointConfigFor(dynamic, endpoint)); + + it('resolves to classic unless configured, leaving the merged config unchanged', () => { + const merged = mergeFileConfig({ + fileTokenLimit: 5, + endpoints: { [EModelEndpoint.openAI]: { fileLimit: 3 }, default: { fileLimit: 4 } }, + }); + + expect(resolveLLMDeliveryPolicy(undefined)).toBe('classic'); + expect(resolveLLMDeliveryPolicy(null)).toBe('classic'); + expect(resolveLLMDeliveryPolicy({})).toBe('classic'); + expect(endpointConfigFor(undefined, EModelEndpoint.openAI).llmDeliveryPolicy).toBeUndefined(); + expect(policyFor(undefined, EModelEndpoint.openAI)).toBe('classic'); + expect(policyFor({ endpoints: { default: {} } }, EModelEndpoint.openAI)).toBe('classic'); + expect(baseFileConfig.endpoints.default).not.toHaveProperty('llmDeliveryPolicy'); + expect(mergeFileConfig(undefined)).not.toHaveProperty('llmDeliveryPolicy'); + expect(merged).not.toHaveProperty('llmDeliveryPolicy'); + expect(merged.endpoints[EModelEndpoint.openAI]).not.toHaveProperty('llmDeliveryPolicy'); + expect(merged.endpoints.default).not.toHaveProperty('llmDeliveryPolicy'); + expect(merged.endpoints[EModelEndpoint.agents]).not.toHaveProperty('llmDeliveryPolicy'); + }); + + it('accepts the setting at the top level and on an endpoint', () => { + expect( + fileConfigSchema.safeParse({ + llmDeliveryPolicy: 'automatic', + endpoints: { openAI: { llmDeliveryPolicy: 'classic' } }, + }).success, + ).toBe(true); + expect(fileConfigSchema.safeParse({ llmDeliveryPolicy: 'auto' }).success).toBe(false); + expect( + fileConfigSchema.safeParse({ endpoints: { openAI: { llmDeliveryPolicy: true } } }).success, + ).toBe(false); + }); + + const endpointConfigured: DynamicConfig = { + endpoints: { [EModelEndpoint.openAI]: { llmDeliveryPolicy: 'automatic' } }, + }; + const defaultConfigured: DynamicConfig = { + endpoints: { default: { llmDeliveryPolicy: 'automatic' } }, + }; + const overriding: DynamicConfig = { + llmDeliveryPolicy: 'automatic', + endpoints: { + [EModelEndpoint.openAI]: { llmDeliveryPolicy: 'classic' }, + default: { llmDeliveryPolicy: 'classic' }, + MyGateway: { llmDeliveryPolicy: 'automatic' }, + }, + }; + + it.each<[string, DynamicConfig, string, 'automatic' | 'classic']>([ + [ + 'reaches an endpoint configured for it', + endpointConfigured, + EModelEndpoint.openAI, + 'automatic', + ], + [ + 'stays classic on an endpoint beside the configured one', + endpointConfigured, + EModelEndpoint.anthropic, + 'classic', + ], + [ + 'is inherited from the top level', + { llmDeliveryPolicy: 'automatic' }, + EModelEndpoint.anthropic, + 'automatic', + ], + [ + 'is inherited from the default endpoint by a custom endpoint', + defaultConfigured, + 'MyGateway', + 'automatic', + ], + [ + 'is inherited from the default endpoint by a known endpoint', + defaultConfigured, + EModelEndpoint.bedrock, + 'automatic', + ], + [ + 'lets an endpoint turn off what the top level sets', + overriding, + EModelEndpoint.openAI, + 'classic', + ], + [ + 'lets the default endpoint turn off what the top level sets', + overriding, + EModelEndpoint.anthropic, + 'classic', + ], + [ + 'lets an endpoint turn on what the default endpoint turns off', + overriding, + 'MyGateway', + 'automatic', + ], + ])('%s', (_case, dynamic, endpoint, expected) => { + expect(policyFor(dynamic, endpoint)).toBe(expected); + }); + + it('keeps classic wherever the legacy chooser is on', () => { + expect( + resolveLLMDeliveryPolicy({ llmDeliveryPolicy: 'automatic', legacyFileUploadUX: true }), + ).toBe('classic'); + expect( + resolveLLMDeliveryPolicy({ llmDeliveryPolicy: 'automatic', legacyFileUploadUX: false }), + ).toBe('automatic'); + expect( + policyFor( + { llmDeliveryPolicy: 'automatic', legacyFileUploadUX: true }, + EModelEndpoint.openAI, + ), + ).toBe('classic'); + + const dynamic: DynamicConfig = { + llmDeliveryPolicy: 'automatic', + endpoints: { [EModelEndpoint.openAI]: { legacyFileUploadUX: true } }, + }; + expect(policyFor(dynamic, EModelEndpoint.openAI)).toBe('classic'); + expect(policyFor(dynamic, EModelEndpoint.anthropic)).toBe('automatic'); + }); +}); + describe('agent attachment context limits', () => { it('keeps the turn-memory ceiling separate from agent upload storage', () => { expect(baseFileConfig.fileContextSizeLimit).toBe(128 * 1024 * 1024); diff --git a/packages/data-provider/src/file-config.ts b/packages/data-provider/src/file-config.ts index fadeb3352a5..b9c961bafef 100644 --- a/packages/data-provider/src/file-config.ts +++ b/packages/data-provider/src/file-config.ts @@ -638,6 +638,22 @@ export const defaultLLMDeliveryPathSchema = z.object({ export type TDefaultLLMDeliveryPathConfig = z.infer; type TDeliveryPathOverrides = NonNullable; +export const LLMDeliveryPolicy = z.enum(['classic', 'automatic']); +export type TLLMDeliveryPolicy = z.infer; + +/** + * How unified attachments are read for an endpoint. The compatibility default lives only here, + * because zod defaults never reach the merged runtime config; `legacyFileUploadUX` keeps its + * chooser semantics by forcing `classic`. + */ +export function resolveLLMDeliveryPolicy( + config?: Pick | null, +): TLLMDeliveryPolicy { + return config?.llmDeliveryPolicy === 'automatic' && config.legacyFileUploadUX !== true + ? 'automatic' + : 'classic'; +} + export const endpointFileConfigSchema = z.object({ disabled: z.boolean().optional(), fileLimit: z.number().min(0).optional(), @@ -647,6 +663,7 @@ export const endpointFileConfigSchema = z.object({ defaultLLMDeliveryPath: defaultLLMDeliveryPathSchema.optional(), legacyFileUploadUX: z.boolean().optional(), textFallbackWithoutTools: z.boolean().optional(), + llmDeliveryPolicy: LLMDeliveryPolicy.optional(), }); const skillFileConfigSchema = z.object({ @@ -690,6 +707,7 @@ export const fileConfigSchema = z.object({ defaultLLMDeliveryPath: defaultLLMDeliveryPathSchema.optional(), legacyFileUploadUX: z.boolean().optional(), textFallbackWithoutTools: z.boolean().optional(), + llmDeliveryPolicy: LLMDeliveryPolicy.optional(), }); export type TFileConfig = z.infer; @@ -1046,6 +1064,7 @@ function mergeWithDefault( legacyFileUploadUX: endpointConfig.legacyFileUploadUX ?? defaultConfig.legacyFileUploadUX, textFallbackWithoutTools: endpointConfig.textFallbackWithoutTools ?? defaultConfig.textFallbackWithoutTools, + llmDeliveryPolicy: endpointConfig.llmDeliveryPolicy ?? defaultConfig.llmDeliveryPolicy, }; } @@ -1138,6 +1157,7 @@ export function getEndpointFileConfig(params: { legacyFileUploadUX: mergedFileConfig.legacyFileUploadUX ?? baseDefaultConfig.legacyFileUploadUX, textFallbackWithoutTools: mergedFileConfig.textFallbackWithoutTools ?? baseDefaultConfig.textFallbackWithoutTools, + llmDeliveryPolicy: mergedFileConfig.llmDeliveryPolicy ?? baseDefaultConfig.llmDeliveryPolicy, }; const userDefaultConfig = mergedFileConfig.endpoints.default; const defaultConfig = userDefaultConfig @@ -1278,6 +1298,10 @@ function buildMergedFileConfig(dynamic: DynamicFileConfig | undefined): FileConf mergedConfig.textFallbackWithoutTools = dynamic.textFallbackWithoutTools; } + if (dynamic.llmDeliveryPolicy !== undefined) { + mergedConfig.llmDeliveryPolicy = dynamic.llmDeliveryPolicy; + } + if (dynamic.serverFileSizeLimit !== undefined) { mergedConfig.serverFileSizeLimit = mbToBytes(dynamic.serverFileSizeLimit); } @@ -1404,6 +1428,10 @@ function buildMergedFileConfig(dynamic: DynamicFileConfig | undefined): FileConf if (dynamicEndpoint.textFallbackWithoutTools !== undefined) { mergedEndpoint.textFallbackWithoutTools = dynamicEndpoint.textFallbackWithoutTools; } + + if (dynamicEndpoint.llmDeliveryPolicy !== undefined) { + mergedEndpoint.llmDeliveryPolicy = dynamicEndpoint.llmDeliveryPolicy; + } } return mergedConfig; diff --git a/packages/data-provider/src/index.ts b/packages/data-provider/src/index.ts index 3238b8ddab6..5dbae5bb8b4 100644 --- a/packages/data-provider/src/index.ts +++ b/packages/data-provider/src/index.ts @@ -9,6 +9,7 @@ export * from './langchain'; export * from './filters'; export * from './file-config'; export * from './resolve-llm-delivery-path'; +export * from './reading'; /* messages */ export * from './messages'; export * from './previews'; diff --git a/packages/data-provider/src/messages.spec.ts b/packages/data-provider/src/messages.spec.ts index d453f6b8562..7fcb1009fe3 100644 --- a/packages/data-provider/src/messages.spec.ts +++ b/packages/data-provider/src/messages.spec.ts @@ -2,7 +2,13 @@ import type { SummaryContentPart } from './types/content'; import type { ParentMessage } from './messages'; import type { TFile } from './types/files'; import type { TMessage } from './types'; -import { buildTree, findMessageById, isCompactedLeaf, isUserInitiatedCompaction } from './messages'; +import { + buildTree, + findMessageById, + hydrateMessageFile, + isCompactedLeaf, + isUserInitiatedCompaction, +} from './messages'; import { ContentTypes } from './types/runs'; const msg = (messageId: string, parentMessageId: string, over: Partial = {}): TMessage => @@ -147,6 +153,28 @@ describe('buildTree', () => { expect(tree?.[0].files?.[0]).toBe(file); }); + it('restores separate code and text delivery paths when the same workbook is reused', () => { + const refreshed = { + file_id: 'workbook', + filename: 'sales.xlsx', + llmDeliveryPath: 'text', + preview: 'fresh-preview', + } as TFile; + const codeFile = { file_id: refreshed.file_id, llmDeliveryPath: 'none' as const }; + const textFile = { file_id: refreshed.file_id, llmDeliveryPath: 'text' as const }; + const messages = [ + msg('code-turn', '', { files: [codeFile] }), + msg('text-turn', 'code-turn', { files: [textFile] }), + ]; + + const restored = buildTree({ messages, fileMap: { workbook: refreshed } }); + + expect(restored?.[0].files?.[0]).toEqual({ ...refreshed, ...codeFile }); + expect(asParent(restored?.[0]).children[0].files?.[0]).toBe(refreshed); + expect(refreshed.llmDeliveryPath).toBe('text'); + expect(messages[0].files?.[0]).toBe(codeFile); + }); + describe('memoization', () => { const chain = () => [ msg('u1', '00000000-0000-0000-0000-000000000000', { isCreatedByUser: true }), @@ -199,6 +227,38 @@ describe('buildTree', () => { }); }); +describe('hydrateMessageFile', () => { + const stored = { file_id: 'file', filename: 'report.pdf', llmDeliveryPath: 'provider' } as TFile; + + it('returns the stored record when the message copy names no path or the same path', () => { + expect(hydrateMessageFile({ file_id: stored.file_id }, { file: stored })).toBe(stored); + expect( + hydrateMessageFile( + { file_id: stored.file_id, llmDeliveryPath: 'provider' }, + { file: stored }, + ), + ).toBe(stored); + }); + + it("carries the message copy's differing delivery path onto the stored record", () => { + const copy = { + file_id: stored.file_id, + filename: 'stub.pdf', + llmDeliveryPath: 'text' as const, + }; + expect(hydrateMessageFile(copy, { file: stored })).toEqual({ + ...stored, + llmDeliveryPath: 'text', + }); + expect(stored.llmDeliveryPath).toBe('provider'); + }); + + it('returns the message copy itself when no stored record backs it', () => { + const copy = { file_id: 'missing', llmDeliveryPath: 'text' as const }; + expect(hydrateMessageFile(copy, { file: stored })).toBe(copy); + }); +}); + describe('isCompactedLeaf', () => { const summary = (overrides: Record = {}) => ({ type: ContentTypes.SUMMARY, diff --git a/packages/data-provider/src/messages.ts b/packages/data-provider/src/messages.ts index b424f648907..c348bef90bd 100644 --- a/packages/data-provider/src/messages.ts +++ b/packages/data-provider/src/messages.ts @@ -24,6 +24,24 @@ export function stripReasoningLabelMetadata(part: TMessageContentParts): TMessag export type ParentMessage = TMessage & { children: TMessage[]; depth: number }; +type MessageFile = NonNullable[number]; + +/** + * The stored record stands in for a message's file, but the delivery path that message's turn + * used lives only on the message's copy (a workbook kept for Run Code may have been read as + * text), so it is carried over for the preview. + */ +export function hydrateMessageFile(file: MessageFile, fileMap: Record): MessageFile { + const stored = fileMap[file.file_id ?? '']; + if (stored == null) { + return file; + } + if (file.llmDeliveryPath == null || file.llmDeliveryPath === stored.llmDeliveryPath) { + return stored; + } + return { ...stored, llmDeliveryPath: file.llmDeliveryPath }; +} + /** * Memoizes built trees per messages-array identity. The same query data feeds * several independent `select`s (ChatView plus the branch-tail helpers), which @@ -93,7 +111,7 @@ export function buildTree({ }; if (message.files && fileMap) { - extendedMessage.files = message.files.map((file) => fileMap[file.file_id ?? ''] ?? file); + extendedMessage.files = message.files.map((file) => hydrateMessageFile(file, fileMap)); } messageMap[message.messageId] = extendedMessage; diff --git a/packages/data-provider/src/reading.spec.ts b/packages/data-provider/src/reading.spec.ts new file mode 100644 index 00000000000..b4f67b088d8 --- /dev/null +++ b/packages/data-provider/src/reading.spec.ts @@ -0,0 +1,179 @@ +import type { DirectContentEntry } from './reading'; +import { + excelFileTypes, + fullMimeTypesList, + retrievalMimeTypesList, + fileConfig as baseFileConfig, + codeInterpreterMimeTypesList, +} from './file-config'; +import { + AFTER_FAILURE, + READING_ORDER, + isOriginalBacked, + isTextOnlyRecord, + categorizeForReading, + allocateDirectContent, +} from './reading'; +import { FileSources } from './types/files'; + +const TABULAR_TYPES = [ + ...excelFileTypes, + 'application/vnd.oasis.opendocument.spreadsheet', + 'text/csv', + 'application/csv', + 'text/tab-separated-values', + 'application/x-parquet', + 'application/vnd.apache.parquet', +]; + +const DOCUMENT_TYPES = [ + 'application/pdf', + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + 'application/vnd.openxmlformats-officedocument.presentationml.presentation', + 'application/msword', + 'application/zip', + 'application/epub+zip', + 'application/json', + 'text/markdown', + 'text/plain', + 'message/rfc822', + 'application/vnd.oasis.opendocument.text', +]; + +const MEDIA_TYPES = ['image/png', 'image/jpeg', 'audio/mpeg', 'video/mp4']; + +const entry = ( + fileId: string, + bytes: number, + textChars = 0, + counts = true, +): DirectContentEntry => ({ fileId, bytes, textChars, counts }); + +const sorted = (ids: ReadonlySet): string[] => Array.from(ids).sort(); + +describe('allocateDirectContent', () => { + it('admits candidates first-fit in the order given', () => { + const overflow = allocateDirectContent([], [entry('a', 60), entry('b', 50), entry('c', 30)], { + bytes: 100, + }); + expect(sorted(overflow)).toEqual(['b']); + }); + + it('charges committed content before any candidate', () => { + const overflow = allocateDirectContent( + [entry('history', 70)], + [entry('a', 60), entry('b', 50), entry('c', 30)], + { bytes: 100 }, + ); + expect(sorted(overflow)).toEqual(['a', 'b']); + }); + + it('admits a candidate that lands exactly on the limit', () => { + expect(allocateDirectContent([entry('h', 40)], [entry('a', 60)], { bytes: 100 }).size).toBe(0); + }); + + it('judges each limit dimension on its own', () => { + const candidates = [entry('a', 10, 500), entry('b', 10, 600), entry('c', 10, 100)]; + expect(sorted(allocateDirectContent([], candidates, { textChars: 1000 }))).toEqual(['b']); + expect(sorted(allocateDirectContent([], candidates, { count: 2 }))).toEqual(['c']); + expect(sorted(allocateDirectContent([], candidates, { bytes: 25 }))).toEqual(['c']); + }); + + it('does not charge the count for a file that does not count', () => { + const candidates = [entry('a', 1), entry('output', 1, 0, false), entry('b', 1)]; + expect(sorted(allocateDirectContent([entry('h', 1)], candidates, { count: 3 }))).toEqual([]); + expect(sorted(allocateDirectContent([entry('h', 1)], candidates, { count: 2 }))).toEqual(['b']); + }); + + it('treats a missing or zero limit as unlimited', () => { + const candidates = [entry('a', 10_000, 10_000), entry('b', 10_000, 10_000)]; + expect(allocateDirectContent([], candidates, {}).size).toBe(0); + expect(allocateDirectContent([], candidates, { count: 0, bytes: 0, textChars: 0 }).size).toBe( + 0, + ); + expect(sorted(allocateDirectContent([], candidates, { bytes: 0, textChars: 15_000 }))).toEqual([ + 'b', + ]); + }); +}); + +describe('categorizeForReading', () => { + it.each(TABULAR_TYPES)('classifies %s as tabular', (mimeType) => { + expect(categorizeForReading(mimeType)).toBe('tabular'); + }); + + it.each(DOCUMENT_TYPES)('classifies %s as a document', (mimeType) => { + expect(categorizeForReading(mimeType)).toBe('document'); + }); + + it.each(MEDIA_TYPES)('classifies %s as media', (mimeType) => { + expect(categorizeForReading(mimeType)).toBe('media'); + }); + + it('ignores parameters and case, as browsers send both', () => { + expect(categorizeForReading('Text/CSV; charset=utf-8')).toBe('tabular'); + expect(categorizeForReading(' APPLICATION/VND.MS-EXCEL ')).toBe('tabular'); + expect(categorizeForReading('IMAGE/PNG')).toBe('media'); + expect(categorizeForReading('text/plain; charset=utf-8')).toBe('document'); + }); + + it('treats an unknown or missing type as a document', () => { + expect(categorizeForReading('')).toBe('document'); + expect(categorizeForReading('application/octet-stream')).toBe('document'); + }); +}); + +describe('tabular classification stays inside admission', () => { + it('admits every known type it classifies as tabular', () => { + const known = [ + ...fullMimeTypesList, + ...codeInterpreterMimeTypesList, + ...retrievalMimeTypesList, + ...TABULAR_TYPES, + ]; + const unadmitted = known.filter( + (mimeType) => + categorizeForReading(mimeType) === 'tabular' && + !baseFileConfig.checkType(mimeType, baseFileConfig.endpoints.default.supportedMimeTypes), + ); + expect(unadmitted).toEqual([]); + }); +}); + +describe('record sources', () => { + it('treats a record without a source as holding the original', () => { + expect(isOriginalBacked({})).toBe(true); + expect(isOriginalBacked({ source: null })).toBe(true); + }); + + it('marks only a text source as text-only', () => { + expect(isTextOnlyRecord({ source: FileSources.text })).toBe(true); + expect(isTextOnlyRecord({ source: FileSources.local })).toBe(false); + expect(isTextOnlyRecord({})).toBe(false); + }); +}); + +describe('reading rule data', () => { + it('orders each category over all four readers exactly once', () => { + Object.values(READING_ORDER).forEach((order) => { + expect(order.slice().sort()).toEqual(['code', 'provider', 'search', 'text']); + }); + }); + + it('never retries a direct reader after a capacity or encode-time failure', () => { + const direct = ['provider', 'text']; + Object.values(AFTER_FAILURE).forEach((map) => { + expect(map.native_rejected?.some((reader) => direct.includes(reader))).toBe(false); + expect(map.aggregate_overflow?.some((reader) => direct.includes(reader))).toBe(false); + expect(map.native_capacity?.includes('provider') ?? false).toBe(false); + }); + }); + + it('tries File Search before text after a native capacity failure', () => { + Object.values(AFTER_FAILURE).forEach((map) => { + const order = map.native_capacity ?? []; + expect(order.indexOf('search')).toBe(0); + expect(order.indexOf('text')).toBe(order.length - 1); + }); + }); +}); diff --git a/packages/data-provider/src/reading.ts b/packages/data-provider/src/reading.ts new file mode 100644 index 00000000000..8514430dc93 --- /dev/null +++ b/packages/data-provider/src/reading.ts @@ -0,0 +1,248 @@ +import type { TurnDeliveryFile } from './resolve-llm-delivery-path'; +import type { TDefaultLLMDeliveryPath } from './file-config'; +import { excelMimeTypes } from './file-config'; +import { FileSources } from './types/files'; + +/** A reader the automatic policy can give an attachment to on a turn. */ +export type ReaderKind = 'provider' | 'text' | 'search' | 'code'; + +/** The automatic reading categories; images, audio and video stay on their classic route. */ +export type ReadingCategory = 'tabular' | 'document'; + +/** Why a reader was passed over. Stable log and notice codes. */ +export type SkipReason = + | 'code_unavailable' + | 'native_unsupported' + | 'native_capacity' + | 'native_rejected' + | 'aggregate_overflow' + | 'text_exceeds' + | 'text_unavailable' + | 'search_unavailable'; + +/** Why a file kept its classic route, named after the eligibility gate that kept it. */ +export type ClassicReason = + | 'classic_policy' + | 'legacy_record' + | 'explicit_destination' + | 'unmarked_record' + | 'not_message_attachment' + | 'text_only_record' + | 'configured_route' + | 'consumers_unknown' + | 'no_file_tools' + | 'media_category'; + +export type ReadingReason = + | ClassicReason + | 'code_preferred' + | 'code_selected' + | 'native_supported' + | 'text_fits' + | 'search_selected' + | SkipReason + | 'no_reader'; + +export type CodeEligibility = + | 'eligible' + | 'no_run_code' + | 'incompatible' + | 'text_only_record' + | 'unstreamable' + | 'declined'; + +export interface ReaderSkip { + reader: ReaderKind; + reason: SkipReason; +} + +/** One attachment's reading on one turn. Recomputed from its inputs and never stored. */ +export interface FileReading { + /** Turn-copy value; always inside the stored vocabulary. */ + path: TDefaultLLMDeliveryPath | undefined; + /** What classic routing answers for the same inputs (inspection view, diagnostics). */ + classicPath: TDefaultLLMDeliveryPath | undefined; + automatic: boolean; + reader: ReaderKind | 'unavailable' | 'unresolved'; + /** + * The skip that best explains the reading, or the selected reader's own reason when nothing was + * skipped. A limit the file hit outranks a reader that is merely missing; among missing readers + * the category's preferred one names the reason. + */ + reason: ReadingReason; + skipped: ReadonlyArray; + code: CodeEligibility; + category: ReadingCategory | 'media'; + /** The reader is text, but the text must first be derived from the retained original. */ + needsText: boolean; +} + +/** Plain-data verdicts the host supplies for one file on this request. */ +export interface ReadingEvidence { + /** Whether the encoder would emit this type for the provider, and within its size limit. */ + native?: 'fits' | 'capacity' | 'unsupported'; + /** The encoder rejected the file in skip mode on this request. */ + rejected?: 'capacity' | 'integrity' | 'unsupported'; + /** Judged only when the record carries text. */ + text?: 'fits' | 'exceeds'; + /** Direct content did not fit the request's first-fit allocation. */ + overflow?: true; + /** Text derivation failed on this request. */ + textFailed?: true; + /** Final pass only: whether File Search will actually receive the file. */ + search?: 'reachable' | 'unreachable'; +} + +/** The turn's evidence seam: synchronous, memoized, and side-effect free for the decision. */ +export interface TurnReadingInputs { + judge(file: TurnDeliveryFile): ReadingEvidence; + canDerive: boolean; +} + +/** The built-in extractors text can be derived with, without a RAG or OCR call. */ +export type BuiltInTextPlan = 'document_parser' | 'native_text'; + +/** How the text a reader may need was obtained; written only under the automatic policy. */ +export interface TextDerivation { + outcome: 'deferred' | 'complete' | 'failed'; + extractor?: BuiltInTextPlan | 'configured'; + reason?: + | 'too_large' + | 'expansion_limit' + | 'parser' + | 'empty' + | 'no_extractor' + | 'extractor_unavailable' + | 'original_missing'; + at?: number; +} + +/** Each category's reader order: the tabular row, and the document rows judged by capability. */ +export const READING_ORDER: Readonly> = { + tabular: ['code', 'provider', 'text', 'search'], + document: ['provider', 'text', 'search', 'code'], +}; + +/** + * Where a walk continues after a reader fails for this reason. Reasons not listed continue the + * current queue: the category order, or the list a previous failure switched to. A direct reader + * is never retried after a capacity or encode-time failure, and text follows a native capacity + * failure only after File Search, and only when it genuinely fits. + */ +export const AFTER_FAILURE: Readonly< + Record>> +> = { + tabular: { + native_capacity: ['search', 'text'], + aggregate_overflow: ['search'], + text_exceeds: ['search'], + native_rejected: ['search'], + }, + document: { + native_capacity: ['search', 'code', 'text'], + aggregate_overflow: ['search', 'code'], + text_exceeds: ['search', 'code'], + native_rejected: ['search', 'code'], + }, +}; + +export const READER_PATH: Readonly> = { + provider: 'provider', + text: 'text', + search: 'none', + code: 'none', +}; + +/** Sources whose original can be streamed back from storage, so a tool or extractor can read it. */ +export const ORIGINAL_BACKED_SOURCES: ReadonlySet = new Set([ + FileSources.local, + FileSources.s3, + FileSources.cloudfront, + FileSources.azure_blob, + FileSources.firebase, +]); + +/** + * A missing source reads as the schema default, `local`, as the encoders and agent file binding + * read it. The provision service refuses a missing source today, so it must coerce the same way + * when it adopts this set. + */ +export const isOriginalBacked = (file: Pick): boolean => + ORIGINAL_BACKED_SOURCES.has(file.source ?? FileSources.local); + +/** A record that holds only extracted text, with no original behind it. */ +export const isTextOnlyRecord = (file: Pick): boolean => + file.source === FileSources.text; + +/** Classification only; admission stays with `supportedMimeTypes`. */ +const TABULAR_PATTERNS: readonly RegExp[] = [ + excelMimeTypes, + /^application\/vnd\.oasis\.opendocument\.spreadsheet$/, + /^(text|application)\/csv$/, + /^text\/tab-separated-values$/, + /^application\/(x-parquet|vnd\.apache\.parquet)$/, +]; + +const MEDIA_MIME_PATTERN = /^(image|audio|video)\//; + +export function categorizeForReading(mimeType: string): ReadingCategory | 'media' { + const type = mimeType.split(';', 1)[0].trim().toLowerCase(); + if (MEDIA_MIME_PATTERN.test(type)) { + return 'media'; + } + return TABULAR_PATTERNS.some((pattern) => pattern.test(type)) ? 'tabular' : 'document'; +} + +export interface DirectContentEntry { + fileId: string; + /** Whether the file counts toward the attachment count limit. */ + counts: boolean; + bytes: number; + textChars: number; +} + +/** A missing or zero limit leaves that dimension unlimited. */ +export interface DirectContentLimits { + count?: number; + bytes?: number; + textChars?: number; +} + +type DirectContentUsage = Required; + +const chargeDirectContent = ( + usage: DirectContentUsage, + entry: DirectContentEntry, +): DirectContentUsage => ({ + count: usage.count + (entry.counts ? 1 : 0), + bytes: usage.bytes + entry.bytes, + textChars: usage.textChars + entry.textChars, +}); + +const withinLimit = (value: number, limit?: number): boolean => !limit || value <= limit; + +/** + * First-fit allocation of direct model content in the given (stable) candidate order, after + * charging `committed`. Returns the ids that do not fit, which must take another reader. + */ +export function allocateDirectContent( + committed: readonly DirectContentEntry[], + candidates: readonly DirectContentEntry[], + limits: DirectContentLimits, +): ReadonlySet { + const fits = (usage: DirectContentUsage): boolean => + withinLimit(usage.count, limits.count) && + withinLimit(usage.bytes, limits.bytes) && + withinLimit(usage.textChars, limits.textChars); + const overflow = new Set(); + let used = committed.reduce(chargeDirectContent, { count: 0, bytes: 0, textChars: 0 }); + for (const entry of candidates) { + const next = chargeDirectContent(used, entry); + if (fits(next)) { + used = next; + continue; + } + overflow.add(entry.fileId); + } + return overflow; +} diff --git a/packages/data-provider/src/resolve-llm-delivery-path.spec.ts b/packages/data-provider/src/resolve-llm-delivery-path.spec.ts index ab3728cedd8..49178731b95 100644 --- a/packages/data-provider/src/resolve-llm-delivery-path.spec.ts +++ b/packages/data-provider/src/resolve-llm-delivery-path.spec.ts @@ -1,8 +1,18 @@ import type { + UploadReading, TurnFileConsumers, TurnDeliveryFile, + UploadReadingInput, TurnDeliveryRouting, } from './resolve-llm-delivery-path'; +import type { + ReaderKind, + FileReading, + ClassicReason, + BuiltInTextPlan, + ReadingEvidence, + TurnReadingInputs, +} from './reading'; import type { TDefaultLLMDeliveryPathConfig } from './file-config'; import type { EndpointFileConfig } from './types/files'; import type { TEndpoint } from './config'; @@ -18,8 +28,19 @@ import { resolveDefaultLLMDeliveryPath, resolveUploadLLMDeliveryPath, SYSTEM_LLM_DELIVERY_DEFAULTS, + decideFileReading, + decideUploadReading, + judgeCodeEligibility, + selectBuiltInTextPlan, + isAutomaticReadingRecord, + resolveClassicTurnLLMDeliveryPath, } from './resolve-llm-delivery-path'; -import { mergeFileConfig, supportedMimeTypes, getEndpointFileConfig } from './file-config'; +import { + excelFileTypes, + mergeFileConfig, + supportedMimeTypes, + getEndpointFileConfig, +} from './file-config'; import { EToolResources } from './types/tools'; function resolveTurnLLMDeliveryPath({ @@ -1081,3 +1102,1230 @@ describe('resolveTurnLLMDeliveryPath', () => { ).toBe('none'); }); }); + +const XLSX = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'; +const DOCX = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'; +const PPTX = 'application/vnd.openxmlformats-officedocument.presentationml.presentation'; +const ODS = 'application/vnd.oasis.opendocument.spreadsheet'; +const TSV = 'text/tab-separated-values'; +const PARQUET = 'application/vnd.apache.parquet'; +const MSWORD = 'application/msword'; +const PDF = 'application/pdf'; +const ZIP = 'application/zip'; + +const NO_TOOLS: TurnFileConsumers = { executeCode: false, fileSearch: false }; +const CODE_ONLY: TurnFileConsumers = { executeCode: true, fileSearch: false }; +const SEARCH_ONLY: TurnFileConsumers = { executeCode: false, fileSearch: true }; +const BOTH_TOOLS: TurnFileConsumers = { executeCode: true, fileSearch: true }; + +const automaticConfig: EndpointFileConfig = { llmDeliveryPolicy: 'automatic' }; + +function automaticRouting( + endpoint: string, + overrides: Partial = {}, +): Partial { + return { + endpoint, + endpointConfig: automaticConfig, + fileConfig: mergeFileConfig(undefined), + sttConfigured: true, + ...overrides, + }; +} + +const withEvidence = (evidence: ReadingEvidence = {}, canDerive = false): TurnReadingInputs => ({ + judge: () => evidence, + canDerive, +}); + +function attachment(type: string, overrides: Partial = {}): TurnDeliveryFile { + return { + file_id: 'file_1', + type, + bytes: 2048, + source: 'local', + context: 'message_attachment', + llmDeliveryPath: 'none', + metadata: { destinationChosen: false }, + ...overrides, + }; +} + +const decide = ( + file: TurnDeliveryFile, + consumers: TurnFileConsumers | undefined, + routing: Partial | undefined = automaticRouting('openAI'), +): FileReading => decideFileReading({ routing, file, consumers }); + +const skippedReaders = (reading: FileReading): ReaderKind[] => + reading.skipped.map(({ reader }) => reader); + +describe('judgeCodeEligibility', () => { + const file = attachment(XLSX); + + it('names the first reason Run Code cannot read the record', () => { + expect(judgeCodeEligibility(file, undefined)).toBe('no_run_code'); + expect(judgeCodeEligibility(file, SEARCH_ONLY)).toBe('no_run_code'); + expect(judgeCodeEligibility({ ...file, source: 'text' }, CODE_ONLY)).toBe('text_only_record'); + expect(judgeCodeEligibility({ ...file, type: 'audio/mpeg' }, CODE_ONLY)).toBe('incompatible'); + expect(judgeCodeEligibility({ ...file, type: eml }, CODE_ONLY)).toBe('incompatible'); + expect(judgeCodeEligibility({ ...file, source: 'openai' }, CODE_ONLY)).toBe('unstreamable'); + expect( + judgeCodeEligibility({ ...file, metadata: { destinationChosen: true } }, CODE_ONLY), + ).toBe('declined'); + }); + + it('counts an enabled Run Code before the sandbox holds a copy', () => { + expect(judgeCodeEligibility(file, CODE_ONLY)).toBe('eligible'); + expect(judgeCodeEligibility({ ...file, source: undefined }, BOTH_TOOLS)).toBe('eligible'); + expect( + judgeCodeEligibility( + { ...file, metadata: { destinationChosen: true, codeEnvRef: codeRef } }, + CODE_ONLY, + ), + ).toBe('eligible'); + }); + + it('judges the stored type the sandbox receives', () => { + const converted = { ...file, type: 'audio/mpeg', metadata: { routingMimeType: XLSX } }; + expect(judgeCodeEligibility(converted, CODE_ONLY)).toBe('incompatible'); + }); +}); + +describe('decideFileReading', () => { + describe('eligibility gate', () => { + const eligible = attachment(XLSX, { text: 'region,total' }); + const withConfig = (endpointConfig: EndpointFileConfig) => + automaticRouting('openAI', { endpointConfig: { ...automaticConfig, ...endpointConfig } }); + const cases: Array< + [ + ClassicReason, + Partial | undefined, + TurnDeliveryFile, + TurnFileConsumers | undefined, + ] + > = [ + ['legacy_record', undefined, eligible, CODE_ONLY], + [ + 'legacy_record', + automaticRouting('openAI'), + { ...eligible, llmDeliveryPath: null }, + CODE_ONLY, + ], + ['classic_policy', automaticRouting('openAI', { endpointConfig: {} }), eligible, CODE_ONLY], + ['classic_policy', withConfig({ llmDeliveryPolicy: 'classic' }), eligible, CODE_ONLY], + ['classic_policy', withConfig({ legacyFileUploadUX: true }), eligible, CODE_ONLY], + [ + 'explicit_destination', + automaticRouting('openAI'), + { ...eligible, metadata: { destinationChosen: true } }, + CODE_ONLY, + ], + ['unmarked_record', automaticRouting('openAI'), { ...eligible, metadata: {} }, CODE_ONLY], + ['unmarked_record', automaticRouting('openAI'), { ...eligible, metadata: null }, CODE_ONLY], + [ + 'not_message_attachment', + automaticRouting('openAI'), + { ...eligible, context: 'agents' }, + CODE_ONLY, + ], + [ + 'not_message_attachment', + automaticRouting('openAI'), + { ...eligible, context: 'execute_code' }, + CODE_ONLY, + ], + [ + 'not_message_attachment', + automaticRouting('openAI'), + { ...eligible, context: undefined }, + CODE_ONLY, + ], + ['text_only_record', automaticRouting('openAI'), { ...eligible, source: 'text' }, CODE_ONLY], + [ + 'configured_route', + withConfig({ defaultLLMDeliveryPath: { overrides: { [XLSX]: 'text' } } }), + eligible, + CODE_ONLY, + ], + [ + 'configured_route', + withConfig({ defaultLLMDeliveryPath: { overrides: { 'application/*': 'none' } } }), + eligible, + NO_TOOLS, + ], + [ + 'configured_route', + automaticRouting('openAI', { + fileConfig: { endpoints: {}, defaultLLMDeliveryPath: { fallback: 'text' } }, + }), + eligible, + CODE_ONLY, + ], + ['consumers_unknown', automaticRouting('openAI'), eligible, undefined], + ['no_file_tools', automaticRouting('openAI'), eligible, NO_TOOLS], + ['no_file_tools', automaticRouting('bedrock'), attachment(XLSX), NO_TOOLS], + ['no_file_tools', automaticRouting('openAI'), attachment(PPTX, { text: 'slides' }), NO_TOOLS], + ['no_file_tools', automaticRouting('openAI'), attachment('audio/mpeg'), NO_TOOLS], + [ + 'media_category', + automaticRouting('openAI'), + attachment('image/png', { llmDeliveryPath: 'provider' }), + CODE_ONLY, + ], + ['media_category', automaticRouting('openAI'), attachment('audio/mpeg'), SEARCH_ONLY], + ['media_category', automaticRouting('google'), attachment('video/mp4'), BOTH_TOOLS], + ]; + + it.each(cases)('keeps the classic route: %s', (reason, routing, file, consumers) => { + const reading = decideFileReading({ routing, file, consumers }); + + expect(reading).toMatchObject({ reason, automatic: false, skipped: [], needsText: false }); + expect(reading.path).toBe(resolveClassicTurnLLMDeliveryPath(routing, file, consumers)); + expect(reading.classicPath).toBe(reading.path); + }); + + it('checks the gates in order', () => { + const configured = automaticRouting('openAI', { + endpointConfig: { + ...automaticConfig, + defaultLLMDeliveryPath: { overrides: { [XLSX]: 'text' } }, + }, + }); + let file = attachment(XLSX, { + source: 'text', + context: 'agents', + metadata: { destinationChosen: true }, + }); + const reasonOf = ( + routing: Partial, + consumers?: TurnFileConsumers, + ): FileReading['reason'] => decideFileReading({ routing, file, consumers }).reason; + + expect(reasonOf(configured)).toBe('explicit_destination'); + file = { ...file, metadata: {} }; + expect(reasonOf(configured)).toBe('unmarked_record'); + file = { ...file, metadata: { destinationChosen: false } }; + expect(reasonOf(configured)).toBe('not_message_attachment'); + file = { ...file, context: 'message_attachment' }; + expect(reasonOf(configured)).toBe('text_only_record'); + file = { ...file, source: 'local' }; + expect(reasonOf(configured)).toBe('configured_route'); + expect(reasonOf(automaticRouting('openAI'))).toBe('consumers_unknown'); + expect(reasonOf(automaticRouting('openAI'), NO_TOOLS)).toBe('no_file_tools'); + expect(reasonOf(automaticRouting('openAI'), CODE_ONLY)).toBe('code_preferred'); + file = { ...file, type: 'image/png' }; + expect(reasonOf(automaticRouting('openAI'), NO_TOOLS)).toBe('no_file_tools'); + expect(reasonOf(automaticRouting('openAI'), CODE_ONLY)).toBe('media_category'); + }); + }); + + describe('tabular order: code, provider, text, search', () => { + it('prefers Run Code whenever it can read the file', () => { + expect(decide(attachment(XLSX, { text: 'region,total' }), BOTH_TOOLS)).toMatchObject({ + path: 'none', + reader: 'code', + reason: 'code_preferred', + automatic: true, + skipped: [], + code: 'eligible', + category: 'tabular', + classicPath: 'text', + }); + }); + + it.each([...excelFileTypes, ODS, 'text/csv', 'application/csv', TSV])( + 'gives %s to Run Code', + (mimeType) => { + expect(decide(attachment(mimeType), CODE_ONLY)).toMatchObject({ + category: 'tabular', + reader: 'code', + reason: 'code_preferred', + }); + }, + ); + + it('falls back to native delivery where the provider takes the type', () => { + const reading = decide(attachment(XLSX), SEARCH_ONLY, automaticRouting('bedrock')); + expect(reading).toMatchObject({ + reader: 'provider', + path: 'provider', + reason: 'code_unavailable', + }); + expect(skippedReaders(reading)).toEqual(['code']); + }); + + it('then to complete stored text', () => { + const routing = automaticRouting('openAI', { reading: withEvidence({ text: 'fits' }) }); + const reading = decide(attachment(XLSX, { text: 'region,total' }), SEARCH_ONLY, routing); + expect(reading).toMatchObject({ reader: 'text', path: 'text', needsText: false }); + expect(reading.skipped).toEqual([ + { reader: 'code', reason: 'code_unavailable' }, + { reader: 'provider', reason: 'native_unsupported' }, + ]); + }); + + it('then to File Search', () => { + const reading = decide(attachment(XLSX), SEARCH_ONLY); + expect(reading).toMatchObject({ reader: 'search', path: 'none' }); + expect(skippedReaders(reading)).toEqual(['code', 'provider', 'text']); + }); + + it('reports the file unavailable when no reader can take it', () => { + const routing = automaticRouting('openAI', { + reading: withEvidence({ search: 'unreachable' }), + }); + const reading = decide(attachment(XLSX), SEARCH_ONLY, routing); + expect(reading).toMatchObject({ + reader: 'unavailable', + path: 'none', + reason: 'code_unavailable', + }); + expect(skippedReaders(reading)).toEqual(['code', 'provider', 'text', 'search']); + }); + + it('keeps the classic route when no file tool is loaded', () => { + const reading = decide(attachment(XLSX, { text: 'region,total' }), NO_TOOLS); + expect(reading).toMatchObject({ + reader: 'text', + path: 'text', + classicPath: 'text', + reason: 'no_file_tools', + automatic: false, + skipped: [], + }); + }); + }); + + describe('document order: provider, text, search, code', () => { + it('sends a document natively where the provider takes it', () => { + const routing = automaticRouting('openAI', { reading: withEvidence({ native: 'fits' }) }); + expect( + decide(attachment(PDF, { llmDeliveryPath: 'provider' }), BOTH_TOOLS, routing), + ).toMatchObject({ + reader: 'provider', + path: 'provider', + reason: 'native_supported', + skipped: [], + category: 'document', + }); + }); + + it('then complete stored text', () => { + const reading = decide(attachment(DOCX, { text: 'parsed' }), BOTH_TOOLS); + expect(reading).toMatchObject({ reader: 'text', path: 'text', reason: 'native_unsupported' }); + expect(decide(attachment('text/markdown', { text: '# notes' }), CODE_ONLY)).toMatchObject({ + reader: 'text', + reason: 'native_unsupported', + }); + }); + + it('then File Search, before Run Code', () => { + const reading = decide(attachment(DOCX), BOTH_TOOLS); + expect(reading).toMatchObject({ reader: 'search', path: 'none' }); + expect(skippedReaders(reading)).toEqual(['provider', 'text']); + }); + + it('then Run Code', () => { + const reading = decide(attachment(DOCX), CODE_ONLY); + expect(reading).toMatchObject({ reader: 'code', path: 'none', reason: 'text_unavailable' }); + expect(skippedReaders(reading)).toEqual(['provider', 'text', 'search']); + }); + + it('keeps an email export with stored text on its text route', () => { + const mail = attachment(eml, { llmDeliveryPath: 'text', text: 'mail body' }); + const reading = decide(mail, BOTH_TOOLS); + expect(reading).toMatchObject({ + reader: 'text', + path: 'text', + code: 'incompatible', + category: 'document', + automatic: true, + }); + expect(reading.path).toBe(reading.classicPath); + }); + + it('sends a Word 97 document natively on Bedrock', () => { + const doc = attachment(MSWORD); + expect(decide(doc, CODE_ONLY, automaticRouting('bedrock'))).toMatchObject({ + category: 'document', + reader: 'provider', + path: 'provider', + reason: 'native_supported', + }); + expect(decide(doc, CODE_ONLY, automaticRouting('openAI'))).toMatchObject({ + reader: 'code', + path: 'none', + }); + }); + + it('tries File Search before Run Code for a presentation', () => { + const searched = decide(attachment(PPTX), BOTH_TOOLS); + expect(searched).toMatchObject({ reader: 'search', path: 'none' }); + expect(skippedReaders(searched)).toEqual(['provider', 'text']); + + const coded = decide(attachment(PPTX), CODE_ONLY); + expect(coded).toMatchObject({ reader: 'code', path: 'none' }); + expect(skippedReaders(coded)).toEqual(['provider', 'text', 'search']); + }); + + it('leaves archives and columnar data to Run Code or nothing', () => { + expect(decide(attachment(ZIP), CODE_ONLY)).toMatchObject({ reader: 'code', path: 'none' }); + expect(decide(attachment(ZIP), SEARCH_ONLY)).toMatchObject({ reader: 'unavailable' }); + expect(decide(attachment(ZIP), NO_TOOLS)).toMatchObject({ + reader: 'unavailable', + path: 'none', + reason: 'no_file_tools', + automatic: false, + }); + expect(decide(attachment(PARQUET), CODE_ONLY)).toMatchObject({ + reader: 'code', + reason: 'code_preferred', + }); + expect(decide(attachment(PARQUET), SEARCH_ONLY)).toMatchObject({ + reader: 'unavailable', + path: 'none', + }); + expect(decide(attachment('application/x-parquet'), NO_TOOLS)).toMatchObject({ + reader: 'unavailable', + path: 'none', + reason: 'no_file_tools', + automatic: false, + }); + }); + }); + + describe('evidence', () => { + /* The walk runs only with a file tool loaded, so "no tool can take it" is a loaded File + * Search that will not receive the file: the row is then walked to its end. */ + const unreachable = { search: 'unreachable' } as const; + + it.each<[string, string, ReadingEvidence]>([ + ['native capacity', 'openAI', { native: 'capacity' }], + ['capacity discovered by encoding', 'anthropic', { native: 'fits', rejected: 'capacity' }], + ])( + 'sends a document over %s to search, then code, then complete fitting text last', + (_name, endpoint, evidence) => { + const pdf = attachment(PDF, { llmDeliveryPath: 'provider', text: 'extracted' }); + const routing = automaticRouting(endpoint, { + reading: withEvidence({ ...evidence, text: 'fits' }), + }); + + expect(decide(pdf, BOTH_TOOLS, routing)).toMatchObject({ + reader: 'search', + reason: 'native_capacity', + }); + expect(decide(pdf, CODE_ONLY, routing)).toMatchObject({ + reader: 'code', + reason: 'native_capacity', + }); + const unreached = automaticRouting(endpoint, { + reading: withEvidence({ ...evidence, text: 'fits', ...unreachable }), + }); + const text = decide(pdf, SEARCH_ONLY, unreached); + expect(text).toMatchObject({ + reader: 'text', + path: 'text', + reason: 'native_capacity', + needsText: false, + }); + expect(skippedReaders(text)).toEqual(['provider', 'search', 'code']); + + const exceeding = automaticRouting(endpoint, { + reading: withEvidence({ ...evidence, text: 'exceeds', ...unreachable }), + }); + expect(decide(pdf, SEARCH_ONLY, exceeding)).toMatchObject({ + reader: 'unavailable', + path: 'none', + reason: 'native_capacity', + }); + }, + ); + + it('sends a spreadsheet over native capacity to File Search before its text', () => { + const sheet = attachment(XLSX, { text: 'region,total' }); + const routing = automaticRouting('bedrock', { + reading: withEvidence({ native: 'capacity', text: 'fits' }), + }); + + const searched = decide(sheet, SEARCH_ONLY, routing); + expect(searched).toMatchObject({ reader: 'search', reason: 'native_capacity' }); + expect(skippedReaders(searched)).toEqual(['code', 'provider']); + + const unreached = automaticRouting('bedrock', { + reading: withEvidence({ native: 'capacity', text: 'fits', ...unreachable }), + }); + const text = decide(sheet, SEARCH_ONLY, unreached); + expect(text).toMatchObject({ reader: 'text', path: 'text', reason: 'native_capacity' }); + expect(skippedReaders(text)).toEqual(['code', 'provider', 'search']); + expect(decide(sheet, CODE_ONLY, routing)).toMatchObject({ reader: 'code', skipped: [] }); + }); + + it('derives a capacity fallback while keeping integrity failures away from text', () => { + const pdf = attachment(PDF, { llmDeliveryPath: 'provider' }); + const routing = automaticRouting('anthropic', { + reading: withEvidence({ native: 'fits', rejected: 'capacity', ...unreachable }, true), + }); + expect(decide(pdf, SEARCH_ONLY, routing)).toMatchObject({ + reader: 'text', + reason: 'native_capacity', + needsText: true, + }); + + const invalid = automaticRouting('anthropic', { + reading: withEvidence({ native: 'fits', rejected: 'integrity', ...unreachable }, true), + }); + expect(decide(pdf, SEARCH_ONLY, invalid)).toMatchObject({ + reader: 'unavailable', + reason: 'native_rejected', + needsText: false, + }); + }); + + it('preserves unsupported encoding as a capability failure with a fitting-text fallback', () => { + const routing = automaticRouting('anthropic', { + reading: withEvidence({ native: 'fits', rejected: 'unsupported', text: 'fits' }), + }); + expect(decide(attachment(PDF, { text: 'extracted' }), BOTH_TOOLS, routing)).toMatchObject({ + reader: 'text', + reason: 'native_unsupported', + }); + }); + + it('moves past a provider the encoder would skip', () => { + const routing = automaticRouting('anthropic', { + reading: withEvidence({ native: 'unsupported' }), + }); + const reading = decide(attachment(PDF, { text: 'extracted' }), BOTH_TOOLS, routing); + expect(reading).toMatchObject({ reader: 'text', reason: 'native_unsupported' }); + }); + + it('never offers a direct reader after an encode-time rejection', () => { + const routing = automaticRouting('bedrock', { + reading: withEvidence({ rejected: 'integrity', text: 'fits' }), + }); + const unreached = automaticRouting('bedrock', { + reading: withEvidence({ rejected: 'integrity', text: 'fits', ...unreachable }), + }); + const pdf = attachment(PDF, { text: 'extracted' }); + + const unavailable = decide(pdf, SEARCH_ONLY, unreached); + expect(unavailable).toMatchObject({ reader: 'unavailable', reason: 'native_rejected' }); + expect(skippedReaders(unavailable)).toEqual(['provider', 'search', 'code']); + expect(decide(pdf, SEARCH_ONLY, routing)).toMatchObject({ + reader: 'search', + reason: 'native_rejected', + }); + + const sheet = decide(attachment(XLSX, { text: 'region,total' }), SEARCH_ONLY, unreached); + expect(sheet.reader).toBe('unavailable'); + expect(skippedReaders(sheet)).toEqual(['code', 'provider', 'search']); + }); + + it('sends text over the limit to search and code instead of truncating it', () => { + const routing = automaticRouting('openAI', { reading: withEvidence({ text: 'exceeds' }) }); + const docx = attachment(DOCX, { text: 'long' }); + + const coded = decide(docx, CODE_ONLY, routing); + expect(coded).toMatchObject({ reader: 'code', reason: 'text_exceeds' }); + expect(coded.skipped).toContainEqual({ reader: 'text', reason: 'text_exceeds' }); + const unreached = automaticRouting('openAI', { + reading: withEvidence({ text: 'exceeds', ...unreachable }), + }); + expect(decide(docx, SEARCH_ONLY, unreached)).toMatchObject({ + reader: 'unavailable', + path: 'none', + }); + + const csv = decide(attachment('text/csv', { text: 'long' }), SEARCH_ONLY, routing); + expect(csv).toMatchObject({ reader: 'search' }); + expect(skippedReaders(csv)).toEqual(['code', 'provider', 'text']); + }); + + it('gives overflowed direct content to search and code only', () => { + const routing = automaticRouting('openAI', { + reading: withEvidence({ overflow: true, text: 'fits' }), + }); + const unreached = automaticRouting('openAI', { + reading: withEvidence({ overflow: true, text: 'fits', ...unreachable }), + }); + const pdf = attachment(PDF, { llmDeliveryPath: 'provider', text: 'extracted' }); + + const unavailable = decide(pdf, SEARCH_ONLY, unreached); + expect(unavailable).toMatchObject({ reader: 'unavailable', path: 'none' }); + expect(unavailable.skipped[0]).toEqual({ reader: 'provider', reason: 'aggregate_overflow' }); + expect(skippedReaders(unavailable)).toEqual(['provider', 'search', 'code']); + expect(decide(pdf, SEARCH_ONLY, routing)).toMatchObject({ + reader: 'search', + reason: 'aggregate_overflow', + }); + + const docx = decide(attachment(DOCX, { text: 'parsed' }), CODE_ONLY, routing); + expect(docx).toMatchObject({ reader: 'code' }); + expect(docx.skipped).toContainEqual({ reader: 'text', reason: 'aggregate_overflow' }); + }); + + it('names the limit the file hit rather than a reader that was missing', () => { + const exceeds = automaticRouting('openAI', { reading: withEvidence({ text: 'exceeds' }) }); + const unreached = automaticRouting('openAI', { + reading: withEvidence({ text: 'exceeds', ...unreachable }), + }); + + const largeCsv = decide(attachment('text/csv', { text: 'long' }), SEARCH_ONLY, unreached); + expect(largeCsv).toMatchObject({ reader: 'unavailable', reason: 'text_exceeds' }); + expect(skippedReaders(largeCsv)).toEqual(['code', 'provider', 'text', 'search']); + + const hugeLog = decide(attachment('text/plain', { text: 'long' }), SEARCH_ONLY, exceeds); + expect(hugeLog).toMatchObject({ reader: 'search', reason: 'text_exceeds' }); + expect(hugeLog.skipped[0]).toEqual({ reader: 'provider', reason: 'native_unsupported' }); + }); + + it('names the preferred reader that was missing when no limit was hit', () => { + const unreached = automaticRouting('openAI', { reading: withEvidence(unreachable) }); + expect(decide(attachment(PARQUET), SEARCH_ONLY)).toMatchObject({ + reader: 'unavailable', + reason: 'code_unavailable', + }); + expect(decide(attachment(PPTX), SEARCH_ONLY, unreached)).toMatchObject({ + reader: 'unavailable', + reason: 'text_unavailable', + }); + expect(decide(attachment(DOCX, { text: 'parsed' }), BOTH_TOOLS)).toMatchObject({ + reader: 'text', + reason: 'native_unsupported', + }); + }); + + it('continues the row when File Search will not receive the file', () => { + const routing = automaticRouting('openAI', { + reading: withEvidence({ search: 'unreachable' }), + }); + const reading = decide(attachment(PPTX), BOTH_TOOLS, routing); + expect(reading).toMatchObject({ reader: 'code', path: 'none' }); + expect(reading.skipped).toContainEqual({ reader: 'search', reason: 'search_unavailable' }); + expect(decide(attachment(XLSX), SEARCH_ONLY, routing)).toMatchObject({ + reader: 'unavailable', + }); + }); + + it('asks the turn for evidence once per decision', () => { + const judge = jest.fn((): ReadingEvidence => ({ native: 'capacity' })); + decide( + attachment(PDF), + BOTH_TOOLS, + automaticRouting('openAI', { reading: { judge, canDerive: false } }), + ); + expect(judge).toHaveBeenCalledTimes(1); + }); + }); + + describe('text derived from the original', () => { + const deferred = attachment(XLSX, { + metadata: { destinationChosen: false, textDerivation: { outcome: 'deferred' } }, + }); + const deriving = (evidence: ReadingEvidence = {}) => + automaticRouting('openAI', { reading: withEvidence(evidence, true) }); + + const unreachable = { search: 'unreachable' } as const; + + it('asks for text when a deriver is wired and code cannot read the file', () => { + expect(decide(deferred, SEARCH_ONLY, deriving(unreachable))).toMatchObject({ + reader: 'text', + path: 'text', + needsText: true, + automatic: true, + }); + expect(decide(deferred, CODE_ONLY, deriving())).toMatchObject({ + reader: 'code', + needsText: false, + }); + }); + + it('derives the text its classic route names when no file tool is loaded', () => { + expect(decide(deferred, NO_TOOLS, deriving())).toMatchObject({ + reader: 'text', + path: 'text', + classicPath: 'text', + needsText: true, + automatic: false, + reason: 'no_file_tools', + }); + expect(decide(deferred, NO_TOOLS, automaticRouting('openAI'))).toMatchObject({ + path: 'none', + classicPath: 'text', + reader: 'unavailable', + needsText: false, + reason: 'no_file_tools', + }); + }); + + it('does not ask without a deriver, an original, an extractor, or after a failure', () => { + const failed = { + ...deferred, + metadata: { destinationChosen: false, textDerivation: { outcome: 'failed' as const } }, + }; + const unreadable = [ + decide( + deferred, + SEARCH_ONLY, + automaticRouting('openAI', { reading: withEvidence(unreachable) }), + ), + decide({ ...deferred, source: 'openai' }, SEARCH_ONLY, deriving(unreachable)), + decide({ ...deferred, type: PPTX }, SEARCH_ONLY, deriving(unreachable)), + decide(failed, SEARCH_ONLY, deriving(unreachable)), + decide(deferred, SEARCH_ONLY, deriving({ textFailed: true, ...unreachable })), + ]; + unreadable.forEach((reading) => { + expect(reading).toMatchObject({ reader: 'unavailable', needsText: false }); + expect(reading.skipped).toContainEqual({ reader: 'text', reason: 'text_unavailable' }); + }); + + /* With no reading inputs at all, File Search is reachable and takes the file; text is + * still passed over rather than asked for. */ + const bare = decide(deferred, SEARCH_ONLY, automaticRouting('openAI')); + expect(bare).toMatchObject({ reader: 'search', needsText: false }); + expect(bare.skipped).toContainEqual({ reader: 'text', reason: 'text_unavailable' }); + expect(decide(deferred, SEARCH_ONLY, deriving({ textFailed: true }))).toMatchObject({ + reader: 'search', + }); + }); + + it('uses stored text rather than deriving it again', () => { + expect(decide({ ...deferred, text: 'region,total' }, SEARCH_ONLY, deriving())).toMatchObject({ + reader: 'text', + needsText: false, + automatic: true, + }); + expect(decide({ ...deferred, text: 'region,total' }, NO_TOOLS, deriving())).toMatchObject({ + reader: 'text', + needsText: false, + automatic: false, + }); + }); + }); + + describe('marked records under classic routing', () => { + const marked = attachment(XLSX, { + metadata: { destinationChosen: false, textDerivation: { outcome: 'deferred' } }, + }); + const classicRouting = (reading?: TurnReadingInputs) => + automaticRouting('openAI', { endpointConfig: {}, reading }); + + it('derives the text a classic text route needs', () => { + expect(decide(marked, NO_TOOLS, classicRouting(withEvidence({}, true)))).toMatchObject({ + path: 'text', + classicPath: 'text', + reader: 'text', + needsText: true, + automatic: false, + reason: 'classic_policy', + }); + }); + + it('delivers nothing rather than empty text when it cannot', () => { + const failed = { + ...marked, + metadata: { destinationChosen: false, textDerivation: { outcome: 'failed' as const } }, + }; + const readings = [ + decide(marked, NO_TOOLS, classicRouting()), + decide(marked, NO_TOOLS, classicRouting(withEvidence({}, false))), + decide(failed, NO_TOOLS, classicRouting(withEvidence({}, true))), + decide(marked, NO_TOOLS, classicRouting(withEvidence({ textFailed: true }, true))), + decide({ ...marked, source: 'openai' }, NO_TOOLS, classicRouting(withEvidence({}, true))), + ]; + readings.forEach((reading) => { + expect(reading).toMatchObject({ path: 'none', classicPath: 'text', needsText: false }); + }); + expect(decide(marked, CODE_ONLY, classicRouting()).reader).toBe('code'); + }); + + it('leaves every record classic routing wrote as it was', () => { + const unmarked = attachment(XLSX); + expect(decide(unmarked, NO_TOOLS, classicRouting()).path).toBe('text'); + expect(decide({ ...marked, text: 'region,total' }, NO_TOOLS, classicRouting())).toMatchObject( + { + path: 'text', + needsText: false, + }, + ); + expect(decide({ ...marked, type: ZIP }, NO_TOOLS, classicRouting()).path).toBe('none'); + }); + }); + + describe('automatic counterparts of classic pins', () => { + it('reads a classic-era spreadsheet with Run Code when it runs, and as text when not', () => { + const sheet = attachment(XLSX, { llmDeliveryPath: 'text', text: 'region,total' }); + expect(decide(sheet, CODE_ONLY)).toMatchObject({ path: 'none', classicPath: 'text' }); + expect(decide(sheet, SEARCH_ONLY)).toMatchObject({ + path: 'text', + classicPath: 'text', + automatic: true, + }); + expect(decide(sheet, NO_TOOLS)).toMatchObject({ + path: 'text', + classicPath: 'text', + automatic: false, + reason: 'no_file_tools', + }); + }); + + it('reads stored text as a primary reader without the fallback flag', () => { + const slides = attachment(PPTX, { text: 'slide text' }); + expect(decide(slides, BOTH_TOOLS)).toMatchObject({ path: 'text', classicPath: 'none' }); + expect(decide(slides, NO_TOOLS)).toMatchObject({ + path: 'none', + classicPath: 'none', + reason: 'no_file_tools', + }); + }); + + it('keeps an explicit none override with the flag off', () => { + const routing = automaticRouting('openAI', { + endpointConfig: { + ...automaticConfig, + defaultLLMDeliveryPath: { overrides: { 'text/csv': 'none' } }, + }, + }); + expect(decide(attachment('text/csv', { text: 'a,b' }), NO_TOOLS, routing)).toMatchObject({ + path: 'none', + reason: 'configured_route', + }); + }); + }); +}); + +describe('isAutomaticReadingRecord', () => { + const config = automaticRouting('openAI'); + const record = attachment(XLSX); + + it('accepts an inferred message attachment under the automatic policy', () => { + expect(isAutomaticReadingRecord(config, record)).toBe(true); + expect(isAutomaticReadingRecord(config, { ...record, source: undefined })).toBe(true); + expect(isAutomaticReadingRecord(config, { ...record, source: 'openai' })).toBe(true); + expect(isAutomaticReadingRecord(config, { ...record, llmDeliveryPath: 'text' })).toBe(true); + }); + + it.each<[string, Partial | undefined, TurnDeliveryFile]>([ + ['no endpoint config', undefined, record], + ['the classic policy', { endpointConfig: {} }, record], + [ + 'the legacy chooser', + { endpointConfig: { ...automaticConfig, legacyFileUploadUX: true } }, + record, + ], + ['a record predating routing', config, { ...record, llmDeliveryPath: null }], + ['an explicit destination', config, { ...record, metadata: { destinationChosen: true } }], + ['a missing marker', config, { ...record, metadata: {} }], + ['no metadata', config, { ...record, metadata: null }], + ['an agent resource', config, { ...record, context: 'agents' }], + ['a run artifact', config, { ...record, context: 'run_artifact' }], + ['no context', config, { ...record, context: undefined }], + ['a text-only record', config, { ...record, source: 'text' }], + ['media', config, { ...record, type: 'image/png' }], + ])('rejects %s', (_name, gateConfig, file) => { + expect(isAutomaticReadingRecord(gateConfig, file)).toBe(false); + }); + + it('rejects a type a configured route names, which the decision keeps classic', () => { + const pdf = attachment(PDF, { llmDeliveryPath: 'provider', bytes: 20 * 1024 * 1024 }); + const configured = automaticRouting('openAI', { + endpointConfig: { + ...automaticConfig, + fileSizeLimit: 5 * 1024 * 1024, + defaultLLMDeliveryPath: { overrides: { [PDF]: 'provider' } }, + }, + }); + const globallyConfigured = automaticRouting('openAI', { + fileConfig: { endpoints: {}, defaultLLMDeliveryPath: { fallback: 'text' } }, + }); + + expect(isAutomaticReadingRecord(configured, pdf)).toBe(false); + expect(decide(pdf, BOTH_TOOLS, configured)).toMatchObject({ + automatic: false, + reason: 'configured_route', + path: 'provider', + }); + expect(isAutomaticReadingRecord(globallyConfigured, pdf)).toBe(false); + expect(isAutomaticReadingRecord(config, pdf)).toBe(true); + }); + + it('judges media by the type routing saw before conversion', () => { + const converted = { + ...record, + type: 'image/png', + metadata: { destinationChosen: false, routingMimeType: 'text/csv' }, + }; + const heic = { + ...record, + type: 'text/csv', + metadata: { destinationChosen: false, routingMimeType: 'image/heic' }, + }; + expect(isAutomaticReadingRecord(config, converted)).toBe(true); + expect(isAutomaticReadingRecord(config, heic)).toBe(false); + }); +}); + +describe('decideUploadReading', () => { + const base: UploadReadingInput = { + mimeType: XLSX, + endpoint: 'openAI', + endpointConfig: automaticConfig, + sttConfigured: true, + isMessageAttachment: true, + extractionRequiredForInspection: false, + }; + const classicFields = { + codePreferred: false, + needsCodeAvailability: false, + keepOriginalOnExtractionFailure: false, + deferredMarker: false, + }; + + it.each<[string, Partial, Partial]>([ + [ + 'a spreadsheet Run Code can read', + { codePossible: true }, + { + path: 'none', + policy: 'automatic', + category: 'tabular', + reason: 'code_preferred', + codePreferred: true, + needsCodeAvailability: false, + keepOriginalOnExtractionFailure: false, + deferredMarker: true, + }, + ], + [ + 'a spreadsheet before code availability is known', + {}, + { + path: 'text', + reason: 'automatic_default', + codePreferred: false, + needsCodeAvailability: true, + keepOriginalOnExtractionFailure: true, + }, + ], + [ + 'a spreadsheet where Run Code is not possible', + { codePossible: false }, + { + path: 'text', + reason: 'code_unavailable', + codePreferred: false, + keepOriginalOnExtractionFailure: true, + }, + ], + [ + 'csv', + { mimeType: 'text/csv', codePossible: true }, + { path: 'none', codePreferred: true, deferredMarker: true }, + ], + [ + 'tsv', + { mimeType: TSV, codePossible: true }, + { path: 'none', codePreferred: true, deferredMarker: true }, + ], + [ + 'ods', + { mimeType: ODS, codePossible: true }, + { path: 'none', codePreferred: true, deferredMarker: true }, + ], + [ + 'parquet, which no built-in extractor reads', + { mimeType: PARQUET, codePossible: true }, + { path: 'none', codePreferred: true, deferredMarker: false }, + ], + [ + 'a spreadsheet an inspection policy must extract', + { codePossible: true, extractionRequiredForInspection: true }, + { path: 'text', reason: 'inspection_requires_text', ...classicFields }, + ], + [ + 'a spreadsheet an inspection policy must extract, before code availability is known', + { extractionRequiredForInspection: true }, + { path: 'text', reason: 'inspection_requires_text', ...classicFields }, + ], + [ + 'a PDF an inspection policy must extract', + { mimeType: PDF, endpoint: 'azureOpenAI', extractionRequiredForInspection: true }, + { path: 'text', reason: 'automatic_default', ...classicFields }, + ], + [ + 'a natively sent PDF under an inspection policy', + { mimeType: PDF, extractionRequiredForInspection: true }, + { path: 'provider', reason: 'automatic_default', ...classicFields }, + ], + [ + 'a Word document an inspection policy must extract', + { mimeType: DOCX, extractionRequiredForInspection: true }, + { path: 'text', reason: 'automatic_default', ...classicFields }, + ], + [ + 'a PDF on a document-capable endpoint', + { mimeType: PDF, codePossible: true }, + { + path: 'provider', + category: 'document', + reason: 'automatic_default', + keepOriginalOnExtractionFailure: true, + }, + ], + [ + 'a PDF the endpoint cannot take natively', + { mimeType: PDF, endpoint: 'azureOpenAI' }, + { path: 'text', reason: 'automatic_default', keepOriginalOnExtractionFailure: true }, + ], + [ + 'a Word document', + { mimeType: DOCX }, + { path: 'text', reason: 'automatic_default', keepOriginalOnExtractionFailure: true }, + ], + [ + 'a Word document on Bedrock', + { mimeType: DOCX, endpoint: 'bedrock' }, + { path: 'provider', reason: 'automatic_default', keepOriginalOnExtractionFailure: true }, + ], + [ + 'an archive', + { mimeType: ZIP, codePossible: true }, + { + path: 'none', + reason: 'automatic_default', + codePreferred: false, + keepOriginalOnExtractionFailure: true, + deferredMarker: false, + }, + ], + [ + 'an image', + { mimeType: 'image/png', codePossible: true }, + { path: 'provider', category: 'media', reason: 'automatic_default', ...classicFields }, + ], + [ + 'an explicit Run Code upload', + { toolResource: 'execute_code', codePossible: true }, + { path: 'none', reason: 'explicit_destination', ...classicFields }, + ], + [ + 'an explicit context upload', + { toolResource: 'context' }, + { path: 'text', reason: 'explicit_destination', ...classicFields }, + ], + [ + 'a permanent agent upload', + { isMessageAttachment: false, codePossible: true }, + { path: 'text', reason: 'agent_resource', ...classicFields }, + ], + [ + 'an upload on a classic endpoint', + { endpointConfig: {}, codePossible: true }, + { path: 'text', policy: 'classic', reason: 'classic_policy', ...classicFields }, + ], + [ + 'an upload through the legacy chooser', + { endpointConfig: { ...automaticConfig, legacyFileUploadUX: true }, codePossible: true }, + { path: 'provider', policy: 'classic', reason: 'classic_policy', ...classicFields }, + ], + [ + 'a type an endpoint route names', + { + endpointConfig: { + ...automaticConfig, + defaultLLMDeliveryPath: { overrides: { [XLSX]: 'text' } }, + }, + codePossible: true, + }, + { path: 'text', reason: 'configured_route', ...classicFields }, + ], + [ + 'a type a global route names', + { + fileConfig: { endpoints: {}, defaultLLMDeliveryPath: { fallback: 'provider' } }, + codePossible: true, + }, + { path: 'provider', reason: 'configured_route', ...classicFields }, + ], + ])('decides %s', (_name, overrides, expected) => { + expect(decideUploadReading({ ...base, ...overrides })).toMatchObject(expected); + }); + + it('never changes an upload route under the classic policy', () => { + const mimeTypes = [ + XLSX, + ODS, + 'text/csv', + TSV, + PARQUET, + PDF, + DOCX, + PPTX, + ZIP, + 'image/png', + 'audio/mpeg', + ]; + const configs: EndpointFileConfig[] = [ + {}, + { llmDeliveryPolicy: 'classic' }, + { ...automaticConfig, legacyFileUploadUX: true }, + ]; + const departures: string[] = []; + mimeTypes.forEach((mimeType) => + ['openAI', 'bedrock', 'azureOpenAI', 'agents'].forEach((endpoint) => + configs.forEach((endpointConfig) => + [undefined, true, false].forEach((codePossible) => + [true, false].forEach((isMessageAttachment) => + [true, false].forEach((extractionRequiredForInspection) => { + const input = { + ...base, + mimeType, + endpoint, + endpointConfig, + codePossible, + isMessageAttachment, + extractionRequiredForInspection, + }; + const reading = decideUploadReading(input); + const classicPath = resolveUploadLLMDeliveryPath(input); + const unchanged = + reading.policy === 'classic' && + reading.path === classicPath && + !reading.codePreferred && + !reading.needsCodeAvailability && + !reading.keepOriginalOnExtractionFailure && + !reading.deferredMarker; + if (!unchanged) { + departures.push(`${mimeType} on ${endpoint}: ${reading.path} vs ${classicPath}`); + } + }), + ), + ), + ), + ), + ); + expect(departures).toEqual([]); + }); +}); + +describe('selectBuiltInTextPlan', () => { + it.each<[string, BuiltInTextPlan | null]>([ + [XLSX, 'document_parser'], + ['application/vnd.ms-excel', 'document_parser'], + ['application/x-dos_ms_excel', 'document_parser'], + [ODS, 'document_parser'], + [PDF, 'document_parser'], + [DOCX, 'document_parser'], + ['application/vnd.oasis.opendocument.text', 'document_parser'], + ['text/csv', 'native_text'], + [TSV, 'native_text'], + ['application/csv', 'native_text'], + ['application/json', 'native_text'], + ['text/markdown', 'native_text'], + [eml, 'native_text'], + [PPTX, null], + [MSWORD, null], + [ZIP, null], + [PARQUET, null], + ['application/epub+zip', null], + ['image/png', null], + ['audio/mpeg', null], + ])('plans %s as %s', (mimeType, plan) => { + expect(selectBuiltInTextPlan(mimeType)).toBe(plan); + }); +}); + +describe('classic equivalence', () => { + const mimeTypes = [ + ...excelFileTypes, + ODS, + 'text/csv', + 'application/csv', + TSV, + 'application/x-parquet', + PARQUET, + PDF, + DOCX, + PPTX, + MSWORD, + ZIP, + 'application/epub+zip', + 'application/json', + 'text/markdown', + eml, + 'image/png', + 'audio/mpeg', + 'video/mp4', + ]; + const endpoints = [ + 'openAI', + 'anthropic', + 'bedrock', + 'google', + 'azureOpenAI', + 'agents', + 'MyGateway', + ]; + const consumerSets: Array = [ + undefined, + NO_TOOLS, + CODE_ONLY, + SEARCH_ONLY, + BOTH_TOOLS, + ]; + const shapes: Array> = [ + { text: 'stored text' }, + { embedded: true }, + { llmDeliveryPath: 'text', text: 'stored text' }, + { llmDeliveryPath: 'provider' }, + { text: 'stored text', metadata: { destinationChosen: true } }, + { text: 'stored text', metadata: null }, + { llmDeliveryPath: undefined, text: 'stored text' }, + ]; + const overrides: EndpointFileConfig = { + textFallbackWithoutTools: true, + defaultLLMDeliveryPath: { + overrides: { 'text/csv': 'none', 'application/pdf': 'text', 'image/*': 'none' }, + }, + }; + + it.each<[string, EndpointFileConfig]>([ + ['unset', {}], + ['classic', { llmDeliveryPolicy: 'classic' }], + ])('keeps every turn route identical with the policy %s', (_name, policy) => { + const mismatches: string[] = []; + let combinations = 0; + const configs: EndpointFileConfig[] = [{}, overrides]; + configs.forEach((config) => + endpoints.forEach((endpoint) => { + const routing: Partial = { + endpoint, + endpointConfig: { ...config, ...policy }, + fileConfig: mergeFileConfig(undefined), + sttConfigured: true, + reading: withEvidence({ native: 'capacity', text: 'exceeds', overflow: true }, true), + }; + mimeTypes.forEach((mimeType) => + shapes.forEach((shape) => { + const file = attachment(mimeType, shape); + consumerSets.forEach((consumers) => { + combinations += 1; + const classicPath = resolveClassicTurnLLMDeliveryPath(routing, file, consumers); + const reading = decideFileReading({ routing, file, consumers }); + const turnPath = resolveStoredTurnPath(routing, file, consumers); + if (reading.path !== classicPath || turnPath !== classicPath || reading.automatic) { + mismatches.push( + `${mimeType} on ${endpoint}: ${String(reading.path)} vs ${String(classicPath)}`, + ); + } + }); + }), + ); + }), + ); + + expect(combinations).toBe(2 * endpoints.length * mimeTypes.length * shapes.length * 5); + expect(mismatches).toEqual([]); + }); +}); diff --git a/packages/data-provider/src/resolve-llm-delivery-path.ts b/packages/data-provider/src/resolve-llm-delivery-path.ts index 6b41d378fc8..dce6b8f1c08 100644 --- a/packages/data-provider/src/resolve-llm-delivery-path.ts +++ b/packages/data-provider/src/resolve-llm-delivery-path.ts @@ -1,4 +1,22 @@ -import type { TDefaultLLMDeliveryPath, TDefaultLLMDeliveryPathConfig } from './file-config'; +import type { + ReaderKind, + ReaderSkip, + SkipReason, + FileReading, + ClassicReason, + ReadingReason, + TextDerivation, + BuiltInTextPlan, + CodeEligibility, + ReadingCategory, + ReadingEvidence, + TurnReadingInputs, +} from './reading'; +import type { + TLLMDeliveryPolicy, + TDefaultLLMDeliveryPath, + TDefaultLLMDeliveryPathConfig, +} from './file-config'; import type { EndpointFileConfig, FileConfig, RegexLike } from './types/files'; import type { CodeEnvReferenceSet } from './codeEnvRef'; import type { TEndpoint } from './config'; @@ -6,7 +24,9 @@ import { retrievalMimeTypes, isExplicitMimeConfig, isBedrockDocumentType, + documentParserMimeTypes, codeInterpreterMimeTypes, + resolveLLMDeliveryPolicy, fileConfig as baseFileConfig, } from './file-config'; import { @@ -16,9 +36,18 @@ import { isMediaSupportedProvider, isDocumentSupportedProvider, } from './schemas'; +import { + READER_PATH, + AFTER_FAILURE, + READING_ORDER, + isTextOnlyRecord, + isOriginalBacked, + categorizeForReading, +} from './reading'; import { normalizeEndpointName } from './utils'; import { EToolResources } from './types/tools'; import { getCodeEnvRefs } from './codeEnvRef'; +import { FileContext } from './types/files'; /** * The native provider a custom endpoint declares, when it declares one. A custom endpoint @@ -144,18 +173,26 @@ export function hasTextExtractionPath(mimeType: string): boolean { } /** - * Resolves the default file path destination for a given mime type. - * Resolution chain: endpoint overrides -> endpoint fallback -> global overrides -> global fallback -> system defaults. + * The built-in extractor that can turn a stored type into text, or null when none can. Only + * built-in readers count, so text for a file meant for a tool never costs a RAG or OCR call. */ -export function resolveDefaultLLMDeliveryPath( +export function selectBuiltInTextPlan(mimeType: string): BuiltInTextPlan | null { + if (documentParserMimeTypes.some((pattern) => pattern.test(mimeType))) { + return 'document_parser'; + } + return isNativelyReadableText(mimeType) ? 'native_text' : null; +} + +/** + * The route an explicit endpoint or global `defaultLLMDeliveryPath` assigns a type, or undefined + * when neither names it: endpoint overrides, endpoint fallback, global overrides, then global + * fallback, with an exact type ahead of its wildcard at each level. + */ +export function matchConfiguredDeliveryPath( mimeType: string, endpointConfig?: TDefaultLLMDeliveryPathConfig, globalConfig?: TDefaultLLMDeliveryPathConfig, - endpoint?: string, - useResponsesApi?: boolean, - sttConfigured?: boolean, - supportedMimeTypes?: RegexLike[], -): TDefaultLLMDeliveryPath { +): TDefaultLLMDeliveryPath | undefined { const wildcard = mimeType.split('/')[0] + '/*'; if (endpointConfig?.overrides) { @@ -184,6 +221,28 @@ export function resolveDefaultLLMDeliveryPath( return globalConfig.fallback; } + return undefined; +} + +/** + * Resolves the default file path destination for a given mime type. + * Resolution chain: endpoint overrides -> endpoint fallback -> global overrides -> global fallback -> system defaults. + */ +export function resolveDefaultLLMDeliveryPath( + mimeType: string, + endpointConfig?: TDefaultLLMDeliveryPathConfig, + globalConfig?: TDefaultLLMDeliveryPathConfig, + endpoint?: string, + useResponsesApi?: boolean, + sttConfigured?: boolean, + supportedMimeTypes?: RegexLike[], +): TDefaultLLMDeliveryPath { + const configured = matchConfiguredDeliveryPath(mimeType, endpointConfig, globalConfig); + if (configured != null) { + return configured; + } + + const wildcard = mimeType.split('/')[0] + '/*'; const systemDefault = (SYSTEM_LLM_DELIVERY_DEFAULTS.overrides[mimeType] ?? SYSTEM_LLM_DELIVERY_DEFAULTS.overrides[wildcard] ?? SYSTEM_LLM_DELIVERY_DEFAULTS.fallback) as TDefaultLLMDeliveryPath; @@ -249,20 +308,8 @@ export function resolveDefaultLLMDeliveryPath( return systemDefault; } -/** - * Delivery path for an upload that named no tool resource. The legacy chooser makes the - * destination explicit, so nothing is inferred there. - */ -export function resolveDefaultUploadLLMDeliveryPath({ - mimeType, - endpointConfig, - fileConfig, - endpoint, - endpointProvider, - useResponsesApi, - sttConfigured, -}: { - mimeType: string; +/** The endpoint a delivery route is resolved against, at upload or on a turn. */ +export interface DeliveryRouteInputs { endpointConfig?: EndpointFileConfig; fileConfig?: FileConfig; endpoint?: string; @@ -271,10 +318,24 @@ export function resolveDefaultUploadLLMDeliveryPath({ endpointProvider?: string | null; useResponsesApi?: boolean; sttConfigured?: boolean; -}): TDefaultLLMDeliveryPath { - if (endpointConfig?.legacyFileUploadUX === true) { - return 'provider'; - } +} + +/** + * The configured or capability-gated route for a type on an endpoint, without the legacy + * chooser. Where no configured route names the type, this is the system default the endpoint's + * encoders can actually deliver. + */ +export function resolveSystemDeliveryPath( + mimeType: string, + { + endpointConfig, + fileConfig, + endpoint, + endpointProvider, + useResponsesApi, + sttConfigured, + }: DeliveryRouteInputs, +): TDefaultLLMDeliveryPath { /* The media opt-in exists for OpenAI-format parts, so an endpoint known to run as * something else — a custom endpoint declaring `provider: anthropic` — keeps the * capability gate it had, where audio still reaches transcription. */ @@ -290,6 +351,20 @@ export function resolveDefaultUploadLLMDeliveryPath({ ); } +/** + * Delivery path for an upload that named no tool resource. The legacy chooser makes the + * destination explicit, so nothing is inferred there. + */ +export function resolveDefaultUploadLLMDeliveryPath({ + mimeType, + ...routing +}: DeliveryRouteInputs & { mimeType: string }): TDefaultLLMDeliveryPath { + if (routing.endpointConfig?.legacyFileUploadUX === true) { + return 'provider'; + } + return resolveSystemDeliveryPath(mimeType, routing); +} + /** Delivery path for an upload, honoring an explicitly chosen tool resource. */ export function resolveUploadLLMDeliveryPath({ toolResource, @@ -300,15 +375,9 @@ export function resolveUploadLLMDeliveryPath({ useResponsesApi, endpointProvider, sttConfigured, -}: { +}: DeliveryRouteInputs & { toolResource?: string | null; mimeType: string; - endpointConfig?: EndpointFileConfig; - fileConfig?: FileConfig; - endpoint?: string; - endpointProvider?: string | null; - useResponsesApi?: boolean; - sttConfigured?: boolean; }): TDefaultLLMDeliveryPath { if (toolResource === EToolResources.context || toolResource === EToolResources.ocr) { return 'text'; @@ -364,6 +433,15 @@ export interface TurnFileConsumers { fileSearch: boolean; } +/** + * Whether a file tool is loaded for the turn. The automatic policy decides readers only then; + * without Run Code or File Search every attachment keeps its classic route, so a deployment + * that enables the policy sees no change in conversations that attach no tool. + */ +export function hasFileToolConsumer(consumers?: TurnFileConsumers | null): boolean { + return consumers?.executeCode === true || consumers?.fileSearch === true; +} + /** * Whether the record shows this file reached a tool's own store: vectors for file search, a * sandbox pointer for code execution. Nothing else writes either, so their presence is proof @@ -419,11 +497,17 @@ export interface TurnDeliveryRouting { endpointProvider?: string; useResponsesApi?: boolean; sttConfigured: boolean; + /** This turn's reading evidence for the automatic policy; absent where the host supplies none. */ + reading?: TurnReadingInputs; } /** The fields of an attachment record that decide its delivery on a turn. */ export interface TurnDeliveryFile { + file_id?: string; type?: string; + bytes?: number; + source?: string | null; + context?: string | null; text?: string | null; /** Stored as an upload-time inference, so any string may be read back. */ llmDeliveryPath?: string | null; @@ -436,6 +520,7 @@ export interface TurnDeliveryFile { destinationChosen?: boolean; /** Vector namespaces holding this file, written as each embedding succeeds. */ embeddedEntities?: string[]; + textDerivation?: TextDerivation; } & CodeEnvReferenceSet) | null; } @@ -449,7 +534,7 @@ export function hasInferredLLMDeliveryPath(file: TurnDeliveryFile): boolean { } /** - * Delivery path for one attachment on one agent's turn. + * Classic delivery path for one attachment on one agent's turn. * * A record predating routing and a destination the user chose keep what they stored. An * inferred route re-resolves against the endpoint handling the turn. A `none` route leaves @@ -464,7 +549,7 @@ export function hasInferredLLMDeliveryPath(file: TurnDeliveryFile): boolean { * that never received the file. Run Code is judged by the tool set: see * {@link hasTurnFileConsumer}. */ -export function resolveTurnLLMDeliveryPath( +export function resolveClassicTurnLLMDeliveryPath( routing: Partial | undefined, file: TurnDeliveryFile, consumers?: TurnFileConsumers, @@ -473,8 +558,7 @@ export function resolveTurnLLMDeliveryPath( return isLLMDeliveryPath(file.llmDeliveryPath) ? file.llmDeliveryPath : undefined; } const { endpointConfig } = routing; - /* Conversion changes the stored type, so use the type routing originally saw. */ - const mimeType = file.metadata?.routingMimeType ?? file.type ?? ''; + const mimeType = getRoutingMimeType(file); const path = resolveUploadLLMDeliveryPath({ mimeType, ...routing }); const hasFallbackText = typeof file.text === 'string' && file.text.length > 0; if ( @@ -489,6 +573,498 @@ export function resolveTurnLLMDeliveryPath( return path; } +/** + * Delivery path for one attachment on one agent's turn: the name every turn reader resolves. + * + * Returns the path of {@link decideFileReading}. Under classic routing a record the automatic + * policy never marked gets exactly the classic path from that decision, so it is returned from + * {@link resolveClassicTurnLLMDeliveryPath} directly, without classifying the file. + */ +export function resolveTurnLLMDeliveryPath( + routing: Partial | undefined, + file: TurnDeliveryFile, + consumers?: TurnFileConsumers, +): TDefaultLLMDeliveryPath | undefined { + if ( + resolveLLMDeliveryPolicy(routing?.endpointConfig) !== 'automatic' && + file.metadata?.textDerivation == null + ) { + return resolveClassicTurnLLMDeliveryPath(routing, file, consumers); + } + return decideFileReading({ routing, file, consumers }).path; +} + +/** + * Whether Run Code can read this record on this turn. Depends only on consumers and the record: + * an enabled Run Code with no sandbox copy yet is eligible, since its first call uploads it. + */ +export function judgeCodeEligibility( + file: TurnDeliveryFile, + consumers?: TurnFileConsumers, +): CodeEligibility { + if (consumers?.executeCode !== true) { + return 'no_run_code'; + } + if (isTextOnlyRecord(file)) { + return 'text_only_record'; + } + /* The sandbox receives the stored bytes, so compatibility is judged on the stored type. */ + if (!canToolResourceConsume(EToolResources.execute_code, file.type ?? '')) { + return 'incompatible'; + } + if (!isOriginalBacked(file)) { + return 'unstreamable'; + } + if ( + file.metadata?.destinationChosen === true && + !hasToolResourceProvisioning(file, EToolResources.execute_code) + ) { + return 'declined'; + } + return 'eligible'; +} + +export interface FileReadingInput { + routing: Partial | undefined; + file: TurnDeliveryFile; + consumers?: TurnFileConsumers; +} + +type ReaderVerdict = 'ok' | 'needs_text' | SkipReason; + +type ReadingBase = Pick; + +interface ReadingWalk { + routing: Partial; + file: TurnDeliveryFile; + consumers: TurnFileConsumers; + mimeType: string; + category: ReadingCategory; + code: CodeEligibility; + evidence: ReadingEvidence; +} + +const SEARCH_ONLY: TurnFileConsumers = { executeCode: false, fileSearch: true }; + +const SELECTED_REASON: Readonly> = { + code: 'code_selected', + provider: 'native_supported', + text: 'text_fits', + search: 'search_selected', +}; + +const hasStoredText = (file: TurnDeliveryFile): boolean => + typeof file.text === 'string' && file.text.length > 0; + +/** Whether text can be derived from the retained original on this request. */ +function canDeriveText( + routing: Partial | undefined, + file: TurnDeliveryFile, + evidence: ReadingEvidence, +): boolean { + return ( + routing?.reading?.canDerive === true && + isOriginalBacked(file) && + file.metadata?.textDerivation?.outcome !== 'failed' && + evidence.textFailed !== true && + selectBuiltInTextPlan(file.type ?? '') != null + ); +} + +function judgeProvider({ mimeType, routing, evidence }: ReadingWalk): ReaderVerdict { + if ( + resolveSystemDeliveryPath(mimeType, routing) !== 'provider' || + evidence.native === 'unsupported' + ) { + return 'native_unsupported'; + } + if (evidence.rejected === 'capacity') { + return 'native_capacity'; + } + if (evidence.rejected === 'unsupported') { + return 'native_unsupported'; + } + if (evidence.rejected === 'integrity') { + return 'native_rejected'; + } + if (evidence.native === 'capacity') { + return 'native_capacity'; + } + return evidence.overflow === true ? 'aggregate_overflow' : 'ok'; +} + +function judgeText({ routing, file, evidence }: ReadingWalk): ReaderVerdict { + if (!hasStoredText(file)) { + return canDeriveText(routing, file, evidence) ? 'needs_text' : 'text_unavailable'; + } + if (evidence.text === 'exceeds') { + return 'text_exceeds'; + } + return evidence.overflow === true ? 'aggregate_overflow' : 'ok'; +} + +function judgeSearch({ consumers, file, evidence }: ReadingWalk): ReaderVerdict { + const reachable = + consumers.fileSearch && + canToolResourceConsume(EToolResources.file_search, file.type ?? '') && + isOriginalBacked(file) && + evidence.search !== 'unreachable'; + return reachable ? 'ok' : 'search_unavailable'; +} + +const READER_JUDGES: Readonly ReaderVerdict>> = { + code: ({ code }) => (code === 'eligible' ? 'ok' : 'code_unavailable'), + provider: judgeProvider, + text: judgeText, + search: judgeSearch, +}; + +/** Skip reasons naming a limit the file hit, most significant first. */ +const LIMIT_SKIPS: readonly SkipReason[] = [ + 'native_capacity', + 'native_rejected', + 'aggregate_overflow', + 'text_exceeds', +]; + +/** + * The skip that best explains a reading. A limit the file hit outranks a reader that is merely + * missing, and among missing readers the first the walk judged wins, so the category's preferred + * reader names the reason. The provider not taking the type, true of most documents, explains a + * reading only when nothing else does. + */ +function significantSkip(skipped: readonly ReaderSkip[]): SkipReason | undefined { + const limit = LIMIT_SKIPS.find((reason) => skipped.some((skip) => skip.reason === reason)); + if (limit != null) { + return limit; + } + return (skipped.find(({ reason }) => reason !== 'native_unsupported') ?? skipped[0])?.reason; +} + +function selectedReason( + reader: ReaderKind, + category: ReadingCategory, + skipped: readonly ReaderSkip[], +): ReadingReason { + const skip = significantSkip(skipped); + if (skip != null) { + return skip; + } + return reader === 'code' && category === 'tabular' ? 'code_preferred' : SELECTED_REASON[reader]; +} + +/** + * Judges each reader of the category in order and returns the first that can read the file. A + * failure may redirect the rest of the walk, and every reader is judged at most once, so the walk + * ends after at most one judgment per reader. + */ +function walkReaders(walk: ReadingWalk, base: ReadingBase): FileReading { + const skipped: ReaderSkip[] = []; + const tried = new Set(); + let queue = READING_ORDER[walk.category]; + for (let i = 0; i < queue.length; i++) { + const reader = queue[i]; + if (tried.has(reader)) { + continue; + } + tried.add(reader); + const verdict = READER_JUDGES[reader](walk); + if (verdict === 'ok' || verdict === 'needs_text') { + return { + ...base, + skipped, + path: READER_PATH[reader], + reader, + reason: selectedReason(reader, walk.category, skipped), + needsText: verdict === 'needs_text', + }; + } + skipped.push({ reader, reason: verdict }); + const next = AFTER_FAILURE[walk.category][verdict]; + if (next != null) { + queue = next; + i = -1; + } + } + return { + ...base, + skipped, + path: 'none', + reader: 'unavailable', + reason: significantSkip(skipped) ?? 'no_reader', + needsText: false, + }; +} + +/** The reader a classic route amounts to, for diagnostics only. */ +function classicReader( + path: TDefaultLLMDeliveryPath | undefined, + code: CodeEligibility, + mimeType: string, + file: TurnDeliveryFile, + consumers?: TurnFileConsumers, +): FileReading['reader'] { + if (path == null) { + return 'unresolved'; + } + if (path !== 'none') { + return path; + } + if (code === 'eligible') { + return 'code'; + } + return consumers?.fileSearch === true && hasTurnFileConsumer(mimeType, SEARCH_ONLY, file) + ? 'search' + : 'unavailable'; +} + +/** + * The classic route, except for a record the automatic policy marked and left without text: its + * classic `text` route either derives that text first or, when it cannot, delivers nothing rather + * than an empty text part. + */ +function classicReading( + base: ReadingBase, + { routing, file, consumers }: FileReadingInput, + mimeType: string, + reason: ClassicReason, +): FileReading { + const reading = { ...base, skipped: [], reason, needsText: false }; + const derivation = file.metadata?.textDerivation; + if (base.classicPath !== 'text' || derivation == null || hasStoredText(file)) { + const reader = classicReader(base.classicPath, base.code, mimeType, file, consumers); + return { ...reading, path: base.classicPath, reader }; + } + const evidence = routing?.reading?.judge(file) ?? {}; + if (derivation.outcome === 'deferred' && canDeriveText(routing, file, evidence)) { + return { ...reading, path: 'text', reader: 'text', needsText: true }; + } + return { + ...reading, + path: 'none', + reader: classicReader('none', base.code, mimeType, file, consumers), + }; +} + +/** The endpoint file configuration the eligibility gate reads. */ +type ReadingConfig = Pick; + +/** The type routing saw at upload; conversion rewrites the stored type. */ +export function getRoutingMimeType(file: TurnDeliveryFile): string { + return file.metadata?.routingMimeType ?? file.type ?? ''; +} + +/** + * Eligibility gates the record and the endpoint's file configuration decide alone: the first that + * keeps the file on its classic route, or null when none does. + */ +function recordGateReason( + config: ReadingConfig, + file: TurnDeliveryFile, + mimeType: string, +): ClassicReason | null { + if (file.llmDeliveryPath == null) { + return 'legacy_record'; + } + if (resolveLLMDeliveryPolicy(config.endpointConfig) !== 'automatic') { + return 'classic_policy'; + } + if (file.metadata?.destinationChosen === true) { + return 'explicit_destination'; + } + if (file.metadata?.destinationChosen !== false) { + return 'unmarked_record'; + } + if (file.context !== FileContext.message_attachment) { + return 'not_message_attachment'; + } + if (isTextOnlyRecord(file)) { + return 'text_only_record'; + } + const configured = matchConfiguredDeliveryPath( + mimeType, + config.endpointConfig?.defaultLLMDeliveryPath, + config.fileConfig?.defaultLLMDeliveryPath, + ); + return configured != null ? 'configured_route' : null; +} + +/** + * Whether the automatic policy reads this record once the turn's consumers are known: every + * eligibility gate of {@link decideFileReading} except the consumer one. The endpoint filter asks + * this before consumers exist, so it agrees with the decision on every file the decision judges. + */ +export function isAutomaticReadingRecord( + config: ReadingConfig | undefined, + file: TurnDeliveryFile, +): boolean { + const mimeType = getRoutingMimeType(file); + return ( + config != null && + recordGateReason(config, file, mimeType) == null && + categorizeForReading(mimeType) !== 'media' + ); +} + +type EligibleReading = Pick; + +/** The eligibility gate in precedence order: why a file keeps its classic route, or what the walk reads. */ +function gateFileReading( + { routing, file, consumers }: FileReadingInput, + mimeType: string, + category: ReadingCategory | 'media', +): ClassicReason | EligibleReading { + if (routing == null) { + return 'legacy_record'; + } + const recordReason = recordGateReason(routing, file, mimeType); + if (recordReason != null) { + return recordReason; + } + if (consumers == null) { + return 'consumers_unknown'; + } + if (!hasFileToolConsumer(consumers)) { + return 'no_file_tools'; + } + if (category === 'media') { + return 'media_category'; + } + return { routing, consumers, category }; +} + +/** + * How one attachment is read on one agent's turn. Classic routing answers unless every + * eligibility gate passes; then the category's readers are judged in order against the turn's + * consumers and evidence. The returned path always stays inside the stored vocabulary. + */ +export function decideFileReading(input: FileReadingInput): FileReading { + const { routing, file, consumers } = input; + const mimeType = getRoutingMimeType(file); + const category = categorizeForReading(mimeType); + const base: ReadingBase = { + classicPath: resolveClassicTurnLLMDeliveryPath(routing, file, consumers), + category, + code: judgeCodeEligibility(file, consumers), + automatic: false, + }; + const gate = gateFileReading(input, mimeType, category); + if (typeof gate === 'string') { + return classicReading(base, input, mimeType, gate); + } + const evidence = gate.routing.reading?.judge(file) ?? {}; + return walkReaders( + { ...gate, file, mimeType, code: base.code, evidence }, + { ...base, automatic: true }, + ); +} + +export interface UploadReadingInput extends DeliveryRouteInputs { + toolResource?: string | null; + mimeType: string; + isMessageAttachment: boolean; + /** + * The upload preflight would defer an extracted-text fail-close only for the context route, so + * that route's extraction is the inspection and a failed extraction must reject the upload. + */ + extractionRequiredForInspection: boolean; + /** Undefined until the caller resolves it; asked for only when the decision needs it. */ + codePossible?: boolean; +} + +export type UploadReadingReason = + | 'explicit_destination' + | 'classic_policy' + | 'agent_resource' + | 'configured_route' + | 'automatic_default' + | 'inspection_requires_text' + | 'code_unavailable' + | 'code_preferred'; + +export interface UploadReading { + path: TDefaultLLMDeliveryPath; + policy: TLLMDeliveryPolicy; + category: ReadingCategory | 'media'; + reason: UploadReadingReason; + /** Run Code will read the file, so nothing is extracted at upload. */ + codePreferred: boolean; + /** Resolve `codePossible`, then decide again. */ + needsCodeAvailability: boolean; + /** + * An extraction failure keeps the original, marked failed, instead of rejecting the upload. + * Never set where an inspection policy relies on that extraction. + */ + keepOriginalOnExtractionFailure: boolean; + /** Write `textDerivation: { outcome: 'deferred' }`, since a built-in extractor can derive text later. */ + deferredMarker: boolean; +} + +/** + * How an upload is read, decided before any extraction. Under the automatic policy a tabular + * message attachment is left to Run Code when code is possible at upload, and other message + * attachments outside the media rows keep their original when extraction fails, unless an + * inspection policy relies on that extraction; everything else stays classic. + */ +export function decideUploadReading(input: UploadReadingInput): UploadReading { + const path = resolveUploadLLMDeliveryPath(input); + const policy = resolveLLMDeliveryPolicy(input.endpointConfig); + const category = categorizeForReading(input.mimeType); + const classic = ( + reason: UploadReadingReason, + keepOriginalOnExtractionFailure = false, + ): UploadReading => ({ + path, + policy, + category, + reason, + codePreferred: false, + needsCodeAvailability: false, + keepOriginalOnExtractionFailure, + deferredMarker: false, + }); + + if (input.toolResource != null) { + return classic('explicit_destination'); + } + if (policy !== 'automatic') { + return classic('classic_policy'); + } + if (!input.isMessageAttachment) { + return classic('agent_resource'); + } + const configured = matchConfiguredDeliveryPath( + input.mimeType, + input.endpointConfig?.defaultLLMDeliveryPath, + input.fileConfig?.defaultLLMDeliveryPath, + ); + if (configured != null) { + return classic('configured_route'); + } + const keepOriginal = category !== 'media' && !input.extractionRequiredForInspection; + if ( + category !== 'tabular' || + !canToolResourceConsume(EToolResources.execute_code, input.mimeType) + ) { + return classic('automatic_default', keepOriginal); + } + if (input.extractionRequiredForInspection) { + return classic('inspection_requires_text'); + } + if (input.codePossible === undefined) { + return { ...classic('automatic_default', keepOriginal), needsCodeAvailability: true }; + } + if (!input.codePossible) { + return classic('code_unavailable', keepOriginal); + } + return { + ...classic('code_preferred'), + path: 'none', + codePreferred: true, + deferredMarker: selectBuiltInTextPlan(input.mimeType) != null, + }; +} + /** Why an upload cannot be accepted, when nothing would be able to read it. */ export type UploadRejection = 'no-agent-resource' | 'context-disabled' | 'no-consumer'; diff --git a/packages/data-provider/src/types/files.ts b/packages/data-provider/src/types/files.ts index 50c6e4dbe92..a3a72b763a5 100644 --- a/packages/data-provider/src/types/files.ts +++ b/packages/data-provider/src/types/files.ts @@ -1,5 +1,6 @@ -import type { TDefaultLLMDeliveryPathConfig } from '../file-config'; +import type { TDefaultLLMDeliveryPathConfig, TLLMDeliveryPolicy } from '../file-config'; import type { CodeEnvRef, CodeEnvRefMap } from '../codeEnvRef'; +import type { TextDerivation } from '../reading'; import { EToolResources } from './tools'; export enum FileSources { @@ -55,6 +56,9 @@ export type EndpointFileConfig = { /** Delivers the text extracted at upload for a file routed to tools (`none`) on a turn that * runs no tool able to read it. Off by default, which leaves such a file out of the prompt. */ textFallbackWithoutTools?: boolean; + /** How unified attachments are read. Absent means `classic`; read it through + * `resolveLLMDeliveryPolicy`, which also lets `legacyFileUploadUX` force `classic`. */ + llmDeliveryPolicy?: TLLMDeliveryPolicy; }; export type FileConfig = { @@ -96,6 +100,9 @@ export type FileConfig = { /** Delivers the text extracted at upload for a file routed to tools (`none`) on a turn that * runs no tool able to read it. Off by default, which leaves such a file out of the prompt. */ textFallbackWithoutTools?: boolean; + /** How unified attachments are read. Absent means `classic`; read it through + * `resolveLLMDeliveryPolicy`, which also lets `legacyFileUploadUX` force `classic`. */ + llmDeliveryPolicy?: TLLMDeliveryPolicy; }; export type FileConfigInput = { @@ -132,6 +139,9 @@ export type FileConfigInput = { /** Delivers the text extracted at upload for a file routed to tools (`none`) on a turn that * runs no tool able to read it. Off by default, which leaves such a file out of the prompt. */ textFallbackWithoutTools?: boolean; + /** How unified attachments are read. Absent means `classic`; read it through + * `resolveLLMDeliveryPolicy`, which also lets `legacyFileUploadUX` force `classic`. */ + llmDeliveryPolicy?: TLLMDeliveryPolicy; }; /** The immutable origin of a file explicitly published from an agent execution. */ @@ -215,6 +225,8 @@ export type TFile = { destinationChosen?: boolean; /** The type the delivery route was resolved against, when conversion changed it. */ routingMimeType?: string; + /** How the text a reader may need was obtained; written only under the automatic policy. */ + textDerivation?: TextDerivation; }; llmDeliveryPath?: 'provider' | 'text' | 'none'; createdAt?: string | Date; diff --git a/packages/data-schemas/src/methods/file.spec.ts b/packages/data-schemas/src/methods/file.spec.ts index 4a867c5dcc9..72d0c1a339c 100644 --- a/packages/data-schemas/src/methods/file.spec.ts +++ b/packages/data-schemas/src/methods/file.spec.ts @@ -2,6 +2,7 @@ import mongoose from 'mongoose'; import { v4 as uuidv4 } from 'uuid'; import { MongoMemoryServer } from 'mongodb-memory-server'; import { EToolResources, FileContext } from 'librechat-data-provider'; +import type { TextDerivation } from 'librechat-data-provider'; import type { IChatProject, IMongoFile } from '~/types'; import { _resetStrictCache } from '~/models/plugins/tenantIsolation'; import { tenantStorage, runAsSystem } from '~/config/tenantContext'; @@ -1926,6 +1927,249 @@ describe('File Methods', () => { }); }); + describe('saveFileTextDerivation', () => { + const deferred: TextDerivation = { outcome: 'deferred', at: 1_000 }; + const complete: TextDerivation = { + outcome: 'complete', + extractor: 'document_parser', + at: 2_000, + }; + const failed: TextDerivation = { + outcome: 'failed', + extractor: 'document_parser', + reason: 'parser', + at: 3_000, + }; + const codeEnvRef = { + kind: 'user' as const, + id: 'user-1', + storage_session_id: 'session-1', + file_id: 'code-file-1', + executionProfile: 'default' as const, + }; + + const seedFile = async ( + overrides: Partial = {}, + ): Promise<{ file_id: string; user: string }> => { + const file_id = uuidv4(); + const user = overrides.user ?? new mongoose.Types.ObjectId(); + await fileMethods.createFile({ + file_id, + filename: 'workbook.xlsx', + filepath: '/uploads/workbook.xlsx', + type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + bytes: 2048, + context: FileContext.message_attachment, + llmDeliveryPath: 'none', + metadata: { + destinationChosen: false, + routingMimeType: 'application/vnd.ms-excel', + codeEnvRefs: { default: codeEnvRef }, + textDerivation: deferred, + }, + ...overrides, + user, + }); + return { file_id, user: user.toString() }; + }; + + it('round-trips the marker through createFile on the strict metadata path', async () => { + const { file_id } = await seedFile(); + + const stored = await fileMethods.findFileById(file_id); + + expect(stored?.metadata?.textDerivation).toEqual(deferred); + expect(stored?.metadata?.destinationChosen).toBe(false); + expect(stored?.metadata?.routingMimeType).toBe('application/vnd.ms-excel'); + }); + + it('saves derived text and the marker on a deferred record without text', async () => { + const { file_id, user } = await seedFile(); + + const saved = await fileMethods.saveFileTextDerivation( + { file_id, text: 'Region,Total\nNorth,42', textDerivation: complete }, + { user }, + ); + + expect(saved).toBe(true); + const stored = await fileMethods.findFileById(file_id); + expect(stored?.text).toBe('Region,Total\nNorth,42'); + expect(stored?.llmDeliveryPath).toBe('none'); + expect(stored?.metadata).toEqual({ + destinationChosen: false, + routingMimeType: 'application/vnd.ms-excel', + codeEnvRefs: { default: codeEnvRef }, + textDerivation: complete, + }); + }); + + it.each([ + ['empty', ''], + ['null', null], + ])('applies when the deferred record holds %s text', async (_label, text) => { + const { file_id, user } = await seedFile(); + await File.collection.updateOne({ file_id }, { $set: { text } }); + + const saved = await fileMethods.saveFileTextDerivation( + { file_id, text: 'derived', textDerivation: complete }, + { user }, + ); + + expect(saved).toBe(true); + const stored = await fileMethods.findFileById(file_id); + expect(stored?.text).toBe('derived'); + expect(stored?.metadata?.textDerivation).toEqual(complete); + }); + + it('caches derived text on an unmarked record without text', async () => { + const { file_id, user } = await seedFile({ metadata: { destinationChosen: false } }); + + const saved = await fileMethods.saveFileTextDerivation( + { file_id, text: 'derived', textDerivation: complete }, + { user }, + ); + + expect(saved).toBe(true); + const stored = await fileMethods.findFileById(file_id); + expect(stored?.text).toBe('derived'); + expect(stored?.metadata?.textDerivation).toEqual(complete); + }); + + it.each([ + ['complete', complete], + ['failed', failed], + ])('leaves a %s record untouched', async (_label, textDerivation) => { + const { file_id, user } = await seedFile({ + metadata: { destinationChosen: false, textDerivation }, + }); + + const saved = await fileMethods.saveFileTextDerivation( + { file_id, text: 'derived', textDerivation: complete }, + { user }, + ); + + expect(saved).toBe(false); + const stored = await fileMethods.findFileById(file_id); + expect(stored?.text).toBeUndefined(); + expect(stored?.metadata?.textDerivation).toEqual(textDerivation); + }); + + it('leaves a deferred record that already holds text untouched', async () => { + const { file_id, user } = await seedFile({ text: 'extracted at upload' }); + + const saved = await fileMethods.saveFileTextDerivation( + { file_id, text: 'derived', textDerivation: complete }, + { user }, + ); + + expect(saved).toBe(false); + const stored = await fileMethods.findFileById(file_id); + expect(stored?.text).toBe('extracted at upload'); + expect(stored?.metadata?.textDerivation).toEqual(deferred); + }); + + it('enforces the owner scope', async () => { + const { file_id, user } = await seedFile(); + const otherUser = new mongoose.Types.ObjectId().toString(); + + const denied = await fileMethods.saveFileTextDerivation( + { file_id, text: 'derived', textDerivation: complete }, + { user: otherUser }, + ); + const unscoped = await fileMethods.saveFileTextDerivation( + { file_id, text: 'derived', textDerivation: complete }, + { user: '' }, + ); + + expect(denied).toBe(false); + expect(unscoped).toBe(false); + expect((await fileMethods.findFileById(file_id))?.text).toBeUndefined(); + + const allowed = await fileMethods.saveFileTextDerivation( + { file_id, text: 'derived', textDerivation: complete }, + { user }, + ); + expect(allowed).toBe(true); + }); + + it('enforces the tenant scope', async () => { + const { file_id, user } = await tenantStorage.run({ tenantId: 'tenant-a' }, () => + seedFile({ tenantId: 'tenant-a' }), + ); + + const denied = await fileMethods.saveFileTextDerivation( + { file_id, text: 'derived', textDerivation: complete }, + { user, tenantId: 'tenant-b' }, + ); + + expect(denied).toBe(false); + expect((await fileMethods.findFileById(file_id))?.text).toBeUndefined(); + + const allowed = await tenantStorage.run({ tenantId: 'tenant-a' }, () => + fileMethods.saveFileTextDerivation( + { file_id, text: 'derived', textDerivation: complete }, + { user, tenantId: 'tenant-a' }, + ), + ); + expect(allowed).toBe(true); + expect((await fileMethods.findFileById(file_id))?.text).toBe('derived'); + }); + + it('keeps the upload TTL, the retention deadline and the content timestamp', async () => { + const { file_id, user } = await seedFile(); + const expiresAt = new Date(Date.now() + 30 * 60_000); + const expiredAt = new Date(Date.now() + 30 * 24 * 3_600_000); + const updatedAt = new Date('2020-01-01T00:00:00.000Z'); + await File.collection.updateOne({ file_id }, { $set: { expiresAt, expiredAt, updatedAt } }); + + const saved = await fileMethods.saveFileTextDerivation( + { file_id, text: 'derived', textDerivation: complete }, + { user }, + ); + + expect(saved).toBe(true); + const stored = await fileMethods.findFileById(file_id); + expect(stored?.expiresAt).toEqual(expiresAt); + expect(stored?.expiredAt).toEqual(expiredAt); + expect(stored?.updatedAt).toEqual(updatedAt); + }); + + it('saves a failed marker without text, after which the record takes no text', async () => { + const { file_id, user } = await seedFile(); + + const saved = await fileMethods.saveFileTextDerivation( + { file_id, textDerivation: failed }, + { user }, + ); + const later = await fileMethods.saveFileTextDerivation( + { file_id, text: 'derived', textDerivation: complete }, + { user }, + ); + + expect(saved).toBe(true); + expect(later).toBe(false); + const stored = await fileMethods.findFileById(file_id); + expect(stored?.text).toBeUndefined(); + expect(stored?.metadata?.textDerivation).toEqual(failed); + expect(stored?.metadata?.destinationChosen).toBe(false); + }); + + it('rejects a marker outside the outcome vocabulary', async () => { + const { file_id, user } = await seedFile(); + const outcome: string = 'abandoned'; + + await expect( + fileMethods.saveFileTextDerivation( + { file_id, textDerivation: { outcome: outcome as TextDerivation['outcome'] } }, + { user }, + ), + ).rejects.toThrow(/not a valid enum value/); + + const stored = await fileMethods.findFileById(file_id); + expect(stored?.metadata?.textDerivation).toEqual(deferred); + }); + }); + describe('updateFilesUsage', () => { it('should update usage for multiple files', async () => { const userId = new mongoose.Types.ObjectId(); diff --git a/packages/data-schemas/src/methods/file.ts b/packages/data-schemas/src/methods/file.ts index 2c74b1c9764..3539ebbf3d0 100644 --- a/packages/data-schemas/src/methods/file.ts +++ b/packages/data-schemas/src/methods/file.ts @@ -11,6 +11,7 @@ import type { CodeFileCommitData, RunArtifactRunScope, PublishRunArtifactInput, + FileTextDerivationUpdate, } from '~/types/file'; import type { IChatProjectDocument } from '~/types'; import { tenantSafeBulkWrite } from '~/utils/tenantBulkWrite'; @@ -354,6 +355,10 @@ export function createFileMethods(mongoose: typeof import('mongoose')): { user?: string; tenantId?: string | null; }) => Promise; + saveFileTextDerivation: ( + update: FileTextDerivationUpdate, + scope: { user: string; tenantId?: string | null }, + ) => Promise; deleteFile: (file_id: string) => Promise; deleteFiles: (file_ids: string[], user?: string) => Promise<{ deletedCount?: number }>; deleteFileByFilter: (filter: FilterQuery) => Promise; @@ -1250,6 +1255,50 @@ export function createFileMethods(mongoose: typeof import('mongoose')): { }).lean(); } + /** + * Saves text derived at turn time onto a record whose upload deferred extraction. It applies + * only while the record is still marked `deferred` and holds no text, so a record that was + * never deferred, or one another derivation already settled, is left as it is. The text comes + * from the unchanged original, so the upload TTL, the retention deadline and the content + * timestamp all stay where they are. + * + * @param update - The file, the derived text when there is any, and the marker to record + * @param scope - Owner scope; a mismatch leaves the record unchanged + * @returns Whether a record was updated + */ + async function saveFileTextDerivation( + update: FileTextDerivationUpdate, + scope: { user: string; tenantId?: string | null }, + ): Promise { + if (!scope.user) { + return false; + } + const File = mongoose.models.File as Model; + const { file_id, text, textDerivation } = update; + /* A deferred record, or an unmarked one, takes a derivation while it stores no text. */ + const filter = withOwnerScope( + { + file_id, + $and: [ + { + $or: [ + { 'metadata.textDerivation.outcome': 'deferred' }, + { 'metadata.textDerivation': { $exists: false } }, + ], + }, + { $or: [{ text: { $exists: false } }, { text: null }, { text: '' }] }, + ], + }, + { userId: scope.user, tenantId: scope.tenantId }, + ); + const result = await File.updateOne( + filter, + { $set: { ...(text != null && { text }), 'metadata.textDerivation': textDerivation } }, + { timestamps: false, runValidators: true }, + ); + return result.modifiedCount > 0; + } + /** * Removes deleted file ids from the owner's Chat Projects so a project never * keeps a dangling reference, bumping the revision of every project it changes. @@ -1570,6 +1619,7 @@ export function createFileMethods(mongoose: typeof import('mongoose')): { updateFileCodeEnvRef, addFileEmbeddedEntity, updateFileUsage, + saveFileTextDerivation, deleteFile, deleteFiles, deleteFileByFilter, diff --git a/packages/data-schemas/src/schema/file.ts b/packages/data-schemas/src/schema/file.ts index f8445ad9dcc..276b31966ad 100644 --- a/packages/data-schemas/src/schema/file.ts +++ b/packages/data-schemas/src/schema/file.ts @@ -1,6 +1,6 @@ import mongoose, { Schema } from 'mongoose'; import { FileContext, FileSources } from 'librechat-data-provider'; -import type { RunFileProvenance } from 'librechat-data-provider'; +import type { RunFileProvenance, TextDerivation } from 'librechat-data-provider'; import type { IMongoFile } from '~/types'; import { codeEnvRefMapSchema, codeEnvRefSchema } from './codeEnvRef'; @@ -19,6 +19,16 @@ const runFileProvenanceSchema = new Schema( { _id: false }, ); +const textDerivationSchema = new Schema( + { + outcome: { type: String, enum: ['deferred', 'complete', 'failed'], required: true }, + extractor: { type: String }, + reason: { type: String }, + at: { type: Number }, + }, + { _id: false }, +); + const file: Schema = new Schema( { user: { @@ -177,6 +187,13 @@ const file: Schema = new Schema( type: String, default: undefined, }, + /** How the text a reader may need was obtained, under the automatic reading policy. + * A `deferred` or unmarked record without text accepts text derived at turn + * time; a failed marker is kept only by a deferred record. */ + textDerivation: { + type: textDerivationSchema, + default: undefined, + }, }, llmDeliveryPath: { /* What upload time inferred about delivery, from the endpoint and MIME type it saw. diff --git a/packages/data-schemas/src/types/file.ts b/packages/data-schemas/src/types/file.ts index 012f9bbb072..95e012cdd24 100644 --- a/packages/data-schemas/src/types/file.ts +++ b/packages/data-schemas/src/types/file.ts @@ -4,6 +4,7 @@ import type { CodeEnvRefMap, FileContext, RunFileProvenance, + TextDerivation, TFile, } from 'librechat-data-provider'; @@ -82,6 +83,14 @@ export type RunArtifactClaim = { file?: RunArtifactFile; }; +/** What a turn-time text derivation writes onto the record whose upload deferred extraction. */ +export interface FileTextDerivationUpdate { + file_id: string; + /** The derived text; absent when only a failed marker is recorded. */ + text?: string; + textDerivation: TextDerivation; +} + export interface IMongoFile extends Omit { user: Types.ObjectId; conversationId?: string; @@ -161,6 +170,8 @@ export interface IMongoFile extends Omit { destinationChosen?: boolean; /** The type the delivery route was resolved against, when conversion changed it. */ routingMimeType?: string; + /** How the text a reader may need was obtained, under the automatic reading policy. */ + textDerivation?: TextDerivation; }; /** Upload-time inference, not a durable contract. See the schema field for why. */ llmDeliveryPath?: string;