From fbde5d4781717d211a610e35138d399886dd3f49 Mon Sep 17 00:00:00 2001 From: Joe Licata Date: Tue, 6 Oct 2026 14:44:04 -0400 Subject: [PATCH 1/2] =?UTF-8?q?=F0=9F=A7=B2=20feat:=20Automatic=20Reading?= =?UTF-8?q?=20Policy=20for=20Unified=20Attachments?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds an opt-in `llmDeliveryPolicy: automatic` under `fileConfig` (global, `endpoints.default`, or per endpoint); the default `classic` keeps today's routing. On a turn that loads Run Code or File Search, the policy decides each attachment's reader from its category, the current provider's existing validators and limits, and the tools actually loaded: spreadsheets (xlsx, xls, ods, csv, tsv, parquet) go to Run Code with no upload-time extraction; other documents go to the provider within its limits, then complete extracted text when it fits, then File Search, then Run Code; a document too large to send directly tries File Search and Run Code before text. Without a file tool, and under the classic policy, every attachment keeps its classic route and limits unchanged. Explicit destinations, `legacyFileUploadUX` and `defaultLLMDeliveryPath` still win. Code access is independent of the primary reader: a compatible original the model reads natively or as text is also advertised to Run Code at a stable path and copied into the sandbox only when code first runs. The model receives a per-turn inventory of how each attachment can be read and which it cannot. Text is derived from the retained original only when a reader needs it, under the deployment's text allowlist, and cached on the record (`metadata.textDerivation`). A pending File Search file is advertised and indexed before the first search; a failed preparation fails the turn with the stable `file_search_preparation_failed` error. Native size limits resolve in one place, a provider's capacity rejection moves the file to the next reader instead of dropping it, and a resumed turn charges checkpoint files by the resuming agent's decision. The UI is unchanged except that the preview dialog no longer treats Office Open XML types as XML text, which rendered a code-kept workbook as raw ZIP bytes; such a file shows the existing unavailable state. The e2e mock lane gains fake-model, code and RAG fixtures that record what the model, the sandbox and the vector store received, and an automatic-upload spec that proves the policy end to end. --- api/app/clients/BaseClient.js | 72 +- api/app/clients/specs/BaseClient.test.js | 227 ++- .../__tests__/request.resumeMetadata.spec.js | 110 +- .../agents/__tests__/resume.spec.js | 110 ++ api/server/controllers/agents/client.js | 114 +- api/server/controllers/agents/client.test.js | 359 ++++- api/server/controllers/agents/openai.js | 1 + api/server/routes/files/files.agents.test.js | 67 + .../services/Endpoints/agents/addedConvo.js | 6 + .../Endpoints/agents/addedConvo.spec.js | 15 + .../services/Endpoints/agents/initialize.js | 25 + .../Endpoints/agents/initialize.spec.js | 18 + api/server/services/Files/process.js | 234 +-- api/server/services/Files/process.spec.js | 216 +++ api/server/services/Files/provision.js | 1 + api/server/services/ToolService.js | 7 +- .../services/__tests__/ToolService.spec.js | 74 +- api/test/__mocks__/logger.js | 2 + .../__tests__/FilePreviewDialog.test.ts | 9 + .../Chat/Messages/Content/preview.ts | 5 +- .../Messages/Content/Error/registry.ts | 1 + .../Messages/Content/__tests__/Error.spec.tsx | 12 + .../Files/__tests__/equivalence.spec.tsx | 449 ++++++ client/src/locales/en/translation.json | 1 + client/src/routes/Search.tsx | 3 +- client/src/utils/thread.ts | 3 +- e2e/config/librechat.e2e.yaml | 33 + e2e/setup/fake-code-server.js | 181 ++- e2e/setup/fake-model.js | 191 ++- e2e/setup/fake-rag-server.js | 39 +- e2e/setup/global-setup.ts | 2 + e2e/setup/uploads.ts | 167 +++ e2e/specs/mock/automatic-upload.spec.ts | 792 +++++++++++ e2e/specs/mock/file-provisioning.spec.ts | 21 +- e2e/specs/mock/helpers.ts | 174 ++- e2e/specs/mock/run-files-delivery.spec.ts | 11 +- librechat.example.yaml | 6 + .../initialize.reading.integration.test.ts | 330 +++++ .../src/agents/__tests__/initialize.test.ts | 638 ++++++++- .../agents/__tests__/run-codeTools.test.ts | 207 ++- packages/api/src/agents/attachments.test.ts | 222 ++- packages/api/src/agents/attachments.ts | 187 ++- packages/api/src/agents/discovery.spec.ts | 31 + packages/api/src/agents/discovery.ts | 6 + .../api/src/agents/files/delivery.spec.ts | 474 ++++++- packages/api/src/agents/files/delivery.ts | 367 ++++- packages/api/src/agents/files/encode.spec.ts | 389 ++++- packages/api/src/agents/files/encode.ts | 152 +- .../api/src/agents/hitl/protection.spec.ts | 107 ++ packages/api/src/agents/hitl/protection.ts | 25 +- packages/api/src/agents/initialize.ts | 292 +++- packages/api/src/agents/resources.test.ts | 2 + packages/api/src/agents/resources.ts | 4 +- packages/api/src/agents/run.ts | 168 +-- .../agents/steering/__tests__/media.spec.ts | 267 +++- packages/api/src/agents/steering/media.ts | 92 +- packages/api/src/files/code/priming.ts | 3 +- packages/api/src/files/context.spec.ts | 132 +- packages/api/src/files/context.ts | 60 +- packages/api/src/files/documents/crud.ts | 3 +- .../api/src/files/encode/document.spec.ts | 492 ++++++- packages/api/src/files/encode/document.ts | 192 ++- packages/api/src/files/encode/utils.spec.ts | 71 +- packages/api/src/files/encode/utils.ts | 41 +- packages/api/src/files/filter.spec.ts | 152 +- packages/api/src/files/filter.ts | 49 +- packages/api/src/files/index.ts | 1 + .../api/src/files/provision/callback.spec.ts | 225 ++- packages/api/src/files/provision/callback.ts | 79 +- .../api/src/files/provision/service.spec.ts | 62 + packages/api/src/files/provision/service.ts | 53 +- packages/api/src/files/reading/derive.spec.ts | 398 ++++++ packages/api/src/files/reading/derive.ts | 286 ++++ .../api/src/files/reading/diagnostics.spec.ts | 120 ++ packages/api/src/files/reading/diagnostics.ts | 102 ++ .../api/src/files/reading/history.spec.ts | 557 ++++++++ packages/api/src/files/reading/history.ts | 237 ++++ packages/api/src/files/reading/index.ts | 7 + .../api/src/files/reading/inventory.spec.ts | 709 ++++++++++ packages/api/src/files/reading/inventory.ts | 467 ++++++ packages/api/src/files/reading/native.spec.ts | 678 +++++++++ packages/api/src/files/reading/native.ts | 396 ++++++ packages/api/src/files/reading/settle.spec.ts | 513 +++++++ packages/api/src/files/reading/settle.ts | 188 +++ packages/api/src/files/reading/turn.spec.ts | 1024 ++++++++++++++ packages/api/src/files/reading/turn.ts | 767 ++++++++++ packages/api/src/files/reading/upload.spec.ts | 513 +++++++ packages/api/src/files/reading/upload.ts | 293 ++++ .../api/src/files/upload/fallback.spec.ts | 257 +++- packages/api/src/files/upload/fallback.ts | 194 ++- packages/api/src/files/validation.spec.ts | 364 ++++- packages/api/src/files/validation.ts | 198 +-- packages/api/src/index.ts | 1 + packages/api/src/types/files.ts | 19 +- packages/api/src/utils/message.ts | 17 +- packages/api/src/utils/text.spec.ts | 45 + packages/api/src/utils/text.ts | 6 +- .../data-provider/src/file-config.spec.ts | 132 ++ packages/data-provider/src/file-config.ts | 28 + packages/data-provider/src/index.ts | 1 + packages/data-provider/src/messages.spec.ts | 62 +- packages/data-provider/src/messages.ts | 20 +- packages/data-provider/src/reading.spec.ts | 179 +++ packages/data-provider/src/reading.ts | 248 ++++ .../src/resolve-llm-delivery-path.spec.ts | 1250 ++++++++++++++++- .../src/resolve-llm-delivery-path.ts | 652 ++++++++- packages/data-provider/src/types/files.ts | 14 +- .../data-schemas/src/methods/file.spec.ts | 244 ++++ packages/data-schemas/src/methods/file.ts | 50 + packages/data-schemas/src/schema/file.ts | 19 +- packages/data-schemas/src/types/file.ts | 11 + 111 files changed, 19731 insertions(+), 868 deletions(-) create mode 100644 client/src/hooks/Files/__tests__/equivalence.spec.tsx create mode 100644 e2e/setup/uploads.ts create mode 100644 e2e/specs/mock/automatic-upload.spec.ts create mode 100644 packages/api/src/agents/__tests__/initialize.reading.integration.test.ts create mode 100644 packages/api/src/files/reading/derive.spec.ts create mode 100644 packages/api/src/files/reading/derive.ts create mode 100644 packages/api/src/files/reading/diagnostics.spec.ts create mode 100644 packages/api/src/files/reading/diagnostics.ts create mode 100644 packages/api/src/files/reading/history.spec.ts create mode 100644 packages/api/src/files/reading/history.ts create mode 100644 packages/api/src/files/reading/index.ts create mode 100644 packages/api/src/files/reading/inventory.spec.ts create mode 100644 packages/api/src/files/reading/inventory.ts create mode 100644 packages/api/src/files/reading/native.spec.ts create mode 100644 packages/api/src/files/reading/native.ts create mode 100644 packages/api/src/files/reading/settle.spec.ts create mode 100644 packages/api/src/files/reading/settle.ts create mode 100644 packages/api/src/files/reading/turn.spec.ts create mode 100644 packages/api/src/files/reading/turn.ts create mode 100644 packages/api/src/files/reading/upload.spec.ts create mode 100644 packages/api/src/files/reading/upload.ts create mode 100644 packages/data-provider/src/reading.spec.ts create mode 100644 packages/data-provider/src/reading.ts diff --git a/api/app/clients/BaseClient.js b/api/app/clients/BaseClient.js index f02af652d4b..8a097ba8790 100644 --- a/api/app/clients/BaseClient.js +++ b/api/app/clients/BaseClient.js @@ -10,10 +10,15 @@ const { extractFileContext, getReferencedQuotes, applyTurnDelivery, + prepareTurnFiles, + getTurnTextOptions, encodeAndFormatAudios, encodeAndFormatVideos, getTransactionsConfig, encodeAndFormatDocuments, + encodeNativeDocuments, + recordNativeRejections, + prepareMessageAttachments, getLangfuseTraceMessageFields, isContentFilterError, assertModelBoundProviderContent, @@ -859,7 +864,7 @@ class BaseClient { if (this.options.resendFiles !== false && this.authorizedHistoricalFiles == null) { const historicalFileState = collectModelBoundHistoricalFileIdState(modelBoundStoredMessages); this.modelBoundHistoricalFileIdsOverflowed ||= historicalFileState.overflowed; - const files = this.resolveTurnAttachments( + const files = await this.prepareTurnAttachments( await getOwnerHistoricalFiles(historicalFileState.fileIds, this.options.req?.user), ); this.authorizedHistoricalFiles = new Map( @@ -1739,17 +1744,24 @@ class BaseClient { } async addDocuments(message, attachments) { - const documentResult = await encodeAndFormatDocuments( - this.options.req, + const documentResult = await encodeNativeDocuments( attachments, - { - provider: this.options.agent?.provider ?? this.options.endpoint, - endpoint: this.options.agent?.endpoint ?? this.options.endpoint, - useResponsesApi: this.usesResponsesApi(), - model: this.modelOptions?.model ?? this.model, - }, - getStrategyFunctions, + this.options.agent, + (files, onValidationFailure) => + encodeAndFormatDocuments( + this.options.req, + files, + { + provider: this.options.agent?.provider ?? this.options.endpoint, + endpoint: this.options.agent?.endpoint ?? this.options.endpoint, + useResponsesApi: this.usesResponsesApi(), + model: this.modelOptions?.model ?? this.model, + onValidationFailure, + }, + getStrategyFunctions, + ), ); + recordNativeRejections(this.getConversationAgents(), documentResult.rejected); message.documents = documentResult.documents && documentResult.documents.length ? documentResult.documents @@ -1757,6 +1769,11 @@ class BaseClient { return documentResult.files; } + /** The agents whose messages carry this conversation's encoded attachments. */ + getConversationAgents() { + return [this.options.agent]; + } + async addVideos(message, attachments) { const videoResult = await encodeAndFormatVideos( this.options.req, @@ -1800,6 +1817,7 @@ class BaseClient { attachments: textAttachments, req: this.options?.req, tokenCountFn: (text) => countTokens(text), + ...getTurnTextOptions(this.options.agent?.deliveryRouting), }); if (fileContext) { @@ -1825,6 +1843,26 @@ class BaseClient { }); } + /** {@link resolveTurnAttachments}, deriving text the turn's reading needs first. */ + prepareTurnAttachments(files, fileConsumers = this.options.agent?.fileConsumers) { + return prepareTurnFiles({ + routing: this.options.agent?.deliveryRouting, + files, + consumers: fileConsumers, + signal: this.options.abortController?.signal, + }); + } + + processMessageAttachments(message, files, fileConsumers, contextFiles) { + return prepareMessageAttachments({ + client: this, + message, + files, + consumers: fileConsumers, + contextFiles, + }); + } + getAttachmentDeliveryPath(file, fileConsumers = this.options.agent?.fileConsumers) { return resolveTurnLLMDeliveryPath(this.options.agent?.deliveryRouting, file, fileConsumers); } @@ -1949,7 +1987,7 @@ class BaseClient { const historicalFileState = collectModelBoundHistoricalFileIdState(_messages); this.modelBoundHistoricalFileIdsOverflowed ||= historicalFileState.overflowed; const authorizedFilesById = new Map(); - const files = this.resolveTurnAttachments( + const files = await this.prepareTurnAttachments( await getOwnerHistoricalFiles(historicalFileState.fileIds, this.options.req?.user), ); const nonSteerReplayFileIds = collectModelBoundHistoricalFileIdState( @@ -2066,16 +2104,10 @@ class BaseClient { return message; } - const [, processedFiles] = await Promise.all([ - this.addFileContextToMessage(message, contextFiles), - this.processAttachments(message, contextFiles), - ]); + const processedFiles = await this.processMessageAttachments(message, contextFiles); - const processedFileIds = new Set( - (processedFiles ?? []).map((file) => file?.file_id).filter(Boolean), - ); - this.message_file_map[message.messageId] = contextFiles.filter( - (file) => processedFileIds.has(file?.file_id) && isModelBoundAttachmentFile(file), + this.message_file_map[message.messageId] = (processedFiles ?? []).filter( + isModelBoundAttachmentFile, ); return message; }; diff --git a/api/app/clients/specs/BaseClient.test.js b/api/app/clients/specs/BaseClient.test.js index cd6ac2c34f1..a8ef61b1be9 100644 --- a/api/app/clients/specs/BaseClient.test.js +++ b/api/app/clients/specs/BaseClient.test.js @@ -1,3 +1,6 @@ +const fs = require('fs'); +const os = require('os'); +const path = require('path'); const { Constants, ContentTypes, EModelEndpoint } = require('librechat-data-provider'); const BaseClientClass = require('../BaseClient'); const { @@ -8,6 +11,8 @@ const { getPrivateTextInspectionTokens, assertModelBoundContent, resolveTurnDeliveryRouting, + buildTurnReadingContext, + buildMessageFiles, buildSteerMedia, Tokenizer, } = require('@librechat/api'); @@ -3059,6 +3064,34 @@ describe('BaseClient', () => { expect(userSave[0].files).toHaveLength(1); expect(userSave[0].files[0].file_id).toBe('file-abc'); }); + + test('saves the user row with the files buildMessageFiles derives from the request and attachments', async () => { + const brief = { + user: 'user-1', + file_id: 'brief', + filename: 'brief.pdf', + filepath: '/uploads/brief.pdf', + type: 'application/pdf', + bytes: 2048, + source: 'local', + text: 'extracted text', + _id: 'mongo-brief', + }; + const requestFiles = [{ file_id: 'brief' }, { file_id: 'scan' }]; + TestClient.options.req = { body: { files: requestFiles } }; + TestClient.options.attachments = [brief]; + TestClient.saveMessageToDatabase = jest.fn().mockResolvedValue({ message: {} }); + + await TestClient.sendMessage('Hello'); + + const userSave = TestClient.saveMessageToDatabase.mock.calls.find( + ([msg]) => msg.isCreatedByUser, + ); + const expected = buildMessageFiles(requestFiles, [brief]); + expect(expected).toHaveLength(1); + expect(userSave[0].files).toEqual(expected); + expect(TestClient.options.attachments).toEqual([brief]); + }); }); describe('addPreviousAttachments authorization', () => { @@ -3481,6 +3514,70 @@ describe('BaseClient', () => { expect(JSON.stringify(secondMessage)).not.toContain('second-forged'); }); + describe('text a deferred file needs on a later turn', () => { + const deferredWorkbook = { + file_id: 'deferred-xlsx', + filename: 'quarterly.xlsx', + filepath: '/uploads/quarterly.xlsx', + source: 'local', + type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + bytes: 2048, + user: 'user-1', + context: 'message_attachment', + llmDeliveryPath: 'none', + metadata: { destinationChosen: false, textDerivation: { outcome: 'deferred', at: 1 } }, + }; + const derivedMarker = { outcome: 'complete', extractor: 'document_parser', at: 2 }; + + test('prepares the replayed file once and hands the prepared copy to the limit check and file context', async () => { + const deriveText = jest.fn(async () => ({ + status: 'derived', + text: 'Q1,1200', + textDerivation: derivedMarker, + })); + getFiles.mockResolvedValueOnce([deferredWorkbook]); + TestClient.options.req.config = { + fileConfig: { + endpoints: { [EModelEndpoint.openAI]: { llmDeliveryPolicy: 'automatic' } }, + }, + }; + TestClient.options.agent = { + provider: EModelEndpoint.openAI, + fileConsumers: { executeCode: false, fileSearch: false }, + }; + const routing = resolveTurnDeliveryRouting({ + agent: TestClient.options.agent, + config: TestClient.options.req.config, + }); + routing.reading = buildTurnReadingContext({ + routing, + provider: EModelEndpoint.openAI, + fileTokenLimit: 100000, + configuredFileSizeLimit: undefined, + countTokens: (text) => text.length, + deriveText, + }); + TestClient.options.agent.deliveryRouting = routing; + TestClient.assertHistoricalAttachmentLimits = jest.fn(async (files) => files); + const prepareTurnAttachments = jest.spyOn(TestClient, 'prepareTurnAttachments'); + + const [message] = await TestClient.addPreviousAttachments([ + { messageId: 'msg-xlsx', text: 'Totals?', files: [{ file_id: 'deferred-xlsx' }] }, + ]); + + const prepared = { + ...deferredWorkbook, + text: 'Q1,1200', + llmDeliveryPath: 'text', + metadata: { ...deferredWorkbook.metadata, textDerivation: derivedMarker }, + }; + expect(prepareTurnAttachments).toHaveBeenCalledTimes(1); + expect(TestClient.assertHistoricalAttachmentLimits).toHaveBeenCalledWith([prepared]); + expect(message.fileContext).toBe('Q1,1200'); + expect(deferredWorkbook.text).toBeUndefined(); + }); + }); + test('extracts historical file context while encoding provider attachments', async () => { getFiles.mockResolvedValueOnce([ownerFile]); const fileContext = deferred(); @@ -3503,8 +3600,7 @@ describe('BaseClient', () => { return messages; }); - await Promise.resolve(); - await Promise.resolve(); + await new Promise((resolve) => setImmediate(resolve)); expect(TestClient.addFileContextToMessage).toHaveBeenCalledTimes(1); expect(TestClient.processAttachments).toHaveBeenCalledTimes(1); @@ -4535,6 +4631,133 @@ describe('BaseClient', () => { }); }); +describe('BaseClient attachment text under a reading context', () => { + const longNote = { + file_id: 'long-note', + filename: 'notes.txt', + source: 'text', + type: 'text/plain', + text: 'quarterly revenue grew '.repeat(200), + }; + + beforeEach(() => { + jest.spyOn(Tokenizer, 'initEncoding').mockResolvedValue(undefined); + jest.spyOn(Tokenizer, 'getTokenCount').mockImplementation((text) => text.length); + }); + + afterEach(() => { + jest.restoreAllMocks(); + }); + + /** The truncation notice is emitted only when the turn reading context's text options + * reach `extractFileContext`; the notice wording itself belongs to packages/api. */ + test('extracts file context with the text options of the turn reading context', async () => { + const client = initializeFakeClient(apiKey, { modelOptions: { model: 'gpt-4o-mini' } }, []); + const config = { + fileConfig: { endpoints: { [EModelEndpoint.openAI]: { llmDeliveryPolicy: 'automatic' } } }, + }; + client.options.req = { body: { fileTokenLimit: 20 }, config }; + client.options.agent = { + provider: EModelEndpoint.openAI, + fileConsumers: { executeCode: false, fileSearch: false }, + }; + const routing = resolveTurnDeliveryRouting({ agent: client.options.agent, config }); + routing.reading = buildTurnReadingContext({ + routing, + provider: EModelEndpoint.openAI, + fileTokenLimit: 20, + configuredFileSizeLimit: undefined, + countTokens: (text) => text.length, + }); + client.options.agent.deliveryRouting = routing; + const message = {}; + + await client.addFileContextToMessage(message, [longNote], client.options.agent.fileConsumers); + + expect(message.fileContext).toContain('# "notes.txt"'); + expect(message.fileContext).toContain( + '[Truncated: only the beginning of "notes.txt" fits; the rest is omitted.]', + ); + }); +}); + +describe('BaseClient native documents under a reading context', () => { + const brokenPdf = { + user: 'user1', + file_id: 'broken-pdf', + filename: 'brief.pdf', + filepath: '/uploads/brief.pdf', + type: 'application/pdf', + bytes: 18, + source: 'local', + context: 'message_attachment', + llmDeliveryPath: 'provider', + metadata: { destinationChosen: false }, + }; + const rejection = [{ file_id: 'broken-pdf', reason: 'integrity' }]; + let uploads; + + beforeAll(() => { + uploads = fs.mkdtempSync(path.join(os.tmpdir(), 'baseclient-documents-')); + fs.writeFileSync(path.join(uploads, 'brief.pdf'), 'not a pdf document'); + }); + + afterAll(() => { + fs.rmSync(uploads, { recursive: true, force: true }); + }); + + const routeBrokenPdf = () => { + const client = initializeFakeClient(apiKey, { modelOptions: { model: 'claude-sonnet-4' } }, []); + const config = { + paths: { uploads }, + fileConfig: { endpoints: { [EModelEndpoint.anthropic]: { llmDeliveryPolicy: 'automatic' } } }, + }; + client.options.req = { config, user: { id: 'user1' } }; + client.options.agent = { + provider: EModelEndpoint.anthropic, + endpoint: EModelEndpoint.anthropic, + fileConsumers: { executeCode: false, fileSearch: true }, + currentRequestAttachments: [brokenPdf], + }; + const routing = resolveTurnDeliveryRouting({ agent: client.options.agent, config }); + routing.reading = buildTurnReadingContext({ + routing, + provider: EModelEndpoint.anthropic, + fileTokenLimit: 100000, + configuredFileSizeLimit: undefined, + countTokens: (text) => text.length, + }); + client.options.agent.deliveryRouting = routing; + return { client, routing }; + }; + + test('leaves a rejected document out of the forwarded files and records it on the reading context', async () => { + const { client, routing } = routeBrokenPdf(); + const recordRejections = jest.spyOn(routing.reading, 'recordRejections'); + const message = {}; + + const files = await client.addDocuments(message, [brokenPdf]); + + expect(files).toEqual([]); + expect(message.documents).toBeUndefined(); + expect(recordRejections).toHaveBeenCalledWith(rejection); + }); + + test('records the rejection on every agent the conversation agents list', async () => { + const { client } = routeBrokenPdf(); + const { routing: handoffRouting } = routeBrokenPdf(); + const recordRejections = jest.spyOn(handoffRouting.reading, 'recordRejections'); + client.getConversationAgents = () => [ + client.options.agent, + { deliveryRouting: handoffRouting }, + ]; + + await client.addDocuments({}, [brokenPdf]); + + expect(recordRejections).toHaveBeenCalledWith(rejection); + }); +}); + describe('BaseClient compaction turns', () => { const compactionOptions = { modelOptions: { model: 'gpt-4o-mini', temperature: 0 } }; const compactionHistory = [ diff --git a/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js b/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js index 1d0790a95aa..4476a453497 100644 --- a/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js +++ b/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js @@ -299,7 +299,8 @@ jest.mock('@librechat/api', () => ({ return { text, fileIds: fileIds.sort() }; }), getViolationInfo: (...args) => mockGetViolationInfo(...args), - buildMessageFiles: jest.fn(() => []), + /** Real, because the files the final user-message save carries are under test. */ + buildMessageFiles: jest.requireActual('@librechat/api').buildMessageFiles, resolveTitleTiming: jest.fn(() => 'immediate'), createConvoPersistenceSignal: jest.requireActual('@librechat/api').createConvoPersistenceSignal, recoverTurnMessageReference: jest.requireActual('@librechat/api').recoverTurnMessageReference, @@ -3771,6 +3772,91 @@ describe('ResumableAgentController resume metadata', () => { } } + it('persists the user message without file text and fails the turn when search preparation fails', async () => { + const api = jest.requireActual('@librechat/api'); + const file = { + user: 'user-123', + file_id: 'search-pdf', + filename: 'search.pdf', + filepath: '/uploads/search.pdf', + type: 'application/pdf', + bytes: 3 * 1024 * 1024, + object: 'file', + embedded: false, + usage: 0, + source: 'local', + context: 'message_attachment', + llmDeliveryPath: 'none', + metadata: { destinationChosen: false }, + text: 'complete cached text was not sent', + }; + const preparation = new Map(); + const agent = { + id: 'agent-1', + fileConsumers: { executeCode: false, fileSearch: true }, + currentRequestAttachments: [file], + provisionState: { + codeEnvFiles: [], + vectorDBFiles: [file], + searchPreparation: preparation, + aliveFileIds: new Set(), + agentScopedFileIds: new Set(), + }, + }; + const refs = [{ file_id: file.file_id }]; + const message = { + messageId: 'user-message', + parentMessageId: 'prior-response', + conversationId, + text: 'Search the attached PDF.', + files: api.buildMessageFiles(refs, [file]), + }; + const provisionFiles = api.createProvisionFilesCallback({ + req: { user: { id: 'user-123' } }, + agentToolContexts: new Map([[agent.id, { provisionState: agent.provisionState }]]), + provisionToCodeEnv: jest.fn(), + provisionToVectorDB: jest.fn().mockResolvedValue({ embedded: false, fileUpdate: null }), + updateFile: jest.fn(), + updateCodeEnvRef: jest.fn(), + addEmbeddedEntity: jest.fn(), + }); + const client = { + options: { attachments: [file], agent }, + sendMessage: jest.fn(async (_text, options) => { + options.onStart(message, 'user-message_'); + await provisionFiles(['file_search'], agent.id); + }), + }; + mockGenerationJobManager.completeJob.mockImplementation( + async (_streamId, error, _createdAt, options) => { + await options.beforeErrorPublication(); + const saved = mockSaveMessage.mock.calls.find( + ([, row]) => row.messageId === message.messageId, + )?.[1]; + expect(saved.files[0]).toMatchObject({ file_id: file.file_id }); + expect(saved.files[0]).not.toHaveProperty('text'); + expect(saved.files[0]).not.toHaveProperty('reading'); + expect(error).toBe(new api.FileSearchPreparationError().message); + return true; + }, + ); + + await AgentController( + createFailedRequest({ text: message.text, files: refs }), + createResumableResponse(), + jest.fn(), + jest.fn().mockResolvedValue({ client }), + null, + ); + await flushBackgroundGeneration(); + expect(mockGenerationJobManager.completeJob).toHaveBeenCalled(); + const saved = mockSaveMessage.mock.calls.find( + ([, row]) => row.messageId === message.messageId, + )?.[1]; + expect(saved.files[0]).not.toHaveProperty('reading'); + expect(agent.provisionState.vectorDBFiles).toEqual([file]); + }); + it.each(['history', 'exact-model'])( 'does not persist a protected turn rejected by %s policy during terminal recovery', async (boundary) => { @@ -3895,6 +3981,28 @@ describe('ResumableAgentController resume metadata', () => { ); }); + it('saves an edited resubmission that fails early with the file refs the request carried', async () => { + const initializeClient = jest.fn().mockRejectedValue(new Error('model unavailable')); + const replayedFile = { + file_id: 'book', + filename: 'book.xlsx', + type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + }; + + await AgentController( + createFailedRequest({ files: [replayedFile] }), + createResumableResponse(), + jest.fn(), + initializeClient, + null, + ); + + const [, savedUser] = mockSaveMessage.mock.calls.find( + ([, message]) => message?.messageId === 'user-message', + ); + expect(savedUser.files).toEqual([replayedFile]); + }); + it('allows a follow-up to chain from the persisted failed response', async () => { const initializeClient = jest.fn().mockRejectedValue(new Error('model unavailable')); await AgentController( diff --git a/api/server/controllers/agents/__tests__/resume.spec.js b/api/server/controllers/agents/__tests__/resume.spec.js index b5baf0c4a71..f3d36626db7 100644 --- a/api/server/controllers/agents/__tests__/resume.spec.js +++ b/api/server/controllers/agents/__tests__/resume.spec.js @@ -3655,6 +3655,116 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => { }); }); + /** A resumed turn whose only attachment File Search prepares with `outcome`. */ + const prepareResumedSearchTurn = (outcome) => { + const { + resolveTurnDeliveryRouting, + buildTurnReadingContext, + buildMessageFiles, + createProvisionFilesCallback, + FileSearchPreparationError, + } = jest.requireActual('@librechat/api'); + const file = { + user: USER_ID, + file_id: 'search-brief', + filename: 'search-brief.pdf', + filepath: '/uploads/search-brief.pdf', + type: 'application/pdf', + bytes: 15 * 1024 * 1024, + source: 'local', + context: 'message_attachment', + llmDeliveryPath: 'none', + metadata: { destinationChosen: false }, + text: 'cached text must not reach status', + }; + const preparation = new Map(); + const provisionState = { + codeEnvFiles: [], + vectorDBFiles: [file], + aliveFileIds: new Set(), + agentScopedFileIds: new Set(), + searchPreparation: preparation, + }; + const agent = { + id: AGENT_ID, + provider: 'openAI', + endpoint: 'openAI', + fileConsumers: { executeCode: false, fileSearch: true }, + currentRequestAttachments: [file], + provisionState, + }; + agent.deliveryRouting = resolveTurnDeliveryRouting({ + agent, + config: { fileConfig: { endpoints: { openAI: { llmDeliveryPolicy: 'automatic' } } } }, + }); + agent.deliveryRouting.reading = buildTurnReadingContext({ + routing: agent.deliveryRouting, + provider: 'openAI', + fileTokenLimit: 100000, + configuredFileSizeLimit: undefined, + countTokens: (text) => text.length, + }); + agent.deliveryRouting.reading.setSearchEvidence({ + queued: [file.file_id], + registered: [], + preparation, + }); + const refs = buildMessageFiles([{ file_id: file.file_id }], [file]); + mockGetMessages.mockResolvedValue([{ files: refs }]); + mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob()); + const context = { agentId: AGENT_ID, provisionState }; + const provisionFiles = createProvisionFilesCallback({ + req: { user: { id: USER_ID } }, + agentToolContexts: new Map([[AGENT_ID, context]]), + provisionToVectorDB: async () => ({ embedded: outcome !== 'failure' }), + provisionToCodeEnv: jest.fn(), + updateFile: jest.fn(), + updateCodeEnvRef: jest.fn(), + addEmbeddedEntity: jest.fn(), + }); + const client = makeClient({ options: { attachments: [file], agent } }); + client.resumeCompletion.mockImplementation(async () => { + await provisionFiles(['file_search'], AGENT_ID); + }); + mockInitializeClient.mockResolvedValue({ client, userMCPAuthMap: {} }); + mockSaveMessage.mockImplementation(async (_ctx, message) => message); + return { file, refs, FileSearchPreparationError }; + }; + + it('fails the resumed turn with FileSearchPreparationError when search preparation fails', async () => { + const { file, FileSearchPreparationError } = prepareResumedSearchTurn('failure'); + + const response = await post(approveBody()); + expect(response.status).toBe(200); + await settled; + await flush(); + + expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith( + CONVO_ID, + new FileSearchPreparationError().message, + 1000, + ); + expect(JSON.stringify(mockGenerationJobManager.completeJob.mock.calls)).not.toContain( + file.text, + ); + expect(mockGenerationJobManager.publishTerminalClaim).not.toHaveBeenCalled(); + }); + + it('finalizes the resumed turn with the restored refs as they are, without file text, once search preparation succeeds', async () => { + const { file, refs } = prepareResumedSearchTurn('success'); + + const response = await post(approveBody()); + expect(response.status).toBe(200); + await settled; + await flush(); + + expect(mockGenerationJobManager.publishTerminalClaim).toHaveBeenCalledTimes(1); + const [, finalEvent] = mockGenerationJobManager.publishTerminalClaim.mock.calls[0]; + expect(finalEvent.requestMessage.files).toEqual(refs); + expect(finalEvent.requestMessage.files[0]).not.toHaveProperty('reading'); + expect(JSON.stringify(finalEvent)).not.toContain(file.text); + }); + it('persists the response, claims terminal ownership, emits done, finishes, and prunes', async () => { mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob()); await post(approveBody()); diff --git a/api/server/controllers/agents/client.js b/api/server/controllers/agents/client.js index 746cb5b7e4e..33c64fee324 100644 --- a/api/server/controllers/agents/client.js +++ b/api/server/controllers/agents/client.js @@ -150,6 +150,10 @@ const { createAgentMemoryCallback, assertAgentAttachmentLimits, assertAgentAttachmentTopology, + allocateTurnAttachmentsWithHistory, + allocateTurnAttachmentsWithRetainedContext, + admitNativeFallbackAttachments, + retainNativeAttachmentTopology, collectHistoricalAttachmentIds, admitSteerAttachmentHistory, rollbackSteerAttachmentHistory, @@ -159,6 +163,7 @@ const { buildAgentScopedContext, buildAgentScopedAttachmentMap, resolveScopedTurnAttachments, + prepareScopedTurnCandidates, buildAgentContextAttachmentsByAgentId, buildSkillPrimeContentParts, buildInitialToolSessions, @@ -407,6 +412,11 @@ class AgentClient extends BaseClient { ); } + /** The primary agent and the handoff agents that share its conversation payload. */ + getConversationAgents() { + return [this.options.agent, ...(this.agentConfigs?.values() ?? [])]; + } + async addDocuments(message, attachments) { const memoryContext = { req: this.options.req, @@ -422,6 +432,10 @@ class AgentClient extends BaseClient { } } + admitPreparedAttachments(files, fileConsumers) { + return admitNativeFallbackAttachments(this, files, fileConsumers); + } + async processAttachments(message, attachments, fileConsumers) { const modelBoundAttachments = this.getModelBoundAttachmentsForEndpoint(attachments); const processableAttachments = this.getProcessableAttachmentsForEndpoint( @@ -469,6 +483,7 @@ class AgentClient extends BaseClient { attachmentsByAgentId, sharedRunAttachmentIds: sharedAttachmentIds, req: this.options.req, + consumers: this.options.agent?.fileConsumers, endpoint: this.options.agent?.endpoint ?? this.options.endpoint ?? EModelEndpoint.agents, endpointType: this.options.endpointType, endpointsByAgentId, @@ -505,6 +520,7 @@ class AgentClient extends BaseClient { historicalFileIds, scopedAttachmentsByAgentId, req: this.options.req, + consumers: this.options.agent?.fileConsumers, endpoint: this.options.agent?.endpoint ?? this.options.endpoint, endpointType: this.options.endpointType, endpointsByAgentId: this.turnAttachmentEndpointsByAgentId, @@ -527,7 +543,34 @@ class AgentClient extends BaseClient { } async assertHistoricalAttachmentLimits(historicalAttachments) { - const currentAttachments = this.options.attachments ? await this.options.attachments : []; + const agents = collectReachableAgents([ + this.options.agent, + ...(this.agentConfigs?.values() ?? []), + ]); + const endpointsByAgentId = new Map( + agents + .filter((agent) => agent?.id) + .map((agent) => [ + agent.id, + { + endpoint: agent.endpoint, + endpointType: agent === this.options.agent ? this.options.endpointType : undefined, + }, + ]), + ); + const currentAttachments = await allocateTurnAttachmentsWithHistory({ + agent: this.options.agent, + historical: historicalAttachments, + current: this.options.attachments ? await this.options.attachments : [], + req: this.options.req, + endpoint: this.options.agent?.endpoint ?? this.options.endpoint, + endpointType: this.options.endpointType, + endpointsByAgentId, + onAllocated: (files) => { + this.options.attachments = files; + }, + signal: this.options.abortController?.signal, + }); const compatibleHistoricalAttachments = this.getModelBoundAttachmentsForEndpoint(historicalAttachments); const compatibleCurrentAttachments = @@ -539,36 +582,28 @@ class AgentClient extends BaseClient { ); const sharedAttachments = [...compatibleHistoricalAttachments, ...compatibleCurrentAttachments]; const sharedAttachmentIds = collectFileIds(sharedAttachments); - const agents = collectReachableAgents([ - this.options.agent, - ...(this.agentConfigs?.values() ?? []), - ]); const scopedAttachmentMap = this.getFilteredScopedAttachmentMap( sharedAttachmentIds, this.options.agentContextAttachmentsByAgentId, agents, ); - const endpointsByAgentId = new Map( - agents - .filter((agent) => agent?.id) - .map((agent) => [ - agent.id, - { - endpoint: agent.endpoint, - endpointType: agent === this.options.agent ? this.options.endpointType : undefined, - }, - ]), - ); assertAgentAttachmentTopology({ sharedAttachments, historicalFileIds: this.turnHistoricalAttachmentIds, scopedAttachmentsByAgentId: scopedAttachmentMap, req: this.options.req, + consumers: this.options.agent?.fileConsumers, endpoint: this.options.agent?.endpoint ?? this.options.endpoint, endpointType: this.options.endpointType, endpointsByAgentId, }); this.assertTurnAttachmentLimits(sharedAttachments, [...scopedAttachmentMap.values()].flat()); + retainNativeAttachmentTopology( + this, + sharedAttachments, + scopedAttachmentMap, + endpointsByAgentId, + ); return compatibleHistoricalAttachments; } @@ -2474,9 +2509,6 @@ class AgentClient extends BaseClient { files: [...modelBoundFileContexts], }); const requestAttachmentsSource = this.options.attachments; - const requestAttachments = requestAttachmentsSource ? await requestAttachmentsSource : []; - const modelBoundRequestAttachments = - this.getModelBoundAttachmentsForEndpoint(requestAttachments); const retainedHistoricalFileContexts = this.options.resendFiles === false ? orderedMessages @@ -2489,13 +2521,29 @@ class AgentClient extends BaseClient { bytes: Buffer.byteLength(message.fileContext, 'utf8'), })) : []; + const requestAttachments = await allocateTurnAttachmentsWithRetainedContext({ + agent: this.options.agent, + resendFiles: this.options.resendFiles, + retained: retainedHistoricalFileContexts, + current: requestAttachmentsSource ? await requestAttachmentsSource : [], + req: this.options.req, + endpoint: this.options.agent?.endpoint ?? this.options.endpoint, + endpointType: this.options.endpointType, + endpointsByAgentId, + onAllocated: (files) => { + this.options.attachments = files; + }, + signal: this.options.abortController?.signal, + }); + const modelBoundRequestAttachments = + this.getModelBoundAttachmentsForEndpoint(requestAttachments); const sharedAttachmentFiles = [ ...Object.values(this.message_file_map ?? {}).flat(), ...(this.modelBoundHistoricalSteerFiles ?? []), ...modelBoundRequestAttachments, ]; const sharedRunAttachmentIds = collectFileIds(sharedAttachmentFiles); - this.options.agentContextAttachmentsByAgentId = resolveScopedTurnAttachments({ + const scopedInputs = { agents: allAgents, sharedConversationAgentIds: [this.options.agent.id, ...(this.agentConfigs?.keys() ?? [])], resendFiles: this.options.resendFiles, @@ -2504,12 +2552,23 @@ class AgentClient extends BaseClient { requestAttachments, sharedRunAttachmentIds, attachmentsByAgentId: this.options.agentContextAttachmentsByAgentId, + }; + const scopedCandidates = await prepareScopedTurnCandidates({ + ...scopedInputs, + appConfig: this.options.req?.config, + endpointsByAgentId, + signal: this.options.abortController?.signal, + }); + this.options.agentContextAttachmentsByAgentId = resolveScopedTurnAttachments({ + ...scopedInputs, + ...scopedCandidates, }); const scopedAttachmentMap = buildAgentScopedAttachmentMap({ agentIds: allAgents.map(({ agentId }) => agentId), attachmentsByAgentId: this.options.agentContextAttachmentsByAgentId, sharedRunAttachmentIds, req: this.options.req, + consumers: this.options.agent?.fileConsumers, endpoint: this.options.agent?.endpoint ?? this.options.endpoint ?? EModelEndpoint.agents, endpointType: this.options.endpointType, endpointsByAgentId, @@ -2528,6 +2587,7 @@ class AgentClient extends BaseClient { historicalFileIds: this.turnHistoricalAttachmentIds, scopedAttachmentsByAgentId: scopedAttachmentMap, req: this.options.req, + consumers: this.options.agent?.fileConsumers, endpoint: this.options.agent?.endpoint ?? this.options.endpoint, endpointType: this.options.endpointType, endpointsByAgentId, @@ -2585,10 +2645,12 @@ class AgentClient extends BaseClient { }; } - const [, files] = await Promise.all([ - this.addFileContextToMessage(latestMessage, modelBoundRequestAttachments), - this.processAttachments(latestMessage, attachments), - ]); + const files = await this.processMessageAttachments( + latestMessage, + attachments, + undefined, + modelBoundRequestAttachments, + ); this.options.attachments = files; } else { @@ -5592,6 +5654,10 @@ class AgentClient extends BaseClient { checkpointNamespace: this.checkpointNamespace, agents, files: [...modelBoundAgentFiles, ...dynamicToolContexts], + delivery: { + routing: this.options.agent?.deliveryRouting, + consumers: this.options.agent?.fileConsumers, + }, }, { getAgentCheckpointer, diff --git a/api/server/controllers/agents/client.test.js b/api/server/controllers/agents/client.test.js index 3d30076aef9..a7fed44fc7d 100644 --- a/api/server/controllers/agents/client.test.js +++ b/api/server/controllers/agents/client.test.js @@ -10,6 +10,18 @@ const mockHasDurableAgentInterruptCheckpoint = jest.fn().mockResolvedValue(true) const mockBuildAgentScopedContext = jest.fn((...args) => jest.requireActual('@librechat/api').buildAgentScopedContext(...args), ); +const mockPrepareScopedTurnCandidates = jest.fn((...args) => + jest.requireActual('@librechat/api').prepareScopedTurnCandidates(...args), +); +const mockResolveScopedTurnAttachments = jest.fn((...args) => + jest.requireActual('@librechat/api').resolveScopedTurnAttachments(...args), +); +const mockAllocateTurnAttachmentsWithHistory = jest.fn((...args) => + jest.requireActual('@librechat/api').allocateTurnAttachmentsWithHistory(...args), +); +const mockAllocateTurnAttachmentsWithRetainedContext = jest.fn((...args) => + jest.requireActual('@librechat/api').allocateTurnAttachmentsWithRetainedContext(...args), +); const mockFormatAgentMessages = jest.fn(() => ({ messages: [], indexTokenCountMap: {}, @@ -276,6 +288,11 @@ jest.mock('@librechat/agents', () => ({ jest.mock('@librechat/api', () => ({ ...jest.requireActual('@librechat/api'), buildAgentScopedContext: (...args) => mockBuildAgentScopedContext(...args), + prepareScopedTurnCandidates: (...args) => mockPrepareScopedTurnCandidates(...args), + resolveScopedTurnAttachments: (...args) => mockResolveScopedTurnAttachments(...args), + allocateTurnAttachmentsWithHistory: (...args) => mockAllocateTurnAttachmentsWithHistory(...args), + allocateTurnAttachmentsWithRetainedContext: (...args) => + mockAllocateTurnAttachmentsWithRetainedContext(...args), checkAccess: jest.fn(), createRun: (...args) => mockCreateRun(...args), countFormattedMessageTokens: jest.fn(() => 42), @@ -5565,6 +5582,7 @@ describe('AgentClient - titleConvo', () => { null, {}, ); + await new Promise((resolve) => setImmediate(resolve)); expect(client.contextHandlers.createContext).toHaveBeenCalledTimes(1); expect(client.useMemory).toHaveBeenCalledTimes(1); @@ -5623,7 +5641,7 @@ describe('AgentClient - titleConvo', () => { expect(client.processAttachments).not.toHaveBeenCalled(); requestAttachments.resolve([requestFile]); - await Promise.resolve(); + await new Promise((resolve) => setImmediate(resolve)); expect(mockBuildAgentScopedContext).toHaveBeenCalledTimes(1); const scopedContextArgs = mockBuildAgentScopedContext.mock.calls[0][0]; @@ -6604,6 +6622,59 @@ describe('AgentClient - titleConvo', () => { expect(() => client.admitSteerAttachments([current], 'later-steer')).not.toThrow(); }); + it('rolls back prepared native fallback copies after steer extraction fails with File Search loaded but unreachable', async () => { + const { buildTurnReadingContext } = jest.requireActual('@librechat/api'); + mockReq.config.fileConfig = { endpoints: { openAI: { llmDeliveryPolicy: 'automatic' } } }; + /** Automatic reading needs a loaded file tool; text is the fallback once File Search + * cannot reach the file. */ + mockAgent.fileConsumers = { executeCode: false, fileSearch: true }; + mockAgent.deliveryRouting = jest.requireActual('@librechat/api').resolveTurnDeliveryRouting({ + agent: mockAgent, + config: mockReq.config, + }); + const context = buildTurnReadingContext({ + routing: mockAgent.deliveryRouting, + provider: EModelEndpoint.openAI, + fileTokenLimit: 1000, + configuredFileSizeLimit: undefined, + countTokens: (text) => text.length, + }); + context.setSearchEvidence({ queued: [], registered: [], preparation: new Map() }); + mockAgent.deliveryRouting.reading = context; + const file = { + ...makeUploadedFile('steered-pdf', 'steered.pdf', 'application/pdf'), + context: 'message_attachment', + llmDeliveryPath: 'provider', + text: 'Complete PDF fallback.', + metadata: { destinationChosen: false }, + }; + client.turnSharedAttachmentFiles = []; + client.turnHistoricalAttachmentIds = new Set(); + client.turnAttachmentEndpointsByAgentId = new Map([ + [mockAgent.id, { endpoint: mockAgent.endpoint }], + ]); + client.attachmentMemoryContext = { attachments: [] }; + client.admitSteerAttachments([file], 'failed-native-steer'); + context.recordRejections([{ file_id: file.file_id, reason: 'capacity' }]); + client.addFileContextToMessage = jest.fn().mockRejectedValue(new Error('extraction failed')); + + await expect(client.processMessageAttachments({}, [file])).rejects.toThrow( + 'extraction failed', + ); + expect(client.turnSharedAttachmentFiles[0]).toMatchObject({ llmDeliveryPath: 'text' }); + expect(client.admittedSteerAttachments.get('failed-native-steer')[0]).toBe( + client.turnSharedAttachmentFiles[0], + ); + client.rollbackSteerAttachmentAdmission('failed-native-steer'); + + expect(client.turnSharedAttachmentFiles).toEqual([]); + expect(client.attachmentMemoryContext.attachments).toEqual([]); + expect(client.admittedSteerAttachments.has('failed-native-steer')).toBe(false); + expect(() => + client.admitSteerAttachments([makeTextFile('later', 'later.txt', 'later')], 'later'), + ).not.toThrow(); + }); + it('keeps committed resubmissions counted when a later duplicate steer fails', () => { mockReq.config.fileConfig = { endpoints: { openAI: { fileLimit: 1 } } }; const historical = makeTextFile('history', 'history.txt', 'history'); @@ -6684,6 +6755,185 @@ describe('AgentClient - titleConvo', () => { expect(client.processAttachments).not.toHaveBeenCalled(); }); + describe('history allocation under the automatic policy', () => { + const historicalPdf = { + ...makeUploadedFile('history-1', 'one.pdf', 'application/pdf'), + bytes: 600_000, + context: 'message_attachment', + llmDeliveryPath: 'provider', + metadata: { destinationChosen: false }, + }; + const currentPdf = { + ...makeUploadedFile('current-1', 'two.pdf', 'application/pdf'), + bytes: 600_000, + context: 'message_attachment', + llmDeliveryPath: 'provider', + metadata: { destinationChosen: false }, + }; + const historyMessages = [ + { messageId: 'msg-1', isCreatedByUser: true, files: [{ file_id: historicalPdf.file_id }] }, + ]; + + const readAutomatically = (endpointConfig, deriveText) => { + const { resolveTurnDeliveryRouting, buildTurnReadingContext } = + jest.requireActual('@librechat/api'); + client.options.resendFiles = true; + mockReq.config.fileConfig = { endpoints: { openAI: endpointConfig } }; + mockAgent.fileConsumers = { executeCode: false, fileSearch: true }; + mockAgent.deliveryRouting = resolveTurnDeliveryRouting({ + agent: mockAgent, + config: mockReq.config, + }); + mockAgent.deliveryRouting.reading = buildTurnReadingContext({ + routing: mockAgent.deliveryRouting, + provider: EModelEndpoint.openAI, + model: 'gpt-4', + fileTokenLimit: 1000, + configuredFileSizeLimit: undefined, + countTokens: (text) => text.length, + deriveText, + }); + client.options.attachments = [currentPdf]; + require('~/models').getFiles.mockResolvedValue([historicalPdf]); + client.addFileContextToMessage = jest.fn(); + client.processAttachments = jest.fn(); + return mockAgent.deliveryRouting.reading; + }; + + it('allocates history beside the current attachments and adopts the copies it hands back', async () => { + readAutomatically({ fileLimit: 10, totalSizeLimit: 1, llmDeliveryPolicy: 'automatic' }); + + await expect(client.addPreviousAttachments(historyMessages)).resolves.toEqual( + expect.any(Array), + ); + + expect(mockAllocateTurnAttachmentsWithHistory).toHaveBeenCalledTimes(1); + expect(mockAllocateTurnAttachmentsWithHistory).toHaveBeenCalledWith( + expect.objectContaining({ + agent: mockAgent, + historical: [expect.objectContaining({ file_id: historicalPdf.file_id })], + current: [currentPdf], + onAllocated: expect.any(Function), + }), + ); + const [rerouted] = client.options.attachments; + expect(rerouted).not.toBe(currentPdf); + expect(rerouted).toEqual( + expect.objectContaining({ file_id: currentPdf.file_id, llmDeliveryPath: 'none' }), + ); + expect(client.processAttachments).toHaveBeenCalledWith( + expect.anything(), + [historicalPdf], + undefined, + ); + }); + + it('reserves repeated historical steer copies before native fallback is extracted with File Search loaded but unreachable', async () => { + const reading = readAutomatically( + { fileLimit: 10, llmDeliveryPolicy: 'automatic' }, + async () => ({ + status: 'derived', + text: 'complete', + textDerivation: { outcome: 'complete', extractor: 'document_parser' }, + }), + ); + mockReq.config.fileConfig.fileContextCharLimit = 10; + /** Automatic reading needs a loaded file tool; the text fallback follows once File + * Search cannot reach the replayed file. */ + reading.setSearchEvidence({ queued: [], registered: [], preparation: new Map() }); + const file = { + ...historicalPdf, + metadata: { destinationChosen: false, textDerivation: { outcome: 'deferred' } }, + }; + client.options.attachments = []; + require('~/models').getFiles.mockResolvedValue([file]); + client.processAttachments = jest.fn(async (_message, files) => { + expect(client.turnSharedAttachmentFiles).toHaveLength(3); + reading.recordRejections([{ file_id: file.file_id, reason: 'capacity' }]); + return files; + }); + + await client.addPreviousAttachments([ + { + messageId: 'repeated-history', + isCreatedByUser: true, + files: [{ file_id: file.file_id }], + content: [ + { type: ContentTypes.STEER, files: [{ file_id: file.file_id }] }, + { type: ContentTypes.STEER, files: [{ file_id: file.file_id }] }, + ], + }, + ]); + + expect(client.message_file_map['repeated-history']).toEqual([]); + expect(client.addFileContextToMessage).toHaveBeenCalledWith( + expect.anything(), + [expect.objectContaining({ file_id: file.file_id, llmDeliveryPath: 'none' })], + undefined, + ); + }); + + describe('without file replay', () => { + const retainedHistory = { + messageId: 'historical-context', + parentMessageId: null, + sender: 'User', + text: 'Earlier context.', + isCreatedByUser: true, + fileContext: 'r'.repeat(500), + }; + const latestMessage = { + messageId: 'msg-1', + parentMessageId: 'historical-context', + sender: 'User', + text: 'Continue.', + isCreatedByUser: true, + }; + const DOCX_TYPE = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'; + const notes = { + ...makeUploadedFile('notes', 'notes.docx', DOCX_TYPE), + context: 'message_attachment', + llmDeliveryPath: 'text', + text: 'n'.repeat(600), + metadata: { destinationChosen: false }, + }; + + it('allocates the current file against the retained text and adopts the copy it hands back', async () => { + readAutomatically({ llmDeliveryPolicy: 'automatic' }); + client.options.resendFiles = false; + mockReq.config.fileConfig.fileContextCharLimit = 1000; + client.options.attachments = [notes]; + client.processAttachments = jest.fn(async (_message, attachments) => attachments); + + await expect( + client.buildMessages([retainedHistory, latestMessage], 'msg-1', {}), + ).resolves.toEqual(expect.objectContaining({ prompt: expect.any(Array) })); + + expect(mockAllocateTurnAttachmentsWithRetainedContext).toHaveBeenCalledTimes(1); + expect(mockAllocateTurnAttachmentsWithRetainedContext).toHaveBeenCalledWith( + expect.objectContaining({ + agent: mockAgent, + resendFiles: false, + retained: [expect.objectContaining({ text: retainedHistory.fileContext })], + current: [notes], + onAllocated: expect.any(Function), + }), + ); + expect(client.options.attachments).toEqual([ + expect.objectContaining({ file_id: 'notes', llmDeliveryPath: 'none' }), + ]); + }); + }); + }); + + it('lists the primary and handoff agents as the conversation agents', () => { + const handoff = { id: 'handoff' }; + client.agentConfigs = new Map([['handoff', handoff]]); + + expect(client.getConversationAgents()).toEqual([client.options.agent, handoff]); + client.agentConfigs = undefined; + }); + it('rejects combined historical and scoped bytes before history is encoded', async () => { mockAgent.endpoint = 'Moonshot'; client.options.endpointType = EModelEndpoint.custom; @@ -6873,6 +7123,32 @@ describe('AgentClient - titleConvo', () => { expect(client.message_file_map['msg-1']).toEqual([toolOnly, admitted]); }); + it.each([{ disabled: true }, { supportedMimeTypes: ['^image/png$'] }])( + 'excludes rejected legacy text from current context (%j)', + async (endpointConfig) => { + mockReq.config.fileConfig = { endpoints: { openAI: endpointConfig } }; + const rejected = { + ...makeUploadedFile('rejected-legacy', 'rejected.txt', 'text/plain'), + text: 'Rejected text must stay out of the prompt.', + }; + client.options.attachments = [rejected]; + client.addFileContextToMessage = jest.fn(); + client.processAttachments = jest.fn(async (_message, files) => files); + + await client.buildMessages( + [{ messageId: 'msg-1', sender: 'User', text: 'Continue.', isCreatedByUser: true }], + 'msg-1', + {}, + ); + + expect(client.addFileContextToMessage).toHaveBeenCalledWith( + expect.objectContaining({ messageId: 'msg-1' }), + [], + ); + expect(client.processAttachments).toHaveBeenCalledWith(expect.anything(), []); + }, + ); + it('defers current attachment persistence until model admission', () => { client.modelBoundCurrentFiles = [makeTextFile('pending', 'pending.txt', 'context')]; @@ -7372,6 +7648,87 @@ describe('AgentClient - titleConvo', () => { }, ); + it('derives text a handoff receiver needs before resolving its scoped attachments', async () => { + const { resolveTurnDeliveryRouting, buildTurnReadingContext } = + jest.requireActual('@librechat/api'); + const workbook = { + ...makeUploadedFile( + 'deferred-xlsx', + 'quarterly.xlsx', + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + ), + context: 'message_attachment', + llmDeliveryPath: 'none', + metadata: { destinationChosen: false, textDerivation: { outcome: 'deferred', at: 1 } }, + }; + const derivedMarker = { outcome: 'complete', extractor: 'document_parser', at: 2 }; + const deriveText = jest.fn(async () => ({ + status: 'derived', + text: 'Q1 total 1200', + textDerivation: derivedMarker, + })); + client.options.req.config.fileConfig = { + endpoints: { default: { llmDeliveryPolicy: 'automatic' } }, + }; + mockAgent.deliveryRouting = resolveTurnDeliveryRouting({ + agent: mockAgent, + config: client.options.req.config, + }); + mockAgent.fileConsumers = { executeCode: true, fileSearch: false }; + const handoffAgent = { + id: 'handoff-agent', + endpoint: EModelEndpoint.openAI, + provider: EModelEndpoint.openAI, + instructions: 'Handoff instructions', + model_parameters: { model: 'gpt-4' }, + tools: [], + fileConsumers: { executeCode: false, fileSearch: false }, + }; + handoffAgent.deliveryRouting = resolveTurnDeliveryRouting({ + agent: handoffAgent, + config: client.options.req.config, + }); + handoffAgent.deliveryRouting.reading = buildTurnReadingContext({ + routing: handoffAgent.deliveryRouting, + provider: EModelEndpoint.openAI, + fileTokenLimit: 100000, + configuredFileSizeLimit: undefined, + countTokens: (text) => text.length, + deriveText, + }); + client.agentConfigs = new Map([['handoff-agent', handoffAgent]]); + client.options.resendFiles = true; + client.options.attachments = []; + client.authorizedHistoricalFiles = new Map([[workbook.file_id, workbook]]); + client.message_file_map = {}; + + await client.buildMessages( + [ + { + messageId: 'msg-1', + sender: 'User', + text: 'Total it', + isCreatedByUser: true, + files: [{ file_id: workbook.file_id }], + }, + ], + 'msg-1', + {}, + ); + + expect(mockPrepareScopedTurnCandidates).toHaveBeenCalledTimes(1); + expect(mockResolveScopedTurnAttachments).toHaveBeenCalledTimes(1); + expect(mockPrepareScopedTurnCandidates.mock.invocationCallOrder[0]).toBeLessThan( + mockResolveScopedTurnAttachments.mock.invocationCallOrder[0], + ); + const [{ candidates }] = mockResolveScopedTurnAttachments.mock.calls[0]; + expect(candidates.get(workbook.file_id)).toEqual( + expect.objectContaining({ text: 'Q1 total 1200' }), + ); + expect(handoffAgent.additional_instructions).toContain('Q1 total 1200'); + expect(mockAgent.additional_instructions ?? '').not.toContain('Q1 total 1200'); + }); + it('places request context inline and applies each agent context doc only once', async () => { const requestFile = makeTextFile('request-file', 'request.txt', 'Shared request context'); const primaryContext = makeTextFile( diff --git a/api/server/controllers/agents/openai.js b/api/server/controllers/agents/openai.js index c768bdefbcf..02ec6085399 100644 --- a/api/server/controllers/agents/openai.js +++ b/api/server/controllers/agents/openai.js @@ -1094,6 +1094,7 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => { agentIds: contextAgents.map(({ id }) => id), attachmentsByAgentId: buildAgentContextAttachmentsByAgentId(contextAgents), req, + consumers: primaryConfig.fileConsumers, endpoint: primaryConfig.endpoint, endpointsByAgentId: new Map( contextAgents.map((runAgent) => [runAgent.id, { endpoint: runAgent.endpoint }]), diff --git a/api/server/routes/files/files.agents.test.js b/api/server/routes/files/files.agents.test.js index d1afe540567..cb72ab1399f 100644 --- a/api/server/routes/files/files.agents.test.js +++ b/api/server/routes/files/files.agents.test.js @@ -973,6 +973,73 @@ describe('File Routes - Agent Files Endpoint', () => { readFile.mockRestore(); }); + describe('upload preflight under each delivery policy', () => { + const xlsx = { + originalname: 'quarterly.xlsx', + mimetype: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + bytes: Buffer.from([0x50, 0x4b, 0x03, 0x04, 0, 255, 0, 1, 2, 3]), + }; + /** [label, filters, status the opaque workbook bytes draw before processing] */ + const cases = [ + ['no filters', undefined, 200], + [ + 'uninspectable block on content', + { files: { pii: { fields: ['content'], uninspectable: 'block' } } }, + 400, + ], + [ + 'uninspectable block on extracted text only', + { files: { pii: { fields: ['extracted_text'], uninspectable: 'block' } } }, + 200, + ], + ]; + + it.each(cases)( + 'answers an xlsx message attachment under %s identically', + async (_label, filters, expectedStatus) => { + const { originalname, mimetype, bytes } = xlsx; + await createAgent({ + id: agentCustomId, + name: 'Test Agent', + provider: 'openai', + model: 'gpt-4', + author: authorId, + }); + const readFile = jest.spyOn(fs, 'readFile').mockResolvedValue(bytes); + const send = async (llmDeliveryPolicy) => { + processAgentFileUpload.mockClear(); + const testApp = createAppWithUser( + authorId, + SystemRoles.USER, + { + fileConfig: llmDeliveryPolicy ? { llmDeliveryPolicy } : {}, + ...(filters ? { filters } : {}), + }, + { originalname, mimetype }, + ); + const response = await request(testApp).post('/files').send({ + endpoint: 'agents', + agent_id: agentCustomId, + message_file: 'true', + file_id: uuidv4(), + }); + return { + status: response.status, + body: response.body, + processed: processAgentFileUpload.mock.calls.length, + }; + }; + + const classic = await send(undefined); + const automatic = await send('automatic'); + readFile.mockRestore(); + + expect(classic.status).toBe(expectedStatus); + expect(automatic).toEqual(classic); + }, + ); + }); + it('detects opaque content after a long printable file prefix', async () => { await createAgent({ id: agentCustomId, diff --git a/api/server/services/Endpoints/agents/addedConvo.js b/api/server/services/Endpoints/agents/addedConvo.js index e598fe46c4a..cf413f7a427 100644 --- a/api/server/services/Endpoints/agents/addedConvo.js +++ b/api/server/services/Endpoints/agents/addedConvo.js @@ -70,6 +70,8 @@ const loadAddedAgent = (params) => * Resolver for the added agent's own `instructionsPrompt` link, forwarded verbatim. * @param {boolean} [params.recordLinkedPromptUsage] - Forwarded to `initializeAgent`; * defaults to `true` there when omitted. + * @param {import('@librechat/api').FileTextDeriver} [params.deriveText] + * @param {import('@librechat/api').TextDerivationPersister} [params.persistDerivation] * @param {AbortSignal} [params.signal] - Owning run cancellation signal. * @returns {Promise<{userMCPAuthMap: Object|undefined}>} The updated userMCPAuthMap */ @@ -104,6 +106,8 @@ const processAddedConvo = async ({ toolIntentsAvailable, statefulSessionsAvailable, memoryAvailable, + deriveText, + persistDerivation, signal, }) => { const addedConvo = endpointOption.addedConvo; @@ -210,6 +214,8 @@ const processAddedConvo = async ({ memoryAvailable, skillStates, defaultActiveOnShare, + deriveText, + persistDerivation, signal, }, { diff --git a/api/server/services/Endpoints/agents/addedConvo.spec.js b/api/server/services/Endpoints/agents/addedConvo.spec.js index 8224eb4f5d3..dbe1fa8cbad 100644 --- a/api/server/services/Endpoints/agents/addedConvo.spec.js +++ b/api/server/services/Endpoints/agents/addedConvo.spec.js @@ -120,6 +120,21 @@ describe('processAddedConvo', () => { ); }); + it.each([ + ['present', { deriveText: jest.fn(), persistDerivation: jest.fn() }], + ['absent', { deriveText: undefined, persistDerivation: undefined }], + ])( + 'shares the request text deriver and persister with the parallel agent as supplied (%s)', + async (_label, derivation) => { + await processAddedConvo(baseParams(derivation)); + + expect(mockInitializeAgent).toHaveBeenCalledWith( + expect.objectContaining(derivation), + expect.anything(), + ); + }, + ); + /** The added convo re-hydrates the same conversation's prior-turn files, so a * denied `FILE_SEARCH` grant has to travel with it — otherwise the parallel * agent primes the search files the primary just skipped. `undefined` stays diff --git a/api/server/services/Endpoints/agents/initialize.js b/api/server/services/Endpoints/agents/initialize.js index b82704399ce..7f13eca06ec 100644 --- a/api/server/services/Endpoints/agents/initialize.js +++ b/api/server/services/Endpoints/agents/initialize.js @@ -9,6 +9,8 @@ const { loadSkillStates, resolveInitializationProjectContext, initializeAgent, + createFileTextDeriver, + createDerivationPersister, primeInvokedSkillsForProfiles, validateAgentModel, extractManualSkills, @@ -69,6 +71,7 @@ const { MAX_SUBAGENT_GRAPH_NODES, MAX_SUBAGENT_RUN_CONFIGS, isEphemeralAgentId, + mergeFileConfig, resolveAllowedStatefulCodeEnvironments, } = require('librechat-data-provider'); const { @@ -96,6 +99,7 @@ const { } = require('./skillDeps'); const { loadCodeApiKey, + openStoredFile, provisionToCodeEnv, provisionToVectorDB, checkSessionsAlive, @@ -771,6 +775,16 @@ const initializeClientWithProvider = async ({ skillsCapabilityEnabled, ephemeralSkillsToggle, }); + const deriveText = createFileTextDeriver({ + req, + openStoredFile, + filters: appConfig?.filters, + textMimeTypes: mergeFileConfig(appConfig?.fileConfig).text?.supportedMimeTypes, + }); + const persistDerivation = createDerivationPersister(db.saveFileTextDerivation, { + user: req.user.id, + tenantId: req.user.tenantId, + }); const primaryConfig = await initializeAgent( { useChatProjectContext: true, @@ -799,6 +813,8 @@ const initializeClientWithProvider = async ({ skillStates, defaultActiveOnShare, manualSkills, + deriveText, + persistDerivation, signal, }, { @@ -809,6 +825,7 @@ const initializeClientWithProvider = async ({ getConvoFiles: db.getConvoFiles, getAccessibleMcpServerNames, updateFilesUsage: db.updateFilesUsage, + saveFileTextDerivation: db.saveFileTextDerivation, getUserKeyValues: db.getUserKeyValues, getUserCodeFiles: db.getUserCodeFiles, getDeferredProvisionFiles: db.getDeferredProvisionFiles, @@ -891,6 +908,8 @@ const initializeClientWithProvider = async ({ statefulSessionsAvailable, allowedStatefulCodeEnvironments, memoryAvailable, + deriveText, + persistDerivation, }, { getAgent: db.getAgent, @@ -904,6 +923,7 @@ const initializeClientWithProvider = async ({ getConvoFiles: db.getConvoFiles, getAccessibleMcpServerNames, updateFilesUsage: db.updateFilesUsage, + saveFileTextDerivation: db.saveFileTextDerivation, getUserKeyValues: db.getUserKeyValues, getUserCodeFiles: db.getUserCodeFiles, getDeferredProvisionFiles: db.getDeferredProvisionFiles, @@ -979,6 +999,8 @@ const initializeClientWithProvider = async ({ toolIntentsAvailable, statefulSessionsAvailable, memoryAvailable, + deriveText, + persistDerivation, signal, }); @@ -1437,6 +1459,8 @@ const initializeClientWithProvider = async ({ memoryAvailable, skillStates, defaultActiveOnShare, + deriveText, + persistDerivation, signal: context.signal, }, { @@ -1447,6 +1471,7 @@ const initializeClientWithProvider = async ({ getConvoFiles: db.getConvoFiles, getAccessibleMcpServerNames, updateFilesUsage: db.updateFilesUsage, + saveFileTextDerivation: db.saveFileTextDerivation, getUserKeyValues: db.getUserKeyValues, getUserCodeFiles: db.getUserCodeFiles, getDeferredProvisionFiles: db.getDeferredProvisionFiles, diff --git a/api/server/services/Endpoints/agents/initialize.spec.js b/api/server/services/Endpoints/agents/initialize.spec.js index 9922136450d..fd759cdd36c 100644 --- a/api/server/services/Endpoints/agents/initialize.spec.js +++ b/api/server/services/Endpoints/agents/initialize.spec.js @@ -483,6 +483,24 @@ describe('initializeClient — processAgent ACL gate', () => { expect(resolveUpstreamTokenProvider).not.toHaveBeenCalled(); }); + it('passes the primary request derivation dependencies to parallel initialization', async () => { + mockInitializeAgent.mockResolvedValue(makePrimaryConfig([])); + + await initializeClient({ + req: makeReq(), + res: {}, + signal: new AbortController().signal, + endpointOption: makeEndpointOption(), + }); + + const { deriveText, persistDerivation } = mockInitializeAgent.mock.calls[0][0]; + expect(deriveText).toEqual(expect.any(Function)); + expect(persistDerivation).toEqual(expect.any(Function)); + expect(processAddedConvo).toHaveBeenCalledWith( + expect.objectContaining({ deriveText, persistDerivation }), + ); + }); + it('forwards the linked-instructions resolver and records usage by default', async () => { mockInitializeAgent.mockResolvedValue(makePrimaryConfig([])); diff --git a/api/server/services/Files/process.js b/api/server/services/Files/process.js index 35117ab248e..b59ce31513a 100644 --- a/api/server/services/Files/process.js +++ b/api/server/services/Files/process.js @@ -37,6 +37,13 @@ const { getFileExtractionLogDetails, getUploadExtractedTextPlan, resolveUploadFallbackText, + resolveUploadReading, + resolveUploadCodePossible, + orderToolsForReading, + acquireUploadText, + getUploadReadingMetadata, + logUploadReading, + ExtractorUnavailableError, UPLOAD_EXTRACTED_TEXT_PLANS, MAX_STORED_EXTRACTED_TEXT_BYTES, inspectContent, @@ -794,19 +801,6 @@ const processAgentFileUpload = async ({ req, res, metadata, sseStream }) => { } } - const llmDeliveryPath = resolveUploadLLMDeliveryPath({ - toolResource: tool_resource, - mimeType: file.mimetype, - endpointConfig, - fileConfig, - endpoint, - endpointProvider: getCustomEndpointProvider(appConfig?.endpoints?.custom, endpoint), - useResponsesApi: isResponsesApiUpload(metadata.useResponsesApi ?? req.body?.useResponsesApi), - sttConfigured: isSpeechProviderConfigured(appConfig?.speech?.stt), - }); - - /* Destination and acceptability are one decision, made by shared policy rather than - * rebuilt here. `agentTools` is undefined when no agent record backs the upload. */ /* Only a permanent agent upload can land on a context resource, so the capability is * looked up only there and the common attachment path pays nothing for it. */ const contextEnabled = @@ -819,18 +813,39 @@ const processAgentFileUpload = async ({ req, res, metadata, sseStream }) => { metadata.agentTools?.some( (tool) => tool === EToolResources.execute_code || tool === EToolResources.file_search, ); - const toolRoleGrants = hasRoleGatedConsumer - ? await resolveToolRoleGrants({ - req, - getRoleByName: db.getRoleByName, - context: 'fileUpload', - }) - : undefined; + const loadToolRoleGrants = () => + resolveToolRoleGrants({ req, getRoleByName: db.getRoleByName, context: 'fileUpload' }); + const toolRoleGrants = hasRoleGatedConsumer ? await loadToolRoleGrants() : undefined; + const enabledAgentTools = await filterEnabledConsumers(req, metadata.agentTools, toolRoleGrants); + + const uploadReading = await resolveUploadReading({ + toolResource: tool_resource, + mimeType: file.mimetype, + endpointConfig, + fileConfig, + endpoint, + endpointProvider: getCustomEndpointProvider(appConfig?.endpoints?.custom, endpoint), + useResponsesApi: isResponsesApiUpload(metadata.useResponsesApi ?? req.body?.useResponsesApi), + sttConfigured: isSpeechProviderConfigured(appConfig?.speech?.stt), + isMessageAttachment: messageAttachment, + filters: appConfig?.filters, + resolveCodePossible: () => + resolveUploadCodePossible({ + agentTools: enabledAgentTools, + checkCodeCapability: () => checkCapability(req, AgentCapabilities.execute_code), + getRunCodeGrant: async () => (await loadToolRoleGrants())?.runCode, + }), + }); + let llmDeliveryPath = uploadReading.path; + let textDerivation; + + /* Destination and acceptability are one decision, made by shared policy rather than + * rebuilt here. `agentTools` is undefined when no agent record backs the upload. */ const destination = resolveUploadDestination({ toolResource: tool_resource, deliveryPath: llmDeliveryPath, mimeType: file.mimetype, - agentTools: await filterEnabledConsumers(req, metadata.agentTools, toolRoleGrants), + agentTools: orderToolsForReading(uploadReading, enabledAgentTools), hasAgent: agent_id != null, isMessageAttachment: messageAttachment, /* A message attachment can acquire a file-tool consumer later in the same draft or @@ -1005,6 +1020,7 @@ const processAgentFileUpload = async ({ req, res, metadata, sseStream }) => { updatingUserId: req?.user?.id, }); } + logUploadReading(file_id, uploadReading); const result = await db.createFile(fileInfo, true); sendUploadSuccess(res, sseStream, 'Agent file uploaded and processed successfully', result); }; @@ -1052,92 +1068,118 @@ const processAgentFileUpload = async ({ req, res, metadata, sseStream }) => { } }; - if (shouldUseConfiguredOCR && !(await checkCapability(req, AgentCapabilities.ocr))) { - throw new Error('OCR capability is not enabled for Agents'); - } - - if (shouldUseOCR) { - const ocrResult = await extractInspectableFileText({ - filters: appConfig?.filters, - extract: resolveDocumentText, - }); - if (ocrResult) { - const { text } = ocrResult; - return await createTextFile({ text }); + const ocrCapable = + !shouldUseConfiguredOCR || (await checkCapability(req, AgentCapabilities.ocr)); + const acquireContextText = async () => { + if (!ocrCapable) { + throw new ExtractorUnavailableError( + 'OCR capability is not enabled for Agents', + 'extractor_unavailable', + ); } - throw new Error( - `Unable to extract text from "${file.originalname}". The document may be image-based and requires an OCR service to process.`, - ); - } - const shouldUseSTT = fileConfig.checkType( - file.mimetype, - fileConfig.stt?.supportedMimeTypes || [], - ); + if (shouldUseOCR) { + const ocrResult = await extractInspectableFileText({ + filters: appConfig?.filters, + extract: resolveDocumentText, + }); + if (ocrResult) { + const { text } = ocrResult; + return { text }; + } + throw new Error( + `Unable to extract text from "${file.originalname}". The document may be image-based and requires an OCR service to process.`, + ); + } - if (shouldUseSTT) { - const sttService = await STTService.getInstance(); - const { text } = await processAudioFile({ req, file, sttService }); - return await createTextFile({ text, isTranscript: true }); - } + const shouldUseSTT = fileConfig.checkType( + file.mimetype, + fileConfig.stt?.supportedMimeTypes || [], + ); - const shouldUseText = fileConfig.checkType( - file.mimetype, - fileConfig.text?.supportedMimeTypes || [], - ); + if (shouldUseSTT) { + const sttService = await STTService.getInstance(); + const { text } = await processAudioFile({ req, file, sttService }); + return { text, isTranscript: true }; + } - if (!shouldUseText) { - throw new Error(`File type ${file.mimetype} is not supported for text parsing.`); - } + const shouldUseText = fileConfig.checkType( + file.mimetype, + fileConfig.text?.supportedMimeTypes || [], + ); - /** - * A document type the admin routed to configured text extraction: prefer RAG `/text`, but fall - * back to the built-in document parser (not raw native text) when RAG is unavailable, so a - * transient outage doesn't degrade a docx/pdf to unreadable bytes. Only the RAG extraction is - * inside the fallback catch: a downstream persistence failure (size guard, DB, agent-resource - * mutation) must surface as itself, not trigger a second extraction attempt. - */ - if (shouldUseConfiguredText) { - let configuredText; - try { - configuredText = await parseText({ req, file, file_id, allowNativeFallback: false }); - } catch (err) { - const { errorMetadata } = getExtractionLogDetails(err); - logger.warn( - `[processAgentFileUpload] Configured RAG text extraction unavailable for ${extractionFileLabel}, using built-in document parser:`, - errorMetadata, + if (!shouldUseText) { + throw new ExtractorUnavailableError( + `File type ${file.mimetype} is not supported for text parsing.`, + 'no_extractor', ); - const documentText = await extractInspectableFileText({ - filters: appConfig?.filters, - extract: resolveDocumentText, - }); - if (!documentText) { - throw new Error( - `Unable to extract text from "${file.originalname}". RAG text extraction was unavailable and the built-in parser produced no result.`, + } + + /** + * A document type the admin routed to configured text extraction: prefer RAG `/text`, but fall + * back to the built-in document parser (not raw native text) when RAG is unavailable, so a + * transient outage doesn't degrade a docx/pdf to unreadable bytes. Only the RAG extraction is + * inside the fallback catch: a downstream persistence failure (size guard, DB, agent-resource + * mutation) must surface as itself, not trigger a second extraction attempt. + */ + if (shouldUseConfiguredText) { + let configuredText; + try { + configuredText = await parseText({ req, file, file_id, allowNativeFallback: false }); + } catch (err) { + const { errorMetadata } = getExtractionLogDetails(err); + logger.warn( + `[processAgentFileUpload] Configured RAG text extraction unavailable for ${extractionFileLabel}, using built-in document parser:`, + errorMetadata, ); + const documentText = await extractInspectableFileText({ + filters: appConfig?.filters, + extract: resolveDocumentText, + }); + if (!documentText) { + throw new Error( + `Unable to extract text from "${file.originalname}". RAG text extraction was unavailable and the built-in parser produced no result.`, + ); + } + const { text } = documentText; + return { text }; } - const { text } = documentText; - return await createTextFile({ text }); + return { text: configuredText.text }; } - return await createTextFile({ text: configuredText.text }); - } - /* The native reader decodes whatever bytes it is given as UTF-8, which is meaningful - * only for types that are already text. For anything else, a raster image on a - * deployment without OCR being the case in point, it would store mojibake as the - * file's text, so a real extractor is required and its absence surfaces as an error - * rather than as nonsense content. */ - const { text } = await extractInspectableFileText({ + /* The native reader decodes whatever bytes it is given as UTF-8, which is meaningful + * only for types that are already text. For anything else, a raster image on a + * deployment without OCR being the case in point, it would store mojibake as the + * file's text, so a real extractor is required and its absence surfaces as an error + * rather than as nonsense content. */ + const { text } = await extractInspectableFileText({ + filters: appConfig?.filters, + extract: () => + parseText({ + req, + file, + file_id, + allowNativeFallback: isNativelyReadableText(file.mimetype), + }), + }); + return { text }; + }; + + /* Only the acquisition is wrapped: inspection, the content policy, storage and the + * record in `createTextFile` fail the upload as before. A returned marker keeps the + * original as plain storage below. */ + const acquired = await acquireUploadText({ + reading: uploadReading, + acquire: acquireContextText, + fileId: file_id, filters: appConfig?.filters, - extract: () => - parseText({ - req, - file, - file_id, - allowNativeFallback: isNativelyReadableText(file.mimetype), - }), }); - return await createTextFile({ text }); + if (acquired.textDerivation == null) { + return await createTextFile(acquired); + } + textDerivation = acquired.textDerivation; + llmDeliveryPath = acquired.path; + effectiveToolResource = undefined; } /* Extracted before storage, which may move the temporary upload the extractors read. */ @@ -1149,6 +1191,8 @@ const processAgentFileUpload = async ({ req, res, metadata, sseStream }) => { isMessageAttachment: messageAttachment, endpointConfig, filters: appConfig?.filters, + reading: uploadReading, + textDerivation, }); // Dual storage pattern for RAG files: Storage + Vector DB @@ -1339,6 +1383,7 @@ const processAgentFileUpload = async ({ req, res, metadata, sseStream }) => { * it later. Conversion changes `type`, so without this the second answer is drawn * from a format the administrator never configured a route for. */ ...(storedType !== file.mimetype ? { routingMimeType: file.mimetype } : {}), + ...getUploadReadingMetadata(uploadReading, textDerivation), }, type: storedType, embedded, @@ -1352,6 +1397,7 @@ const processAgentFileUpload = async ({ req, res, metadata, sseStream }) => { ...retentionExpiry, }; + logUploadReading(file_id, uploadReading, textDerivation); const result = await db.createFile(fileInfo, true); sendUploadSuccess(res, sseStream, 'Agent file uploaded and processed successfully', result); diff --git a/api/server/services/Files/process.spec.js b/api/server/services/Files/process.spec.js index 33ec48b7861..59ffa3a4a3e 100644 --- a/api/server/services/Files/process.spec.js +++ b/api/server/services/Files/process.spec.js @@ -7,6 +7,7 @@ jest.mock('@librechat/data-schemas', () => ({ })); jest.mock('@librechat/agents', () => ({ + ...jest.requireActual('@librechat/agents'), Providers: { XAI: 'xai', DEEPSEEK: 'deepseek', @@ -120,7 +121,16 @@ jest.mock('@librechat/api', () => { return UPLOAD_EXTRACTED_TEXT_PLANS.documentParser; }, ); + const actualApi = jest.requireActual('@librechat/api'); return { + resolveUploadReading: jest.fn(actualApi.resolveUploadReading), + resolveUploadCodePossible: jest.fn(actualApi.resolveUploadCodePossible), + orderToolsForReading: jest.fn(actualApi.orderToolsForReading), + acquireUploadText: jest.fn(actualApi.acquireUploadText), + getUploadReadingMetadata: jest.fn(actualApi.getUploadReadingMetadata), + logUploadReading: jest.fn(actualApi.logUploadReading), + ExtractorUnavailableError: actualApi.ExtractorUnavailableError, + UninspectableFileError: actualApi.UninspectableFileError, sanitizeFilename: jest.fn((n) => n), /** Grants both; these specs vary the capability set, not the role. */ resolveToolRoleGrants: jest.fn(async () => ({ runCode: true, fileSearch: true })), @@ -3377,3 +3387,209 @@ describe('fallback text for uploads left to tools', () => { ); }); }); + +describe('automatic reading at upload', () => { + const { logUploadReading } = require('@librechat/api'); + const AUTOMATIC_ENDPOINT = 'Custom Provider'; + let parseDocument; + let storeFile; + + const useEndpointConfig = (endpointConfig) => + mergeFileConfig.mockReturnValue({ + ...makeFileConfig(), + endpoints: { [AUTOMATIC_ENDPOINT]: endpointConfig }, + }); + + beforeEach(() => { + jest.clearAllMocks(); + mockRes.status.mockReturnThis(); + mockRes.json.mockReturnValue({}); + checkCapability.mockResolvedValue(true); + inspectContent.mockReturnValue(null); + useEndpointConfig({ llmDeliveryPolicy: 'automatic' }); + parseDocument = jest.fn().mockResolvedValue({ text: 'region,total', bytes: 12 }); + storeFile = jest.fn().mockResolvedValue({ + bytes: 42, + filename: 'upload.bin', + filepath: '/uploads/upload.bin', + }); + getStrategyFunctions.mockImplementation((source) => ({ + handleFileUpload: source === FileSources.document_parser ? parseDocument : storeFile, + })); + }); + + const upload = ({ mimetype = XLSX_MIME, filters, metadata, ocrConfig } = {}) => { + const req = makeReq({ mimetype, filters, ocrConfig }); + req.body.endpoint = EModelEndpoint.agents; + return processAgentFileUpload({ + req, + res: mockRes, + metadata: { + message_file: 'true', + file_id: 'file-auto', + effectiveEndpoint: AUTOMATIC_ENDPOINT, + ...metadata, + }, + }); + }; + + const storedRecord = () => { + expect(db.createFile).toHaveBeenCalledTimes(1); + return db.createFile.mock.calls[0][0]; + }; + + test('leaves an ephemeral workbook to Run Code without parsing it', async () => { + const { resolveUploadFallbackText } = require('@librechat/api'); + + await upload(); + + storedRecord(); + expect(parseDocument).not.toHaveBeenCalled(); + expect(storeFile).toHaveBeenCalledTimes(1); + expect(checkCapability).toHaveBeenCalledWith(expect.anything(), AgentCapabilities.execute_code); + expect(resolveUploadFallbackText).toHaveBeenCalledWith( + expect.objectContaining({ + deliveryPath: 'none', + reading: expect.objectContaining({ codePreferred: true }), + textDerivation: undefined, + }), + ); + expect(logUploadReading).toHaveBeenCalledTimes(1); + expect(logUploadReading).toHaveBeenCalledWith( + 'file-auto', + expect.objectContaining({ policy: 'automatic', codePreferred: true }), + undefined, + ); + }); + + test('defers a workbook for a saved agent whose narrowed tools include Run Code', async () => { + const { orderToolsForReading } = require('@librechat/api'); + await upload({ + metadata: { + agent_id: 'agent-abc', + agentTools: [EToolResources.file_search, EToolResources.execute_code], + }, + }); + + storedRecord(); + expect(parseDocument).not.toHaveBeenCalled(); + expect(orderToolsForReading).toHaveLastReturnedWith([ + EToolResources.execute_code, + EToolResources.file_search, + ]); + expect(uploadVectors).not.toHaveBeenCalled(); + expect(db.addAgentResourceFile).not.toHaveBeenCalled(); + }); + + test('defers a document whose configured OCR capability is off, for a built-in reader later', async () => { + mergeFileConfig.mockReturnValue({ + ...makeFileConfig({ ocrSupportedMimeTypes: [DOCX_MIME] }), + endpoints: { [AUTOMATIC_ENDPOINT]: { llmDeliveryPolicy: 'automatic' } }, + }); + checkCapability.mockImplementation( + async (_req, capability) => capability !== AgentCapabilities.ocr, + ); + + await upload({ mimetype: DOCX_MIME, ocrConfig: { strategy: FileSources.mistral_ocr } }); + + storedRecord(); + expect(checkCapability).toHaveBeenCalledWith(expect.anything(), AgentCapabilities.ocr); + expect(parseDocument).not.toHaveBeenCalled(); + expect(storeFile).toHaveBeenCalledTimes(1); + }); + + test.each([ + [ + 'the Run Code capability is off', + () => + checkCapability.mockImplementation( + async (_req, capability) => capability !== AgentCapabilities.execute_code, + ), + {}, + ], + ['the endpoint stays classic', () => useEndpointConfig({}), {}], + ])('extracts the workbook at upload when %s', async (_label, arrange, options) => { + arrange(); + + await upload(options); + + const record = storedRecord(); + expect(parseDocument).toHaveBeenCalledTimes(1); + expect(record.llmDeliveryPath).toBe('text'); + expect(record.text).toBe('region,total'); + expect(record.metadata.textDerivation).toBeUndefined(); + expect(logUploadReading).toHaveBeenCalledTimes(1); + }); + + test.each([ + [ + 'keeps the original alone, deferred, when the parser fails unrecognized', + 'automatic', + 'Document parser crashed', + { outcome: 'deferred', reason: 'parser', at: expect.any(Number) }, + ], + [ + 'keeps the original alone, marked failed, when the parser names its failure', + 'automatic', + 'No text found in document', + { outcome: 'failed', extractor: 'document_parser', reason: 'empty', at: expect.any(Number) }, + ], + ['still fails a classic upload whose parser fails', 'classic', 'Document parser crashed', null], + ])('%s', async (_label, policy, failure, textDerivation) => { + const { resolveUploadFallbackText } = require('@librechat/api'); + if (policy === 'classic') { + useEndpointConfig({}); + } + parseDocument.mockRejectedValue(new Error(failure)); + + const result = upload({ mimetype: DOCX_MIME }); + + if (textDerivation === null) { + await expect(result).rejects.toThrow(failure); + expect(db.createFile).not.toHaveBeenCalled(); + return; + } + await result; + const record = storedRecord(); + expect(parseDocument).toHaveBeenCalledTimes(1); + expect(storeFile).toHaveBeenCalledWith(expect.objectContaining({ file_id: 'file-auto' })); + expect(record.llmDeliveryPath).toBe('none'); + expect(record.text).toBeUndefined(); + expect(record.metadata.textDerivation).toEqual(textDerivation); + expect(resolveUploadFallbackText).toHaveBeenCalledWith( + expect.objectContaining({ + deliveryPath: 'none', + textDerivation: record.metadata.textDerivation, + }), + ); + }); + + test('still fails the upload when storage fails after a kept original', async () => { + parseDocument.mockRejectedValue(new Error('Document parser crashed')); + storeFile.mockRejectedValue(new Error('storage unavailable')); + + await expect(upload({ mimetype: DOCX_MIME })).rejects.toThrow('storage unavailable'); + expect(db.createFile).not.toHaveBeenCalled(); + }); + + test('still answers a content-policy finding in extracted text with a 400', async () => { + inspectContent.mockReturnValue({ source: 'file', field: 'extracted_text' }); + + await upload({ + mimetype: DOCX_MIME, + filters: { + files: { + pii: { + fields: ['extracted_text'], + customPatterns: [{ id: 'region', label: 'region', regex: 'region' }], + }, + }, + }, + }); + + expect(parseDocument).toHaveBeenCalledTimes(1); + expect(mockRes.status).toHaveBeenCalledWith(400); + expect(storeFile).not.toHaveBeenCalled(); + expect(db.createFile).not.toHaveBeenCalled(); + }); +}); diff --git a/api/server/services/Files/provision.js b/api/server/services/Files/provision.js index 32311cc500e..8538ca10db6 100644 --- a/api/server/services/Files/provision.js +++ b/api/server/services/Files/provision.js @@ -16,6 +16,7 @@ const getService = () => module.exports = { loadCodeApiKey: (...args) => getService().loadCodeApiKey(...args), + openStoredFile: (...args) => getService().openStoredFile(...args), provisionToCodeEnv: (...args) => getService().provisionToCodeEnv(...args), provisionToVectorDB: (...args) => getService().provisionToVectorDB(...args), checkCodeEnvFileAlive: (...args) => getService().checkCodeEnvFileAlive(...args), diff --git a/api/server/services/ToolService.js b/api/server/services/ToolService.js index 632d884eca7..31a3cb21b29 100644 --- a/api/server/services/ToolService.js +++ b/api/server/services/ToolService.js @@ -1570,14 +1570,16 @@ async function loadToolDefinitionsWrapper({ } } + let primedSearchFiles; if (hasFileSearch && tool_resources) { try { - const { toolContext } = await primeSearchFiles({ + const { toolContext, files } = await primeSearchFiles({ req, tool_resources, agentId: agent.id, agentResourceType, }); + primedSearchFiles = files; if (toolContext) { dynamicToolContextMap[Tools.file_search] = toolContext; } @@ -1586,6 +1588,8 @@ async function loadToolDefinitionsWrapper({ '[loadToolDefinitionsWrapper] Error priming search files:', getSafeErrorMetadata(error), ); + /* Nothing was primed, so no resource file may be advertised as reachable by search. */ + primedSearchFiles = []; } } @@ -1630,6 +1634,7 @@ async function loadToolDefinitionsWrapper({ mcpToolAliases, actionsEnabled, primedCodeFiles, + primedSearchFiles, oauthActionToolNames, codeExecutionContext: resolvedCodeExecutionContext, repositoryInstructionSource: createRepositoryInstructionSource({ diff --git a/api/server/services/__tests__/ToolService.spec.js b/api/server/services/__tests__/ToolService.spec.js index 82f98c1dc7f..80093e79857 100644 --- a/api/server/services/__tests__/ToolService.spec.js +++ b/api/server/services/__tests__/ToolService.spec.js @@ -1040,6 +1040,54 @@ describe('ToolService - Action Capability Gating', () => { expect(JSON.stringify(errorSpy.mock.calls)).not.toContain(rawValue); errorSpy.mockRestore(); }); + + const primed = [{ file_id: 'file-indexed', filename: 'indexed.pdf', fromAgent: false }]; + const primedNote = `- Note: Use the ${Tools.file_search} tool to find relevant information within:`; + const emptyNote = `- Note: Semantic search is available through the ${Tools.file_search} tool but no files are currently loaded.`; + + it.each([ + [ + 'primed', + () => + mockPrimeSearchFiles.mockResolvedValueOnce({ files: primed, toolContext: primedNote }), + primed, + primedNote, + ], + [ + 'none', + () => mockPrimeSearchFiles.mockResolvedValueOnce({ files: [], toolContext: emptyNote }), + [], + emptyNote, + ], + [ + 'failed', + () => mockPrimeSearchFiles.mockRejectedValueOnce(new Error('search priming failed')), + [], + undefined, + ], + ])( + 'returns the primed search files beside the file search note when priming is %s', + async (_outcome, arrange, files, toolContext) => { + arrange(); + const req = createMockReq([AgentCapabilities.file_search]); + mockGetEndpointsConfig.mockResolvedValue( + createEndpointsConfig([AgentCapabilities.file_search]), + ); + + const result = await loadAgentTools({ + req, + res: {}, + agent: { id: 'agent_search', tools: [Tools.file_search] }, + tool_resources: { + [EToolResources.file_search]: { file_ids: ['file-indexed'] }, + }, + definitionsOnly: true, + }); + + expect(result.primedSearchFiles).toEqual(files); + expect(result.dynamicToolContextMap[Tools.file_search]).toBe(toolContext); + }, + ); }); describe('loadAgentTools (definitionsOnly=true) — action tool filtering', () => { @@ -3236,15 +3284,9 @@ describe('ToolService - Action Capability Gating', () => { codeWorkspaces: [{ environmentId: 'personal-machine', workspaceId: 'project-a' }], }; mockGetEndpointsConfig.mockResolvedValue(createEndpointsConfig(capabilities)); - mockResolveCodeExecutionContext.mockReturnValueOnce({ - baseUrl: 'http://attached-code.test/v1', - codeSessionKey: 'execute_code:stateful:attached', - executionProfile: 'stateful', - statefulSessions: true, - environmentType: 'attached', - environmentId: 'personal-machine', - bridgeWorkerId: 'worker-abc', - }); + req.config.endpoints[EModelEndpoint.agents].statefulCodeSessions = { + environments: [attachedEnvironment()], + }; mockCreateLaneGitRecorder.mockClear(); await loadToolsForExecution({ @@ -3255,6 +3297,7 @@ describe('ToolService - Action Capability Gating', () => { tools: [Tools.execute_code], stateful_code_sessions: true, stateful_code_environment: 'agent-user', + code_environment_id: 'personal-machine', }, conversationId: 'resolved-convo', toolNames: [AgentConstants.BASH_TOOL], @@ -3294,15 +3337,9 @@ describe('ToolService - Action Capability Gating', () => { codeWorkspaces: [{ environmentId: 'personal-machine', workspaceId: 'project-a' }], }; mockGetEndpointsConfig.mockResolvedValue(createEndpointsConfig(capabilities)); - mockResolveCodeExecutionContext.mockReturnValueOnce({ - baseUrl: 'http://attached-code.test/v1', - codeSessionKey: 'execute_code:stateful:attached', - executionProfile: 'stateful', - statefulSessions: true, - environmentType: 'attached', - environmentId: 'personal-machine', - bridgeWorkerId: 'worker-abc', - }); + req.config.endpoints[EModelEndpoint.agents].statefulCodeSessions = { + environments: [attachedEnvironment()], + }; mockCreateLaneGitRecorder.mockClear(); await loadToolsForExecution({ @@ -3313,6 +3350,7 @@ describe('ToolService - Action Capability Gating', () => { tools: [Tools.execute_code], stateful_code_sessions: true, stateful_code_environment: 'agent-user', + code_environment_id: 'personal-machine', }, conversationId: 'resolved-convo', toolNames: [AgentConstants.BASH_TOOL], diff --git a/api/test/__mocks__/logger.js b/api/test/__mocks__/logger.js index 699a94883f1..734102c5a19 100644 --- a/api/test/__mocks__/logger.js +++ b/api/test/__mocks__/logger.js @@ -24,6 +24,8 @@ jest.mock('winston', () => { warn: jest.fn(), debug: jest.fn(), error: jest.fn(), + /** Real winston derives `isEnabled` from its levels; the delivery log gates on it. */ + isDebugEnabled: jest.fn(() => false), }), transports: { Console: jest.fn(), diff --git a/client/src/components/Chat/Messages/Content/__tests__/FilePreviewDialog.test.ts b/client/src/components/Chat/Messages/Content/__tests__/FilePreviewDialog.test.ts index c8c0c9edb22..f76f7390706 100644 --- a/client/src/components/Chat/Messages/Content/__tests__/FilePreviewDialog.test.ts +++ b/client/src/components/Chat/Messages/Content/__tests__/FilePreviewDialog.test.ts @@ -22,6 +22,15 @@ describe('FilePreviewDialog text-source behavior', () => { expect(getDownloadFilename('report.pdf', 'file-3', FileSources.local)).toBe('report.pdf'); }); + it('does not decode an Office Open XML package as text', () => { + const xlsx = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'; + const docx = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'; + expect(getPreviewKind('quarterly.xlsx', xlsx, FileSources.local)).toBe(false); + expect(getPreviewKind('brief.docx', docx, FileSources.local)).toBe(false); + expect(getPreviewKind('feed.xml', 'application/xml', FileSources.local)).toBe('text'); + expect(getPreviewKind('logo.svg', 'image/svg+xml', FileSources.local)).toBe('text'); + }); + it('uses stored extracted text when the delivery path is text', () => { expect(shouldUseExtractedTextPreview('text')).toBe(true); expect(shouldUseExtractedTextPreview('provider')).toBe(false); diff --git a/client/src/components/Chat/Messages/Content/preview.ts b/client/src/components/Chat/Messages/Content/preview.ts index 45f64fa1d68..fdbe9dae61f 100644 --- a/client/src/components/Chat/Messages/Content/preview.ts +++ b/client/src/components/Chat/Messages/Content/preview.ts @@ -51,6 +51,9 @@ export function isExtractedTextPreviewLoading( return !isError && (isInitialLoading || status === 'pending'); } +/** XML proper and XML-based formats such as SVG; Office Open XML packages are zipped binaries. */ +const isXmlMime = (mime: string): boolean => mime === 'application/xml' || mime.endsWith('+xml'); + function getPreviewKindByMime(mime?: string): PreviewKind { if (!mime) { return false; @@ -61,7 +64,7 @@ function getPreviewKindByMime(mime?: string): PreviewKind { if ( mime.startsWith('text/') || mime.includes('json') || - mime.includes('xml') || + isXmlMime(mime) || mime.includes('javascript') || mime.includes('typescript') || mime.includes('yaml') || diff --git a/client/src/components/Messages/Content/Error/registry.ts b/client/src/components/Messages/Content/Error/registry.ts index 3f878637a03..727d872cff9 100644 --- a/client/src/components/Messages/Content/Error/registry.ts +++ b/client/src/components/Messages/Content/Error/registry.ts @@ -23,6 +23,7 @@ export const errorCopy: Record = { [ErrorTypes.MODELS_NOT_LOADED]: 'com_error_models_not_loaded', [ErrorTypes.GOOGLE_TOOL_CONFLICT]: 'com_error_google_tool_conflict', [ErrorTypes.GOOGLE_VIDEO_UNPROCESSABLE]: 'com_error_google_video_unprocessable', + file_search_preparation_failed: 'com_error_file_search_preparation_failed', [ErrorTypes.RESOURCE_RECOVERY_REQUIRED]: 'com_error_resource_recovery_required', [ErrorTypes.STATEFUL_CODE_ENVIRONMENT_NOT_ALLOWED]: 'com_error_stateful_code_environment_not_allowed', diff --git a/client/src/components/Messages/Content/__tests__/Error.spec.tsx b/client/src/components/Messages/Content/__tests__/Error.spec.tsx index 8212906edfe..3b97e59cc13 100644 --- a/client/src/components/Messages/Content/__tests__/Error.spec.tsx +++ b/client/src/components/Messages/Content/__tests__/Error.spec.tsx @@ -211,6 +211,18 @@ describe('Error — reader-facing provider and fallback copy', () => { expectReadable(); }); + it.each([ + '{"code":"file_search_preparation_failed"}', + 'File Search could not prepare the attached files. Try sending the message again.\n' + + '{"code":"file_search_preparation_failed"}', + ])('localizes a search preparation failure from the existing error boundary: %s', (payload) => { + renderError(payload, providerMessage); + + expect(screen.getByText(catalog.com_error_file_search_preparation_failed)).toBeInTheDocument(); + expect(document.body.textContent).not.toContain('{'); + expectReadable(); + }); + it.each([ ['MODEL_NOT_FOUND', 'com_error_model_not_found'], ['MODEL_RATE_LIMIT', 'com_error_model_rate_limit'], diff --git a/client/src/hooks/Files/__tests__/equivalence.spec.tsx b/client/src/hooks/Files/__tests__/equivalence.spec.tsx new file mode 100644 index 00000000000..d367f156270 --- /dev/null +++ b/client/src/hooks/Files/__tests__/equivalence.spec.tsx @@ -0,0 +1,449 @@ +import React from 'react'; +import { RecoilRoot } from 'recoil'; +import { useForm } from 'react-hook-form'; +import { ToastContext } from '@librechat/client'; +import { act, renderHook, waitFor } from '@testing-library/react'; +import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; +import { + Constants, + FileSources, + FileContext, + EModelEndpoint, + mergeFileConfig, +} from 'librechat-data-provider'; +import type { + FileConfig, + TFileUpload, + TConversation, + UploadMutationOptions, +} from 'librechat-data-provider'; +import type { SharePointFile } from '~/data-provider/Files/sharepoint'; +import type { AttachEntry } from '~/hooks/Input/useAttachItems'; +import type { ChatFormValues, ExtendedFile } from '~/common'; +import type { ChatContract } from '~/hooks/Chat/contract'; +import { UploadModalProvider, useUploadModalContext } from '~/Providers/UploadModalContext'; +import useAttachExisting from '~/hooks/Files/useAttachExisting'; +import { DragDropProvider } from '~/Providers/DragDropContext'; +import { ChatFormProvider } from '~/Providers/ChatFormContext'; +import useAttachTarget from '~/hooks/Input/useAttachTarget'; +import useAttachItems from '~/hooks/Input/useAttachItems'; +import { ChatContext } from '~/Providers/ChatContext'; +import useTextarea from '~/hooks/Input/useTextarea'; +import useDragHelpers from '../useDragHelpers'; +import { isUnifiedUploadMode } from '~/utils'; + +/** + * Every way a file reaches the composer in unified mode has to arrive at the server as the same + * request: no destination, marked as a message file, so the server decides how it is read. The + * hooks run for real down to the upload mutation; only the network, the browser's drag source and + * the hooks that have nothing to do with uploads are replaced. + */ + +type RawFileConfig = Parameters[0]; +type ChatFiles = Pick< + ChatContract, + 'index' | 'conversation' | 'isSubmitting' | 'files' | 'setFiles' | 'setFilesLoading' +>; +type Upload = { body: FormData; options: UploadMutationOptions }; +type TestChat = { chat: ChatFiles; staged: () => Map }; +type Attached = { body: FormData; staged: ExtendedFile | undefined; chat: TestChat }; + +const mockShowToast = jest.fn(); +let mockUploads: Upload[] = []; +let mockFileConfig: RawFileConfig; +let mockDrop: ((item: { files: File[] }) => void) | undefined; + +jest.mock('~/data-provider', () => ({ + useGetFileConfig: ({ select }: { select?: (data: RawFileConfig) => FileConfig } = {}) => ({ + data: select != null ? select(mockFileConfig) : mockFileConfig, + isSuccess: true, + isError: false, + isPaused: false, + }), + useGetEndpointsQuery: () => ({ data: undefined }), + useGetAgentByIdQuery: () => ({ data: undefined }), + useGetStartupConfig: () => ({ data: { sharePointFilePickerEnabled: true } }), + useGraphTokenQuery: () => ({ data: { access_token: 'graph-token' } }), + useInteractionHealthCheck: () => async () => true, + useUploadFileMutation: (options: UploadMutationOptions) => ({ + mutate: (body: FormData) => { + mockUploads.push({ body, options }); + }, + }), +})); + +/** The Graph download is an external API; the bytes it returns are what matter here. */ +jest.mock('~/data-provider/Files', () => ({ + useSharePointBatchDownload: () => ({ + mutateAsync: async ({ files }: { files: SharePointFile[] }) => + files.map((file) => new File(['%PDF-'], file.name, { type: 'application/pdf' })), + }), +})); + +/** jsdom cannot originate a native file drag, so the drop target's handler is taken directly. */ +jest.mock('react-dnd', () => ({ + useDrop: (spec: () => { drop: (item: { files: File[] }) => void }) => { + mockDrop = spec().drop; + return [{ canDrop: false, isOver: false }, () => undefined]; + }, +})); + +/* Barrels narrowed to the real modules these hooks use, so loading them does not pull in the + * whole application. */ +jest.mock('~/Providers', () => ({ + ...jest.requireActual( + '~/Providers/AgentsMapContext', + ), + ...jest.requireActual( + '~/Providers/ChatFormContext', + ), + ...jest.requireActual( + '~/Providers/DragDropContext', + ), + ...jest.requireActual( + '~/Providers/UploadModalContext', + ), +})); + +jest.mock('~/hooks/Agents', () => ({ + useAgentCapabilities: jest.requireActual( + '~/hooks/Agents/useAgentCapabilities', + ).default, + useGetAgentsConfig: jest.requireActual( + '~/hooks/Agents/useGetAgentsConfig', + ).default, +})); + +jest.mock('~/hooks', () => ({ + useLocalize: + jest.requireActual('~/hooks/useLocalize').default, + useAgentCapabilities: jest.requireActual( + '~/hooks/Agents/useAgentCapabilities', + ).default, + useGetAgentsConfig: jest.requireActual( + '~/hooks/Agents/useGetAgentsConfig', + ).default, + useAgentToolPermissions: jest.requireActual< + typeof import('~/hooks/Agents/useAgentToolPermissions') + >('~/hooks/Agents/useAgentToolPermissions').default, +})); + +jest.mock('~/hooks/Files', () => ({ + useFileHandlingNoChatContext: jest.requireActual( + '~/hooks/Files/useFileHandling', + ).useFileHandlingNoChatContext, +})); + +/* Composer concerns the paste path renders alongside, none of which touch an upload. */ +jest.mock('~/hooks/Messages/useLatestMessage', () => ({ useLatestMessageMeta: () => undefined })); +jest.mock('~/hooks/Input/useComposerBindings', () => ({ + __esModule: true, + default: () => ({ + shortcutsEnabled: false, + submitOverride: undefined, + yieldedChords: new Set(), + }), +})); +jest.mock('~/hooks/AuthContext', () => ({ useAuthContext: () => ({ user: undefined }) })); + +const conversation = { + conversationId: 'convo-1', + endpoint: EModelEndpoint.agents, + endpointType: EModelEndpoint.agents, + agent_id: Constants.EPHEMERAL_AGENT_ID, +} as TConversation; + +/** The fields every unified upload sends, whichever way the file arrived. */ +const UNIFIED_UPLOAD = { + endpoint: EModelEndpoint.agents, + endpointType: EModelEndpoint.agents, + conversationId: 'convo-1', + message_file: 'true', + agent_id: Constants.EPHEMERAL_AGENT_ID, +}; + +const report = () => new File(['%PDF-'], 'report.pdf', { type: 'application/pdf' }); + +/** What every entry point tells the user when the endpoint refuses the file's type. */ +const REFUSED_TYPE_TOAST = { + message: 'Unsupported file type: application/pdf', + status: 'error', + duration: 5000, +}; + +type Toast = { status?: string }; +const errorToasts = (): Toast[] => + mockShowToast.mock.calls.flatMap(([toast]: [Toast]) => (toast.status === 'error' ? [toast] : [])); + +function createChat(): TestChat { + let staged = new Map(); + const chat: ChatFiles = { + index: 0, + conversation, + isSubmitting: false, + files: staged, + setFiles: (action) => { + staged = typeof action === 'function' ? action(staged) : action; + }, + setFilesLoading: () => undefined, + }; + return { chat, staged: () => staged }; +} + +function providers({ chat }: TestChat) { + const queryClient = new QueryClient(); + return function Providers({ children }: { children: React.ReactNode }) { + const form = useForm(); + return ( + + + + + + + {children} + + + + + + + ); + }; +} + +/** The attach half of the composer palette, wired the way `Palette` wires it. */ +function usePalette({ chat }: TestChat) { + const target = useAttachTarget(conversation, false); + const attach = useAttachItems({ + agentId: conversation.agent_id, + endpoint: conversation.endpoint, + endpointType: target.endpointType, + endpointFileConfig: target.endpointFileConfig, + useResponsesApi: target.useResponsesApi, + conversationId: conversation.conversationId ?? Constants.NEW_CONVO, + conversation, + files: chat.files, + setFiles: chat.setFiles, + setFilesLoading: chat.setFilesLoading, + isUnifiedMode: isUnifiedUploadMode(target.endpointFileConfig, target.canAttach), + }); + const attachExisting = useAttachExisting({ + files: chat.files, + setFiles: chat.setFiles, + conversation, + endpoint: conversation.endpoint, + endpointType: target.endpointType, + endpointFileConfig: target.endpointFileConfig, + }); + return { attach, attachExisting }; +} + +/** What the server learns from an upload, less the per-upload id and the bytes. */ +const uploadFields = (body: FormData): Record => + Object.fromEntries( + Array.from(body.entries()).flatMap<[string, string]>(([key, value]) => + key === 'file_id' || key === 'file' ? [] : [[key, String(value)]], + ), + ); + +/** The fields the send payload copies off a staged file (`useChatFunctions`). */ +const messageFile = (file: ExtendedFile | undefined) => ({ + file_id: file?.file_id, + filepath: file?.filepath, + filename: file?.filename, + type: file?.type, + llmDeliveryPath: file?.llmDeliveryPath, + height: file?.height, + width: file?.width, +}); + +async function settled(test: TestChat): Promise { + await waitFor(() => expect(mockUploads).toHaveLength(1)); + const { body } = mockUploads[0]; + return { body, staged: test.staged().get(String(body.get('file_id'))), chat: test }; +} + +type Dropped = TestChat & { chooserOpen: () => boolean }; + +async function startDrop(file: File): Promise { + const test = createChat(); + const { result } = renderHook( + () => ({ drag: useDragHelpers(), modal: useUploadModalContext() }), + { wrapper: providers(test) }, + ); + act(() => mockDrop?.({ files: [file] })); + return { ...test, chooserOpen: () => result.current.modal.isVisible }; +} + +const pasteEvent = (files: File[]): React.ClipboardEvent => { + const event: Pick, 'preventDefault'> & { + clipboardData: Pick & { files: ArrayLike & Iterable }; + } = { clipboardData: { files, getData: () => '' }, preventDefault: () => undefined }; + return event as React.ClipboardEvent; +}; + +async function startPaste(file: File): Promise { + const test = createChat(); + const { result } = renderHook( + () => + useTextarea({ + textAreaRef: { current: document.createElement('textarea') }, + submitButtonRef: { current: document.createElement('button') }, + setIsScrollable: () => undefined, + enterToSend: true, + }), + { wrapper: providers(test) }, + ); + act(() => result.current.handlePaste(pasteEvent([file]))); + return test; +} + +/** Chooses a palette row, which has to be on offer for the pick to mean anything. */ +function choose(entries: AttachEntry[], id: string) { + const entry = entries.find((candidate) => candidate.id === id); + expect(entry).toBeDefined(); + act(() => entry?.onSelect()); +} + +const changeEvent = (input: HTMLInputElement): React.ChangeEvent => { + const event: Pick, 'target' | 'stopPropagation'> = { + target: input, + stopPropagation: () => undefined, + }; + return event as React.ChangeEvent; +}; + +async function startPick(file: File): Promise { + const test = createChat(); + const { result } = renderHook(() => usePalette(test), { wrapper: providers(test) }); + const input = document.createElement('input'); + input.click = () => undefined; + Object.defineProperty(result.current.attach.inputRef, 'current', { value: input }); + choose(result.current.attach.entries, 'local:unified'); + Object.defineProperty(input, 'files', { value: [file] }); + act(() => result.current.attach.onFileChange(changeEvent(input))); + return test; +} + +async function startSharePoint(file: File): Promise { + const test = createChat(); + const { result } = renderHook(() => usePalette(test), { wrapper: providers(test) }); + choose(result.current.attach.entries, 'sharepoint:unified'); + const picked: SharePointFile = { + id: 'sp-1', + name: file.name, + size: file.size, + webUrl: 'https://contoso.sharepoint.com/report.pdf', + downloadUrl: 'https://graph.microsoft.com/download/sp-1', + driveId: 'drive-1', + itemId: 'item-1', + sharePointItem: {}, + }; + await act(async () => { + await result.current.attach.onSharePointFilesSelected([picked]); + }); + return test; +} + +const drop = async (file: File): Promise => settled(await startDrop(file)); +const paste = async (file: File): Promise => settled(await startPaste(file)); +const pick = async (file: File): Promise => settled(await startPick(file)); +const pickFromSharePoint = async (file: File): Promise => + settled(await startSharePoint(file)); + +beforeAll(() => { + global.URL.createObjectURL = jest.fn(() => 'blob:preview'); + global.URL.revokeObjectURL = jest.fn(); +}); + +beforeEach(() => { + mockUploads = []; + mockDrop = undefined; + mockFileConfig = {}; + mockShowToast.mockClear(); + localStorage.clear(); +}); + +describe('unified upload entry points', () => { + it.each([ + ['drag and drop', drop], + ['a file paste', paste], + ['the local picker', pick], + ['the SharePoint picker', pickFromSharePoint], + ])('uploads through %s with no destination', async (_entry, attach) => { + const { body, staged } = await attach(report()); + expect(uploadFields(body)).toEqual(UNIFIED_UPLOAD); + expect(body.get('file')).toHaveProperty('name', expect.stringMatching(/report\.pdf$/)); + expect(staged).toBeDefined(); + expect(staged?.tool_resource).toBeUndefined(); + }); + + it.each([ + ['drag and drop', startDrop], + ['the SharePoint picker', startSharePoint], + ])( + 'refuses a type the endpoint does not accept through %s the same way', + async (_entry, start) => { + mockFileConfig = { + endpoints: { [EModelEndpoint.agents]: { supportedMimeTypes: ['^image/png$'] } }, + }; + const test = await start(report()); + await waitFor(() => expect(errorToasts()).toEqual([REFUSED_TYPE_TOAST])); + expect(mockUploads).toHaveLength(0); + expect(test.staged().size).toBe(0); + }, + ); + + it.each([ + [{}, false], + [{ legacyFileUploadUX: true }, true], + ])( + 'offers the destination chooser on a drop only where the deployment keeps the legacy menu (%o: %s)', + async (fileConfig, chooserOpen) => { + mockFileConfig = fileConfig; + const test = await startDrop(report()); + await waitFor(() => + expect({ chooserOpen: test.chooserOpen(), uploads: mockUploads.length }).toEqual({ + chooserOpen, + uploads: chooserOpen ? 0 : 1, + }), + ); + }, + ); + + it('stages a re-attached file exactly as the upload that produced it', async () => { + const uploaded = await pick(report()); + const tempId = String(uploaded.body.get('file_id')); + const record: TFileUpload = { + user: 'user-1', + file_id: 'server-file', + temp_file_id: tempId, + bytes: 5, + embedded: false, + filename: 'report.pdf', + filepath: '/uploads/user-1/server-file__report.pdf', + object: 'file', + type: 'application/pdf', + usage: 0, + source: FileSources.local, + context: FileContext.message_attachment, + llmDeliveryPath: 'none', + }; + act(() => mockUploads[0].options.onSuccess?.(record, uploaded.body)); + await waitFor(() => expect(uploaded.chat.staged().get(tempId)?.progress).toBe(1)); + + const reattached = createChat(); + const { result } = renderHook(() => usePalette(reattached), { + wrapper: providers(reattached), + }); + act(() => result.current.attachExisting(record)); + + const fromUpload = uploaded.chat.staged().get(tempId); + const fromLibrary = reattached.staged().get(record.file_id); + expect(fromLibrary).toBeDefined(); + expect(messageFile(fromLibrary)).toEqual(messageFile(fromUpload)); + expect(fromLibrary?.tool_resource).toBeUndefined(); + expect(fromUpload?.tool_resource).toBeUndefined(); + }); +}); diff --git a/client/src/locales/en/translation.json b/client/src/locales/en/translation.json index 1d776fdc72e..3e5aad5efc0 100644 --- a/client/src/locales/en/translation.json +++ b/client/src/locales/en/translation.json @@ -621,6 +621,7 @@ "com_error_file_upload_limit": "You have reached the upload limit of {{0}} files per {{1}}.", "com_error_files_dupe": "Duplicate file detected.", "com_error_files_empty": "Empty files are not allowed.", + "com_error_file_search_preparation_failed": "File Search couldn't prepare the attached files. Send this message again to retry.", "com_error_files_process": "An error occurred while processing the file.", "com_error_files_skipped_dupe": "Skipped duplicate file(s): {{0}}", "com_error_files_skipped_size": "Skipped file(s) over the {{0}} MB size limit: {{1}}", diff --git a/client/src/routes/Search.tsx b/client/src/routes/Search.tsx index 76e214f384c..15a09c0ee1d 100644 --- a/client/src/routes/Search.tsx +++ b/client/src/routes/Search.tsx @@ -3,6 +3,7 @@ import { useAtomValue } from 'jotai'; import throttle from 'lodash/throttle'; import { useRecoilValue } from 'recoil'; import { Search as SearchIcon, SearchX } from 'lucide-react'; +import { hydrateMessageFile } from 'librechat-data-provider'; import { List, CellMeasurer, CellMeasurerCache } from 'react-virtualized'; import { EmptyState, Spinner, useRemScale, useToastContext } from '@librechat/client'; import type { Index, ListRowProps } from 'react-virtualized'; @@ -119,7 +120,7 @@ export default function Search() { } return { ...message, - files: message.files.map((file) => fileMap[file.file_id ?? ''] ?? file), + files: message.files.map((file) => hydrateMessageFile(file, fileMap)), }; }), ) ?? [], diff --git a/client/src/utils/thread.ts b/client/src/utils/thread.ts index 1a53d2c33c0..529d51347ff 100644 --- a/client/src/utils/thread.ts +++ b/client/src/utils/thread.ts @@ -1,3 +1,4 @@ +import { hydrateMessageFile } from 'librechat-data-provider'; import type { TFile, TMessage } from 'librechat-data-provider'; /** Parent key under which parentless (and self-parented) messages are filed. */ @@ -53,7 +54,7 @@ function hydrateFiles(message: TMessage, fileMap?: Record): TMess } return { ...message, - files: message.files.map((file) => fileMap[file.file_id ?? ''] ?? file), + files: message.files.map((file) => hydrateMessageFile(file, fileMap)), }; } diff --git a/e2e/config/librechat.e2e.yaml b/e2e/config/librechat.e2e.yaml index be32ca709e9..e4cdd8d8820 100644 --- a/e2e/config/librechat.e2e.yaml +++ b/e2e/config/librechat.e2e.yaml @@ -99,6 +99,10 @@ mcpServers: # 'provider', so that test's CSV still reaches the model. # - Mock Provider B uses the unified single button with spreadsheet/csv routed # to llmDeliveryPath 'none' (unified-upload.spec.ts). +# - Mock Auto Provider and Mock Auto Small Provider select the automatic reading +# policy with no route overrides (automatic-upload.spec.ts). Auto Small's 1 MB +# fileSizeLimit is the hard limit for new uploads there and the native size +# limit for PDFs uploaded elsewhere. fileConfig: endpoints: 'Mock Run Files': @@ -117,6 +121,11 @@ fileConfig: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': 'none' # Routed to the provider: still delivered to the model AND shown in chat. 'text/markdown': 'provider' + 'Mock Auto Provider': + llmDeliveryPolicy: automatic + 'Mock Auto Small Provider': + llmDeliveryPolicy: automatic + fileSizeLimit: 1 endpoints: # Default capabilities plus run_in_background (off by default upstream) so the @@ -222,6 +231,28 @@ endpoints: titleConvo: false modelDisplayLabel: 'Mock Provider B' + # The automatic reading policy (see fileConfig above). Not mirrored by a model + # spec, so each stays a plain endpoint → model option in the selector. + - name: 'Mock Auto Provider' + apiKey: 'e2e-mock-key-auto' + baseURL: 'http://127.0.0.1:8889/v1' + models: + default: + - 'mock-auto-model' + fetch: false + titleConvo: false + modelDisplayLabel: 'Mock Auto Provider' + + - name: 'Mock Auto Small Provider' + apiKey: 'e2e-mock-key-auto-small' + baseURL: 'http://127.0.0.1:8889/v1' + models: + default: + - 'mock-auto-small-model' + fetch: false + titleConvo: false + modelDisplayLabel: 'Mock Auto Small Provider' + # No model spec mirrors this endpoint's label, so it stays unambiguous in the # selector and gives e2e tests a real ephemeral endpoint → model option. - name: 'Mock Provider C' @@ -299,6 +330,8 @@ modelSpecs: - 'Mock Provider D' - 'Mock Provider E' - 'Mock Provider F' + - 'Mock Auto Provider' + - 'Mock Auto Small Provider' - 'agents' list: - name: 'e2e-mock-provider-a' diff --git a/e2e/setup/fake-code-server.js b/e2e/setup/fake-code-server.js index d441ac6fe15..0c3f17e47e0 100644 --- a/e2e/setup/fake-code-server.js +++ b/e2e/setup/fake-code-server.js @@ -13,6 +13,16 @@ * - `GET /v1/files/:sid` — session liveness (`checkSessionsAlive`). * - `POST /v1/exec` — code execution (`execute_code` tool run). * + * Upload bytes are retained per `fileId` (64 MB in total, oldest evicted first) and + * each debug record carries the bytes' `sha256`. An exec whose code contains + * `E2E_WORKBOOK:` parses the retained upload the exec itself mounted + * under that name (an entry of its `files`, as the sandbox would see it in + * /mnt/data) with the hoisted SheetJS and prints `{ sheets, totals }` (column-B + * sums per sheet); `E2E_CSV:` prints `{ rows, total }` for a CSV or + * TSV upload. An exec that mounted no such file fails the analysis, as a real + * sandbox would find no file. Each exec record lists the files it mounted and + * the one it analyzed, so specs can tie an answer to the bytes the user attached. + * * Uploads are grouped into a deterministic `storage_session_id` per `kind:id`, * mirroring codeapi's sessionKey bucketing so liveness checks resolve. Every * request is recorded and surfaced at `GET /__debug/uploads` so specs can assert @@ -20,17 +30,27 @@ */ const http = require('http'); +const path = require('path'); const busboy = require('busboy'); -const { randomUUID } = require('crypto'); +const { createHash, randomUUID } = require('crypto'); +/** The repository's hoisted SheetJS, the same build LibreChat parses workbooks with. */ +const XLSX = require(path.resolve(__dirname, '../../node_modules/xlsx')); const PORT = parseInt(process.env.E2E_CODE_API_PORT || '8790', 10); const HOST = '127.0.0.1'; +const RETAINED_BYTES_LIMIT = 64 * 1024 * 1024; +const WORKBOOK_TOKEN = /E2E_WORKBOOK:(\S+)/; +const CSV_TOKEN = /E2E_CSV:(\S+)/; /** @type {Map>} */ const sessions = new Map(); -/** @type {Array<{ filename: string; kind: string; id: string; storage_session_id: string; fileId: string; apiKey: string; userId: string; bytes: number }>} */ +/** @type {Array<{ filename: string; kind: string; id: string; storage_session_id: string; fileId: string; apiKey: string; userId: string; bytes: number; sha256: string }>} */ const uploads = []; -/** @type {Array<{ lang: string; codeLength: number; fileCount: number }>} */ +/** Upload bytes by `fileId`, in arrival order so the oldest is evicted first. */ +/** @type {Map} */ +const retained = new Map(); +let retainedBytes = 0; +/** @type {Array<{ lang: string; codeLength: number; fileCount: number; files: Array<{ id: string; name: string }>; analyzedFileId?: string }>} */ const execs = []; /** @type {Map} */ const generated = new Map(); @@ -39,15 +59,45 @@ function sessionIdFor(kind, id) { return `sess-${kind || 'user'}-${id || 'anon'}`; } -function recordUpload({ kind, id, filename, bytes, apiKey, userId }) { - const storage_session_id = sessionIdFor(kind, id); +const sha256 = (content) => createHash('sha256').update(content).digest('hex'); + +function retainUpload(fileId, filename, content) { + retained.set(fileId, { filename, content }); + retainedBytes += content.length; + for (const [retainedId, entry] of retained) { + if (retainedBytes <= RETAINED_BYTES_LIMIT || retainedId === fileId) { + return; + } + retained.delete(retainedId); + retainedBytes -= entry.content.length; + } +} + +function clearRetained() { + retained.clear(); + retainedBytes = 0; +} + +/** Records one received file under its session and keeps its bytes for later execs. */ +function storeUpload({ kind, id, storage_session_id, file, apiKey, userId }) { const fileId = `fid-${randomUUID()}`; if (!sessions.has(storage_session_id)) { sessions.set(storage_session_id, []); } - sessions.get(storage_session_id).push({ fileId, filename }); - uploads.push({ filename, kind, id, storage_session_id, fileId, apiKey, userId, bytes }); - return { storage_session_id, fileId }; + sessions.get(storage_session_id).push({ fileId, filename: file.filename }); + uploads.push({ + filename: file.filename, + kind, + id, + storage_session_id, + fileId, + apiKey, + userId, + bytes: file.bytes, + sha256: sha256(file.content), + }); + retainUpload(fileId, file.filename, file.content); + return fileId; } function sendJson(res, status, body) { @@ -60,18 +110,20 @@ function parseMultipart(req) { const bb = busboy({ headers: req.headers }); /** @type {Record} */ const fields = {}; - /** @type {Array<{ field: string; filename: string; bytes: number }>} */ + /** @type {Array<{ field: string; filename: string; bytes: number; content: Buffer }>} */ const files = []; bb.on('field', (name, value) => { fields[name] = value; }); bb.on('file', (name, stream, info) => { - let bytes = 0; + /** @type {Buffer[]} */ + const chunks = []; stream.on('data', (chunk) => { - bytes += chunk.length; + chunks.push(chunk); }); stream.on('end', () => { - files.push({ field: name, filename: info.filename, bytes }); + const content = Buffer.concat(chunks); + files.push({ field: name, filename: info.filename, bytes: content.length, content }); }); stream.on('error', reject); }); @@ -104,15 +156,14 @@ async function handleUpload(req, res) { sendJson(res, 400, { message: 'error', error: 'no file provided' }); return; } - const apiKey = req.headers['x-api-key'] || ''; - const userId = req.headers['user-id'] || ''; - const { storage_session_id, fileId } = recordUpload({ + const storage_session_id = sessionIdFor(fields.kind, fields.id); + const fileId = storeUpload({ kind: fields.kind, id: fields.id, - filename: files[0].filename, - bytes: files[0].bytes, - apiKey, - userId, + storage_session_id, + file: files[0], + apiKey: req.headers['x-api-key'] || '', + userId: req.headers['user-id'] || '', }); sendJson(res, 200, { message: 'success', @@ -131,20 +182,13 @@ async function handleUploadBatch(req, res) { const userId = req.headers['user-id'] || ''; const storage_session_id = sessionIdFor(fields.kind, fields.id); const responseFiles = files.map((file) => { - const fileId = `fid-${randomUUID()}`; - if (!sessions.has(storage_session_id)) { - sessions.set(storage_session_id, []); - } - sessions.get(storage_session_id).push({ fileId, filename: file.filename }); - uploads.push({ - filename: file.filename, + const fileId = storeUpload({ kind: fields.kind, id: fields.id, storage_session_id, - fileId, + file, apiKey, userId, - bytes: file.bytes, }); return { status: 'success', fileId, filename: file.filename }; }); @@ -193,12 +237,84 @@ function handleRunFileVersion(body, res, label, operation) { }); } +/** The files an exec mounted, as `{ id, name }`. */ +const mountedFiles = (body) => + (Array.isArray(body.files) ? body.files : []).map((file) => ({ + id: String(file?.id ?? ''), + name: String(file?.name ?? ''), + })); + +/** The mounted file of that name whose bytes were uploaded here; nothing else is in the sandbox. */ +function findMountedFile(files, filename) { + const mounted = files.find((file) => file.name === filename && retained.has(file.id)); + return mounted ? { fileId: mounted.id, ...retained.get(mounted.id) } : undefined; +} + +const sumColumnB = (rows) => + rows.reduce((total, row) => (typeof row?.[1] === 'number' ? total + row[1] : total), 0); + +function summarizeWorkbook(content) { + const workbook = XLSX.read(content, { type: 'buffer' }); + const totals = Object.fromEntries( + workbook.SheetNames.map((name) => [ + name, + sumColumnB(XLSX.utils.sheet_to_json(workbook.Sheets[name], { header: 1, raw: true })), + ]), + ); + return { sheets: workbook.SheetNames, totals }; +} + +/** Data rows after the header, and the sum of their numeric column-B cells. */ +function summarizeDelimited(content, filename) { + const lines = content + .toString('utf8') + .split(/\r?\n/) + .filter((line) => line.trim() !== ''); + const separator = + filename.toLowerCase().endsWith('.tsv') || lines[0]?.includes('\t') ? '\t' : ','; + const rows = lines.slice(1).map((line) => line.split(separator)); + const total = rows.reduce((sum, cells) => { + const amount = Number(cells[1]); + return Number.isFinite(amount) ? sum + amount : sum; + }, 0); + return { rows: rows.length, total }; +} + +/** + * Answers the analysis tokens from the upload the exec mounted, or null when the code has none. + * The analyzed upload's id comes back with the output so the exec record can name it. + */ +function analyzeMountedFile(code, files) { + const workbookName = code.match(WORKBOOK_TOKEN)?.[1]; + const csvName = code.match(CSV_TOKEN)?.[1]; + const filename = workbookName ?? csvName; + if (!filename) { + return null; + } + const file = findMountedFile(files, filename); + if (!file) { + const names = files.map((entry) => entry.name).join(', ') || 'none'; + return { + stdout: `E2E file analysis failed: the exec mounted no upload named ${filename} (mounted: ${names})\n`, + }; + } + const summary = workbookName + ? summarizeWorkbook(file.content) + : summarizeDelimited(file.content, filename); + return { stdout: `${JSON.stringify(summary)}\n`, fileId: file.fileId }; +} + async function handleExec(req, res) { const body = await readJson(req); + const code = typeof body.code === 'string' ? body.code : ''; + const files = mountedFiles(body); + const analysis = analyzeMountedFile(code, files); execs.push({ lang: body.lang || '', - codeLength: typeof body.code === 'string' ? body.code.length : 0, - fileCount: Array.isArray(body.files) ? body.files.length : 0, + codeLength: code.length, + fileCount: files.length, + files, + ...(analysis?.fileId != null && { analyzedFileId: analysis.fileId }), }); const versionMarker = body.code?.match( /E2E_RUN_FILE_VERSION:([A-Za-z0-9-]+):(write-v1|inspect|write-v2)/, @@ -207,6 +323,10 @@ async function handleExec(req, res) { handleRunFileVersion(body, res, versionMarker[1], versionMarker[2]); return; } + if (analysis != null) { + sendJson(res, 200, { stdout: analysis.stdout, stderr: '', files: [] }); + return; + } const runFileLabel = body.code?.match(/E2E_RUN_FILE_ARTIFACT:([A-Za-z0-9-]+)/)?.[1]; if (runFileLabel) { const sessionId = `e2e-run-files-${runFileLabel}`; @@ -245,6 +365,7 @@ const server = http.createServer((req, res) => { if (pathname === '/__debug/reset' && req.method === 'POST') { sessions.clear(); generated.clear(); + clearRetained(); uploads.length = 0; execs.length = 0; sendJson(res, 200, { ok: true }); diff --git a/e2e/setup/fake-model.js b/e2e/setup/fake-model.js index 7035ba39328..14d3bae9715 100644 --- a/e2e/setup/fake-model.js +++ b/e2e/setup/fake-model.js @@ -8,7 +8,16 @@ * This exercises the real `Run.create` -> graph -> tool-node pipeline end to end * without a live provider or a standalone HTTP mock server: responses are decided * from the conversation and the agents' advertised tools. + * + * The hook runs inside the server process, so it has no HTTP surface of its own. + * Each model invocation's prompt (system instructions plus messages, inline file + * data left out and document parts listed by file name) is written to + * `e2e/.generated/last-request.json` (override: `E2E_MODEL_REQUEST_LOG`) for the + * most recent runs, which specs read to assert what the model was actually sent. */ +const fs = require('fs'); +const path = require('path'); +const { createHash, randomUUID } = require('crypto'); const { FakeChatModel } = require('@librechat/agents'); const { ChatGenerationChunk } = require('@langchain/core/outputs'); const { AIMessageChunk } = require('@langchain/core/messages'); @@ -185,8 +194,28 @@ const SKILL_DESCRIPTION = 'Use this skill to verify LibreChat skill file authoring in mock end-to-end tests.'; const EDITED_SKILL_DESCRIPTION = 'Use this edited skill to verify LibreChat skill file authoring in mock end-to-end tests.'; +/** + * Prompts that make the model read an attachment with Run Code: the model calls the code tool + * with `` for the file the instructions advertise to Run Code, then answers + * with the tool output verbatim. The fake code server parses the retained upload for the token. + */ +const FILE_ANALYSIS_TARGETS = [ + { marker: 'E2E_ANALYZE_WORKBOOK', token: 'E2E_WORKBOOK:', extensions: ['.xlsx', '.xls', '.ods'] }, + { marker: 'E2E_ANALYZE_CSV', token: 'E2E_CSV:', extensions: ['.csv', '.tsv'] }, +]; +const FILE_ANALYSIS_TOOLS = [BASH_TOOL_NAME, 'execute_code']; +/** A path the queued-file note or the file inventory gives Run Code for an attachment. */ +const ADVERTISED_CODE_PATH = /(?:\/mnt\/data|\$LIBRECHAT_CODE_DATA_DIR)\/(\S+)/g; +const MODEL_REQUEST_LOG_PATH = + process.env.E2E_MODEL_REQUEST_LOG || path.resolve(__dirname, '../.generated/last-request.json'); +const MODEL_REQUEST_LOG_RUNS = 8; +const MODEL_REQUEST_TEXT_LIMIT = 4_000_000; const countedReplies = new Map(); const slowCountedReplies = new Map(); +/** @type {Array<{ sequence: number; conversationId: string | null; userText: string; invocations: Array<{ systemText: string; promptText: string; documentFiles: string[] }> }>} */ +const modelRequestRuns = []; +let modelRequestSequence = 0; +let modelRequestWrite = Promise.resolve(); function messageType(message) { if (typeof message.getType === 'function') { @@ -815,6 +844,73 @@ function replyResponses(text) { return null; } +/** Text a provider would read from a message part; inline file data is left out. */ +function readableText(value, parts = []) { + if (value == null) { + return parts; + } + if (typeof value === 'string') { + if (!value.startsWith('data:')) { + parts.push(value); + } + return parts; + } + if (Array.isArray(value)) { + value.forEach((item) => readableText(item, parts)); + return parts; + } + if (typeof value === 'object' && !ArrayBuffer.isView(value)) { + Object.values(value).forEach((child) => readableText(child, parts)); + } + return parts; +} + +function describePromptMessage(message) { + const text = readableText(message?.content).join('\n'); + const toolCalls = message?.tool_calls?.length ? `\n${JSON.stringify(message.tool_calls)}` : ''; + return `${messageType(message)}: ${text}${toolCalls}`; +} + +function summarizeModelRequest(messages) { + const promptText = messages.map(describePromptMessage).join('\n'); + return { + systemText: collectSystemPromptText(messages), + promptText: + promptText.length > MODEL_REQUEST_TEXT_LIMIT + ? `${promptText.slice(0, MODEL_REQUEST_TEXT_LIMIT)}\n[truncated]` + : promptText, + documentFiles: [...collectProviderFileNames(messages.map((message) => message?.content))], + }; +} + +/** Writes the recent runs in order; a failed write is logged and never fails the run. */ +function persistModelRequestLog() { + const snapshot = JSON.stringify({ runs: modelRequestRuns }); + modelRequestWrite = modelRequestWrite + .then(() => fs.promises.mkdir(path.dirname(MODEL_REQUEST_LOG_PATH), { recursive: true })) + .then(() => fs.promises.writeFile(MODEL_REQUEST_LOG_PATH, snapshot)) + .catch((error) => { + console.warn(`[e2e] fake-model request log not written: ${error?.message ?? error}`); + }); +} + +/** Opens a run's entry in the request log and returns the per-invocation recorder. */ +function startModelRequestLog({ conversationId, userText }) { + modelRequestSequence += 1; + const run = { + sequence: modelRequestSequence, + conversationId: conversationId ?? null, + userText, + invocations: [], + }; + modelRequestRuns.push(run); + modelRequestRuns.splice(0, Math.max(0, modelRequestRuns.length - MODEL_REQUEST_LOG_RUNS)); + return (promptMessages) => { + run.invocations.push(summarizeModelRequest(promptMessages)); + persistModelRequestLog(); + }; +} + /** * Attaches synthetic usage_metadata on a final empty chunk (the OpenAI * streaming pattern) so token-usage SSE events flow end to end in mock runs. @@ -822,11 +918,12 @@ function replyResponses(text) { * instructions included — since the context snapshot calibrates against it. */ class UsageEmittingFakeChatModel extends FakeChatModel { - constructor({ graph, resolveInvocation, resolveOnStream, sleep, ...options }) { + constructor({ graph, resolveInvocation, resolveOnStream, recordRequest, sleep, ...options }) { super({ ...options, sleep }); this.graph = graph; this.resolveInvocation = resolveInvocation; this.resolveOnStream = resolveOnStream; + this.recordRequest = recordRequest; this.streamSleep = sleep ?? CHUNK_DELAY_MS; } @@ -921,6 +1018,13 @@ class UsageEmittingFakeChatModel extends FakeChatModel { async *_streamResponseChunks(messages, options, runManager) { let outputChars = 0; + const { messages: promptMessages } = await getStreamAgentView({ + graph: this.graph, + messages: messages ?? [], + options, + runManager, + }); + this.recordRequest?.(promptMessages); const scriptedResponse = await this.resolveInvocation?.(messages, options, runManager); const dynamicResponse = scriptedResponse ? null @@ -948,12 +1052,6 @@ class UsageEmittingFakeChatModel extends FakeChatModel { outputChars += typeof chunk.text === 'string' ? chunk.text.length : 0; yield chunk; } - const { messages: promptMessages } = await getStreamAgentView({ - graph: this.graph, - messages: messages ?? [], - options, - runManager, - }); const inputChars = promptMessages.reduce( (sum, message) => sum + getContentText(message?.content).length, 0, @@ -980,6 +1078,7 @@ function overrideModel({ disableHumanInTheLoop, resolveInvocation, resolveOnStream, + recordRequest, modelCallbacks, }) { /** The shared mock profile enables approval HITL for its dedicated specs. @@ -1012,6 +1111,7 @@ function overrideModel({ toolCalls, resolveInvocation, resolveOnStream, + recordRequest, }); model.callbacks = modelCallbacks; graph.overrideModel = model; @@ -3384,7 +3484,7 @@ function provisioningToolResponses({ text, toolNames }) { responses: ['', `${FILE_SEARCH_FINAL_TEXT}: ${searchLabel}`], toolCalls: [ { - id: FILE_SEARCH_TOOL_CALL_ID, + id: `${FILE_SEARCH_TOOL_CALL_ID}_${createHash('sha256').update(searchLabel).digest('hex').slice(0, 12)}`, name: FILE_SEARCH_TOOL_NAME, args: { query: `e2e ${searchLabel}` }, type: 'tool_call', @@ -3396,6 +3496,71 @@ function provisioningToolResponses({ text, toolNames }) { return null; } +/** The base name of the first advertised code path with one of `extensions`. */ +function findAdvertisedCodeFile(systemText, extensions) { + return ( + [...systemText.matchAll(ADVERTISED_CODE_PATH)] + .map(([, advertised]) => path.posix.basename(advertised.replace(/[.,;:)"'`]+$/, ''))) + .find((name) => extensions.some((extension) => name.toLowerCase().endsWith(extension))) ?? '' + ); +} + +const fileAnalysisArgs = (toolName, code) => + toolName === BASH_TOOL_NAME ? { command: `# ${code}` } : { lang: 'py', code: `# ${code}` }; + +/** + * Reads an attachment through Run Code. The file name comes from the prompt the model is sent, + * so the call names exactly the path the run advertised, and the answer is the tool output. + */ +function fileAnalysisResponses({ graph, text, toolNames }) { + const target = FILE_ANALYSIS_TARGETS.find(({ marker }) => text.includes(marker)); + if (!target) { + return null; + } + const toolName = FILE_ANALYSIS_TOOLS.find((name) => toolNames.has(name)); + if (!toolName) { + return { + responses: [ + `E2E file analysis unavailable: no code-execution tool advertised (saw ${JSON.stringify([ + ...toolNames, + ])}).`, + ], + }; + } + /** Unique per run, so an analysis earlier in the conversation is never read as this one. */ + const toolCallId = `call_e2e_file_analysis_${randomUUID().slice(0, 8)}`; + return { + responses: [''], + resolveInvocation: async (messages, options, runManager) => { + const result = findLastToolMessage(messages, toolCallId); + if (result) { + return { response: getContentText(result.content) }; + } + const view = await getStreamAgentView({ graph, messages, options, runManager }); + const filename = findAdvertisedCodeFile( + collectSystemPromptText(view.messages), + target.extensions, + ); + if (!filename) { + return { + response: `E2E file analysis failed: no ${target.extensions.join('/')} file was advertised to Run Code.`, + }; + } + return { + response: '', + toolCalls: [ + { + id: toolCallId, + name: toolName, + args: fileAnalysisArgs(toolName, `${target.token}${filename}`), + type: 'tool_call', + }, + ], + }; + }, + }; +} + function codeExecResponses({ filename, toolCallId, finalText, code }, toolNames) { if (!toolNames.has(BASH_TOOL_NAME)) { return { @@ -3514,6 +3679,11 @@ function resolveResponses({ graph, messages, text, toolNames }) { return mcpLegacyActionResponses(toolNames); } + const fileAnalysis = fileAnalysisResponses({ graph, text, toolNames }); + if (fileAnalysis) { + return fileAnalysis; + } + const provisioningTool = provisioningToolResponses({ text, toolNames }); if (provisioningTool) { return provisioningTool; @@ -3735,6 +3905,10 @@ module.exports = function fakeModelHook(run, context) { ) { return; } + const recordRequest = startModelRequestLog({ + conversationId: context?.conversationId, + userText: text, + }); const toolNames = collectToolNames(context?.agents); const handoffScript = parseHandoffScript(text); const { @@ -3777,6 +3951,7 @@ module.exports = function fakeModelHook(run, context) { approvalOutcomeResponses(streamMessages) ?? resolveOnStream?.(streamMessages, streamOptions, runManager) ?? null, + recordRequest, modelCallbacks: context?.modelCallbacks, }); }; diff --git a/e2e/setup/fake-rag-server.js b/e2e/setup/fake-rag-server.js index 20469df2e0b..71f5170acd9 100644 --- a/e2e/setup/fake-rag-server.js +++ b/e2e/setup/fake-rag-server.js @@ -15,6 +15,9 @@ * * Every embed is recorded and surfaced at `GET /__debug/embedded` so specs can * assert a file's bytes actually reached the RAG env, independent of the DB write. + * Embeds and queries share one sequence (`seq`, never reset): an embed takes its + * number once its upload is fully received and a query when it arrives, so a spec + * can show a file was indexed before it was searched. */ const http = require('http'); @@ -23,12 +26,16 @@ const busboy = require('busboy'); const PORT = parseInt(process.env.E2E_RAG_API_PORT || '8791', 10); const HOST = '127.0.0.1'; -/** @type {Array<{ file_id: string; filename: string; entity_id: string; bytes: number; auth: string }>} */ +/** @type {Array<{ seq: number; file_id: string; filename: string; entity_id: string; bytes: number; auth: string }>} */ const embedded = []; -/** @type {Array<{ file_id: string; query: string }>} */ +/** @type {Array<{ seq: number; file_id: string; query: string }>} */ const queries = []; +let sequence = 0; /** @type {string[]} */ const deleted = []; +/** Per-file failures let retry tests leave every other upload unaffected. */ +const embeddingFailures = new Set(); +const failedEmbeds = []; function sendJson(res, status, body) { res.writeHead(status, { 'Content-Type': 'application/json' }); @@ -80,7 +87,13 @@ function readJson(req) { async function handleEmbed(req, res) { const { fields, files } = await parseMultipart(req); + if (embeddingFailures.has(fields.file_id)) { + failedEmbeds.push({ seq: ++sequence, file_id: fields.file_id }); + sendJson(res, 503, { status: false, message: 'E2E embedding unavailable' }); + return; + } embedded.push({ + seq: ++sequence, file_id: fields.file_id || '', filename: files[0]?.filename || '', entity_id: fields.entity_id || '', @@ -91,8 +104,9 @@ async function handleEmbed(req, res) { } async function handleQuery(req, res) { + const seq = ++sequence; const body = await readJson(req); - queries.push({ file_id: body.file_id || '', query: body.query || '' }); + queries.push({ seq, file_id: body.file_id || '', query: body.query || '' }); sendJson(res, 200, []); } @@ -116,7 +130,7 @@ const server = http.createServer((req, res) => { } if (pathname === '/__debug/embedded' && req.method === 'GET') { - sendJson(res, 200, { embedded, queries, deleted }); + sendJson(res, 200, { embedded, queries, deleted, failedEmbeds }); return; } @@ -124,6 +138,23 @@ const server = http.createServer((req, res) => { embedded.length = 0; queries.length = 0; deleted.length = 0; + failedEmbeds.length = 0; + embeddingFailures.clear(); + sendJson(res, 200, { ok: true }); + return; + } + + if (pathname === '/__debug/embedding-failure' && req.method === 'POST') { + const body = await readJson(req); + if (typeof body.file_id !== 'string' || typeof body.enabled !== 'boolean') { + sendJson(res, 400, { message: 'file_id and enabled are required' }); + return; + } + if (body.enabled) { + embeddingFailures.add(body.file_id); + } else { + embeddingFailures.delete(body.file_id); + } sendJson(res, 200, { ok: true }); return; } diff --git a/e2e/setup/global-setup.ts b/e2e/setup/global-setup.ts index c417ebc3409..267a1c996b8 100644 --- a/e2e/setup/global-setup.ts +++ b/e2e/setup/global-setup.ts @@ -1,8 +1,10 @@ import { FullConfig } from '@playwright/test'; +import { writeUploadFixtures } from './uploads'; import authenticate from './authenticate'; import { getE2EUser } from './user'; async function globalSetup(config: FullConfig) { + writeUploadFixtures(); await authenticate(config, getE2EUser()); } diff --git a/e2e/setup/uploads.ts b/e2e/setup/uploads.ts new file mode 100644 index 00000000000..a59bcb38502 --- /dev/null +++ b/e2e/setup/uploads.ts @@ -0,0 +1,167 @@ +import fs from 'fs'; +import path from 'path'; +import * as XLSX from 'xlsx'; + +/** + * Upload fixtures for the automatic reading policy (automatic-upload.spec.ts). The mock lane's + * global setup writes them from the rows below; the totals a spec asserts are stated on their own + * and checked against those rows when the fixtures are written, so an edit to either fails there. + */ +export const UPLOAD_FIXTURE_DIR = path.resolve(__dirname, '../.generated/uploads'); + +/** Sits alone in `Notes!A1`, so it reaches the model only through extracted text. */ +export const WORKBOOK_SENTINEL = 'SENTINEL-7F3A'; + +export type SheetRow = readonly [item: string, amount: number]; + +/** Six-digit amounts, so none can be mistaken for an id, a port or a timestamp fragment. */ +export const Q1_ROWS: readonly SheetRow[] = [ + ['North', 731842], + ['South', 905317], + ['East', 648259], + ['West', 517093], +]; + +export const Q2_ROWS: readonly SheetRow[] = [ + ['North', 812406], + ['South', 693178], + ['East', 574921], + ['West', 786350], +]; + +/** Column-B totals the fake code server computes from the uploaded bytes, as known values. */ +export const WORKBOOK_TOTALS = { + Q1: 2_802_511, + Q2: 2_866_855, + Notes: 0, +} as const; + +const sumAmounts = (rows: readonly SheetRow[]): number => + rows.reduce((total, [, amount]) => total + amount, 0); + +function assertKnownTotals(): void { + const computed = { Q1: sumAmounts(Q1_ROWS), Q2: sumAmounts(Q2_ROWS) }; + if (computed.Q1 !== WORKBOOK_TOTALS.Q1 || computed.Q2 !== WORKBOOK_TOTALS.Q2) { + throw new Error( + `Fixture rows sum to ${JSON.stringify(computed)}, not the known totals ${JSON.stringify( + WORKBOOK_TOTALS, + )}`, + ); + } +} + +export type UploadFixture = { name: string; mimeType: string; path: string }; + +const fixture = (name: string, mimeType: string): UploadFixture => ({ + name, + mimeType, + path: path.join(UPLOAD_FIXTURE_DIR, name), +}); + +export const UPLOAD_FIXTURES = { + xlsx: fixture( + 'quarterly.xlsx', + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + ), + csv: fixture('q1.csv', 'text/csv'), + smallPdf: fixture('small.pdf', 'application/pdf'), + largePdf: fixture('large.pdf', 'application/pdf'), +} as const; + +const SHEET_HEADER = ['item', 'amount']; +const SMALL_PDF_BYTES = 50 * 1024; +const LARGE_PDF_BYTES = 3 * 1024 * 1024; + +const sheetRows = (rows: readonly SheetRow[]): (string | number)[][] => [ + SHEET_HEADER, + ...rows.map(([item, amount]) => [item, amount]), +]; + +function buildWorkbook(): XLSX.WorkBook { + const workbook = XLSX.utils.book_new(); + XLSX.utils.book_append_sheet(workbook, XLSX.utils.aoa_to_sheet(sheetRows(Q1_ROWS)), 'Q1'); + XLSX.utils.book_append_sheet(workbook, XLSX.utils.aoa_to_sheet(sheetRows(Q2_ROWS)), 'Q2'); + XLSX.utils.book_append_sheet(workbook, XLSX.utils.aoa_to_sheet([[WORKBOOK_SENTINEL]]), 'Notes'); + return workbook; +} + +const writeWorkbook = (workbook: XLSX.WorkBook, bookType: XLSX.BookType): Buffer => + XLSX.write(workbook, { type: 'buffer', bookType }) as Buffer; + +const delimited = (rows: readonly SheetRow[], separator: string): string => + `${sheetRows(rows) + .map((cells) => cells.join(separator)) + .join('\n')}\n`; + +/** Deterministic filler bytes for the PDF padding stream. */ +function fillerBytes(length: number): Buffer { + const bytes = Buffer.alloc(length); + let state = 0x2545f491; + for (let index = 0; index < length; index++) { + state = (Math.imul(state, 1103515245) + 12345) >>> 0; + bytes[index] = state >>> 24; + } + return bytes; +} + +const ascii = (value: string): Buffer => Buffer.from(value, 'latin1'); + +const streamBody = (data: Buffer): Buffer => + Buffer.concat([ascii(`<< /Length ${data.length} >>\nstream\n`), data, ascii('\nendstream')]); + +/** A one-page PDF with a correct cross-reference table; the extra stream sets its size. */ +function assemblePdf(title: string, padding: Buffer): Buffer { + const header = ascii('%PDF-1.4\n%\xe2\xe3\xcf\xd3\n'); + const bodies = [ + ascii('<< /Type /Catalog /Pages 2 0 R >>'), + ascii('<< /Type /Pages /Kids [3 0 R] /Count 1 >>'), + ascii( + '<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] ' + + '/Resources << /Font << /F1 5 0 R >> >> /Contents 4 0 R >>', + ), + streamBody(ascii(`BT /F1 18 Tf 72 720 Td (${title}) Tj ET`)), + ascii('<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>'), + streamBody(padding), + ]; + const objects = bodies.map((body, index) => + Buffer.concat([ascii(`${index + 1} 0 obj\n`), body, ascii('\nendobj\n')]), + ); + let cursor = header.length; + const offsets = objects.map((object) => { + const offset = cursor; + cursor += object.length; + return offset; + }); + const xref = [ + 'xref', + `0 ${objects.length + 1}`, + '0000000000 65535 f ', + ...offsets.map((offset) => `${String(offset).padStart(10, '0')} 00000 n `), + 'trailer', + `<< /Size ${objects.length + 1} /Root 1 0 R >>`, + 'startxref', + String(cursor), + '%%EOF', + '', + ].join('\n'); + return Buffer.concat([header, ...objects, ascii(xref)]); +} + +function buildPdf(title: string, targetBytes: number): Buffer { + const baseLength = assemblePdf(title, Buffer.alloc(0)).length; + return assemblePdf(title, fillerBytes(Math.max(0, targetBytes - baseLength))); +} + +/** Writes every upload fixture under `e2e/.generated/uploads/`, replacing earlier copies. */ +export function writeUploadFixtures(): void { + assertKnownTotals(); + fs.mkdirSync(UPLOAD_FIXTURE_DIR, { recursive: true }); + const workbook = buildWorkbook(); + const contents: ReadonlyArray = [ + [UPLOAD_FIXTURES.xlsx, writeWorkbook(workbook, 'xlsx')], + [UPLOAD_FIXTURES.csv, Buffer.from(delimited(Q1_ROWS, ','), 'utf8')], + [UPLOAD_FIXTURES.smallPdf, buildPdf('E2E small PDF', SMALL_PDF_BYTES)], + [UPLOAD_FIXTURES.largePdf, buildPdf('E2E large PDF', LARGE_PDF_BYTES)], + ]; + contents.forEach(([{ path: filePath }, content]) => fs.writeFileSync(filePath, content)); +} diff --git a/e2e/specs/mock/automatic-upload.spec.ts b/e2e/specs/mock/automatic-upload.spec.ts new file mode 100644 index 00000000000..d9973af47c6 --- /dev/null +++ b/e2e/specs/mock/automatic-upload.spec.ts @@ -0,0 +1,792 @@ +import path from 'path'; +import { expect, test } from '@playwright/test'; +import { createHash, randomUUID } from 'crypto'; +import type { TFile, TMessage, TextDerivation } from 'librechat-data-provider'; +import type { Page, Response } from '@playwright/test'; +import type { + AttachFile, + MockEndpoint, + ModelRunRecord, + RagQueryRecord, + RagEmbedRecord, + ModelRequestRecord, +} from './helpers'; +import type { UploadFixture } from '../../setup/uploads'; +import { + MOCK_ENDPOINTS, + NEW_CHAT_PATH, + fetchJson, + getModelRun, + uniqueName, + replyPrompt, + sendMessage, + getCodeExecs, + requestJson, + messagesView, + uploadViaDrop, + getRagQueries, + getAccessToken, + getRagEmbedded, + uploadViaPaste, + enableFileSearch, + attachFileBuffer, + resetProvisioning, + supportsFilePaste, + selectMockEndpoint, + enableCodeInterpreter, + uploadViaUnifiedButton, + getCodeProvisionedUploads, + sendMessageAndWaitForCompletion, +} from './helpers'; +import { + Q1_ROWS, + Q2_ROWS, + UPLOAD_FIXTURES, + WORKBOOK_TOTALS, + WORKBOOK_SENTINEL, +} from '../../setup/uploads'; +import { cleanupAgent, uniqueAgentName } from './agents.helpers'; +import { withMongo } from './db'; + +/** + * The automatic reading policy end to end. + * + * Mock Auto Provider and Mock Auto Small Provider (e2e/config/librechat.e2e.yaml) select + * `llmDeliveryPolicy: automatic` with no route overrides; Auto Small adds a 1 MB per-file limit. + * The fixtures come from e2e/setup/uploads.ts, written by the global setup; the totals asserted + * here are known constants checked against the rows the workbook was built from. Rows draw on up + * to three observers: + * + * - the file record: `/api/files` for the route and marker, MongoDB for the stored text the API + * leaves out, and the user message's files for the delivery path the turn used; + * - the fake model's request log (`e2e/.generated/last-request.json`): the system instructions + * with the file inventory, the message text, and the document parts the model was sent; + * - the fake code and RAG servers: the bytes Run Code received (by sha256) and the files each exec + * mounted, and the File Search embeds and queries in the order they arrived. + * + * `E2E_ANALYZE_WORKBOOK` and `E2E_ANALYZE_CSV` make the fake model run code on the file the + * instructions advertise to Run Code. The fake code server answers only from an upload the exec + * itself mounted, and records which one, so a total on screen means code read the attached bytes. + * + * Mock Provider C keeps the classic policy with no file configuration; A-29 shows classic still + * extracts a workbook at upload and writes none of the automatic policy's state. + */ + +const [, , AUTO_ENDPOINT, AUTO_SMALL_ENDPOINT] = MOCK_ENDPOINTS; +const CLASSIC_ENDPOINT: MockEndpoint = { label: 'Mock Provider C', model: 'mock-model-c' }; +const TEST_TIMEOUT = 120_000; +const MULTI_TURN_TIMEOUT = 180_000; +const TURN_TIMEOUT = 60_000; +const WORKBOOK_RESULT = JSON.stringify({ sheets: ['Q1', 'Q2', 'Notes'], totals: WORKBOOK_TOTALS }); +const CSV_RESULT = JSON.stringify({ rows: Q1_ROWS.length, total: WORKBOOK_TOTALS.Q1 }); +/** The error a turn ends with when File Search could not index its attachments. */ +const PREPARATION_FAILED_ERROR = /File Search couldn't prepare the attached files/; +/** The heading of the model's file inventory, which only the automatic policy writes. */ +const INVENTORY_HEADING = 'how you can read them on this turn'; + +type StoredText = { file_id: string; text?: string | null }; +type AgentRecord = { id: string }; +type ReadingRecord = Pick & + Partial>; +type MessageFileReading = Pick; + +const sha256 = (content: Buffer): string => createHash('sha256').update(content).digest('hex'); + +/** The fixture's bytes and type under a name no earlier upload has used. */ +const renamed = (fixture: UploadFixture, prefix: string): AttachFile => ({ + name: `${uniqueName(prefix)}${path.extname(fixture.name)}`, + mimeType: fixture.mimeType, + path: fixture.path, +}); + +const composerChip = (page: Page, filename: string) => + page.getByTestId('composer-tray').getByRole('button', { name: filename }); + +async function startChat(page: Page, endpoint: MockEndpoint) { + await page.goto(NEW_CHAT_PATH, { timeout: 10_000 }); + await selectMockEndpoint(page, endpoint); + await expect(page.getByRole('button', { name: 'Attach and tools', exact: true })).toBeVisible({ + timeout: 15_000, + }); + await resetProvisioning(page); +} + +/** Waits for an upload to land in the composer and returns the record it created. */ +async function confirmUpload(page: Page, upload: Promise): Promise { + const response = await upload; + expect(response.ok(), `upload returned ${response.status()}`).toBeTruthy(); + const uploaded = (await response.json()) as TFile; + await expect(composerChip(page, uploaded.filename)).toBeVisible({ timeout: 15_000 }); + return uploaded; +} + +async function getFileRecord(page: Page, fileId: string): Promise { + const token = await getAccessToken(page); + const files = await fetchJson(page, '/api/files', token); + const record = files.find((file) => file.file_id === fileId); + if (!record) { + throw new Error(`/api/files does not list ${fileId}`); + } + return record; +} + +/** The stored extracted text, which the file API never returns. */ +function getStoredText(fileId: string): Promise { + return withMongo(async (db) => { + const file = await db + .collection('files') + .findOne({ file_id: fileId }, { projection: { text: 1 } }); + return file?.text ?? undefined; + }); +} + +/** What decides how a record is read, without the derivation timestamp. */ +const readingRecord = ({ llmDeliveryPath, metadata }: TFile): ReadingRecord => ({ + llmDeliveryPath, + outcome: metadata?.textDerivation?.outcome, + extractor: metadata?.textDerivation?.extractor, + reason: metadata?.textDerivation?.reason, +}); + +/** Kept for Run Code at upload: no model route, no extracted text, a deferred marker. */ +async function expectDeferredRecord(page: Page, fileId: string) { + expect(readingRecord(await getFileRecord(page, fileId))).toMatchObject({ + llmDeliveryPath: 'none', + outcome: 'deferred', + }); + expect(await getStoredText(fileId), 'a deferred upload stores no extracted text').toBeFalsy(); +} + +function conversationIdFromPage(page: Page): string { + const conversationId = new URL(page.url()).pathname.split('/c/')[1] ?? ''; + expect(conversationId).toMatch(/^[0-9a-f-]{36}$/i); + return conversationId; +} + +/** The files the latest user message persisted, with the delivery path its turn gave each. */ +async function getLatestUserMessageFiles(page: Page): Promise[]> { + const token = await getAccessToken(page); + const conversationId = encodeURIComponent(conversationIdFromPage(page)); + const messages = await fetchJson(page, `/api/messages/${conversationId}`, token); + return messages.filter((message) => message.isCreatedByUser).pop()?.files ?? []; +} + +function messageFileReading(files: Partial[], fileId: string): MessageFileReading { + const file = files.find((entry) => entry.file_id === fileId); + expect(file, `the user message should carry ${fileId}`).toBeTruthy(); + return { llmDeliveryPath: file?.llmDeliveryPath }; +} + +/** Sends a turn, waits for the stored answer, and returns what the model was sent for it. */ +async function sendTurn(page: Page, prompt: string, minInvocations = 1): Promise { + await sendMessageAndWaitForCompletion(page, prompt, { timeout: TURN_TIMEOUT }); + return getModelRun(prompt, { minInvocations }); +} + +/** A turn where the model calls Run Code and then answers, so two model requests. */ +const sendAnalysisTurn = (page: Page, prompt: string): Promise => + sendTurn(page, prompt, 2); + +/** The turn's first model request: what the model had before any tool ran. */ +const initialRequest = (run: ModelRunRecord): ModelRequestRecord => run.invocations[0]; + +/** The inventory line naming `filename`; the queued-path note never quotes a name. */ +const inventoryLine = (request: ModelRequestRecord, filename: string): string => + request.systemText.split('\n').find((line) => line.includes(JSON.stringify(filename))) ?? ''; + +/** + * No request of the turn carried the workbook: not as a file part, and no cell as text (the + * sentinel or an amount). Inline file data never reaches `promptText`, so the parts are checked + * by name. + */ +function expectNoWorkbookCells(run: ModelRunRecord, filename: string) { + expect(run.invocations.flatMap(({ documentFiles }) => documentFiles)).not.toContain(filename); + const sent = run.invocations.map(({ promptText }) => promptText).join('\n'); + expect(sent).not.toContain(WORKBOOK_SENTINEL); + [...Q1_ROWS, ...Q2_ROWS].forEach(([, amount]) => expect(sent).not.toContain(String(amount))); +} + +/** + * Run Code received exactly this upload's bytes under its name, and an exec mounted that upload; + * with `analyzed`, the exec's analysis token read it. Renamed copies of a fixture share its + * bytes, so the name ties the receipt to this test's upload. + */ +async function expectCodeReceived( + page: Page, + file: AttachFile, + { analyzed = false }: { analyzed?: boolean } = {}, +) { + const expected = sha256(attachFileBuffer(file)); + let fileId: string | undefined; + await expect + .poll( + async () => { + const uploads = await getCodeProvisionedUploads(page); + fileId = uploads.find( + (upload) => upload.filename === file.name && upload.sha256 === expected, + )?.fileId; + return fileId != null; + }, + { timeout: 30_000, message: `Run Code should receive the bytes of ${file.name}` }, + ) + .toBe(true); + const execs = await getCodeExecs(page); + const mounted = execs.filter((exec) => exec.files.some(({ id }) => id === fileId)); + expect(mounted.length, `an exec should mount ${file.name}`).toBeGreaterThan(0); + if (analyzed) { + expect(mounted.map((exec) => exec.analyzedFileId)).toContain(fileId); + } +} + +/** The inventory line a deferred workbook gets while Run Code can still open it. */ +const expectQueuedForCode = (request: ModelRequestRecord, filename: string) => + expect(inventoryLine(request, filename)).toContain( + `read it with Run Code at /mnt/data/${filename}`, + ); + +const latestSeq = (records: ReadonlyArray): number => + records.reduce((latest, { seq }) => Math.max(latest, seq), 0); + +/** The assistant's visible answer carries the code server's result. */ +async function expectAnswer(page: Page, expected: string) { + await expect( + messagesView(page).getByText(expected).filter({ visible: true }).first(), + ).toBeVisible({ timeout: 20_000 }); +} + +async function createAutoAgent(page: Page, name: string, tools: string[]): Promise { + const token = await getAccessToken(page); + const agent = await requestJson(page, { + path: '/api/agents', + token, + method: 'POST', + body: { + name, + provider: AUTO_ENDPOINT.label, + model: AUTO_ENDPOINT.model, + model_parameters: {}, + tools, + }, + }); + return agent.id; +} + +async function setAgentTools(page: Page, agentId: string, tools: string[]) { + const token = await getAccessToken(page); + await requestJson(page, { + path: `/api/agents/${encodeURIComponent(agentId)}`, + token, + method: 'PATCH', + body: { tools }, + }); +} + +async function startAgentChat(page: Page, agentName: string) { + await page.goto(NEW_CHAT_PATH, { timeout: 10_000 }); + const trigger = page.getByRole('button', { name: 'Select a model' }).first(); + await trigger.click(); + await page.getByRole('option', { name: 'My Agents' }).click(); + await page.getByRole('option', { name: agentName }).click(); + await expect(trigger).toContainText(agentName); + await resetProvisioning(page); +} + +/** Reloads the open conversation, so the next turn starts from restored state. */ +async function reloadConversation(page: Page) { + await page.reload({ timeout: 20_000 }); + await expect(page.getByRole('textbox', { name: 'Message input' })).toBeVisible({ + timeout: 20_000, + }); +} + +/** Posts an upload straight to the server, past the client's own size check. */ +async function postUpload( + page: Page, + endpoint: MockEndpoint, + file: AttachFile, + toolResource?: 'execute_code' | 'file_search', +) { + const token = await getAccessToken(page); + return page.request.post('/api/files', { + headers: { Authorization: `Bearer ${token}` }, + multipart: { + endpoint: endpoint.label, + endpointType: 'custom', + file_id: randomUUID(), + message_file: 'true', + ...(toolResource != null && { tool_resource: toolResource }), + file: { name: file.name, mimeType: file.mimeType, buffer: attachFileBuffer(file) }, + }, + }); +} + +/** The stored original's bytes, as the download route serves them. */ +async function downloadOriginal(page: Page, record: TFile): Promise { + const token = await getAccessToken(page); + const response = await page.request.get( + `/api/files/download/${encodeURIComponent(record.user)}/${encodeURIComponent(record.file_id)}`, + { headers: { Authorization: `Bearer ${token}` } }, + ); + expect(response.ok(), `download returned ${response.status()}`).toBeTruthy(); + return response.body(); +} + +/** Attaches an earlier upload from the palette's file list, which re-stages the stored record. */ +async function attachExisting(page: Page, file: TFile) { + await page.getByRole('button', { name: 'Attach and tools', exact: true }).click(); + await page.getByTestId('composer-palette-search').fill(file.filename); + await page.locator(`[data-row-key="file:${file.file_id}"]`).getByRole('button').first().click(); + await expect(composerChip(page, file.filename)).toBeVisible({ timeout: 15_000 }); +} + +async function expectWorkbookTextPreview(page: Page, file: TFile) { + /* The latest message carrying the workbook; earlier turns in the conversation attached it too. */ + await messagesView(page).getByRole('button', { name: file.filename, exact: true }).last().click(); + const dialog = page.getByRole('dialog'); + await expect(dialog).toContainText(WORKBOOK_SENTINEL); + await expect(dialog).not.toContainText('[Content_Types].xml'); + await page.keyboard.press('Escape'); + await expect(dialog).not.toBeVisible(); +} + +/** Fail one original's external indexing until the retry explicitly restores the service. */ +async function setEmbeddingFailure(page: Page, fileId: string, enabled: boolean) { + const port = process.env.E2E_RAG_API_PORT || '8791'; + const response = await page.request.post(`http://127.0.0.1:${port}/__debug/embedding-failure`, { + data: { file_id: fileId, enabled }, + }); + expect(response.ok()).toBe(true); +} + +test.describe('automatic upload reading', () => { + test.describe.configure({ timeout: TEST_TIMEOUT }); + + const CODE_FORMATS = [ + { fixture: UPLOAD_FIXTURES.xlsx, prompt: 'E2E_ANALYZE_WORKBOOK', result: WORKBOOK_RESULT }, + { fixture: UPLOAD_FIXTURES.csv, prompt: 'E2E_ANALYZE_CSV', result: CSV_RESULT }, + ]; + + for (const { fixture, prompt, result } of CODE_FORMATS) { + test(`A-01/A-02: ${fixture.name} is kept for Run Code, parsed there, and never sent as cells`, async ({ + page, + }) => { + await startChat(page, AUTO_ENDPOINT); + await enableCodeInterpreter(page); + + const file = renamed(fixture, 'a02-quarterly'); + const uploaded = await confirmUpload(page, uploadViaUnifiedButton(page, file)); + await expectDeferredRecord(page, uploaded.file_id); + + const run = await sendAnalysisTurn(page, `${prompt} ${uniqueName('a02')}`); + expectQueuedForCode(initialRequest(run), file.name); + expectNoWorkbookCells(run, file.name); + await expectCodeReceived(page, file, { analyzed: true }); + await expectAnswer(page, result); + }); + } + + test('A-15: a workbook goes to Run Code and a small PDF to the provider in one message', async ({ + page, + }) => { + await startChat(page, AUTO_ENDPOINT); + await enableCodeInterpreter(page); + + const workbook = renamed(UPLOAD_FIXTURES.xlsx, 'a15-quarterly'); + const pdf = renamed(UPLOAD_FIXTURES.smallPdf, 'a15-small'); + await confirmUpload(page, uploadViaUnifiedButton(page, workbook)); + await confirmUpload(page, uploadViaUnifiedButton(page, pdf)); + + const run = await sendAnalysisTurn(page, `E2E_ANALYZE_WORKBOOK ${uniqueName('a15')}`); + const request = initialRequest(run); + expect(request.documentFiles).toContain(pdf.name); + expect(inventoryLine(request, pdf.name)).toContain('Run Code can also open it'); + expectQueuedForCode(request, workbook.name); + expectNoWorkbookCells(run, workbook.name); + await expectCodeReceived(page, workbook, { analyzed: true }); + await expectCodeReceived(page, pdf); + await expectAnswer(page, WORKBOOK_RESULT); + }); + + test('A-16: enabling Run Code later reads an extracted workbook with code, across reloads', async ({ + page, + }) => { + test.setTimeout(MULTI_TURN_TIMEOUT); + const agentName = uniqueAgentName('E2E Auto Later Code'); + let agentId: string | undefined; + try { + await page.goto(NEW_CHAT_PATH, { timeout: 10_000 }); + agentId = await createAutoAgent(page, agentName, []); + await startAgentChat(page, agentName); + + /* Without Run Code at upload, the workbook is extracted as it is under classic. */ + const workbook = renamed(UPLOAD_FIXTURES.xlsx, 'a16-quarterly'); + const uploaded = await confirmUpload(page, uploadViaUnifiedButton(page, workbook)); + const record = await getFileRecord(page, uploaded.file_id); + expect(record.llmDeliveryPath).toBe('text'); + expect(record.metadata?.textDerivation).toBeUndefined(); + expect(await getStoredText(uploaded.file_id)).toContain(WORKBOOK_SENTINEL); + + const firstTurn = await sendTurn(page, replyPrompt(uniqueName('a16-turn1'))); + expect(initialRequest(firstTurn).promptText).toContain(WORKBOOK_SENTINEL); + + await setAgentTools(page, agentId, ['execute_code']); + await reloadConversation(page); + await resetProvisioning(page); + const secondTurn = await sendAnalysisTurn( + page, + `E2E_ANALYZE_WORKBOOK ${uniqueName('a16-turn2')}`, + ); + expectNoWorkbookCells(secondTurn, workbook.name); + await expectCodeReceived(page, workbook, { analyzed: true }); + await expectAnswer(page, WORKBOOK_RESULT); + + await reloadConversation(page); + const thirdTurn = await sendTurn(page, replyPrompt(uniqueName('a16-turn3'))); + expect(initialRequest(thirdTurn).promptText).not.toContain(WORKBOOK_SENTINEL); + expect(readingRecord(await getFileRecord(page, uploaded.file_id))).toEqual( + readingRecord(record), + ); + } finally { + await cleanupAgent(page, agentId); + } + }); + + test('A-17: disabling Run Code derives the workbook text once for later turns, across reloads', async ({ + page, + }) => { + test.setTimeout(MULTI_TURN_TIMEOUT); + const agentName = uniqueAgentName('E2E Auto Drop Code'); + let agentId: string | undefined; + try { + await page.goto(NEW_CHAT_PATH, { timeout: 10_000 }); + agentId = await createAutoAgent(page, agentName, ['execute_code']); + await startAgentChat(page, agentName); + + const workbook = renamed(UPLOAD_FIXTURES.xlsx, 'a17-quarterly'); + const uploaded = await confirmUpload(page, uploadViaUnifiedButton(page, workbook)); + await expectDeferredRecord(page, uploaded.file_id); + + const firstTurn = await sendAnalysisTurn( + page, + `E2E_ANALYZE_WORKBOOK ${uniqueName('a17-turn1')}`, + ); + expectNoWorkbookCells(firstTurn, workbook.name); + await expectCodeReceived(page, workbook, { analyzed: true }); + await expectAnswer(page, WORKBOOK_RESULT); + + await setAgentTools(page, agentId, []); + await reloadConversation(page); + const secondTurn = await sendTurn(page, replyPrompt(uniqueName('a17-turn2'))); + const secondRequest = initialRequest(secondTurn); + expect(secondRequest.promptText).toContain(WORKBOOK_SENTINEL); + /* Run Code is gone, so nothing may still say the workbook can be opened there. */ + expect(secondRequest.systemText).not.toContain(`/mnt/data/${workbook.name}`); + expect(inventoryLine(secondRequest, workbook.name)).not.toContain('Run Code can'); + await expect + .poll(async () => readingRecord(await getFileRecord(page, uploaded.file_id)).outcome, { + timeout: 15_000, + message: 'the derived text should be saved onto the deferred record', + }) + .toBe('complete'); + const derivation = (await getFileRecord(page, uploaded.file_id)).metadata?.textDerivation; + expect(derivation?.at, 'the derivation should be stamped when it was saved').toEqual( + expect.any(Number), + ); + expect(await getStoredText(uploaded.file_id)).toContain(WORKBOOK_SENTINEL); + + /* Derived once: a later turn reads the saved text and leaves the derivation, stamp and + * all, as it was. */ + await reloadConversation(page); + await attachExisting(page, uploaded); + const thirdTurn = await sendTurn(page, replyPrompt(uniqueName('a17-turn3'))); + expect(initialRequest(thirdTurn).promptText).toContain(WORKBOOK_SENTINEL); + expect((await getFileRecord(page, uploaded.file_id)).metadata?.textDerivation).toEqual( + derivation, + ); + expect(messageFileReading(await getLatestUserMessageFiles(page), uploaded.file_id)).toEqual({ + llmDeliveryPath: 'text', + }); + await expectWorkbookTextPreview(page, uploaded); + expect(await downloadOriginal(page, uploaded)).toEqual(attachFileBuffer(workbook)); + } finally { + await cleanupAgent(page, agentId); + } + }); + + test('A-06/A-13: a PDF over the small endpoint limit is searched there, and refused as a new upload', async ({ + page, + }) => { + test.setTimeout(MULTI_TURN_TIMEOUT); + await startChat(page, AUTO_ENDPOINT); + await sendTurn(page, replyPrompt(uniqueName('a06-open'))); + + /* Uploaded where it fits natively; the conversation keeps it across the model switch. */ + const pdf = renamed(UPLOAD_FIXTURES.largePdf, 'a06-large'); + const uploaded = await confirmUpload(page, uploadViaUnifiedButton(page, pdf)); + expect(uploaded.llmDeliveryPath).toBe('provider'); + + await selectMockEndpoint(page, AUTO_SMALL_ENDPOINT); + await expect(composerChip(page, pdf.name)).toBeVisible(); + await enableFileSearch(page); + + /* Read as late as possible before the turn, so an index built at upload has had time to + * land: nothing may hold the PDF yet, and everything after this mark happened in the turn. */ + const beforeTurn = await getRagEmbedded(page); + expect(beforeTurn.map((entry) => entry.file_id)).not.toContain(uploaded.file_id); + const mark = Math.max(latestSeq(beforeTurn), latestSeq(await getRagQueries(page))); + + const run = await sendTurn(page, `E2E_FILE_SEARCH:${uniqueName('a06')}`); + const request = initialRequest(run); + expect(request.documentFiles).not.toContain(pdf.name); + const line = inventoryLine(request, pdf.name); + expect(line).toContain('too large to send directly'); + expect(line).toContain('file_search'); + expect(line).toContain('it is indexed when you first search'); + + /* Embedded once, during the turn, and before any search of it. */ + let embeds: RagEmbedRecord[] = []; + await expect + .poll( + async () => { + embeds = (await getRagEmbedded(page)).filter( + (entry) => entry.file_id === uploaded.file_id, + ); + return embeds.length; + }, + { timeout: 30_000, message: 'the PDF should be indexed for File Search during the turn' }, + ) + .toBe(1); + let searches: RagQueryRecord[] = []; + await expect + .poll(async () => { + searches = (await getRagQueries(page)).filter( + (query) => query.file_id === uploaded.file_id, + ); + return searches.length; + }) + .toBeGreaterThan(0); + const [embed] = embeds; + expect(embed.seq).toBeGreaterThan(mark); + expect(embed.seq).toBeLessThan(Math.min(...searches.map(({ seq }) => seq))); + + /* A fresh conversation receives this indexed original through message resources.files, + * not an agent's configured file_ids. It must remain searchable without another embed. */ + await page.goto(NEW_CHAT_PATH); + await selectMockEndpoint(page, AUTO_ENDPOINT); + await sendTurn(page, replyPrompt(uniqueName('a06-reuse-open'))); + await attachExisting(page, uploaded); + await selectMockEndpoint(page, AUTO_SMALL_ENDPOINT); + if ( + !(await page.getByRole('button', { name: 'Remove File Search', exact: true }).isVisible()) + ) { + await enableFileSearch(page); + } + const reuseMark = latestSeq(await getRagQueries(page)); + const reusedRun = await sendTurn(page, `E2E_FILE_SEARCH:${uniqueName('a06-reuse')}`); + expect(initialRequest(reusedRun).documentFiles).not.toContain(pdf.name); + expect(inventoryLine(initialRequest(reusedRun), pdf.name)).toContain('file_search'); + expect(messageFileReading(await getLatestUserMessageFiles(page), uploaded.file_id)).toEqual({ + llmDeliveryPath: 'none', + }); + expect( + (await getRagEmbedded(page)).filter((entry) => entry.file_id === uploaded.file_id), + ).toHaveLength(1); + expect( + (await getRagQueries(page)).some( + (query) => query.file_id === uploaded.file_id && query.seq > reuseMark, + ), + ).toBe(true); + await reloadConversation(page); + expect(messageFileReading(await getLatestUserMessageFiles(page), uploaded.file_id)).toEqual({ + llmDeliveryPath: 'none', + }); + + /* Refused as before this policy existed, on every route: the size check throws inside the + * upload handler, which answers with its generic error, and no tool receives the bytes. The + * small PDF on the same route shows size decides. */ + const direct = renamed(UPLOAD_FIXTURES.largePdf, 'a13-large'); + const largeSha = sha256(attachFileBuffer(direct)); + for (const toolResource of [undefined, 'execute_code', 'file_search'] as const) { + const rejected = await postUpload(page, AUTO_SMALL_ENDPOINT, direct, toolResource); + const route = toolResource ?? 'message file'; + expect(rejected.status(), `${route} upload status`).toBe(500); + expect(await rejected.json(), `${route} upload body`).toEqual({ + message: 'Error processing file', + }); + } + expect((await getCodeProvisionedUploads(page)).map((upload) => upload.sha256)).not.toContain( + largeSha, + ); + expect((await getRagEmbedded(page)).map((entry) => entry.filename)).not.toContain(direct.name); + const control = await postUpload( + page, + AUTO_SMALL_ENDPOINT, + renamed(UPLOAD_FIXTURES.smallPdf, 'a13-small'), + ); + expect(control.ok(), await control.text()).toBe(true); + const token = await getAccessToken(page); + const files = await fetchJson(page, '/api/files', token); + expect(files.map((file) => file.filename)).not.toContain(direct.name); + }); + + test('failed indexing fails the turn with a retryable error, and retry reuses the original', async ({ + page, + }) => { + test.setTimeout(MULTI_TURN_TIMEOUT); + await startChat(page, AUTO_ENDPOINT); + await sendTurn(page, replyPrompt(uniqueName('search-failure-open'))); + const pdf = renamed(UPLOAD_FIXTURES.largePdf, 'search-retry'); + const uploaded = await confirmUpload(page, uploadViaUnifiedButton(page, pdf)); + await selectMockEndpoint(page, AUTO_SMALL_ENDPOINT); + await enableFileSearch(page); + await setEmbeddingFailure(page, uploaded.file_id, true); + try { + const response = await sendMessage(page, `E2E_FILE_SEARCH:${uniqueName('search-failure')}`); + expect(response.ok()).toBe(true); + /* The turn ends with the retryable preparation error rather than a search over an index + * that was never built, and the message keeps its attachment. */ + await expect(messagesView(page).getByText(PREPARATION_FAILED_ERROR)).toBeVisible({ + timeout: TURN_TIMEOUT, + }); + expect(messageFileReading(await getLatestUserMessageFiles(page), uploaded.file_id)).toEqual({ + llmDeliveryPath: 'none', + }); + expect( + (await getRagQueries(page)).filter((query) => query.file_id === uploaded.file_id), + ).toHaveLength(0); + const port = process.env.E2E_RAG_API_PORT || '8791'; + const debugResponse = await page.request.get(`http://127.0.0.1:${port}/__debug/embedded`); + const debug = (await debugResponse.json()) as { failedEmbeds: { file_id: string }[] }; + expect(debug.failedEmbeds.some((entry) => entry.file_id === uploaded.file_id)).toBe(true); + await reloadConversation(page); + await expect(messagesView(page).getByText(PREPARATION_FAILED_ERROR)).toBeVisible(); + + await setEmbeddingFailure(page, uploaded.file_id, false); + await selectMockEndpoint(page, AUTO_ENDPOINT); + await attachExisting(page, uploaded); + await selectMockEndpoint(page, AUTO_SMALL_ENDPOINT); + const recovered = await sendTurn(page, `E2E_FILE_SEARCH:${uniqueName('search-recovery')}`); + expect(inventoryLine(initialRequest(recovered), pdf.name)).toContain('file_search'); + expect(messageFileReading(await getLatestUserMessageFiles(page), uploaded.file_id)).toEqual({ + llmDeliveryPath: 'none', + }); + const embeds = (await getRagEmbedded(page)).filter( + (entry) => entry.file_id === uploaded.file_id, + ); + const queries = (await getRagQueries(page)).filter( + (entry) => entry.file_id === uploaded.file_id, + ); + expect(embeds).toHaveLength(1); + expect(queries.length).toBeGreaterThan(0); + expect(embeds[0].seq).toBeLessThan(Math.min(...queries.map(({ seq }) => seq))); + } finally { + await setEmbeddingFailure(page, uploaded.file_id, false); + } + }); + + test('A-10: with no file tool loaded, a spreadsheet reads as it does under classic routing', async ({ + page, + }) => { + await startChat(page, AUTO_ENDPOINT); + + const csv = renamed(UPLOAD_FIXTURES.csv, 'a10-sheet'); + const uploaded = await confirmUpload(page, uploadViaUnifiedButton(page, csv)); + await expectDeferredRecord(page, uploaded.file_id); + + /* Neither Run Code nor File Search is loaded, so the automatic policy stands aside: the + * text is derived from the stored original and sent the way classic routing sends extracted + * text, under the same limits, and no inventory tells the model anything about the file. */ + const run = await sendTurn(page, replyPrompt(uniqueName('a10'))); + const request = initialRequest(run); + expect(request.promptText).toContain(`${Q1_ROWS[0][0]},${Q1_ROWS[0][1]}`); + expect(request.systemText).not.toContain(INVENTORY_HEADING); + expect(inventoryLine(request, csv.name)).toBe(''); + expect(messageFileReading(await getLatestUserMessageFiles(page), uploaded.file_id)).toEqual({ + llmDeliveryPath: 'text', + }); + await expect + .poll(async () => readingRecord(await getFileRecord(page, uploaded.file_id)).outcome, { + timeout: 15_000, + message: 'the derived text should be saved onto the deferred record', + }) + .toBe('complete'); + + /* The record and its stored original are untouched by the derivation. */ + const record = await getFileRecord(page, uploaded.file_id); + expect(sha256(await downloadOriginal(page, record))).toBe(sha256(attachFileBuffer(csv))); + }); + + test('A-26: palette, drop, paste and attach-existing give the same automatic reading', async ({ + page, + }) => { + await startChat(page, AUTO_ENDPOINT); + await enableCodeInterpreter(page); + + const picked = await confirmUpload( + page, + uploadViaUnifiedButton(page, renamed(UPLOAD_FIXTURES.xlsx, 'a26-palette')), + ); + const dropped = await confirmUpload( + page, + uploadViaDrop(page, renamed(UPLOAD_FIXTURES.xlsx, 'a26-drop')), + ); + /* A browser that exposes no files on a synthetic paste event skips the third chip. */ + const pasted = (await supportsFilePaste(page)) + ? await confirmUpload(page, uploadViaPaste(page, renamed(UPLOAD_FIXTURES.xlsx, 'a26-paste'))) + : undefined; + if (pasted) { + expect(pasted.filename).toMatch(/^clipboard_\d+_a26-paste-/); + } + const uploads = [picked, dropped, ...(pasted ? [pasted] : [])]; + const pickedRecord = readingRecord(await getFileRecord(page, picked.file_id)); + expect(pickedRecord).toMatchObject({ llmDeliveryPath: 'none', outcome: 'deferred' }); + for (const { file_id } of uploads) { + expect(readingRecord(await getFileRecord(page, file_id))).toEqual(pickedRecord); + } + + const firstRun = await sendTurn(page, replyPrompt(uniqueName('a26-first'))); + const firstFiles = await getLatestUserMessageFiles(page); + const pickedReading = messageFileReading(firstFiles, picked.file_id); + expect(pickedReading).toEqual({ llmDeliveryPath: 'none' }); + for (const { filename, file_id } of uploads) { + expect(inventoryLine(initialRequest(firstRun), filename)).toContain('read it with Run Code'); + expect(messageFileReading(firstFiles, file_id)).toEqual(pickedReading); + } + + await attachExisting(page, picked); + const existingRun = await sendTurn(page, replyPrompt(uniqueName('a26-existing'))); + expect(inventoryLine(initialRequest(existingRun), picked.filename)).toContain( + 'read it with Run Code', + ); + const existingFiles = await getLatestUserMessageFiles(page); + expect(messageFileReading(existingFiles, picked.file_id)).toEqual(pickedReading); + expect(readingRecord(await getFileRecord(page, picked.file_id))).toEqual(pickedRecord); + }); + + test('A-29: a classic endpoint still extracts at upload and stores no marker or inventory', async ({ + page, + }) => { + test.setTimeout(MULTI_TURN_TIMEOUT); + /* Run Code is on, so a classic upload would be deferred if the policy gate broke. */ + await startChat(page, CLASSIC_ENDPOINT); + await enableCodeInterpreter(page); + const workbook = renamed(UPLOAD_FIXTURES.xlsx, 'a29-quarterly'); + const uploaded = await confirmUpload(page, uploadViaUnifiedButton(page, workbook)); + const record = await getFileRecord(page, uploaded.file_id); + expect(record.llmDeliveryPath).toBe('text'); + expect(record.metadata?.textDerivation).toBeUndefined(); + expect(await getStoredText(uploaded.file_id)).toContain(WORKBOOK_SENTINEL); + + const run = await sendTurn(page, replyPrompt(uniqueName('a29'))); + const request = initialRequest(run); + expect(request.promptText).toContain(WORKBOOK_SENTINEL); + expect(request.systemText).not.toContain(INVENTORY_HEADING); + expect(inventoryLine(request, workbook.name)).toBe(''); + expect(messageFileReading(await getLatestUserMessageFiles(page), uploaded.file_id)).toEqual({ + llmDeliveryPath: 'text', + }); + const after = await getFileRecord(page, uploaded.file_id); + expect(readingRecord(after)).toEqual(readingRecord(record)); + expect(after.metadata?.textDerivation).toBeUndefined(); + }); +}); diff --git a/e2e/specs/mock/file-provisioning.spec.ts b/e2e/specs/mock/file-provisioning.spec.ts index 47950e4caa9..4a10deda681 100644 --- a/e2e/specs/mock/file-provisioning.spec.ts +++ b/e2e/specs/mock/file-provisioning.spec.ts @@ -155,19 +155,28 @@ test.describe('file provisioning — lazy (unified upload, at tool-execute)', () ).toBeVisible({ timeout: 15000 }); // The first tool call lazily embeds the attachment and searches it. - await sendMessageAndWaitForCompletion(page, `E2E_FILE_SEARCH:${uniqueName('first')}`); + const firstSearch = uniqueName('first'); + await sendMessageAndWaitForCompletion(page, `E2E_FILE_SEARCH:${firstSearch}`); await expect .poll(async () => (await getRagEmbedded(page)).map((e) => e.file_id), { timeout: 30000 }) .toContain(fileId); await expect - .poll(async () => (await getRagQueries(page)).filter((q) => q.file_id === fileId).length) - .toBe(1); + .poll(async () => + (await getRagQueries(page)).filter((q) => q.file_id === fileId).map((q) => q.query), + ) + .toEqual([`e2e ${firstSearch}`]); // A completed, persisted turn must still expose the embedded file to the next call. - await sendMessageAndWaitForCompletion(page, `E2E_FILE_SEARCH:${uniqueName('second')}`); + const secondSearch = uniqueName('second'); + const secondMessage = `E2E_FILE_SEARCH:${secondSearch}`; + await sendMessageAndWaitForCompletion(page, secondMessage); + // Classic mode adds context from the now-embedded file before invoking the model; + // the fresh tool call must then perform its own search, without another embedding. await expect - .poll(async () => (await getRagQueries(page)).filter((q) => q.file_id === fileId).length) - .toBe(2); + .poll(async () => + (await getRagQueries(page)).filter((q) => q.file_id === fileId).map((q) => q.query), + ) + .toEqual([`e2e ${firstSearch}`, secondMessage, `e2e ${secondSearch}`]); expect((await getRagEmbedded(page)).filter((e) => e.file_id === fileId)).toHaveLength(1); }); }); diff --git a/e2e/specs/mock/helpers.ts b/e2e/specs/mock/helpers.ts index 9c16acef008..1589d47dddb 100644 --- a/e2e/specs/mock/helpers.ts +++ b/e2e/specs/mock/helpers.ts @@ -1,3 +1,5 @@ +import fs from 'fs'; +import path from 'path'; import { expect } from '@playwright/test'; import { ContentTypes } from 'librechat-data-provider'; import type { TMessage } from 'librechat-data-provider'; @@ -10,6 +12,10 @@ export const MOCK_REPLY_TEXT = 'E2E mock reply'; export const MOCK_ENDPOINTS = [ { label: 'Mock Provider A', model: 'mock-model-a' }, { label: 'Mock Provider B', model: 'mock-model-b' }, + /** Automatic reading policy, no route overrides (automatic-upload.spec.ts). */ + { label: 'Mock Auto Provider', model: 'mock-auto-model' }, + /** Automatic reading policy with a 1 MB per-file limit. */ + { label: 'Mock Auto Small Provider', model: 'mock-auto-small-model' }, ] as const; export type MockEndpoint = { label: string; model: string }; @@ -406,10 +412,22 @@ export type CodeProvisionRecord = { id: string; storage_session_id: string; fileId: string; + bytes: number; + /** Hex digest of the bytes the code server received. */ + sha256: string; }; -export type RagEmbedRecord = { file_id: string; filename: string; entity_id: string }; -export type RagQueryRecord = { file_id: string; query: string }; +/** One exec the fake code server ran: the files it mounted, and the one an analysis token read. */ +export type CodeExecRecord = { + lang: string; + fileCount: number; + files: Array<{ id: string; name: string }>; + analyzedFileId?: string; +}; + +/** `seq` orders embeds and queries against each other across both logs. */ +export type RagEmbedRecord = { seq: number; file_id: string; filename: string; entity_id: string }; +export type RagQueryRecord = { seq: number; file_id: string; query: string }; /** Every `/query` the fake RAG service received from file_search. */ export async function getRagQueries(page: Page): Promise { @@ -419,14 +437,24 @@ export async function getRagQueries(page: Page): Promise { return body.queries; } -/** Files the fake code server received via /upload (proof they reached the code env). */ -export async function getCodeProvisionedUploads(page: Page): Promise { +/** The fake code server's record of what it received and ran since the last reset. */ +async function getCodeDebug(page: Page): Promise<{ + uploads: CodeProvisionRecord[]; + execs: CodeExecRecord[]; +}> { const response = await page.request.get(`${CODE_API_BASE}/__debug/uploads`); expect(response.ok(), 'fake code server /__debug/uploads should respond').toBeTruthy(); - const body = (await response.json()) as { uploads: CodeProvisionRecord[] }; - return body.uploads; + return response.json(); } +/** Files the fake code server received via /upload (proof they reached the code env). */ +export const getCodeProvisionedUploads = async (page: Page): Promise => + (await getCodeDebug(page)).uploads; + +/** Every exec the fake code server ran since the last reset. */ +export const getCodeExecs = async (page: Page): Promise => + (await getCodeDebug(page)).execs; + /** Files the fake RAG server embedded via /embed (proof they reached the vector DB). */ export async function getRagEmbedded(page: Page): Promise { const response = await page.request.get(`${RAG_API_BASE}/__debug/embedded`); @@ -437,10 +465,78 @@ export async function getRagEmbedded(page: Page): Promise { /** Clear both fake servers' recorded provisioning (call at test start for isolation). */ export async function resetProvisioning(page: Page): Promise { - await Promise.all([ + const [code, rag] = await Promise.all([ page.request.post(`${CODE_API_BASE}/__debug/reset`), page.request.post(`${RAG_API_BASE}/__debug/reset`), ]); + expect(code.ok(), 'fake code server /__debug/reset should succeed').toBeTruthy(); + expect(rag.ok(), 'fake RAG server /__debug/reset should succeed').toBeTruthy(); +} + +/** + * Where the in-process fake model writes the prompts of its most recent runs + * (e2e/setup/fake-model.js). It has no HTTP surface, so the file is its debug endpoint. + */ +export const MODEL_REQUEST_LOG_PATH = + process.env.E2E_MODEL_REQUEST_LOG ?? + path.resolve(__dirname, '../../.generated/last-request.json'); + +/** One model invocation as the provider would have received it. */ +export type ModelRequestRecord = { + /** The system instructions, including the queued code files and the file inventory. */ + systemText: string; + /** Every message's readable text, system first; inline file data is left out. */ + promptText: string; + /** File names of the document and file parts sent to the model. */ + documentFiles: string[]; +}; + +/** One run of the fake model, keyed by the user text that started it. */ +export type ModelRunRecord = { + sequence: number; + conversationId: string | null; + userText: string; + invocations: ModelRequestRecord[]; +}; + +function readModelRuns(): ModelRunRecord[] { + try { + const log = JSON.parse(fs.readFileSync(MODEL_REQUEST_LOG_PATH, 'utf8')) as { + runs?: ModelRunRecord[]; + }; + return log.runs ?? []; + } catch { + return []; + } +} + +/** + * The latest fake-model run whose user text contains `label`, once it has recorded at least + * `minInvocations` model requests. + */ +export async function getModelRun( + label: string, + { minInvocations = 1, timeout = 15_000 }: { minInvocations?: number; timeout?: number } = {}, +): Promise { + let match: ModelRunRecord | undefined; + await expect + .poll( + () => { + match = readModelRuns() + .filter((run) => run.userText.includes(label)) + .pop(); + return match?.invocations.length ?? 0; + }, + { + timeout, + message: `the fake model should record ${minInvocations} request(s) for "${label}"`, + }, + ) + .toBeGreaterThanOrEqual(minInvocations); + if (!match) { + throw new Error(`No fake-model request was recorded for "${label}"`); + } + return match; } /** Shape of a file record as returned by POST /api/files and GET /api/files. */ @@ -453,7 +549,14 @@ export type UploadedFile = { metadata?: { codeEnvRef?: { storage_session_id?: string; file_id?: string } }; }; -export type AttachFile = { name: string; mimeType: string; content: string }; +/** A file to attach: inline text, or a fixture read from disk under the given name. */ +export type AttachFile = + | { name: string; mimeType: string; content: string } + | { name: string; mimeType: string; path: string }; + +/** The bytes the browser receives for an attach helper's file. */ +export const attachFileBuffer = (file: AttachFile): Buffer => + 'path' in file ? fs.readFileSync(file.path) : Buffer.from(file.content, 'utf8'); /** Unique, filesystem-safe name so tests never collide on accumulated fake-server state. */ export const uniqueName = (prefix: string) => @@ -482,8 +585,59 @@ export async function uploadViaUnifiedButton(page: Page, file: AttachFile) { await fileChooser.setFiles({ name: file.name, mimeType: file.mimeType, - buffer: Buffer.from(file.content, 'utf8'), + buffer: attachFileBuffer(file), + }); + /* The palette lists the new record while it closes, which would duplicate the chip's name. */ + await expect(palette).toBeHidden(); + return uploadResponse; +} + +/** A DataTransfer in the page holding one file, as a native drag or a file paste carries it. */ +function createFileTransfer(page: Page, file: AttachFile) { + return page.evaluateHandle( + ({ name, mimeType, base64 }) => { + const bytes = Uint8Array.from(atob(base64), (char) => char.charCodeAt(0)); + const transfer = new DataTransfer(); + transfer.items.add(new File([bytes], name, { type: mimeType })); + return transfer; + }, + { name: file.name, mimeType: file.mimeType, base64: attachFileBuffer(file).toString('base64') }, + ); +} + +/** Drop a file onto the composer with the event sequence a native file drag produces. */ +export async function uploadViaDrop(page: Page, file: AttachFile) { + const input = page.getByRole('textbox', { name: 'Message input' }); + await expect(input).toBeVisible(); + const dataTransfer = await createFileTransfer(page, file); + const uploadResponse = waitForUpload(page); + await input.dispatchEvent('dragenter', { dataTransfer }); + await input.dispatchEvent('dragover', { dataTransfer }); + await input.dispatchEvent('drop', { dataTransfer }); + return uploadResponse; +} + +/** Whether the browser exposes files on a synthetic paste event, which file paste needs. */ +export function supportsFilePaste(page: Page): Promise { + return page.evaluate(() => { + const transfer = new DataTransfer(); + transfer.items.add(new File(['probe'], 'probe.txt', { type: 'text/plain' })); + const event = new ClipboardEvent('paste', { clipboardData: transfer }); + return event.clipboardData?.files.length === 1; }); +} + +/** Paste a file into the composer as the clipboard delivers a copied file. */ +export async function uploadViaPaste(page: Page, file: AttachFile) { + const input = page.getByRole('textbox', { name: 'Message input' }); + await input.click(); + const clipboardData = await createFileTransfer(page, file); + const uploadResponse = waitForUpload(page); + await input.evaluate((element, transfer) => { + element.dispatchEvent( + new ClipboardEvent('paste', { clipboardData: transfer, bubbles: true, cancelable: true }), + ); + }, clipboardData); return uploadResponse; } @@ -525,7 +679,7 @@ export async function uploadViaLegacyOption(page: Page, optionName: string, file await fileChooser.setFiles({ name: file.name, mimeType: file.mimeType, - buffer: Buffer.from(file.content, 'utf8'), + buffer: attachFileBuffer(file), }); return uploadResponse; } diff --git a/e2e/specs/mock/run-files-delivery.spec.ts b/e2e/specs/mock/run-files-delivery.spec.ts index 77d5d04a9c7..715572c15ae 100644 --- a/e2e/specs/mock/run-files-delivery.spec.ts +++ b/e2e/specs/mock/run-files-delivery.spec.ts @@ -7,9 +7,9 @@ import type { AgentDetail } from './agents.helpers'; import { fetchJson, uniqueName, - RAG_API_BASE, messagesView, requestJson, + getRagQueries, getAccessToken, getRagEmbedded, resetProvisioning, @@ -352,12 +352,9 @@ test.describe('run-file delivery and authorization', () => { ).toEqual([input.file_id, file.file_id].sort()); expect(output(messages, label, 'reader_search')).toContain('No content found in the files.'); expect((await getRagEmbedded(page)).map((entry) => entry.file_id)).toContain(file.file_id); - const queryResponse = await page.request.get(`${RAG_API_BASE}/__debug/embedded`); - expect(queryResponse.ok()).toBeTruthy(); - const { queries } = (await queryResponse.json()) as { - queries: Array<{ file_id: string; query: string }>; - }; - expect(queries).toContainEqual({ file_id: file.file_id, query: file.filename }); + expect(await getRagQueries(page)).toContainEqual( + expect.objectContaining({ file_id: file.file_id, query: file.filename }), + ); expect( messages.some((message) => message.attachments?.some((attachment) => diff --git a/librechat.example.yaml b/librechat.example.yaml index 28ff6059494..1b225d2e46b 100644 --- a/librechat.example.yaml +++ b/librechat.example.yaml @@ -1371,6 +1371,12 @@ endpoints: # # Existing files without extracted text are not backfilled. Paused runs retain encoded text # # and conservatively count referenced fallback text against current limits on resume. # textFallbackWithoutTools: false +# # "automatic" lets a turn that loads Run Code or File Search choose each attachment's reader: +# # spreadsheets go to Run Code, documents to the provider within its limits, then text, File +# # Search and Run Code. Without a file tool, and under "classic" (default), routing is unchanged. +# # Explicit destinations and defaultLLMDeliveryPath still win. Set back to classic before a +# # downgrade; deferred workbooks stay readable only by Run Code on older versions. +# llmDeliveryPolicy: classic # fileContextSizeLimit: 128 # Maximum aggregate model-bound attachment size in one agent turn (MB) # fileContextCharLimit: 1000000 # Maximum aggregate extracted-text characters in one agent turn # endpoints: diff --git a/packages/api/src/agents/__tests__/initialize.reading.integration.test.ts b/packages/api/src/agents/__tests__/initialize.reading.integration.test.ts new file mode 100644 index 00000000000..3e87b6a6220 --- /dev/null +++ b/packages/api/src/agents/__tests__/initialize.reading.integration.test.ts @@ -0,0 +1,330 @@ +import os from 'os'; +import fs from 'fs'; +import path from 'path'; +import mongoose from 'mongoose'; +import { Providers } from '@librechat/agents'; +import { MongoMemoryServer } from 'mongodb-memory-server'; +import { createModels, createMethods } from '@librechat/data-schemas'; +import { + Tools, + FileContext, + FileSources, + EModelEndpoint, + decideFileReading, +} from 'librechat-data-provider'; +import type { AppConfig, IMongoFile } from '@librechat/data-schemas'; +import type { Agent, TextDerivation } from 'librechat-data-provider'; +import type { LCTool } from '@librechat/agents'; +import type { ProvisionService } from '~/files/provision/service'; +import type { InitializeAgentDbMethods } from '../initialize'; +import type { ServerRequest } from '~/types'; +import { getTurnReadingContext, createFileTextDeriver } from '~/files/reading'; +import { extractFileContext } from '~/files/context'; +import { initializeAgent } from '../initialize'; + +const XLSX_TYPE = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'; +const SHEET_TEXT = 'Quarter,Total\nQ1,1200\nQ2,1400\nSENTINEL-7F3A'; +const SAMPLE_XLSX = path.join(__dirname, '../../files/documents/sample.xlsx'); +const SAMPLE_TEXT = 'Sheet One:\nData,on,first,sheet\nSecond Sheet:\nData,On\nSecond,Sheet\n'; + +type FileConfigInput = NonNullable; +type OpenStoredFile = ProvisionService['openStoredFile']; +type RoutingFields = Pick; + +interface TurnOptions { + fileIds: string[]; + policy?: 'automatic'; + tools?: string[]; + /** Tools the loader returns definitions for; Run Code registers itself when enabled. */ + loaded?: string[]; + fileConfig?: Omit; + checkSessionsAlive?: InitializeAgentDbMethods['checkSessionsAlive']; + /** Wires the request's text deriver over this storage, as the host does. */ + openStoredFile?: OpenStoredFile; +} + +let mongoServer: MongoMemoryServer; +let methods: ReturnType; +let userId: string; +let storageRoot: string; +const originalApiKey = process.env.OPENAI_API_KEY; + +beforeAll(async () => { + mongoServer = await MongoMemoryServer.create(); + await mongoose.connect(mongoServer.getUri()); + createModels(mongoose); + methods = createMethods(mongoose); + storageRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'reading-integration-')); + process.env.OPENAI_API_KEY = 'sk-reading-integration'; +}, 60000); + +afterAll(async () => { + process.env.OPENAI_API_KEY = originalApiKey; + fs.rmSync(storageRoot, { recursive: true, force: true }); + await mongoose.disconnect(); + await mongoServer.stop(); +}); + +beforeEach(() => { + userId = new mongoose.Types.ObjectId().toString(); +}); + +afterEach(async () => { + await mongoose.connection.db?.dropDatabase(); +}); + +const baseFile = (fileId: string) => ({ + file_id: fileId, + user: new mongoose.Types.ObjectId(userId), + object: 'file' as const, + source: FileSources.local, + context: FileContext.message_attachment, + usage: 0, +}); + +/** + * A spreadsheet the automatic upload path deferred to Run Code: no text and a `deferred` + * marker. Written to the collection directly because the marker is a later commit's schema + * field; reads return it as stored. + */ +async function seedDeferredXlsx(fileId = 'xlsx-deferred'): Promise { + const textDerivation: TextDerivation = { outcome: 'deferred', at: Date.now() }; + await mongoose.models.File.collection.insertOne({ + ...baseFile(fileId), + filename: 'deferred.xlsx', + filepath: `/uploads/${userId}/deferred.xlsx`, + type: XLSX_TYPE, + bytes: 4096, + llmDeliveryPath: 'none', + metadata: { destinationChosen: false, textDerivation }, + }); + return fileId; +} + +/** Writes the real sample workbook where local storage keeps the record's original. */ +async function storeSampleOriginal(fileId: string): Promise { + const [file] = (await methods.getFiles({ file_id: fileId }, null, {})) ?? []; + const target = path.join(storageRoot, file?.filepath ?? fileId); + fs.mkdirSync(path.dirname(target), { recursive: true }); + fs.copyFileSync(SAMPLE_XLSX, target); +} + +/** Local storage: streams a record's original from under the storage root, observed. */ +const localStorage = () => + jest.fn, Parameters>(async (file) => + fs.createReadStream(path.join(storageRoot, file.filepath ?? file.file_id)), + ); + +async function seedPdf(fileId: string, bytes: number): Promise { + await methods.createFile({ + ...baseFile(fileId), + filename: `${fileId}.pdf`, + filepath: `/uploads/${userId}/${fileId}.pdf`, + type: 'application/pdf', + bytes, + llmDeliveryPath: 'provider', + metadata: { destinationChosen: false }, + }); + return fileId; +} + +async function readRoutingFields(fileId: string): Promise { + const [file] = (await methods.getFiles({ file_id: fileId }, null, {})) ?? []; + if (file == null) { + return undefined; + } + const { llmDeliveryPath, text, type, bytes, metadata } = file; + return { llmDeliveryPath, text, type, bytes, metadata }; +} + +function buildDb(options: TurnOptions): InitializeAgentDbMethods { + return { + getFiles: methods.getFiles as InitializeAgentDbMethods['getFiles'], + updateFilesUsage: methods.updateFilesUsage, + getToolFilesByIds: methods.getToolFilesByIds as InitializeAgentDbMethods['getToolFilesByIds'], + getProjectFiles: methods.getProjectFiles as InitializeAgentDbMethods['getProjectFiles'], + getConvoFiles: async () => [], + getUserKey: async () => '', + getUserKeyValues: async () => ({}), + checkSessionsAlive: options.checkSessionsAlive, + saveFileTextDerivation: methods.saveFileTextDerivation, + }; +} + +async function runTurn(options: TurnOptions) { + const { policy, tools = [], loaded = [], fileConfig, openStoredFile } = options; + const appConfig: Pick = { + fileConfig: { + ...fileConfig, + endpoints: { + [EModelEndpoint.openAI]: { ...(policy != null && { llmDeliveryPolicy: policy }) }, + }, + }, + }; + const req = { + user: { id: userId, role: 'USER' }, + config: appConfig, + body: {}, + } as ServerRequest; + const agent = { + id: 'agent-reading', + provider: Providers.OPENAI, + model: 'gpt-4o', + tools, + model_parameters: { model: 'gpt-4o' }, + } as Agent; + const loadTools = jest.fn(async ({ tools: requested }: { tools: string[] }) => ({ + tools: [], + toolContextMap: {}, + toolDefinitions: loaded + .filter((name) => requested.includes(name)) + .map( + (name): LCTool => ({ + name, + description: name, + parameters: { type: 'object', properties: {} }, + }), + ), + })); + const result = await initializeAgent( + { + req, + agent, + loadTools, + requestFiles: options.fileIds.map((file_id) => ({ file_id })) as IMongoFile[], + codeEnvAvailable: true, + fileSearchAvailable: true, + endpointOption: { endpoint: EModelEndpoint.agents }, + allowedProviders: new Set([Providers.OPENAI]), + isInitialAgent: true, + ...(openStoredFile != null && { deriveText: createFileTextDeriver({ req, openStoredFile }) }), + }, + buildDb(options), + ); + return { result, req }; +} + +const pathsById = (files: ReadonlyArray>) => + Object.fromEntries(files.map((file) => [file.file_id, file.llmDeliveryPath])); + +type InitializedAgentResult = Awaited>; + +/** The reader the final decision picked for each request attachment. */ +const readersById = (result: InitializedAgentResult) => + Object.fromEntries( + result.requestAttachments.map((file) => [ + file.file_id, + decideFileReading({ + routing: result.deliveryRouting, + file, + consumers: result.fileConsumers, + }).reader, + ]), + ); + +const injectedText = (files: IMongoFile[], req: ServerRequest) => + extractFileContext({ attachments: files, req, tokenCountFn: (text) => text.length }); + +describe('initializeAgent reading under the automatic policy (MongoDB)', () => { + it('keeps the Run Code reader and injects no text when provisioning fails', async () => { + const fileId = 'xlsx-sandboxed'; + await methods.createFile({ + ...baseFile(fileId), + filename: 'sandboxed.xlsx', + filepath: `/uploads/${userId}/sandboxed.xlsx`, + type: XLSX_TYPE, + bytes: 4096, + text: SHEET_TEXT, + llmDeliveryPath: 'text', + metadata: { + destinationChosen: false, + codeEnvRef: { + kind: 'user', + id: userId, + storage_session_id: 'session-1', + file_id: 'sandbox-file-1', + }, + }, + }); + const checkSessionsAlive = jest.fn().mockRejectedValue(new Error('Code API unreachable')); + + const { result, req } = await runTurn({ + fileIds: [fileId], + policy: 'automatic', + tools: [Tools.execute_code], + checkSessionsAlive, + }); + + expect(checkSessionsAlive).toHaveBeenCalled(); + expect(result.provisionState).toBeUndefined(); + expect(result.fileConsumers?.executeCode).toBe(true); + expect(pathsById(result.requestAttachments)).toEqual({ [fileId]: 'none' }); + expect(await injectedText(result.requestAttachments, req)).toBeFalsy(); + }); + + it('sends the PDFs that fit the context allowance natively and the third to File Search', async () => { + const fileIds = [ + await seedPdf('pdf-a', 400 * 1024), + await seedPdf('pdf-b', 400 * 1024), + await seedPdf('pdf-c', 400 * 1024), + ]; + + const { result } = await runTurn({ + fileIds, + policy: 'automatic', + tools: [Tools.file_search], + loaded: [Tools.file_search], + fileConfig: { fileContextSizeLimit: 1 }, + }); + + expect(pathsById(result.requestAttachments)).toEqual({ + 'pdf-a': 'provider', + 'pdf-b': 'provider', + 'pdf-c': 'none', + }); + expect(readersById(result)).toEqual({ + 'pdf-a': 'provider', + 'pdf-b': 'provider', + 'pdf-c': 'search', + }); + const context = getTurnReadingContext(result.deliveryRouting); + expect(context?.stats().overflow).toBe(1); + const queued = result.provisionState?.vectorDBFiles.map((file) => file.file_id) ?? []; + expect(queued).toContain('pdf-c'); + }); +}); + +describe('initializeAgent text derivation from stored originals (MongoDB)', () => { + it('derives a deferred spreadsheet without Run Code once and keeps the text on the record', async () => { + const fileId = await seedDeferredXlsx(); + await storeSampleOriginal(fileId); + const openStoredFile = localStorage(); + + const { result, req } = await runTurn({ + fileIds: [fileId], + policy: 'automatic', + openStoredFile, + }); + + expect(result.fileConsumers).toEqual({ executeCode: false, fileSearch: false }); + expect(pathsById(result.requestAttachments)).toEqual({ [fileId]: 'text' }); + expect(readersById(result)).toEqual({ [fileId]: 'text' }); + expect(result.requestAttachments[0].text).toBe(SAMPLE_TEXT); + expect(await injectedText(result.requestAttachments, req)).toContain('Data,on,first,sheet'); + expect(openStoredFile).toHaveBeenCalledTimes(1); + expect(await readRoutingFields(fileId)).toMatchObject({ + llmDeliveryPath: 'none', + text: SAMPLE_TEXT, + metadata: { + destinationChosen: false, + textDerivation: { outcome: 'complete', extractor: 'document_parser' }, + }, + }); + + const next = await runTurn({ fileIds: [fileId], policy: 'automatic', openStoredFile }); + + expect(openStoredFile).toHaveBeenCalledTimes(1); + expect(pathsById(next.result.requestAttachments)).toEqual({ [fileId]: 'text' }); + expect(next.result.requestAttachments[0].text).toBe(SAMPLE_TEXT); + }); +}); diff --git a/packages/api/src/agents/__tests__/initialize.test.ts b/packages/api/src/agents/__tests__/initialize.test.ts index 9ac8f6bcf50..3e1b3fe9117 100644 --- a/packages/api/src/agents/__tests__/initialize.test.ts +++ b/packages/api/src/agents/__tests__/initialize.test.ts @@ -50,10 +50,17 @@ import { configSchema, } from 'librechat-data-provider'; import type { IMongoFile } from '@librechat/data-schemas'; -import type { Agent, TFile, FiltersConfig, AgentInstructionsPrompt } from 'librechat-data-provider'; +import type { + Agent, + TFile, + FiltersConfig, + AgentToolResources, + AgentInstructionsPrompt, +} from 'librechat-data-provider'; import type { ServerRequest, InitializeResultBase, EndpointTokenConfig } from '~/types'; import type { ResolveLinkedInstructions } from '../instructions/linked'; import type { InitializeAgentDbMethods } from '../initialize'; +import type { ProvisionState } from '../resources'; import type { CodeExecutionContext } from '../execution'; import type { GraphSubagentHostConfig } from '../discovery'; import type { ProjectFileRecord } from '../../projects/resources'; @@ -143,6 +150,12 @@ jest.mock('../../middleware/modelBoundContent', () => { import { initializeAgent } from '../initialize'; import { primeResources } from '../resources'; import { isFatalAgentInitializationError } from '../errors'; +import { filterFilesByEndpointRuntimeConfig as filterByEndpointPolicy } from '~/files/filter'; +import { assertModelBoundContent } from '../../middleware/modelBoundContent'; +import * as attachments from '../attachments'; +import { ContentFilterError } from '../../middleware/contentFilter'; +import { getTurnReadingContext, createDerivationPersister } from '~/files/reading'; +import type { FileTextDeriver, TextDerivationPersister } from '~/files/reading'; const realUtils = jest.requireActual('~/utils'); @@ -249,6 +262,7 @@ function createMocks(overrides?: { getUserKey: jest.fn().mockResolvedValue('user-1'), getUserKeyValues: jest.fn().mockResolvedValue([]), getToolFilesByIds: jest.fn().mockResolvedValue([]), + saveFileTextDerivation: jest.fn().mockResolvedValue(true), }; return { agent, req, res, loadTools, db }; @@ -6214,3 +6228,625 @@ describe('initializeAgent turn delivery routing', () => { expect(filesOrder).toBeLessThan(toolsOrder); }); }); + +describe('initializeAgent automatic delivery policy', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + const XLSX_TYPE = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'; + + /** A spreadsheet uploaded under classic routing: extracted at upload, route inferred. */ + const classicEraXlsx = () => + ({ + file_id: 'xlsx-file', + filename: 'quarterly.xlsx', + filepath: '/uploads/user-1/quarterly.xlsx', + type: XLSX_TYPE, + bytes: 2048, + source: FileSources.local, + context: FileContext.message_attachment, + text: 'Q1,1200\nQ2,1400', + llmDeliveryPath: 'text', + metadata: { destinationChosen: false }, + }) as IMongoFile; + + const oversizedPdf = () => + ({ + file_id: 'pdf-file', + filename: 'annual.pdf', + filepath: '/uploads/user-1/annual.pdf', + type: 'application/pdf', + bytes: 3 * 1024 * 1024, + source: FileSources.local, + context: FileContext.message_attachment, + llmDeliveryPath: 'provider', + metadata: { destinationChosen: false }, + }) as IMongoFile; + + /** The queued copy priming hands the planner. */ + const queued = (file: IMongoFile): TFile => ({ + file_id: file.file_id, + filename: file.filename, + filepath: file.filepath, + type: file.type, + bytes: file.bytes, + user: 'user-1', + object: 'file', + embedded: false, + usage: 0, + source: FileSources.local, + context: FileContext.message_attachment, + metadata: { destinationChosen: false }, + }); + + const provisionStateFor = ({ + code = [], + search = [], + }: { + code?: IMongoFile[]; + search?: IMongoFile[]; + }): ProvisionState => ({ + codeEnvFiles: code.map(queued), + vectorDBFiles: search.map(queued), + aliveFileIds: new Set(), + agentScopedFileIds: new Set(), + }); + + /** A spreadsheet uploaded under the automatic policy, its extraction left for a later turn. */ + const deferredXlsx = () => + ({ + ...classicEraXlsx(), + text: undefined, + llmDeliveryPath: 'none', + metadata: { destinationChosen: false, textDerivation: { outcome: 'deferred' } }, + }) as IMongoFile; + + async function initializeWith({ + file, + policy, + tools = [], + skillTools, + endpointConfig = {}, + fileContextCharLimit, + contextFiles = [], + deriveText, + saveFileTextDerivation, + persistDerivation, + loaded = {}, + toolResources = {}, + provisionState, + filters, + }: { + file: IMongoFile; + /** What the loader returns beyond the tools the agent lists. */ + loaded?: { + toolNames?: string[]; + dynamicToolContextMap?: Record; + primedSearchFiles?: Array<{ file_id: string; filename: string; fromAgent: boolean }>; + }; + /** The provisioning queues priming hands back. */ + provisionState?: ProvisionState; + toolResources?: AgentToolResources; + filters?: FiltersConfig; + /** The host's text deriver, which also builds a derive-only context under classic. */ + deriveText?: FileTextDeriver; + saveFileTextDerivation?: InitializeAgentDbMethods['saveFileTextDerivation']; + /** The host's request-scoped persister, shared by every agent it initializes. */ + persistDerivation?: TextDerivationPersister; + /** Persistent agent files priming adds to the delivered and agent-context sets. */ + contextFiles?: IMongoFile[]; + policy?: 'automatic' | 'classic'; + tools?: string[]; + /** Tools a manual skill adds; the loader loads none of them. */ + skillTools?: string[]; + endpointConfig?: { fileSizeLimit?: number }; + fileContextCharLimit?: number; + }) { + const { filterFilesByEndpointRuntimeConfig } = jest.requireMock('~/files') as { + filterFilesByEndpointRuntimeConfig: jest.Mock; + }; + filterFilesByEndpointRuntimeConfig.mockImplementation(filterByEndpointPolicy); + const { agent, req, res, loadTools, db } = createMocks(); + agent.tools = tools; + loadTools.mockResolvedValue({ + tools: [], + toolContextMap: {}, + toolDefinitions: (loaded.toolNames ?? []).map((name) => ({ name })), + dynamicToolContextMap: loaded.dynamicToolContextMap, + primedSearchFiles: loaded.primedSearchFiles, + }); + (primeResources as jest.Mock).mockImplementationOnce(async ({ attachments }) => { + const files = await attachments; + return { + attachments: [...files, ...contextFiles], + requestAttachments: files, + agentContextAttachments: contextFiles, + tool_resources: toolResources, + provisionState, + }; + }); + req.config = { + filters, + fileConfig: { + fileContextCharLimit, + endpoints: { + [Providers.OPENAI]: { + ...(policy != null && { llmDeliveryPolicy: policy }), + ...endpointConfig, + }, + }, + }, + } as unknown as ServerRequest['config']; + (db.getFiles as jest.Mock).mockResolvedValueOnce([file]); + + const { Types } = await import('mongoose'); + const skillId = new Types.ObjectId(); + const skillParams = + skillTools == null + ? {} + : { accessibleSkillIds: [skillId], manualSkills: ['spreadsheet-analysis'] }; + const skillDb = + skillTools == null + ? {} + : { + listSkillsByAccess: async () => ({ skills: [], has_more: false, after: null }), + getSkillByName: jest.fn().mockResolvedValue({ + _id: skillId, + name: 'spreadsheet-analysis', + body: 'Analyze spreadsheets.', + author: { toString: () => 'user-1' } as unknown as import('mongoose').Types.ObjectId, + allowedTools: skillTools, + }) as InitializeAgentDbMethods['getSkillByName'], + }; + + return initializeAgent( + { + req, + res, + agent, + loadTools, + requestFiles: [file], + codeEnvAvailable: true, + fileSearchAvailable: true, + endpointOption: { endpoint: EModelEndpoint.agents }, + allowedProviders: new Set([Providers.OPENAI]), + isInitialAgent: true, + deriveText, + persistDerivation, + ...skillParams, + }, + { ...db, ...skillDb, ...(saveFileTextDerivation != null && { saveFileTextDerivation }) }, + ); + } + + /** The delivery routes content inspection saw for one file, per call that inspected it. */ + const inspectionsOf = (fileId: string) => + (assertModelBoundContent as jest.Mock).mock.calls + .map(([input]: [{ files?: IMongoFile[] }]) => + (input.files ?? []) + .filter((file) => file.file_id === fileId) + .map((file) => file.llmDeliveryPath), + ) + .filter((routes) => routes.length > 0); + + it('leaves a classic-era spreadsheet to Run Code, inspecting its classic route without counting or rewriting it', async () => { + const xlsx = classicEraXlsx(); + const stored = structuredClone(xlsx); + + const result = await initializeWith({ + file: xlsx, + policy: 'automatic', + tools: [Tools.execute_code], + fileContextCharLimit: 3, + }); + + expect(result.fileConsumers).toEqual({ executeCode: true, fileSearch: false }); + expect(result.requestAttachments).toEqual([{ ...stored, llmDeliveryPath: 'none' }]); + expect(result.currentRequestAttachments.map((file) => file.llmDeliveryPath)).toEqual(['none']); + expect(xlsx).toEqual(stored); + expect(result.deliveryRouting.reading).toBeDefined(); + expect([...new Set(inspectionsOf(xlsx.file_id).flat())]).toEqual(['text']); + }); + + it('delivers the spreadsheet text, re-running admission and inspection, once the loader drops the Run Code tool', async () => { + const xlsx = classicEraXlsx(); + const admission = jest.spyOn(attachments, 'assertAgentAttachmentLimits'); + + const result = await initializeWith({ + file: xlsx, + policy: 'automatic', + skillTools: [Tools.execute_code], + }); + + expect(result.fileConsumers).toEqual({ executeCode: false, fileSearch: false }); + expect(result.requestAttachments).toEqual([{ ...xlsx, llmDeliveryPath: 'text' }]); + expect(xlsx.llmDeliveryPath).toBe('text'); + expect(admission).toHaveBeenCalledTimes(2); + expect(inspectionsOf(xlsx.file_id)).toHaveLength(2); + }); + + it('leaves the text out rather than failing the turn when it no longer fits the allowance', async () => { + /* File Search is loaded but nothing was provisioned for it, so the overflow finds no + * reader; the policy still leaves the text out instead of failing the turn. */ + const xlsx = classicEraXlsx(); + + const result = await initializeWith({ + file: xlsx, + policy: 'automatic', + tools: [Tools.file_search], + loaded: { toolNames: [Tools.file_search] }, + fileContextCharLimit: 3, + }); + + expect(result.fileConsumers).toEqual({ executeCode: false, fileSearch: true }); + expect(result.requestAttachments).toEqual([{ ...xlsx, llmDeliveryPath: 'none' }]); + expect(getTurnReadingContext(result.deliveryRouting)?.stats()).toMatchObject({ + overflow: 1, + dropped: 0, + }); + }); + + it('fails the turn on the text allowance as classic routing does once the loader drops the only file tool', async () => { + const xlsx = classicEraXlsx(); + + await expect( + initializeWith({ + file: xlsx, + policy: 'automatic', + skillTools: [Tools.execute_code], + fileContextCharLimit: 3, + }), + ).rejects.toMatchObject({ name: 'AgentAttachmentLimitError' }); + }); + + it.each([ + { record: 'classic-era', file: classicEraXlsx, policy: undefined }, + { record: 'classic-era', file: classicEraXlsx, policy: 'classic' as const }, + { record: 'deferred', file: deferredXlsx, policy: undefined }, + ])( + 'keeps classic routing and attaches no reading evidence for a $record spreadsheet when the policy is $policy', + async ({ file, policy }) => { + const xlsx = file(); + + const result = await initializeWith({ file: xlsx, policy, tools: [Tools.execute_code] }); + + expect(result.deliveryRouting.reading).toBeUndefined(); + expect(result.requestAttachments).toEqual([xlsx]); + }, + ); + + it('keeps an oversized PDF the automatic policy reads past the endpoint size limit', async () => { + const pdf = oversizedPdf(); + + const result = await initializeWith({ + file: pdf, + policy: 'automatic', + tools: [Tools.file_search], + loaded: { toolNames: [Tools.file_search] }, + endpointConfig: { fileSizeLimit: 1 }, + provisionState: provisionStateFor({ search: [pdf] }), + }); + + expect(result.fileConsumers).toEqual({ executeCode: false, fileSearch: true }); + expect(result.requestAttachments.map((file) => file.file_id)).toEqual([pdf.file_id]); + expect(result.requestAttachments[0].llmDeliveryPath).toBe('none'); + expect(getTurnReadingContext(result.deliveryRouting)?.stats().dropped).toBe(0); + expect(result.dynamicToolContextMap?.file_inventory).toContain( + `"annual.pdf" (PDF): too large to send directly. Search it with ${Tools.file_search}; it is indexed when you first search.`, + ); + }); + + const refusedImage = () => + ({ + ...oversizedPdf(), + file_id: 'image-file', + filename: 'scan.png', + type: 'image/png', + }) as IMongoFile; + + it.each([ + { + refused: 'an oversized PDF', + file: oversizedPdf, + policy: 'automatic' as const, + dropped: ['pdf-file'], + }, + { refused: 'an oversized PDF', file: oversizedPdf, policy: undefined, dropped: undefined }, + { + refused: 'an image', + file: refusedImage, + policy: 'automatic' as const, + dropped: ['image-file'], + }, + ])( + 'drops $refused the endpoint refuses as classic routing does when no file tool is loaded (policy $policy, recorded $dropped)', + async ({ file, policy, dropped }) => { + const result = await initializeWith({ + file: file(), + policy, + endpointConfig: { fileSizeLimit: 1 }, + }); + + expect(result.fileConsumers).toEqual({ executeCode: false, fileSearch: false }); + expect(result.requestAttachments).toEqual([]); + const context = getTurnReadingContext(result.deliveryRouting); + expect(context?.dropped().map((file) => file.file_id)).toEqual(dropped); + }, + ); + + it('leaves persistent agent files on their stored route when no reading changed', async () => { + /* Agent files are never read by the automatic policy. Re-resolving this stored text route + * to its classic provider route every turn would re-run the full-set checks classic skips. */ + const agentFile = { + file_id: 'agent-pdf', + filename: 'handbook.pdf', + filepath: '/uploads/user-1/handbook.pdf', + type: 'application/pdf', + bytes: 2048, + source: FileSources.local, + context: FileContext.agents, + text: 'handbook text', + llmDeliveryPath: 'text', + metadata: {}, + } as IMongoFile; + + const result = await initializeWith({ + file: classicEraXlsx(), + policy: 'automatic', + contextFiles: [agentFile], + }); + + expect(result.agentContextAttachments).toEqual([agentFile]); + const inspectedAgentFile = (assertModelBoundContent as jest.Mock).mock.calls + .flatMap(([input]: [{ files?: IMongoFile[] }]) => input.files ?? []) + .filter((file) => file.file_id === agentFile.file_id); + expect(inspectedAgentFile).toEqual([]); + }); + + describe('file inventory', () => { + const handbook = { file_id: 'kb', filename: 'handbook.pdf', fromAgent: true }; + const filters: FiltersConfig = { + files: { + pii: { + fields: ['content'], + starterPatterns: [], + customPatterns: [{ id: 'private', label: 'private value', regex: 'PRIVATE-[A-Z]+' }], + }, + }, + }; + it.each([ + ['as the loader primed them', [handbook], ['kb']], + ['as none when the loader primed none', [], []], + ['as unknown when the loader did not report them', undefined, undefined], + ])('records the File Search files %s', async (_case, primedSearchFiles, expected) => { + const result = await initializeWith({ + file: classicEraXlsx(), + policy: 'automatic', + loaded: { primedSearchFiles }, + }); + + expect(result.primedSearchFileIds).toEqual(expected); + }); + + it('reuses an indexed user PDF through the primed search files instead of its cached text', async () => { + const pdf = { + ...oversizedPdf(), + embedded: true, + text: 'Complete cached text', + } as IMongoFile; + const result = await initializeWith({ + file: pdf, + policy: 'automatic', + tools: [Tools.file_search], + endpointConfig: { fileSizeLimit: 1 }, + loaded: { + toolNames: [Tools.file_search], + primedSearchFiles: [{ file_id: pdf.file_id, filename: pdf.filename, fromAgent: false }], + }, + toolResources: { + [EToolResources.file_search]: { files: [queued(pdf)], file_ids: [] }, + }, + }); + + expect(result.provisionState).toBeUndefined(); + expect(result.requestAttachments[0].llmDeliveryPath).toBe('none'); + expect(getTurnReadingContext(result.deliveryRouting)?.judge(pdf).search).toBe('reachable'); + expect(getTurnReadingContext(result.deliveryRouting)?.searchState(pdf.file_id)).toBe('ready'); + expect(result.dynamicToolContextMap?.file_inventory).toContain('Search it with file_search.'); + expect(result.dynamicToolContextMap?.file_inventory).not.toContain('text is included'); + }); + + it('uses both resource representations when an older loader does not report primed files', async () => { + const pdf = { ...oversizedPdf(), embedded: true } as IMongoFile; + const result = await initializeWith({ + file: pdf, + policy: 'automatic', + tools: [Tools.file_search], + endpointConfig: { fileSizeLimit: 1 }, + loaded: { toolNames: [Tools.file_search] }, + toolResources: { + [EToolResources.file_search]: { files: [queued(pdf)], file_ids: ['knowledge-base'] }, + }, + }); + + expect(getTurnReadingContext(result.deliveryRouting)?.searchState(pdf.file_id)).toBe('ready'); + expect(getTurnReadingContext(result.deliveryRouting)?.searchState('knowledge-base')).toBe( + 'ready', + ); + expect(result.requestAttachments[0].llmDeliveryPath).toBe('none'); + }); + + it('honors a loader that explicitly primed no search files', async () => { + const pdf = { ...oversizedPdf(), embedded: true } as IMongoFile; + const result = await initializeWith({ + file: pdf, + policy: 'automatic', + tools: [Tools.file_search], + endpointConfig: { fileSizeLimit: 1 }, + loaded: { toolNames: [Tools.file_search], primedSearchFiles: [] }, + toolResources: { [EToolResources.file_search]: { files: [queued(pdf)] } }, + }); + + expect(getTurnReadingContext(result.deliveryRouting)?.judge(pdf).search).toBe('unreachable'); + expect( + getTurnReadingContext(result.deliveryRouting)?.searchState(pdf.file_id), + ).toBeUndefined(); + expect(result.dynamicToolContextMap?.file_inventory).toContain('cannot be read on this turn'); + }); + + it('plans the queued spreadsheet destination and checks both advertisements', async () => { + const xlsx = classicEraXlsx(); + + const result = await initializeWith({ + file: xlsx, + policy: 'automatic', + tools: [Tools.execute_code], + provisionState: provisionStateFor({ code: [xlsx] }), + filters, + }); + + expect(result.provisionState?.codeEnvDestinations?.get(xlsx.file_id)).toBe('quarterly.xlsx'); + expect(assertModelBoundContent).toHaveBeenCalledWith({ + filters, + files: [{ content: result.dynamicToolContextMap?.queued_code_files }], + }); + expect(assertModelBoundContent).toHaveBeenCalledWith({ + filters, + files: [{ content: result.dynamicToolContextMap?.file_inventory }], + }); + }); + + it('fails the turn when the content policy refuses the inventory', async () => { + const xlsx = () => ({ ...classicEraXlsx(), filename: 'PRIVATE-PLAN.xlsx' }) as IMongoFile; + const params = { tools: [Tools.execute_code], filters }; + + await expect( + initializeWith({ ...params, file: xlsx(), policy: 'automatic' }), + ).rejects.toBeInstanceOf(ContentFilterError); + await expect(initializeWith({ ...params, file: xlsx() })).resolves.toBeDefined(); + }); + }); + + describe('with a host that derives text', () => { + const derived = { + status: 'derived' as const, + text: 'Q1,1200\nQ2,1400', + textDerivation: { + outcome: 'complete' as const, + extractor: 'document_parser' as const, + at: 1, + }, + }; + const deriver = () => + jest.fn, Parameters>(async () => derived); + const saver = () => + jest.fn< + ReturnType>, + Parameters> + >(async () => true); + + it.each([ + ['classic', undefined], + ['automatic', 'automatic' as const], + ])( + 'derives a deferred spreadsheet with no Run Code under %s routing and saves it for the owner', + async (policyName, policy) => { + const xlsx = deferredXlsx(); + const stored = structuredClone(xlsx); + const deriveText = deriver(); + const saveFileTextDerivation = saver(); + + const result = await initializeWith({ + file: xlsx, + policy, + deriveText, + saveFileTextDerivation, + }); + + expect(getTurnReadingContext(result.deliveryRouting)?.policy).toBe(policyName); + expect(deriveText).toHaveBeenCalledTimes(1); + expect(deriveText).toHaveBeenCalledWith( + expect.objectContaining({ file_id: xlsx.file_id }), + undefined, + ); + expect(result.requestAttachments).toEqual([ + { + ...stored, + llmDeliveryPath: 'text', + text: derived.text, + metadata: { destinationChosen: false, textDerivation: derived.textDerivation }, + }, + ]); + expect(saveFileTextDerivation).toHaveBeenCalledTimes(1); + expect(saveFileTextDerivation).toHaveBeenCalledWith( + { file_id: xlsx.file_id, text: derived.text, textDerivation: derived.textDerivation }, + { user: 'user-1', tenantId: undefined }, + ); + expect(xlsx).toEqual(stored); + }, + ); + + it.each([ + ['with stored text', classicEraXlsx()], + ['without stored text', { ...classicEraXlsx(), text: undefined } as IMongoFile], + ])( + 'never derives a record the automatic policy did not mark under classic routing (%s)', + async (_shape, file) => { + const deriveText = deriver(); + const saveFileTextDerivation = saver(); + + const result = await initializeWith({ file, deriveText, saveFileTextDerivation }); + + expect(getTurnReadingContext(result.deliveryRouting)?.policy).toBe('classic'); + expect(deriveText).not.toHaveBeenCalled(); + expect(saveFileTextDerivation).not.toHaveBeenCalled(); + expect(result.requestAttachments).toEqual([file]); + }, + ); + + it('saves through the host persister, once for every agent sharing it', async () => { + const deriveText = deriver(); + const hostSave = saver(); + const ownSave = saver(); + const persistDerivation = createDerivationPersister(hostSave, { user: 'user-1' }); + + for (let agent = 0; agent < 2; agent++) { + await initializeWith({ + file: deferredXlsx(), + policy: 'automatic', + deriveText, + saveFileTextDerivation: ownSave, + persistDerivation, + }); + } + + expect(deriveText).toHaveBeenCalledTimes(2); + expect(ownSave).not.toHaveBeenCalled(); + expect(hostSave).toHaveBeenCalledTimes(1); + expect(hostSave).toHaveBeenCalledWith( + expect.objectContaining({ text: derived.text, textDerivation: derived.textDerivation }), + { user: 'user-1' }, + ); + }); + + it('leaves a deferred spreadsheet to Run Code without deriving it', async () => { + const xlsx = deferredXlsx(); + const deriveText = deriver(); + const saveFileTextDerivation = saver(); + + const result = await initializeWith({ + file: xlsx, + policy: 'automatic', + tools: [Tools.execute_code], + deriveText, + saveFileTextDerivation, + }); + + expect(result.fileConsumers).toEqual({ executeCode: true, fileSearch: false }); + expect(deriveText).not.toHaveBeenCalled(); + expect(saveFileTextDerivation).not.toHaveBeenCalled(); + expect(result.requestAttachments).toEqual([xlsx]); + }); + }); +}); diff --git a/packages/api/src/agents/__tests__/run-codeTools.test.ts b/packages/api/src/agents/__tests__/run-codeTools.test.ts index 286735db7d7..a59ef88e173 100644 --- a/packages/api/src/agents/__tests__/run-codeTools.test.ts +++ b/packages/api/src/agents/__tests__/run-codeTools.test.ts @@ -1,13 +1,16 @@ import { FileContext } from 'librechat-data-provider'; import type { AgentInputs, SubagentTaskConfig, SubagentResolveContext } from '@librechat/agents'; -import type { TFile } from 'librechat-data-provider'; +import type { TFile, FiltersConfig } from 'librechat-data-provider'; +import type { AppConfig } from '@librechat/data-schemas'; import type { HostSubagentTaskConfig } from '~/agents/subagentDelivery'; import type { ProvisionToolContext } from '~/files/provision/callback'; import type { ServerRequest } from '~/types'; import { createProvisionFilesCallback } from '~/files/provision/callback'; import { SUBAGENT_COMPLETION_DELIVERY } from '~/agents/subagentDelivery'; import { mergeCodeFilesIntoContext } from '~/agents/codeFilesSession'; +import { resolveTurnDeliveryRouting } from '~/agents/files/delivery'; import { CHECK_BACKGROUND_TASK_NAME } from '~/agents/background'; +import { ContentFilterError } from '~/middleware/contentFilter'; import { createRun } from '~/agents/run'; /** @@ -98,6 +101,7 @@ async function captureRunConfig( async function captureAgentsRunConfig( agents: Array>, subagentTasks?: SubagentTaskConfig, + appConfig?: AppConfig, ): Promise> { await createRun({ agents: agents as never, @@ -105,6 +109,7 @@ async function captureAgentsRunConfig( streaming: true, streamUsage: true, subagentTasks, + appConfig, }); const createMock = Run.create as jest.Mock; expect(createMock).toHaveBeenCalledTimes(1); @@ -217,6 +222,206 @@ describe('createRun code-tool eager/session wiring', () => { expect(childInput.additional_instructions?.match(/\/mnt\/data\/data\.csv/g)).toHaveLength(1); }); + describe('file inventory', () => { + const INVENTORY_HEADER = 'Attached files and how you can read them on this turn'; + const runsCode = { executeCode: true, fileSearch: false }; + const routingFor = (policy?: 'automatic' | 'classic') => + resolveTurnDeliveryRouting({ + agent: { provider: 'openAI', endpoint: 'openAI' }, + config: { + fileConfig: { + endpoints: { openAI: policy == null ? {} : { llmDeliveryPolicy: policy } }, + }, + }, + }); + const upload = (overrides: Partial = {}): TFile => ({ + file_id: 'shared', + filename: 'data.csv', + filepath: '/uploads/data.csv', + type: 'text/csv', + user: 'user-1', + object: 'file', + bytes: 10, + embedded: false, + usage: 0, + context: FileContext.message_attachment, + llmDeliveryPath: 'none', + metadata: { destinationChosen: false }, + createdAt: '2026-09-01', + ...overrides, + }); + /** Every code path a text advertises, in order. */ + const advertisedPaths = (text: unknown): string[] => + typeof text === 'string' ? (text.match(/\/mnt\/data\/\S+?(?=\.?(?:\s|$))/g) ?? []) : []; + const plannedPaths = (destinations?: Map): string[] => + [...(destinations?.values() ?? [])].map((name) => `/mnt/data/${name}`); + + /** A parent and the lazy child it spawns, both holding colliding queued uploads. */ + function setupRun(policy?: 'automatic' | 'classic', requestFiles: TFile[] = []) { + const shared = upload(); + const fresh = upload({ file_id: 'new', createdAt: '2026-10-01' }); + const child = { + ...makeAgent({ + id: 'child', + deliveryRouting: routingFor(policy), + currentRequestAttachments: [shared, fresh, ...requestFiles], + fileConsumers: runsCode, + }), + dynamicToolContextMap: { file_inventory: 'stale /mnt/data/draft-alias.csv' } as Record< + string, + unknown + >, + provisionState: { + codeEnvFiles: [{ ...shared }, { ...fresh }], + vectorDBFiles: [], + aliveFileIds: new Set(), + agentScopedFileIds: new Set(), + codeEnvDestinations: new Map([ + ['shared', 'draft-alias.csv'], + ['new', 'data.csv'], + ]), + }, + }; + const parent = { + ...makeAgent({ + deliveryRouting: routingFor(policy), + currentRequestAttachments: [shared, ...requestFiles], + fileConsumers: runsCode, + subagents: { enabled: true, allowSelf: false, agent_ids: ['child'] }, + lazySubagentConfigs: [ + { id: 'child', configId: 'child:1', resolve: jest.fn().mockResolvedValue(child) }, + ], + }), + dynamicToolContextMap: {} as Record, + provisionState: { + codeEnvFiles: [{ ...shared }], + vectorDBFiles: [], + aliveFileIds: new Set(), + agentScopedFileIds: new Set(), + codeEnvDestinations: undefined as Map | undefined, + }, + }; + return { parent, child }; + } + + async function runWithChild(policy?: 'automatic' | 'classic') { + const { parent, child } = setupRun(policy); + const config = await captureRunConfig(parent); + const [parentInput] = (config.graphConfig as { agents: AgentInputs[] }).agents; + const resolveInputs = parentInput.subagentConfigs?.[0]?.resolveAgentInputs; + if (!resolveInputs) throw new Error('Missing lazy subagent resolver'); + const childInput = await resolveInputs({ + signal: new AbortController().signal, + } as SubagentResolveContext); + return { parent, child, parentInput, childInput }; + } + + it('re-renders a lazy child’s inventory at the paths its re-plan advertises', async () => { + const { parent, child, parentInput, childInput } = await runWithChild('automatic'); + const childInventory = child.dynamicToolContextMap.file_inventory; + const childPlan = plannedPaths(child.provisionState.codeEnvDestinations); + + expect(childPlan).toHaveLength(2); + expect(childPlan).toContain('/mnt/data/data.csv'); + expect(advertisedPaths(childInventory).sort()).toEqual([...childPlan].sort()); + expect(advertisedPaths(child.dynamicToolContextMap.queued_code_files).sort()).toEqual( + [...childPlan].sort(), + ); + expect(childInventory).not.toContain('draft-alias.csv'); + expect(childInput.additional_instructions).toContain(childInventory); + + const parentInventory = parent.dynamicToolContextMap.file_inventory; + expect(advertisedPaths(parentInventory)).toEqual( + plannedPaths(parent.provisionState.codeEnvDestinations), + ); + expect(parentInput.additional_instructions).toContain(parentInventory); + }); + + it('never tells a child that a request file was sent or included with its task', async () => { + const brief = upload({ + file_id: 'brief', + filename: 'brief.pdf', + type: 'application/pdf', + llmDeliveryPath: 'provider', + }); + const { parent, child } = setupRun('automatic', [brief]); + const eagerChild = { + ...makeAgent({ + id: 'eager-child', + deliveryRouting: routingFor('automatic'), + currentRequestAttachments: [brief], + fileConsumers: runsCode, + }), + dynamicToolContextMap: {} as Record, + }; + Object.assign(parent, { subagentAgentConfigs: [eagerChild] }); + + const config = await captureRunConfig(parent); + const [parentInput] = (config.graphConfig as { agents: AgentInputs[] }).agents; + const resolveInputs = parentInput.subagentConfigs?.find( + (subagent) => subagent.resolveAgentInputs != null, + )?.resolveAgentInputs; + if (!resolveInputs) throw new Error('Missing lazy subagent resolver'); + const childInput = await resolveInputs({ + signal: new AbortController().signal, + } as SubagentResolveContext); + + const sentOrIncluded = /sent with this message|included in this message/; + for (const inventory of [ + child.dynamicToolContextMap.file_inventory, + eagerChild.dynamicToolContextMap.file_inventory, + ]) { + expect(inventory).toContain('"brief.pdf" (PDF): not sent with your task.'); + expect(inventory).not.toMatch(sentOrIncluded); + } + expect(childInput.additional_instructions).not.toMatch(sentOrIncluded); + expect(parent.dynamicToolContextMap.file_inventory).toContain( + '"brief.pdf" (PDF): sent with this message.', + ); + }); + + it.each([undefined, 'classic' as const])( + 'writes no inventory for either agent under %p routing', + async (policy) => { + const { parent, child, parentInput, childInput } = await runWithChild(policy); + + expect(parent.dynamicToolContextMap).not.toHaveProperty('file_inventory'); + expect(child.dynamicToolContextMap).not.toHaveProperty('file_inventory'); + expect(parentInput.additional_instructions).not.toContain(INVENTORY_HEADER); + expect(childInput.additional_instructions).not.toContain(INVENTORY_HEADER); + expect(childInput.additional_instructions).toContain('/mnt/data/data.csv'); + }, + ); + + it('content-checks the inventory with the run’s filters', async () => { + const filters: FiltersConfig = { + files: { + pii: { + fields: ['content'], + starterPatterns: [], + customPatterns: [{ id: 'private', label: 'private value', regex: 'PRIVATE-[A-Z]+' }], + }, + }, + }; + const file = upload({ filename: 'PRIVATE-PLAN.csv' }); + const agent = makeAgent({ + deliveryRouting: routingFor('automatic'), + currentRequestAttachments: [file], + fileConsumers: runsCode, + provisionState: { + codeEnvFiles: [{ ...file }], + vectorDBFiles: [], + aliveFileIds: new Set(), + agentScopedFileIds: new Set(), + }, + }); + + await expect( + captureAgentsRunConfig([agent], undefined, { filters } as AppConfig), + ).rejects.toBeInstanceOf(ContentFilterError); + }); + }); + it('settles a lazy child only after the run has its inputs, so a late cancel is given back', async () => { const controller = new AbortController(); const outcomes: string[] = []; diff --git a/packages/api/src/agents/attachments.test.ts b/packages/api/src/agents/attachments.test.ts index e4a7678143d..29f6df9555e 100644 --- a/packages/api/src/agents/attachments.test.ts +++ b/packages/api/src/agents/attachments.test.ts @@ -1,5 +1,10 @@ import { logger } from '@librechat/data-schemas'; -import { EModelEndpoint, FileContext, FileSources } from 'librechat-data-provider'; +import { + FileContext, + FileSources, + EModelEndpoint, + allocateDirectContent, +} from 'librechat-data-provider'; import type { IMongoFile } from '@librechat/data-schemas'; import type { ServerRequest } from '~/types'; @@ -22,6 +27,8 @@ import { buildAgentContextAttachmentsByAgentId, isModelBoundAttachmentFile, isToolOwnedAttachment, + measureAttachment, + resolveAgentAttachmentLimits, } from './attachments'; import { applyTurnDelivery, resolveTurnDeliveryRouting } from './files/delivery'; @@ -689,6 +696,54 @@ describe('agent attachment helpers', () => { ).rejects.toBeInstanceOf(AgentAttachmentPolicyError); }); + it('holds a shared provider copy to each automatic receiver size limit', async () => { + /* The primary's automatic decision sent this PDF natively within its own 10 MB limit; the + * receiver reads the same encoded message, so its smaller limit must still refuse it. */ + const shared = { + ...makeTextFile('shared-pdf', 'shared.pdf', ''), + source: FileSources.local, + type: 'application/pdf', + bytes: 3 * 1024 * 1024, + context: FileContext.message_attachment, + llmDeliveryPath: 'provider', + metadata: { destinationChosen: false }, + } as IMongoFile; + const textCopy = { + ...shared, + file_id: 'shared-text', + llmDeliveryPath: 'text', + text: 'extracted', + } as IMongoFile; + const fileConfig: NonNullable['fileConfig'] = { + endpoints: { + openAI: { fileSizeLimit: 10, llmDeliveryPolicy: 'automatic' }, + Moonshot: { fileSizeLimit: 1, llmDeliveryPolicy: 'automatic' }, + }, + }; + const req = { body: { fileTokenLimit: 1000 }, config: { fileConfig } } as ServerRequest; + const searches = { executeCode: false, fileSearch: true }; + const shareWith = (sharedAttachments: IMongoFile[], consumers = searches) => + buildAgentScopedContext({ + agentIds: ['primary', 'secondary'], + attachmentsByAgentId: new Map(), + sharedAttachments, + endpointsByAgentId: new Map([ + ['primary', { endpoint: 'openAI' }], + ['secondary', { endpoint: 'Moonshot' }], + ]), + req, + consumers, + }); + + await expect(shareWith([shared])).rejects.toBeInstanceOf(AgentAttachmentPolicyError); + await expect(shareWith([textCopy])).resolves.toBeInstanceOf(Map); + /* Without a file tool the automatic policy reads nothing, so the size limit is admission + * for the text copy as well, as classic routing applies it. */ + await expect( + shareWith([textCopy], { executeCode: false, fileSearch: false }), + ).rejects.toBeInstanceOf(AgentAttachmentPolicyError); + }); + it('does not apply the primary file count limit across disjoint private scopes', async () => { const req = { body: { fileTokenLimit: 1000 }, @@ -890,3 +945,168 @@ describe('files that belong to a tool', () => { ).not.toThrow(); }); }); + +describe('resolveAgentAttachmentLimits', () => { + const MB = 1024 * 1024; + + it('resolves the count, aggregate byte and extracted-text limits of an endpoint', () => { + expect( + resolveAgentAttachmentLimits({ + fileConfig: { + fileContextCharLimit: 40, + endpoints: { agents: { fileLimit: 3, totalSizeLimit: 2 } }, + }, + }), + ).toEqual({ count: 3, bytes: 2 * MB, textChars: 40 }); + }); + + it('reads the request config when no file config is passed, and prefers a passed one', () => { + const req = { config: { fileConfig: { endpoints: { agents: { fileLimit: 4 } } } } }; + expect(resolveAgentAttachmentLimits({ req }).count).toBe(4); + expect( + resolveAgentAttachmentLimits({ + req, + fileConfig: { endpoints: { agents: { fileLimit: 2 } } }, + }).count, + ).toBe(2); + }); + + it('leaves the count unlimited when the count is not enforced', () => { + expect( + resolveAgentAttachmentLimits({ + fileConfig: { endpoints: { agents: { fileLimit: 3 } } }, + enforceAttachmentCount: false, + }).count, + ).toBeUndefined(); + }); + + it('uses the global context size instead of an endpoint aggregate when asked to', () => { + const fileConfig = { + fileContextSizeLimit: 5, + endpoints: { [EModelEndpoint.openAI]: { totalSizeLimit: 1 } }, + }; + expect( + resolveAgentAttachmentLimits({ fileConfig, endpoint: EModelEndpoint.openAI }).bytes, + ).toBe(MB); + expect( + resolveAgentAttachmentLimits({ + fileConfig, + endpoint: EModelEndpoint.openAI, + useGlobalContextSizeLimit: true, + }).bytes, + ).toBe(5 * MB); + }); + + it.each<{ name: string; params: Parameters[0] }>([ + { + name: 'a provider-backed agent on the agents fallback', + params: { + fileConfig: { fileContextCharLimit: 7, endpoints: { agents: { fileLimit: 1 } } }, + endpoint: EModelEndpoint.openAI, + }, + }, + { + name: 'a named custom endpoint on the generic custom config', + params: { + fileConfig: { endpoints: { custom: { fileLimit: 2, totalSizeLimit: 1 } } }, + endpoint: 'Moonshot', + }, + }, + { + name: 'an explicit backing-endpoint aggregate', + params: { + fileConfig: { endpoints: { openAI: { totalSizeLimit: 3, fileLimit: 2 } } }, + endpoint: EModelEndpoint.openAI, + }, + }, + ])('is the limit the assertion enforces for $name', ({ params }) => { + const limits = resolveAgentAttachmentLimits(params); + const over = (limit: number | undefined): number => (limit ?? 0) + 1; + const files = Array.from({ length: over(limits.count) }, (_, index) => ({ + file_id: `file-${index}`, + })); + + expect(() => assertAgentAttachmentLimits({ ...params, attachments: files })).toThrow( + expect.objectContaining({ limitType: 'count', limit: limits.count }), + ); + expect(() => + assertAgentAttachmentLimits({ + ...params, + attachments: [{ file_id: 'large', bytes: over(limits.bytes) }], + }), + ).toThrow(expect.objectContaining({ limitType: 'bytes', limit: limits.bytes })); + expect(() => + assertAgentAttachmentLimits({ + ...params, + attachments: [{ file_id: 'long', text: 'x'.repeat(over(limits.textChars)) }], + }), + ).toThrow(expect.objectContaining({ limitType: 'extracted_text', limit: limits.textChars })); + }); +}); + +describe('measureAttachment', () => { + it('measures bytes, extracted text and whether the file counts', () => { + expect(measureAttachment({ file_id: 'doc', bytes: 12, text: 'hello' })).toEqual({ + fileId: 'doc', + counts: true, + bytes: 12, + textChars: 5, + }); + }); + + it('does not count a replayed historical file toward the attachment count', () => { + const historicalFileIds = new Set(['history']); + expect(measureAttachment({ file_id: 'history' }, { historicalFileIds }).counts).toBe(false); + expect(measureAttachment({ file_id: 'current' }, { historicalFileIds }).counts).toBe(true); + expect(measureAttachment({}, { historicalFileIds })).toEqual({ + fileId: '', + counts: true, + bytes: 0, + textChars: 0, + }); + }); + + it.each([null, undefined, -1, Number.NaN, Number.POSITIVE_INFINITY])( + 'measures unusable byte count %p as zero', + (bytes) => { + expect(measureAttachment({ file_id: 'odd', bytes }).bytes).toBe(0); + }, + ); + + it('matches the totals the attachment stats report', () => { + const files = [ + { file_id: 'history', bytes: 10, text: 'context' }, + { file_id: 'current', bytes: 5, text: 'new' }, + { bytes: -3, text: 'unidentified' }, + ]; + const historicalFileIds = new Set(['history']); + const entries = files.map((file) => measureAttachment(file, { historicalFileIds })); + + expect(collectAgentAttachmentStats(files, { historicalFileIds })).toMatchObject({ + attachmentCount: entries.filter((entry) => entry.counts).length, + totalKnownBytes: entries.reduce((sum, entry) => sum + entry.bytes, 0), + extractedTextChars: entries.reduce((sum, entry) => sum + entry.textChars, 0), + }); + }); + + it('lets a first-fit allocation and the assertion agree on what fits', () => { + const fileConfig = { fileContextCharLimit: 10, endpoints: { agents: { fileLimit: 3 } } }; + const files = [ + { file_id: 'a', bytes: 1, text: 'abcd' }, + { file_id: 'b', bytes: 1, text: 'efghijk' }, + { file_id: 'c', bytes: 1, text: 'lmn' }, + ]; + const overflow = allocateDirectContent( + [], + files.map((file) => measureAttachment(file)), + resolveAgentAttachmentLimits({ fileConfig }), + ); + const admitted = files.filter((file) => !overflow.has(file.file_id)); + + expect(overflow).toEqual(new Set(['b'])); + expect(() => assertAgentAttachmentLimits({ attachments: admitted, fileConfig })).not.toThrow(); + expect(() => assertAgentAttachmentLimits({ attachments: files, fileConfig })).toThrow( + expect.objectContaining({ limitType: 'extracted_text' }), + ); + }); +}); diff --git a/packages/api/src/agents/attachments.ts b/packages/api/src/agents/attachments.ts index 04885b05c2b..43509637066 100644 --- a/packages/api/src/agents/attachments.ts +++ b/packages/api/src/agents/attachments.ts @@ -6,13 +6,18 @@ import { mergeFileConfig, getEndpointFileConfig, } from 'librechat-data-provider'; +import type { + DirectContentEntry, + DirectContentLimits, + TurnFileConsumers, +} from 'librechat-data-provider'; import type { IMongoFile } from '@librechat/data-schemas'; import type { TokenCountFn } from '~/utils/text'; import type { ServerRequest } from '~/types'; import { filterFilesByEndpointRuntimeConfig } from '~/files/filter'; import { AGENT_ATTACHMENT_LIMIT_EXCEEDED } from './errors'; +import { extractFileContext } from '~/files/context'; import { countTokens } from '~/utils/tokenizer'; -import { extractFileContext } from '~/files'; type FileWithId = { file_id?: string | null; @@ -236,6 +241,32 @@ export function isAgentAttachmentLimitError( return error instanceof AgentAttachmentLimitError || error instanceof AgentAttachmentPolicyError; } +const toNonNegativeFinite = (value: number | null | undefined): number | undefined => + typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : undefined; + +/** + * One file's direct-content footprint as the attachment limits measure it. A replayed historical + * file spends the context budgets but not the current submission's count allowance. + */ +export function measureAttachment( + file: Pick, + { historicalFileIds }: { historicalFileIds?: ReadonlySet } = {}, +): DirectContentEntry { + return { + fileId: file.file_id ?? '', + counts: !file.file_id || !historicalFileIds?.has(file.file_id), + bytes: toNonNegativeFinite(file.bytes) ?? 0, + textChars: typeof file.text === 'string' ? file.text.length : 0, + }; +} + +/** A file's direct content as admission charges it on the current request: none unless model-bound. */ +export function measureModelBoundAttachment(file: AttachmentTelemetryFile): DirectContentEntry { + return isModelBoundAttachmentFile(file) + ? measureAttachment(file) + : { fileId: file.file_id ?? '', counts: false, bytes: 0, textChars: 0 }; +} + export function collectAgentAttachmentStats( attachments?: Iterable | null, options: { @@ -256,14 +287,13 @@ export function collectAgentAttachmentStats( if (!file) { continue; } - const extractedTextChars = typeof file.text === 'string' ? file.text.length : 0; + const { counts, bytes, textChars } = measureAttachment(file, options); if (file.file_id && seenFileIds.has(file.file_id)) { if (options.countRepeatedExtractedText === true) { - stats.extractedTextChars += extractedTextChars; + stats.extractedTextChars += textChars; } if (options.countRepeatedBytes === true) { - stats.totalKnownBytes += - Number.isFinite(file.bytes) && Number(file.bytes) >= 0 ? Number(file.bytes) : 0; + stats.totalKnownBytes += bytes; } continue; } @@ -271,21 +301,17 @@ export function collectAgentAttachmentStats( seenFileIds.add(file.file_id); } - const bytes = Number.isFinite(file.bytes) && Number(file.bytes) >= 0 ? Number(file.bytes) : 0; - const pageCount = - Number.isFinite(file.metadata?.pageCount) && Number(file.metadata?.pageCount) >= 0 - ? Number(file.metadata?.pageCount) - : undefined; - if (!file.file_id || !options.historicalFileIds?.has(file.file_id)) { + const pageCount = toNonNegativeFinite(file.metadata?.pageCount); + if (counts) { stats.attachmentCount += 1; } stats.totalKnownBytes += bytes; - stats.extractedTextChars += extractedTextChars; + stats.extractedTextChars += textChars; stats.files.push({ ...(file.file_id && { fileId: file.file_id }), ...(file.type && { mimeType: file.type }), ...(bytes > 0 && { bytes }), - ...(extractedTextChars > 0 && { extractedTextChars }), + ...(textChars > 0 && { extractedTextChars: textChars }), ...(pageCount != null && { pageCount }), }); } @@ -293,35 +319,26 @@ export function collectAgentAttachmentStats( return stats; } -export function assertAgentAttachmentLimits({ - attachments, - req, +/** + * The direct-content limits a turn's attachments are held to on an endpoint: the count, aggregate + * byte and extracted-text limits {@link assertAgentAttachmentLimits} enforces, so an allocation + * against them and the assertion cannot disagree. + */ +export function resolveAgentAttachmentLimits({ fileConfig: providedFileConfig, + req, endpoint = EModelEndpoint.agents, endpointType, - countRepeatedExtractedText = false, - countRepeatedBytes = countRepeatedExtractedText, enforceAttachmentCount = true, - historicalFileIds, useGlobalContextSizeLimit = false, }: { - attachments?: Iterable | null; + fileConfig?: DynamicFileConfig; req?: AgentAttachmentLimitRequest; - fileConfig?: Parameters[0]; endpoint?: string | null; endpointType?: string | null; - countRepeatedExtractedText?: boolean; - countRepeatedBytes?: boolean; enforceAttachmentCount?: boolean; - /** Replayed files consume context budgets, not the current submission's count allowance. */ - historicalFileIds?: ReadonlySet; useGlobalContextSizeLimit?: boolean; -}): AgentAttachmentStats { - const stats = collectAgentAttachmentStats(attachments, { - countRepeatedExtractedText, - countRepeatedBytes, - historicalFileIds, - }); +}): DirectContentLimits { const dynamicFileConfig = providedFileConfig ?? req?.config?.fileConfig; const fileConfig = mergeFileConfig(dynamicFileConfig); const endpointConfig = getEndpointFileConfig({ fileConfig, endpoint, endpointType }); @@ -339,18 +356,57 @@ export function assertAgentAttachmentLimits({ ? undefined : getExplicitEndpointAggregateLimit(dynamicFileConfig, endpoint, endpointType)) ?? fileConfig.fileContextSizeLimit; - const configuredContextCharLimit = dynamicFileConfig?.fileContextCharLimit; - const limits: Array<[AgentAttachmentLimit, number, number | undefined]> = [ - ['count', stats.attachmentCount, enforceAttachmentCount ? configuredFileLimit : undefined], - ['bytes', stats.totalKnownBytes, configuredContextSizeLimit], - [ - 'extracted_text', - stats.extractedTextChars, - configuredContextCharLimit ?? fileConfig.fileContextCharLimit, - ], + return { + count: enforceAttachmentCount ? configuredFileLimit : undefined, + bytes: configuredContextSizeLimit, + textChars: dynamicFileConfig?.fileContextCharLimit ?? fileConfig.fileContextCharLimit, + }; +} + +export function assertAgentAttachmentLimits({ + attachments, + req, + fileConfig, + endpoint, + endpointType, + countRepeatedExtractedText = false, + countRepeatedBytes = countRepeatedExtractedText, + enforceAttachmentCount, + historicalFileIds, + useGlobalContextSizeLimit, +}: { + attachments?: Iterable | null; + req?: AgentAttachmentLimitRequest; + fileConfig?: DynamicFileConfig; + endpoint?: string | null; + endpointType?: string | null; + countRepeatedExtractedText?: boolean; + countRepeatedBytes?: boolean; + enforceAttachmentCount?: boolean; + /** Replayed files consume context budgets, not the current submission's count allowance. */ + historicalFileIds?: ReadonlySet; + useGlobalContextSizeLimit?: boolean; +}): AgentAttachmentStats { + const stats = collectAgentAttachmentStats(attachments, { + countRepeatedExtractedText, + countRepeatedBytes, + historicalFileIds, + }); + const limits = resolveAgentAttachmentLimits({ + fileConfig, + req, + endpoint, + endpointType, + enforceAttachmentCount, + useGlobalContextSizeLimit, + }); + const checks: Array<[AgentAttachmentLimit, number, number | undefined]> = [ + ['count', stats.attachmentCount, limits.count], + ['bytes', stats.totalKnownBytes, limits.bytes], + ['extracted_text', stats.extractedTextChars, limits.textChars], ]; - for (const [limitType, observed, limit] of limits) { + for (const [limitType, observed, limit] of checks) { if (limit != null && limit > 0 && observed > limit) { throw new AgentAttachmentLimitError(limitType, observed, limit); } @@ -367,6 +423,7 @@ export function assertAgentAttachmentTopology({ endpointType, endpointsByAgentId, historicalFileIds, + consumers, }: { sharedAttachments?: IMongoFile[]; scopedAttachmentsByAgentId?: Map; @@ -375,24 +432,23 @@ export function assertAgentAttachmentTopology({ endpointType?: string | null; endpointsByAgentId?: AgentAttachmentEndpointsByAgentId; historicalFileIds?: ReadonlySet; + /** The requesting agent's loaded file tools, which decide whether an oversized record stays. */ + consumers?: TurnFileConsumers | null; }): void { const agentIds = new Set([ ...scopedAttachmentsByAgentId.keys(), - ...(endpointsByAgentId instanceof Map - ? endpointsByAgentId.keys() - : Object.keys(endpointsByAgentId ?? {})), + ...listAgentIds(endpointsByAgentId), ]); for (const agentId of agentIds) { - const agentEndpoint = - endpointsByAgentId instanceof Map - ? endpointsByAgentId.get(agentId) - : endpointsByAgentId?.[agentId]; + const agentEndpoint = getAgentEntry(endpointsByAgentId, agentId); const compatibleSharedAttachments = filterFilesByEndpointRuntimeConfig(req?.config, { files: sharedAttachments, endpoint: agentEndpoint?.endpoint ?? endpoint ?? EModelEndpoint.agents, endpointType: agentEndpoint?.endpointType ?? endpointType, skipTotalSizeLimit: true, preserveTextSources: true, + bindProviderCopies: true, + consumers, }); if (compatibleSharedAttachments.length !== sharedAttachments.length) { throw new AgentAttachmentPolicyError(); @@ -524,6 +580,23 @@ export type AgentAttachmentEndpointsByAgentId = | Map | Record; +/** A collection keyed by agent id, which callers supply as a Map or a plain record. */ +export type AgentKeyed = Map | Record | null | undefined; + +export function getAgentEntry(entries: AgentKeyed, agentId: string): T | undefined { + return entries instanceof Map ? entries.get(agentId) : entries?.[agentId]; +} + +export function listAgentIds(entries: AgentKeyed): string[] { + return entries instanceof Map ? [...entries.keys()] : Object.keys(entries ?? {}); +} + +/** The entries of an agent-keyed collection, without the gaps a record may carry. */ +export function listAgentEntries(entries: AgentKeyed): T[] { + const values = entries instanceof Map ? [...entries.values()] : Object.values(entries ?? {}); + return values.filter((value): value is T => value != null); +} + export function collectFileIds( files?: Array | null, ): Set { @@ -659,10 +732,7 @@ export function getAgentContextAttachments({ return []; } - const attachments: TFile[] = - attachmentsByAgentId instanceof Map - ? (attachmentsByAgentId.get(agentId) ?? []) - : (attachmentsByAgentId[agentId] ?? []); + const attachments: TFile[] = getAgentEntry(attachmentsByAgentId, agentId) ?? []; if (!excludeFileIds || excludeFileIds.size === 0) { return attachments; @@ -679,6 +749,7 @@ export function buildAgentScopedAttachmentMap({ endpoint, endpointType, endpointsByAgentId, + consumers, }: { agentIds: string[]; attachmentsByAgentId: AgentContextAttachmentsByAgentId; @@ -687,12 +758,11 @@ export function buildAgentScopedAttachmentMap({ endpoint?: string | null; endpointType?: string | null; endpointsByAgentId?: AgentAttachmentEndpointsByAgentId; + /** The requesting agent's loaded file tools, which decide whether an oversized record stays. */ + consumers?: TurnFileConsumers | null; }): Map { const entries = Array.from(new Set(agentIds.filter(Boolean))).map((agentId) => { - const agentEndpoint = - endpointsByAgentId instanceof Map - ? endpointsByAgentId.get(agentId) - : endpointsByAgentId?.[agentId]; + const agentEndpoint = getAgentEntry(endpointsByAgentId, agentId); const attachments = getAgentContextAttachments({ agentId, attachmentsByAgentId, @@ -706,6 +776,7 @@ export function buildAgentScopedAttachmentMap({ endpointType: agentEndpoint?.endpointType ?? endpointType, skipTotalSizeLimit: true, preserveTextSources: true, + consumers, }), ] as const; }); @@ -723,6 +794,7 @@ export async function buildAgentScopedContext({ endpoint, endpointType, endpointsByAgentId, + consumers, }: { agentIds: string[]; attachmentsByAgentId: AgentContextAttachmentsByAgentId; @@ -734,6 +806,7 @@ export async function buildAgentScopedContext({ endpoint?: string | null; endpointType?: string | null; endpointsByAgentId?: AgentAttachmentEndpointsByAgentId; + consumers?: TurnFileConsumers | null; }): Promise> { const attachmentEntries = [ ...buildAgentScopedAttachmentMap({ @@ -744,6 +817,7 @@ export async function buildAgentScopedContext({ endpoint, endpointType, endpointsByAgentId, + consumers, }), ]; assertAgentAttachmentTopology({ @@ -754,6 +828,7 @@ export async function buildAgentScopedContext({ endpoint, endpointType, endpointsByAgentId, + consumers, }); const entries = await Promise.all( attachmentEntries.map(async ([agentId, attachments]) => { diff --git a/packages/api/src/agents/discovery.spec.ts b/packages/api/src/agents/discovery.spec.ts index a7fbb6e36ca..75cf4440272 100644 --- a/packages/api/src/agents/discovery.spec.ts +++ b/packages/api/src/agents/discovery.spec.ts @@ -310,6 +310,37 @@ describe('discoverConnectedAgents', () => { /** A handoff agent's provider config can turn native web search on as well, so * dropping the resolver here would leave it ungated one hop later. */ + it('forwards the text deriver and its persister to every handoff initializeAgent call', async () => { + const primaryConfig = makeConfig('A', [{ from: 'A', to: 'B', edgeType: 'handoff' }]); + const getAgent = jest.fn(async () => makeAgent('B', [])); + const deriveText = jest.fn(); + const persistDerivation = jest.fn(); + + await discoverConnectedAgents( + { + req: makeReq(), + res: makeRes(), + primaryConfig, + allowedProviders: new Set(), + modelsConfig: { openai: ['gpt-4o'] }, + loadTools: jest.fn(), + deriveText, + persistDerivation, + }, + { + getAgent, + checkPermission: jest.fn().mockResolvedValue(true), + logViolation: jest.fn(), + db: {} as never, + }, + ); + + expect(mockInitializeAgent).toHaveBeenCalledWith( + expect.objectContaining({ deriveText, persistDerivation }), + expect.anything(), + ); + }); + it('forwards resolveWebSearchGrant to every handoff initializeAgent call', async () => { const primaryConfig = makeConfig('A', [{ from: 'A', to: 'B', edgeType: 'handoff' }]); const getAgent = jest.fn(async () => makeAgent('B', [])); diff --git a/packages/api/src/agents/discovery.ts b/packages/api/src/agents/discovery.ts index 18b9cb4f775..a171b8f6e32 100644 --- a/packages/api/src/agents/discovery.ts +++ b/packages/api/src/agents/discovery.ts @@ -115,6 +115,10 @@ export interface DiscoverConnectedAgentsParams { * the terms its parent did. */ fileSearchAvailable?: InitializeAgentParams['fileSearchAvailable']; + /** The request's text deriver, forwarded so a handoff receiver can read a deferred record as text. */ + deriveText?: InitializeAgentParams['deriveText']; + /** The request's shared derivation persister, so each derived file is written once across agents. */ + persistDerivation?: InitializeAgentParams['persistDerivation']; /** * The caller's `WEB_SEARCH` grant resolver, forwarded so a handoff or subagent * whose provider config turns native web search on is authorized by the same @@ -287,6 +291,8 @@ async function initializeReferencedAgent( statefulSessionsAvailable: params.statefulSessionsAvailable, allowedStatefulCodeEnvironments: params.allowedStatefulCodeEnvironments, memoryAvailable: params.memoryAvailable, + deriveText: params.deriveText, + persistDerivation: params.persistDerivation, signal: params.signal, }, deps.db, diff --git a/packages/api/src/agents/files/delivery.spec.ts b/packages/api/src/agents/files/delivery.spec.ts index 181501f375d..522c8d1406f 100644 --- a/packages/api/src/agents/files/delivery.spec.ts +++ b/packages/api/src/agents/files/delivery.spec.ts @@ -1,9 +1,29 @@ -import type { TurnFileConsumers, TurnDeliveryFile } from 'librechat-data-provider'; +import { FileContext, FileSources } from 'librechat-data-provider'; +import type { + TurnReadingInputs, + TurnFileConsumers, + TurnDeliveryFile, + ReadingEvidence, + FiltersConfig, +} from 'librechat-data-provider'; +import type { AppConfig } from '@librechat/data-schemas'; +import type { + DerivedText, + FileTextDeriver, + TurnReadingFile, + TextDerivationPersister, +} from '~/files/reading/turn'; import { applyTurnDelivery as materializeTurnDelivery, - resolveTurnDeliveryRouting, + prepareScopedTurnCandidates, resolveScopedTurnAttachments, + resolveTurnDeliveryRouting, + toClassicInspectionView, + applyCheckpointDelivery, } from './delivery'; +import * as modelBoundContent from '~/middleware/modelBoundContent'; +import { buildTurnReadingContext } from '~/files/reading/turn'; +import { UninspectableFileError } from '~/protection/files'; function applyTurnDelivery( files: T[], @@ -322,3 +342,453 @@ describe('resolveScopedTurnAttachments', () => { ).toEqual([]); }); }); + +describe('the automatic policy', () => { + const agent = { provider: 'openAI' }; + const automatic = { + fileConfig: { endpoints: { openAI: { llmDeliveryPolicy: 'automatic' as const } } }, + }; + /** The turn's evidence seam, answering every file with the same verdicts. */ + const withEvidence = (evidence: ReadingEvidence) => ({ + ...resolveTurnDeliveryRouting({ agent, config: automatic }), + reading: { judge: () => evidence, canDerive: false } satisfies TurnReadingInputs, + }); + const attachment = { + source: FileSources.local, + context: FileContext.message_attachment, + metadata: { destinationChosen: false }, + }; + /** A PDF too large for the provider, with complete extracted text that fits the prompt. */ + const largePdf = { + ...attachment, + file_id: 'large-pdf', + type: 'application/pdf', + bytes: 40 * 1024 * 1024, + text: 'quarterly totals', + llmDeliveryPath: 'provider', + }; + /** A spreadsheet extracted at upload under classic routing. */ + const csv = { + ...attachment, + file_id: 'csv', + type: 'text/csv', + bytes: 20, + text: 'region,total\nwest,4', + llmDeliveryPath: 'text', + }; + + describe('applyTurnDelivery', () => { + it('gives a classic-era spreadsheet to Run Code and back to text without changing it', () => { + const routing = withEvidence({}); + const files = [csv]; + + expect(materializeTurnDelivery(files, { routing, consumers: runsCode })).toEqual([ + { ...csv, llmDeliveryPath: 'none' }, + ]); + expect(materializeTurnDelivery(files, { routing, consumers: noReader })).toBe(files); + }); + }); + + describe('toClassicInspectionView', () => { + const routing = withEvidence({}); + const strictContent = { + files: { pii: { fields: ['content'], starterPatterns: [], uninspectable: 'block' } }, + } as FiltersConfig; + + it('shows a spreadsheet Run Code reads with the text route classic routing gives it', () => { + const copies = materializeTurnDelivery([csv], { routing, consumers: runsCode }); + + const view = toClassicInspectionView(copies, routing, runsCode); + + expect(view).toEqual([{ ...csv, llmDeliveryPath: 'text' }]); + expect(view).not.toBe(copies); + expect(copies).toEqual([{ ...csv, llmDeliveryPath: 'none' }]); + }); + + it('keeps the inspected content of a rerouted file under a fail-closed content filter', () => { + /* Coverage reads the text route, so the code-routed copy alone would read as + * uninspectable although classic routing inspects and delivers the same text. */ + const copies = materializeTurnDelivery([csv], { routing, consumers: runsCode }); + + expect(() => + modelBoundContent.assertModelBoundContent({ filters: strictContent, files: copies }), + ).toThrow(UninspectableFileError); + expect(() => + modelBoundContent.assertModelBoundContent({ + filters: strictContent, + files: toClassicInspectionView(copies, routing, runsCode), + }), + ).not.toThrow(); + }); + + it('returns its input when no copy differs from its classic route', () => { + const pdf = { ...largePdf, bytes: 20 }; + const copies = materializeTurnDelivery([pdf, csv], { routing, consumers: noReader }); + + expect(toClassicInspectionView(copies, routing, noReader)).toBe(copies); + }); + + it('returns its input under classic routing', () => { + const classic = resolveTurnDeliveryRouting({ agent, config }); + const copies = [{ ...csv, llmDeliveryPath: 'none' }]; + + expect(toClassicInspectionView(copies, classic, runsCode)).toBe(copies); + expect(toClassicInspectionView(copies, undefined, runsCode)).toBe(copies); + }); + }); +}); + +describe('applyCheckpointDelivery', () => { + const agent = { provider: 'openAI' }; + const routing = resolveTurnDeliveryRouting({ + agent, + config: { fileConfig: { endpoints: { openAI: { llmDeliveryPolicy: 'automatic' } } } }, + }); + const csv = { + file_id: 'csv', + type: 'text/csv', + bytes: 20, + source: FileSources.local, + context: FileContext.message_attachment, + text: 'region,total\nwest,4', + llmDeliveryPath: 'none', + metadata: { destinationChosen: false }, + }; + /** A record written before uploads marked whether a destination was chosen. */ + const unmarked = { ...csv, file_id: 'unmarked', metadata: {} }; + + it('charges retained fallback text for every inferred tool file under classic routing', () => { + const classic = resolveTurnDeliveryRouting({ agent, config }); + + expect(applyCheckpointDelivery([csv, unmarked])).toEqual([ + { ...csv, llmDeliveryPath: 'text' }, + { ...unmarked, llmDeliveryPath: 'text' }, + ]); + expect(applyCheckpointDelivery([csv], { routing: classic, consumers: runsCode })).toEqual([ + { ...csv, llmDeliveryPath: 'text' }, + ]); + }); + + it('charges the automatic decision for files it reads and the fallback for the rest', () => { + /* Text the policy withheld for Run Code never reached the checkpoint, so charging it would + * spend the resumed turn's budget on content the model never received. */ + const charged = applyCheckpointDelivery([csv, unmarked], { routing, consumers: runsCode }); + + expect(charged[0]).toBe(csv); + expect(charged[1]).toEqual({ ...unmarked, llmDeliveryPath: 'text' }); + expect(applyCheckpointDelivery([csv], { routing, consumers: noReader })).toEqual([ + { ...csv, llmDeliveryPath: 'text' }, + ]); + expect(csv.llmDeliveryPath).toBe('none'); + }); +}); + +describe('prepareScopedTurnCandidates', () => { + const XLSX = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'; + const automatic = { + fileConfig: { endpoints: { openAI: { llmDeliveryPolicy: 'automatic' as const } } }, + }; + const derived: DerivedText = { + status: 'derived', + text: 'quarter,total\nQ1,42', + textDerivation: { outcome: 'complete', extractor: 'document_parser', at: 1 }, + }; + /** A workbook uploaded under the automatic policy with extraction left for a later turn. */ + const workbook = (file_id: string): TurnReadingFile => ({ + file_id, + filename: `${file_id}.xlsx`, + type: XLSX, + bytes: 2048, + source: FileSources.local, + context: FileContext.message_attachment, + llmDeliveryPath: 'none', + metadata: { destinationChosen: false, textDerivation: { outcome: 'deferred' } }, + }); + const historical = workbook('history-xlsx'); + const requested = workbook('request-xlsx'); + const messages = [{ files: [{ file_id: historical.file_id }] }]; + + /** A receiver whose turn carries a reading context over the given deriver. */ + function receiver( + agentId: string, + { + consumers = noReader, + deriveText, + persistDerivation, + endpoint = 'openAI', + config = automatic, + }: { + consumers?: TurnFileConsumers; + deriveText?: FileTextDeriver; + persistDerivation?: TextDerivationPersister; + endpoint?: string; + config?: Pick; + } = {}, + ) { + const deliveryRouting = resolveTurnDeliveryRouting({ + agent: { provider: endpoint }, + config, + }); + const context = buildTurnReadingContext({ + routing: deliveryRouting, + provider: endpoint, + fileTokenLimit: 100_000, + configuredFileSizeLimit: undefined, + countTokens: (text) => text.length, + deriveText, + persistDerivation, + }); + if (context == null) { + throw new Error('expected a reading context'); + } + deliveryRouting.reading = context; + return { agentId, agent: { deliveryRouting, fileConsumers: consumers }, context }; + } + + const deriver = () => + jest.fn, Parameters>(async () => derived); + + /** The app config a receiver's endpoint file policy is read from. */ + const appConfigWith = (fileConfig: AppConfig['fileConfig']): AppConfig => ({ + config: {}, + fileStrategy: FileSources.local, + imageOutputType: 'png', + fileConfig, + }); + + it('collects nothing when no receiver has a reading context', async () => { + const historicalFiles = new Map([[historical.file_id, historical]]); + const requestAttachments = [requested]; + const routing = resolveTurnDeliveryRouting({ + agent: { provider: 'openAI' }, + config: automatic, + }); + + const prepared = await prepareScopedTurnCandidates({ + agents: [ + { agentId: 'handoff', agent: { deliveryRouting: routing, fileConsumers: noReader } }, + ], + historicalFiles, + requestAttachments, + }); + + expect(prepared).toEqual({}); + }); + + it('derives each candidate once for every receiver that needs its text', async () => { + const firstDeriver = deriver(); + const secondDeriver = deriver(); + const persistDerivation = jest.fn< + ReturnType, + Parameters + >(async () => true); + const first = receiver('first', { deriveText: firstDeriver, persistDerivation }); + const second = receiver('second', { deriveText: secondDeriver }); + const flushSpy = jest.spyOn(first.context, 'flush'); + const inputs = { + agents: [first, second], + sharedConversationAgentIds: ['first', 'second'], + messages, + historicalFiles: new Map([[historical.file_id, historical]]), + requestAttachments: [requested], + sharedRunAttachmentIds: new Set(), + }; + + const prepared = await prepareScopedTurnCandidates(inputs); + + expect(firstDeriver).toHaveBeenCalledTimes(2); + expect(secondDeriver).not.toHaveBeenCalled(); + expect(flushSpy).toHaveBeenCalledTimes(1); + expect(persistDerivation.mock.calls.map(([update]) => update.file_id).sort()).toEqual([ + historical.file_id, + requested.file_id, + ]); + expect(prepared.candidates?.get(historical.file_id)).toMatchObject({ + text: derived.text, + metadata: { textDerivation: derived.textDerivation }, + }); + expect(prepared.candidates?.get(requested.file_id)).toMatchObject({ text: derived.text }); + expect(historical.text).toBeUndefined(); + expect(requested.text).toBeUndefined(); + + const historyWalk = jest.spyOn(modelBoundContent, 'collectModelBoundHistoricalFileIdState'); + const scoped = resolveScopedTurnAttachments({ ...inputs, ...prepared }); + expect(historyWalk).not.toHaveBeenCalled(); + historyWalk.mockRestore(); + for (const agentId of ['first', 'second']) { + expect(scoped.get(agentId)?.map((file) => [file.file_id, file.llmDeliveryPath])).toEqual([ + [historical.file_id, 'text'], + [requested.file_id, 'text'], + ]); + } + }); + + it('derives nothing for a receiver that reads the candidates with Run Code', async () => { + const deriveText = deriver(); + const historicalFiles = new Map([[historical.file_id, historical]]); + const requestAttachments = [requested]; + + const prepared = await prepareScopedTurnCandidates({ + agents: [receiver('handoff', { consumers: runsCode, deriveText })], + historicalFiles, + requestAttachments, + }); + + expect(deriveText).not.toHaveBeenCalled(); + expect(prepared.candidates?.get(historical.file_id)).toBe(historical); + expect(prepared.candidates?.get(requested.file_id)).toBe(requested); + }); + + it('derives the workbook once for a parallel text reader while the primary uses code', async () => { + const deriveText = deriver(); + const persistDerivation = jest.fn< + ReturnType, + Parameters + >(async () => true); + const primary = receiver('primary', { consumers: runsCode, deriveText, persistDerivation }); + const parallel = receiver('parallel', { deriveText, persistDerivation }); + const secondParallel = receiver('second-parallel', { deriveText, persistDerivation }); + const inputs = { + agents: [primary, parallel, secondParallel], + sharedConversationAgentIds: ['primary', 'parallel', 'second-parallel'], + messages: [], + requestAttachments: [requested], + sharedRunAttachmentIds: new Set(), + }; + const prepared = await prepareScopedTurnCandidates(inputs); + const scoped = resolveScopedTurnAttachments({ ...inputs, ...prepared }); + + expect(scoped.get('primary')).toEqual([]); + for (const agentId of ['parallel', 'second-parallel']) { + expect(scoped.get(agentId)).toEqual([ + expect.objectContaining({ + file_id: requested.file_id, + llmDeliveryPath: 'text', + text: derived.text, + }), + ]); + } + expect(deriveText).toHaveBeenCalledTimes(1); + expect(persistDerivation).toHaveBeenCalledTimes(1); + expect(persistDerivation).toHaveBeenCalledWith( + expect.objectContaining({ + file_id: requested.file_id, + text: derived.text, + textDerivation: derived.textDerivation, + }), + ); + expect(requested.text).toBeUndefined(); + expect(requested.metadata?.textDerivation?.outcome).toBe('deferred'); + }); + + it('decides only what each receiver can be offered', async () => { + const deriveText = deriver(); + const outsideDeriver = deriver(); + const scopedFile = workbook('scoped-xlsx'); + const stale = workbook('stale-xlsx'); + + const prepared = await prepareScopedTurnCandidates({ + agents: [ + receiver('handoff', { deriveText }), + receiver('subagent', { deriveText: outsideDeriver }), + ], + sharedConversationAgentIds: ['handoff'], + messages, + historicalFiles: new Map([ + [historical.file_id, historical], + [stale.file_id, stale], + ]), + requestAttachments: [requested, scopedFile], + sharedRunAttachmentIds: new Set([historical.file_id]), + attachmentsByAgentId: { handoff: [scopedFile] }, + }); + + expect(outsideDeriver).not.toHaveBeenCalled(); + expect(deriveText.mock.calls.map(([file]) => file.file_id)).toEqual([requested.file_id]); + expect(prepared.candidates?.get(requested.file_id)?.text).toBe(derived.text); + expect(prepared.candidates?.get(scopedFile.file_id)).toBe(scopedFile); + }); + + it('derives nothing for a receiver whose endpoint refuses the candidate', async () => { + const fileConfig = { + endpoints: { + openAI: { llmDeliveryPolicy: 'automatic' as const, disabled: true }, + anthropic: { + llmDeliveryPolicy: 'automatic' as const, + supportedMimeTypes: ['^application/pdf$'], + }, + }, + }; + const appConfig = appConfigWith(fileConfig); + const disabledDeriver = jest.fn, Parameters>( + async () => ({ status: 'blocked', error: new UninspectableFileError('extracted_text') }), + ); + const allowlistDeriver = deriver(); + + const prepared = await prepareScopedTurnCandidates({ + agents: [ + receiver('disabled', { deriveText: disabledDeriver, config: { fileConfig } }), + receiver('allowlist', { + deriveText: allowlistDeriver, + endpoint: 'anthropic', + config: { fileConfig }, + }), + ], + appConfig, + endpointsByAgentId: new Map([['allowlist', { endpointType: null }]]), + requestAttachments: [requested], + }); + + expect(disabledDeriver).not.toHaveBeenCalled(); + expect(allowlistDeriver).not.toHaveBeenCalled(); + expect(prepared.candidates?.get(requested.file_id)).toBe(requested); + }); + + it('still derives for a receiver whose endpoint accepts the candidate', async () => { + const fileConfig = { + endpoints: { + openAI: { + llmDeliveryPolicy: 'automatic' as const, + supportedMimeTypes: [`^${XLSX}$`], + }, + }, + }; + const deriveText = deriver(); + + const prepared = await prepareScopedTurnCandidates({ + agents: [receiver('handoff', { deriveText, config: { fileConfig } })], + appConfig: appConfigWith(fileConfig), + requestAttachments: [requested], + }); + + expect(deriveText).toHaveBeenCalledTimes(1); + expect(prepared.candidates?.get(requested.file_id)?.text).toBe(derived.text); + }); + + it('marks the text failed on every receiver that needed it, so each moves on', async () => { + const first = receiver('first', { + deriveText: async () => ({ + status: 'failed', + textDerivation: { outcome: 'failed', reason: 'parser' }, + persist: true, + }), + }); + const second = receiver('second', { deriveText: deriver() }); + const inputs = { + agents: [first, second], + sharedConversationAgentIds: ['first', 'second'], + messages: [], + requestAttachments: [requested], + sharedRunAttachmentIds: new Set(), + }; + + const prepared = await prepareScopedTurnCandidates(inputs); + + expect(prepared.candidates?.get(requested.file_id)).toBe(requested); + expect(first.context.judge(requested).textFailed).toBe(true); + expect(second.context.judge(requested).textFailed).toBe(true); + const scoped = resolveScopedTurnAttachments({ ...inputs, ...prepared }); + expect(scoped.get('first')).toEqual([]); + expect(scoped.get('second')).toEqual([]); + }); +}); diff --git a/packages/api/src/agents/files/delivery.ts b/packages/api/src/agents/files/delivery.ts index 97dc85e80ac..30ee430f137 100644 --- a/packages/api/src/agents/files/delivery.ts +++ b/packages/api/src/agents/files/delivery.ts @@ -1,19 +1,31 @@ import { mergeFileConfig, + decideFileReading, getEndpointFileConfig, resolveUseResponsesApi, + resolveLLMDeliveryPolicy, getCustomEndpointProvider, isSpeechProviderConfigured, resolveTurnLLMDeliveryPath, hasInferredLLMDeliveryPath, } from 'librechat-data-provider'; import type { + TDefaultLLMDeliveryPath, TurnDeliveryRouting, TurnDeliveryFile, TurnFileConsumers, } from 'librechat-data-provider'; import type { AppConfig } from '@librechat/data-schemas'; +import type { TurnReadingFile, TextRequest, TurnReader } from '~/files/reading/turn'; +import { + withDerivedText, + needsDerivedText, + deriveRequestedText, + getTurnReadingContext, +} from '~/files/reading/turn'; import { collectModelBoundHistoricalFileIdState } from '~/middleware/modelBoundContent'; +import { filterFilesByEndpointRuntimeConfig } from '~/files/filter'; +import { getAgentEntry } from '~/agents/attachments'; /** The app config a turn's attachment routing reads. */ export type TurnDeliveryConfig = Pick; @@ -56,66 +68,114 @@ export function resolveTurnDeliveryRouting({ }; } +/** The routing an agent's turn delivers by and the tools judged as its readers. */ +export interface TurnDeliveryInputs { + routing?: TurnDeliveryRouting; + consumers?: TurnFileConsumers; +} + +const isAutomaticRouting = (routing?: TurnDeliveryRouting): boolean => + resolveLLMDeliveryPolicy(routing?.endpointConfig) === 'automatic'; + +const withDeliveryPath = ( + file: T, + llmDeliveryPath: TDefaultLLMDeliveryPath | undefined, +): T => + llmDeliveryPath == null || llmDeliveryPath === file.llmDeliveryPath + ? file + : { ...file, llmDeliveryPath }; + +/** Maps each file, returning the input array itself when no file was replaced. */ +function mapTurnCopies(files: T[], map: (file: T) => T): T[] { + let changed = false; + const result = files.map((file) => { + const next = map(file); + changed ||= next !== file; + return next; + }); + return changed ? result : files; +} + /** Materialize the exact turn route before admission without mutating stored records. */ export function applyTurnDelivery( files: T[], - { routing, consumers }: { routing?: TurnDeliveryRouting; consumers?: TurnFileConsumers }, + { routing, consumers }: TurnDeliveryInputs, ): T[] { if (routing == null) { return files; } - let changed = false; - const result = files.map((file) => { - if (!hasInferredLLMDeliveryPath(file)) { - return file; - } - const llmDeliveryPath = resolveTurnLLMDeliveryPath(routing, file, consumers); - if (llmDeliveryPath == null || llmDeliveryPath === file.llmDeliveryPath) { - return file; - } - changed = true; - return { ...file, llmDeliveryPath }; + return mapTurnCopies(files, (file) => + hasInferredLLMDeliveryPath(file) + ? withDeliveryPath(file, resolveTurnLLMDeliveryPath(routing, file, consumers)) + : file, + ); +} + +/** + * The turn copies as classic routing would deliver them, for the full-set content checks. + * + * Inspection coverage keys on the text route, so a file the automatic policy hands to a tool + * instead of the prompt would otherwise lose the text coverage classic routing gives it, and + * rerouting must not change what is inspected. Returns its input under classic routing and + * whenever no copy differs; stored records and the input copies are never changed. + */ +export function toClassicInspectionView( + files: T[], + routing?: TurnDeliveryRouting, + consumers?: TurnFileConsumers, +): T[] { + if (!isAutomaticRouting(routing)) { + return files; + } + return mapTurnCopies(files, (file) => { + const { automatic, classicPath } = decideFileReading({ routing, file, consumers }); + return automatic ? withDeliveryPath(file, classicPath) : file; }); - return changed ? result : files; } +/** Charges text a referenced inferred tool file may have been delivered as fallback. */ +const chargeRetainedFallbackText = (file: T): T => + hasInferredLLMDeliveryPath(file) && file.llmDeliveryPath === 'none' && file.text + ? { ...file, llmDeliveryPath: 'text' } + : file; + /** * Checkpoints replay already encoded content; current tools and policy cannot remove it. * A referenced inferred tool file carrying extracted text may have been delivered as fallback. * Charge that text conservatively even if fallback was since disabled. Explicit destinations * and records predating routing keep their existing accounting; no stored record is changed. + * + * Under the automatic policy a file it reads is charged by the resuming agent's own decision, + * so text that policy withheld for a tool is not charged; the conservative charge still covers + * every file the decision leaves on its classic route. */ -export function applyCheckpointDelivery(files: T[]): T[] { - return files.map((file) => - hasInferredLLMDeliveryPath(file) && file.llmDeliveryPath === 'none' && file.text - ? { ...file, llmDeliveryPath: 'text' } - : file, - ); +export function applyCheckpointDelivery( + files: T[], + { routing, consumers }: TurnDeliveryInputs = {}, +): T[] { + if (!isAutomaticRouting(routing)) { + return files.map(chargeRetainedFallbackText); + } + return files.map((file) => { + const reading = decideFileReading({ routing, file, consumers }); + return reading.automatic + ? withDeliveryPath(file, reading.path) + : chargeRetainedFallbackText(file); + }); } -/** - * A primary agent's tool route cannot decide whether a handoff receives text. Keep - * owner-hydrated candidates until each receiver resolves them, then add only text absent - * from the shared prompt to its scoped context. The existing scoped-context pipeline owns - * endpoint filtering, aggregate admission, inspection and extraction for these copies. - */ -export function resolveScopedTurnAttachments({ - agents, - sharedConversationAgentIds, - resendFiles, - messages, - historicalFiles, - requestAttachments, - sharedRunAttachmentIds, - attachmentsByAgentId, -}: { - agents: readonly { - agentId: string; - agent: { - deliveryRouting?: TurnDeliveryRouting; - fileConsumers?: TurnFileConsumers; - }; - }[]; +/** A receiver of the conversation's scoped attachments, by the routing its turn settled. */ +export interface ScopedTurnAgent { + agentId: string; + agent: { + deliveryRouting?: TurnDeliveryRouting; + fileConsumers?: TurnFileConsumers; + }; +} + +/** The inputs each handoff receiver's scoped attachment context is resolved from. */ +export interface ScopedTurnAttachmentParams { + agents: readonly ScopedTurnAgent[]; /** Only the primary/handoff graph shares root conversation files. */ sharedConversationAgentIds: readonly string[]; resendFiles?: boolean; @@ -124,23 +184,228 @@ export function resolveScopedTurnAttachments; attachmentsByAgentId?: Map | Record; -}): Map { + /** + * The candidates {@link prepareScopedTurnCandidates} already collected from these same inputs, + * so the retained messages are not walked a second time. + */ + candidates?: ReadonlyMap; +} + +/** The inputs that pick the candidates, with the narrowing ones optional. */ +type ScopedCandidateInputs = Pick< + ScopedTurnAttachmentParams, + 'resendFiles' | 'historicalFiles' | 'requestAttachments' +> & + Partial, 'messages' | 'sharedRunAttachmentIds'>>; + +/** The endpoint type a receiver's attachments are filtered under, where the caller knows it. */ +export interface ScopedReceiverEndpoint { + endpointType?: string | null; +} + +/** Each receiver's endpoint type by agent id. */ +export type ScopedReceiverEndpoints = + | Map + | Record; + +/** + * The scoped attachment inputs whose text {@link prepareScopedTurnCandidates} derives. Without the + * narrowing inputs every receiver with a reading context decides every historical and request file. + */ +export type ScopedTurnCandidateParams = ScopedCandidateInputs & + Pick, 'agents' | 'attachmentsByAgentId'> & + Partial, 'sharedConversationAgentIds'>> & { + /** The app config whose endpoint file policy each receiver's scoped files must pass. */ + appConfig?: AppConfig; + /** Each receiver's endpoint type; its endpoint is the one its routing settled. */ + endpointsByAgentId?: ScopedReceiverEndpoints; + signal?: AbortSignal; + }; + +/** + * The scoped attachment candidates by file id, as copies carrying the text the receivers derived; + * absent when no receiver has a reading context, so none were collected. + */ +export interface ScopedTurnCandidates { + candidates?: ReadonlyMap; +} + +/** Historical files still model-bound in the retained messages; every one when none are given. */ +function selectHistoricalFileIds({ + resendFiles, + messages, + historicalFiles, +}: ScopedCandidateInputs): Iterable { + if (resendFiles === false) { + return []; + } + if (messages == null) { + return historicalFiles?.keys() ?? []; + } + return collectModelBoundHistoricalFileIdState(messages).fileIds; +} + +/** Hydrated history and request files no shared prompt carries, once each by file id. */ +function collectScopedCandidates( + inputs: ScopedCandidateInputs, +): Map { + const { historicalFiles, requestAttachments, sharedRunAttachmentIds } = inputs; + const isShared = (fileId: string): boolean => sharedRunAttachmentIds?.has(fileId) === true; const candidates = new Map(); - for (const fileId of collectModelBoundHistoricalFileIdState(resendFiles === false ? [] : messages) - .fileIds) { + for (const fileId of selectHistoricalFileIds(inputs)) { const file = historicalFiles?.get(fileId); - if (file && !sharedRunAttachmentIds.has(fileId)) candidates.set(fileId, file); + if (file && !isShared(fileId)) candidates.set(fileId, file); } for (const file of requestAttachments) { - if (!sharedRunAttachmentIds.has(file.file_id)) candidates.set(file.file_id, file); + if (!isShared(file.file_id)) candidates.set(file.file_id, file); + } + return candidates; +} + +/** + * A receiver that can derive text, with the files its own scoped context already carries and the + * candidates its endpoint's file policy accepts. + */ +interface ScopedTurnReader extends TurnReader { + scopedFileIds: ReadonlySet; + acceptedFileIds: ReadonlySet; +} + +/** A candidate as the endpoint file policy reads it: a missing type or size reads as empty. */ +type PolicyFile = TurnReadingFile & { type: string; bytes: number }; + +const toPolicyFile = (file: TurnReadingFile): PolicyFile => ({ + ...file, + type: file.type ?? '', + bytes: file.bytes ?? 0, +}); + +/** + * The candidates the receiver's endpoint accepts, by the policy the scoped pipeline applies to + * them later: a disabled endpoint, a MIME allowlist and the per-file size limit. A file refused + * there is never derived for that receiver. + */ +function selectAcceptedFileIds( + candidates: PolicyFile[], + routing: TurnDeliveryRouting, + agentId: string, + consumers: TurnFileConsumers | undefined, + { appConfig, endpointsByAgentId }: ScopedTurnCandidateParams, +): ReadonlySet { + const accepted = filterFilesByEndpointRuntimeConfig(appConfig, { + files: candidates, + endpoint: routing.endpoint, + endpointType: getAgentEntry(endpointsByAgentId, agentId)?.endpointType, + skipTotalSizeLimit: true, + preserveTextSources: true, + consumers, + }); + return new Set(accepted.map((file) => file.file_id)); +} + +function selectScopedReaders( + params: ScopedTurnCandidateParams, + candidates: PolicyFile[], +): ScopedTurnReader[] { + const { agents, sharedConversationAgentIds, attachmentsByAgentId } = params; + const sharedAgents = sharedConversationAgentIds && new Set(sharedConversationAgentIds); + return agents.flatMap(({ agentId, agent }) => { + const routing = agent.deliveryRouting; + const context = getTurnReadingContext(routing); + if (routing == null || context == null || sharedAgents?.has(agentId) === false) { + return []; + } + const scoped = getAgentEntry(attachmentsByAgentId, agentId) ?? []; + return [ + { + routing, + consumers: agent.fileConsumers, + context, + scopedFileIds: new Set(scoped.map((file) => file.file_id)), + acceptedFileIds: selectAcceptedFileIds( + candidates, + routing, + agentId, + agent.fileConsumers, + params, + ), + }, + ]; + }); +} + +const needsScopedText = (file: TurnReadingFile, reader: ScopedTurnReader): boolean => + !reader.scopedFileIds.has(file.file_id) && + reader.acceptedFileIds.has(file.file_id) && + needsDerivedText(file, reader); + +/** The candidate with every receiver whose decision needs its text, or nothing when none does. */ +function requestScopedText( + file: T, + readers: readonly ScopedTurnReader[], +): TextRequest[] { + const [first, ...rest] = readers + .filter((reader) => needsScopedText(file, reader)) + .map(({ context }) => context); + return first == null ? [] : [{ file, contexts: [first, ...rest] }]; +} + +/** + * Derives, before handoff receivers resolve their scoped context, the text a receiver's reading + * needs. Each candidate the receiver's endpoint accepts is decided with that receiver's routing + * and readers, and its text is derived once, through the first receiver that needs it, then saved + * where the record allows. + * Returns the candidates, as copies carrying the text, for {@link resolveScopedTurnAttachments} + * to resolve synchronously without collecting them again; nothing when no receiver has a + * reading context. + */ +export async function prepareScopedTurnCandidates( + params: ScopedTurnCandidateParams, +): Promise> { + const { signal } = params; + if (!params.agents.some(({ agent }) => getTurnReadingContext(agent.deliveryRouting) != null)) { + return {}; } + const candidates = collectScopedCandidates(params); + const readers = selectScopedReaders(params, [...candidates.values()].map(toPolicyFile)); + if (readers.length === 0) { + return {}; + } + const requests = [...candidates.values()].flatMap((file) => requestScopedText(file, readers)); + if (requests.length === 0) { + return { candidates }; + } + signal?.throwIfAborted(); + const texts = await deriveRequestedText(requests, signal); + const derivers = new Set(requests.map(({ contexts }) => contexts[0])); + await Promise.all([...derivers].map((context) => context.flush())); + if (texts.size === 0) { + return { candidates }; + } + for (const [fileId, derived] of texts) { + const file = candidates.get(fileId); + if (file != null) { + candidates.set(fileId, withDerivedText(file, derived)); + } + } + return { candidates }; +} + +/** + * A primary agent's tool route cannot decide whether a handoff receives text. Keep + * owner-hydrated candidates until each receiver resolves them, then add only text absent + * from the shared prompt to its scoped context. The existing scoped-context pipeline owns + * endpoint filtering, aggregate admission, inspection and extraction for these copies. + */ +export function resolveScopedTurnAttachments( + params: ScopedTurnAttachmentParams, +): Map { + const { agents, sharedConversationAgentIds, attachmentsByAgentId } = params; + const candidates = params.candidates ?? collectScopedCandidates(params); const sharedAgents = new Set(sharedConversationAgentIds); const result = new Map(); for (const { agentId, agent } of agents) { - const scoped = - attachmentsByAgentId instanceof Map - ? (attachmentsByAgentId.get(agentId) ?? []) - : (attachmentsByAgentId?.[agentId] ?? []); + const scoped = getAgentEntry(attachmentsByAgentId, agentId) ?? []; const files = new Map(scoped.map((file) => [file.file_id, file])); for (const [fileId, file] of sharedAgents.has(agentId) ? candidates : []) { if (files.has(fileId)) continue; diff --git a/packages/api/src/agents/files/encode.spec.ts b/packages/api/src/agents/files/encode.spec.ts index 10c29025be9..9310b55140e 100644 --- a/packages/api/src/agents/files/encode.spec.ts +++ b/packages/api/src/agents/files/encode.spec.ts @@ -1,8 +1,10 @@ import { FileContext, FileSources, ImageDetail } from 'librechat-data-provider'; import type { TFile } from 'librechat-data-provider'; import type { RunFileEncodingAgent, RunFileMessageEncoderDeps } from './encode'; +import type { FileTextDeriver } from '~/files/reading'; import type { ServerRequest } from '~/types'; import { AgentAttachmentLimitError, AgentAttachmentPolicyError } from '../attachments'; +import { buildTurnReadingContext, getTurnReadingContext } from '~/files/reading'; import { resolveTurnDeliveryRouting } from './delivery'; import { createRunFileMessageEncoder } from './encode'; @@ -56,7 +58,10 @@ function setup({ ]), ); const encodeImages = jest.fn(async () => ({ image_urls: [nativeImage] })); - const encodeDocuments = jest.fn(async () => ({ documents: [nativeDocument] })); + const encodeDocuments = jest.fn< + ReturnType, + Parameters + >(async () => ({ documents: [nativeDocument] })); const encodeAudios = jest.fn(async () => ({ audios: [{ type: 'media', data: 'audio' }] })); const encodeVideos = jest.fn(async () => ({ videos: [{ type: 'media', data: 'video' }] })); const extractText = jest.fn< @@ -73,7 +78,7 @@ function setup({ getStrategyFunctions: jest.fn(), extractText, }; - return { ...deps, ...createRunFileMessageEncoder(deps) }; + return { ...deps, encodeDocuments, ...createRunFileMessageEncoder(deps) }; } describe('createRunFileMessageEncoder', () => { @@ -506,4 +511,384 @@ describe('createRunFileMessageEncoder', () => { expect(harness.encodeDocuments).not.toHaveBeenCalled(); expect(harness.extractText).not.toHaveBeenCalled(); }); + + describe('under the automatic policy', () => { + const runsCode = { executeCode: true, fileSearch: false }; + const searches = { executeCode: false, fileSearch: true }; + const noReader = { executeCode: false, fileSearch: false }; + const attachment = { + context: FileContext.message_attachment, + metadata: { destinationChosen: false }, + }; + /** + * File Search is loaded but queued and registered none of the child's files, as a failed + * provisioning leaves it: the walk passes search by, no code tool is loaded, and a document + * the provider rejects falls back to its text or, without any, to unavailable. + */ + const withoutSearchFiles = ( + context: ReturnType, + ): ReturnType => { + context?.setSearchEvidence({ queued: [], registered: [] }); + return context; + }; + + it('admits an oversized file the child reads with Run Code and never counts it', async () => { + /* The endpoint size limit is the provider's capacity for a file the policy reads, and a + * file left to Run Code never reaches the provider, so it passes the limit and spends none + * of the count, byte or text budgets. Classic routing still refuses the same file. */ + const workbook: TFile = { + ...pdf, + ...attachment, + file_id: 'input-xlsx', + filename: 'quarterly.xlsx', + filepath: '/files/quarterly.xlsx', + type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + bytes: 2 * 1024 * 1024, + text: 'quarter,total\nQ1,4', + llmDeliveryPath: 'text', + }; + const limits = { fileSizeLimit: 1, fileLimit: 1 }; + const harness = setup({ + agents: { + automatic: { provider: 'openAI', endpoint: 'automatic', fileConsumers: runsCode }, + classic: { provider: 'openAI', endpoint: 'classic', fileConsumers: runsCode }, + }, + fileConfig: { + fileContextSizeLimit: 1, + fileContextCharLimit: 5, + endpoints: { + automatic: { ...limits, llmDeliveryPolicy: 'automatic' }, + classic: limits, + }, + }, + }); + + await expect( + harness.encode([workbook, { ...workbook, file_id: 'second-xlsx' }], 'automatic'), + ).resolves.toEqual([]); + await expect(harness.encode([workbook], 'classic')).rejects.toBeInstanceOf( + AgentAttachmentPolicyError, + ); + expect(harness.encodeDocuments).not.toHaveBeenCalled(); + expect(harness.extractText).not.toHaveBeenCalled(); + expect(workbook.llmDeliveryPath).toBe('text'); + }); + + it('inspects a file the child reads with Run Code as classic routing would deliver it', async () => { + const csv: TFile = { + ...pdf, + ...attachment, + file_id: 'input-csv', + filename: 'sales.csv', + filepath: '/files/sales.csv', + type: 'text/csv', + text: 'region,total\nwest,4', + llmDeliveryPath: 'text', + }; + const harness = setup({ + agents: { child: { provider: 'openAI', fileConsumers: runsCode } }, + fileConfig: { endpoints: { openAI: { llmDeliveryPolicy: 'automatic' } } }, + }); + harness.req.config!.filters = { + files: { pii: { fields: ['content'], starterPatterns: [], uninspectable: 'block' } }, + }; + + await expect(harness.encode([csv], 'child')).resolves.toEqual([]); + expect(harness.extractText).not.toHaveBeenCalled(); + }); + + it('leaves out a document the child cannot accept and records it for the next decision', async () => { + const brief: TFile = { ...pdf, ...attachment, text: undefined }; + const harness = setup({ + agents: { + automatic: { provider: 'openAI', endpoint: 'automatic', fileConsumers: searches }, + classic: { provider: 'openAI', endpoint: 'classic', fileConsumers: searches }, + }, + fileConfig: { endpoints: { automatic: { llmDeliveryPolicy: 'automatic' } } }, + }); + const child = harness.getAgent('automatic'); + const context = + child && + withoutSearchFiles( + buildTurnReadingContext({ + routing: child.deliveryRouting, + provider: 'openAI', + fileTokenLimit: 100_000, + configuredFileSizeLimit: undefined, + countTokens: (text) => text.length, + }), + ); + if (child == null || context == null) { + throw new Error('expected an automatic reading context'); + } + child.deliveryRouting.reading = context; + harness.encodeDocuments.mockResolvedValueOnce({ + documents: [], + rejected: [{ file_id: brief.file_id, reason: 'capacity' }], + }); + + const [message] = await harness.encode([brief], 'automatic'); + await harness.encode([brief], 'classic'); + + expect(harness.encodeDocuments.mock.calls.map(([, , params]) => params)).toEqual([ + expect.objectContaining({ endpoint: 'automatic', onValidationFailure: 'skip' }), + expect.objectContaining({ endpoint: 'classic', onValidationFailure: 'throw' }), + ]); + expect(context.judge(brief).rejected).toBe('capacity'); + expect(context.stats().rejected).toBe(1); + expect(getTurnReadingContext(harness.getAgent('classic')?.deliveryRouting)).toBeUndefined(); + expect(message.content).toBe( + [ + '- Shared files left out of this message (file contents are not listed here):', + '\t- "report.pdf" (PDF): cannot be read on this turn (too large to send directly). Say so if asked; do not claim to have read it.', + ].join('\n'), + ); + }); + + const capacityHarness = ({ + fileConfig = {}, + deriveText, + }: { + fileConfig?: NonNullable['fileConfig']; + deriveText?: FileTextDeriver; + } = {}) => { + const harness = setup({ + agents: { child: { provider: 'anthropic', fileConsumers: searches } }, + fileConfig: { + ...fileConfig, + endpoints: { + ...fileConfig.endpoints, + anthropic: { ...fileConfig.endpoints?.anthropic, llmDeliveryPolicy: 'automatic' }, + }, + }, + }); + const child = harness.getAgent('child'); + if (child == null) { + throw new Error('expected a child agent'); + } + const context = withoutSearchFiles( + buildTurnReadingContext({ + routing: child.deliveryRouting, + provider: 'anthropic', + fileTokenLimit: 100_000, + configuredFileSizeLimit: undefined, + countTokens: (text) => text.length, + deriveText, + }), + ); + child.deliveryRouting.reading = context; + return { ...harness, context, child }; + }; + + it('actually includes cached text after encode-time capacity rejection, once', async () => { + const harness = capacityHarness(); + const file: TFile = { ...pdf, ...attachment, text: 'Complete cached PDF text.' }; + harness.encodeDocuments.mockResolvedValueOnce({ + documents: [], + rejected: [{ file_id: file.file_id, reason: 'capacity' }], + }); + + const [message] = await harness.encode([file], 'child'); + + const content = JSON.stringify(message.content); + expect(content.match(/Complete cached PDF text\./g)).toHaveLength(1); + expect(content).not.toContain('file_data'); + expect(content).not.toContain('left out'); + expect(harness.extractText).toHaveBeenCalledTimes(1); + expect(harness.context?.judge(file).rejected).toBe('capacity'); + expect(file.llmDeliveryPath).toBe('provider'); + }); + + it('charges a fallback shared with permanent agent context once', async () => { + const harness = capacityHarness({ + fileConfig: { endpoints: { anthropic: { fileLimit: 1 } } }, + }); + const file: TFile = { ...pdf, ...attachment, text: 'Complete overlapping PDF text.' }; + harness.child.agentContextAttachments = [file]; + harness.encodeDocuments.mockResolvedValueOnce({ + documents: [], + rejected: [{ file_id: file.file_id, reason: 'capacity' }], + }); + + const [message] = await harness.encode([file], 'child'); + + expect( + JSON.stringify(message.content).match(/Complete overlapping PDF text\./g), + ).toHaveLength(1); + expect(harness.context?.judge(file).overflow).not.toBe(true); + expect(harness.extractText).toHaveBeenCalledTimes(1); + }); + + it('derives the capacity fallback after native validation and sends the complete result', async () => { + const deriveText = jest.fn, Parameters>(); + deriveText.mockResolvedValue({ + status: 'derived', + text: 'Complete derived PDF text.', + textDerivation: { outcome: 'complete', extractor: 'document_parser', at: 2 }, + }); + const harness = capacityHarness({ deriveText }); + const file: TFile = { + ...pdf, + ...attachment, + text: undefined, + metadata: { destinationChosen: false, textDerivation: { outcome: 'deferred', at: 1 } }, + }; + harness.encodeDocuments.mockResolvedValueOnce({ + documents: [], + rejected: [{ file_id: file.file_id, reason: 'capacity' }], + }); + + const [message] = await harness.encode([file], 'child'); + + expect(deriveText).toHaveBeenCalledTimes(1); + expect(JSON.stringify(message.content).match(/Complete derived PDF text\./g)).toHaveLength(1); + expect(harness.extractText).toHaveBeenCalledTimes(1); + expect(file.text).toBeUndefined(); + }); + + it('omits complete text after an integrity rejection', async () => { + const harness = capacityHarness(); + const file: TFile = { ...pdf, ...attachment, text: 'Cached text must not bypass integrity.' }; + harness.encodeDocuments.mockResolvedValueOnce({ + documents: [], + rejected: [{ file_id: file.file_id, reason: 'integrity' }], + }); + + const [message] = await harness.encode([file], 'child'); + + expect(JSON.stringify(message.content)).not.toContain(file.text); + expect(harness.extractText).not.toHaveBeenCalled(); + expect(message.content).toEqual(expect.stringContaining('could not accept it')); + }); + + it('moves an aggregate-overflowed derived fallback to unavailable before extraction', async () => { + const harness = capacityHarness({ + fileConfig: { fileContextCharLimit: 8 }, + deriveText: async () => ({ + status: 'derived', + text: 'This complete text fits per-file but exceeds the aggregate.', + textDerivation: { outcome: 'complete', extractor: 'document_parser' }, + }), + }); + const file: TFile = { ...pdf, ...attachment, text: undefined }; + harness.encodeDocuments.mockResolvedValueOnce({ + documents: [], + rejected: [{ file_id: file.file_id, reason: 'capacity' }], + }); + + const [message] = await harness.encode([file], 'child'); + + expect(harness.extractText).not.toHaveBeenCalled(); + expect(harness.context?.judge(file).overflow).toBe(true); + expect(JSON.stringify(message.content)).not.toContain('This complete text fits'); + }); + + it('derives the text a file needs inside encode, and validates it without the text', async () => { + /* No file tool is loaded, so the deferred workbook keeps its classic text route, which + * has no text yet and needs it derived. Validation stays synchronous: the copy is not + * model-bound until its text exists, so nothing is counted. */ + const workbook: TFile = { + ...pdf, + ...attachment, + file_id: 'input-xlsx', + filename: 'quarterly.xlsx', + filepath: '/files/quarterly.xlsx', + type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + text: undefined, + llmDeliveryPath: 'none', + metadata: { destinationChosen: false, textDerivation: { outcome: 'deferred' } }, + }; + const harness = setup({ + agents: { child: { provider: 'openAI', fileConsumers: noReader } }, + fileConfig: { endpoints: { openAI: { llmDeliveryPolicy: 'automatic' } } }, + }); + const deriveText = jest.fn, Parameters>( + async () => ({ + status: 'derived', + text: 'quarter,total\nQ1,42', + textDerivation: { outcome: 'complete', extractor: 'document_parser', at: 1 }, + }), + ); + const child = harness.getAgent('child'); + const context = + child && + buildTurnReadingContext({ + routing: child.deliveryRouting, + provider: 'openAI', + fileTokenLimit: 100_000, + configuredFileSizeLimit: undefined, + countTokens: (text) => text.length, + deriveText, + }); + if (child == null || context == null) { + throw new Error('expected an automatic reading context'); + } + child.deliveryRouting.reading = context; + + expect(() => harness.validate([workbook], 'child')).not.toThrow(); + expect(deriveText).not.toHaveBeenCalled(); + + const [message] = await harness.encode([workbook], 'child'); + + expect(deriveText).toHaveBeenCalledTimes(1); + expect(deriveText).toHaveBeenCalledWith( + expect.objectContaining({ file_id: 'input-xlsx' }), + undefined, + ); + expect(harness.extractText).toHaveBeenCalledWith( + expect.objectContaining({ + attachments: [ + expect.objectContaining({ file_id: 'input-xlsx', text: 'quarter,total\nQ1,42' }), + ], + }), + ); + expect(message.content).toEqual(expect.stringContaining('Q1,42')); + expect(workbook.text).toBeUndefined(); + }); + + it('reuses the token count the reading judged and marks any truncation', async () => { + /* The byte length exceeds the 10-token limit, so the judge counts (3 tokens) and the + * text fits. Recounting with the host tokenizer (one per character) would truncate it. */ + const csv: TFile = { + ...pdf, + ...attachment, + file_id: 'input-csv', + filename: 'sales.csv', + filepath: '/files/sales.csv', + type: 'text/csv', + text: 'region,total\nwest,4', + llmDeliveryPath: 'text', + }; + const harness = setup({ + agents: { child: { provider: 'openAI', fileConsumers: searches } }, + fileConfig: { endpoints: { openAI: { llmDeliveryPolicy: 'automatic' } } }, + }); + harness.req.body = { fileTokenLimit: 10 }; + const child = harness.getAgent('child'); + const countTokens = jest.fn(() => 3); + const context = + child && + buildTurnReadingContext({ + routing: child.deliveryRouting, + provider: 'openAI', + fileTokenLimit: 10, + configuredFileSizeLimit: undefined, + countTokens, + }); + if (child == null || context == null) { + throw new Error('expected an automatic reading context'); + } + child.deliveryRouting.reading = context; + + const [message] = await harness.encode([csv], 'child'); + + expect(harness.extractText).toHaveBeenCalledWith( + expect.objectContaining({ markTruncation: true, knownTokenCount: context.knownTokenCount }), + ); + expect(countTokens).toHaveBeenCalledTimes(1); + expect(context.knownTokenCount(csv)).toBe(3); + expect(message.content).toEqual(expect.stringContaining('west,4')); + expect(message.content).toEqual(expect.not.stringContaining('[Truncated')); + }); + }); }); diff --git a/packages/api/src/agents/files/encode.ts b/packages/api/src/agents/files/encode.ts index e04d6cbed86..936e51a79e4 100644 --- a/packages/api/src/agents/files/encode.ts +++ b/packages/api/src/agents/files/encode.ts @@ -13,15 +13,32 @@ import type { TurnFileConsumers, } from 'librechat-data-provider'; import type { BaseMessage } from '@librechat/agents/langchain'; -import type { ServerRequest, StrategyFunctions } from '~/types'; +import type { + ServerRequest, + StrategyFunctions, + DocumentRejection, + NativeValidationMode, +} from '~/types'; +import type { TurnTextOptions } from '~/files/reading'; import type { TokenCountFn } from '~/utils/text'; import { isToolOwnedAttachment, isModelBoundAttachmentFile, + measureModelBoundAttachment, assertAgentAttachmentLimits, + resolveAgentAttachmentLimits, AgentAttachmentPolicyError, } from '../attachments'; +import { + prepareTurnFiles, + settleTurnFiles, + renderLeftOutFiles, + getTurnTextOptions, + encodeNativeDocuments, + recordNativeRejections, +} from '~/files/reading'; import { assertModelBoundContent } from '~/middleware/modelBoundContent'; +import { toClassicInspectionView, applyTurnDelivery } from './delivery'; import { filterFilesByEndpointRuntimeConfig } from '~/files/filter'; import { countTokens } from '~/utils/tokenizer'; @@ -47,10 +64,15 @@ export interface RunFileEncodingParams { imageDetail?: ImageDetail; } -type MediaEncoder = ( +/** The document encoder also learns whether to leave out a file it cannot send. */ +export interface RunFileDocumentEncodingParams extends RunFileEncodingParams { + onValidationFailure?: NativeValidationMode; +} + +type MediaEncoder = ( req: ServerRequest, files: TFile[], - params: RunFileEncodingParams, + params: P, getStrategyFunctions: (source: string) => StrategyFunctions, ) => Promise; @@ -59,15 +81,20 @@ export interface RunFileMessageEncoderDeps { req: ServerRequest; getAgent: (agentId: string) => RunFileEncodingAgent | undefined; encodeImages: MediaEncoder<{ image_urls: ContentBlock[] }>; - encodeDocuments: MediaEncoder<{ documents: ContentBlock[] }>; + encodeDocuments: MediaEncoder< + { documents: ContentBlock[]; rejected?: readonly DocumentRejection[] }, + RunFileDocumentEncodingParams + >; encodeAudios: MediaEncoder<{ audios: ContentBlock[] }>; encodeVideos: MediaEncoder<{ videos: ContentBlock[] }>; getStrategyFunctions: (source: string) => StrategyFunctions; - extractText: (params: { - attachments: TFile[]; - req: ServerRequest; - tokenCountFn: TokenCountFn; - }) => Promise; + extractText: ( + params: { + attachments: TFile[]; + req: ServerRequest; + tokenCountFn: TokenCountFn; + } & TurnTextOptions, + ) => Promise; } export interface RunFileMessageEncoder { @@ -79,11 +106,15 @@ export interface RunFileMessageEncoder { export function createRunFileMessageEncoder( deps: RunFileMessageEncoderDeps, ): RunFileMessageEncoder { - function prepare(files: TFile[], agentId: string) { + function resolveAgent(agentId: string): RunFileEncodingAgent { const agent = deps.getAgent(agentId); if (!agent) { throw new Error('The target agent is not available for shared file delivery.'); } + return agent; + } + + function prepare(files: TFile[], agent: RunFileEncodingAgent) { const { deliveryRouting } = agent; const { endpoint, fileConfig, endpointConfig } = deliveryRouting; const params: RunFileEncodingParams = { @@ -111,6 +142,7 @@ export function createRunFileMessageEncoder( endpoint, skipTotalSizeLimit: true, preserveTextSources: true, + consumers: agent.fileConsumers, }); if (compatibleFiles.length !== sharedFiles.length) { throw new AgentAttachmentPolicyError(); @@ -127,17 +159,26 @@ export function createRunFileMessageEncoder( req: deps.req, endpoint, }); - assertModelBoundContent({ filters: deps.req.config?.filters, files: sharedFiles }); + assertModelBoundContent({ + filters: deps.req.config?.filters, + files: toClassicInspectionView(sharedFiles, deliveryRouting, agent.fileConsumers), + }); return { agent, params, sharedFiles, fileConfig, endpointConfig }; } function validate(files: TFile[], agentId: string): void { - if (files.length > 0) prepare(files, agentId); + if (files.length > 0) prepare(files, resolveAgent(agentId)); } async function encode(files: TFile[], agentId: string): Promise { if (files.length === 0) return []; - const { agent, params, sharedFiles, fileConfig, endpointConfig } = prepare(files, agentId); + const target = resolveAgent(agentId); + const turnFiles = await prepareTurnFiles({ + routing: target.deliveryRouting, + files, + consumers: target.fileConsumers, + }); + const { agent, params, sharedFiles, fileConfig, endpointConfig } = prepare(turnFiles, target); const images: TFile[] = []; const documents: TFile[] = []; const audios: TFile[] = []; @@ -183,21 +224,88 @@ export function createRunFileMessageEncoder( } } - const encodeMedia = (encoder: MediaEncoder, inputs: TFile[], empty: T): Promise => + const encodeMedia = ( + encoder: MediaEncoder, + inputs: TFile[], + empty: T, + encoderParams: P, + ): Promise => inputs.length > 0 - ? encoder(deps.req, inputs, params, deps.getStrategyFunctions) + ? encoder(deps.req, inputs, encoderParams, deps.getStrategyFunctions) : Promise.resolve(empty); + const encodeDocuments = (inputs: TFile[], onValidationFailure: NativeValidationMode) => + encodeMedia( + deps.encodeDocuments, + inputs, + { documents: [] }, + { ...params, onValidationFailure }, + ); const [imageResult, documentResult, audioResult, videoResult, text] = await Promise.all([ - encodeMedia(deps.encodeImages, images, { image_urls: [] }), - encodeMedia(deps.encodeDocuments, documents, { documents: [] }), - encodeMedia(deps.encodeAudios, audios, { audios: [] }), - encodeMedia(deps.encodeVideos, videos, { videos: [] }), + encodeMedia(deps.encodeImages, images, { image_urls: [] }, params), + encodeNativeDocuments(documents, agent, encodeDocuments), + encodeMedia(deps.encodeAudios, audios, { audios: [] }, params), + encodeMedia(deps.encodeVideos, videos, { videos: [] }, params), textFiles.length > 0 - ? deps.extractText({ attachments: textFiles, req: deps.req, tokenCountFn: countTokens }) + ? deps.extractText({ + attachments: textFiles, + req: deps.req, + tokenCountFn: countTokens, + ...getTurnTextOptions(agent.deliveryRouting), + }) : Promise.resolve(undefined), ]); + recordNativeRejections([agent], documentResult.rejected); + const rejectedIds = new Set((documentResult.rejected ?? []).map(({ file_id }) => file_id)); + let fallbackText: string | undefined; + let fallbackFiles: TFile[] = []; + if (rejectedIds.size > 0) { + const sharedIds = new Set(sharedFiles.map((file) => file.file_id)); + const settled = await settleTurnFiles({ + routing: agent.deliveryRouting, + files: sharedFiles, + consumers: agent.fileConsumers, + allocation: { + requestFileIds: sharedFiles + .filter((file) => rejectedIds.has(file.file_id)) + .map((file) => file.file_id), + limits: resolveAgentAttachmentLimits({ req: deps.req, endpoint: params.endpoint }), + measure: measureModelBoundAttachment, + committedExtra: applyTurnDelivery( + (agent.agentContextAttachments ?? []).filter((file) => !sharedIds.has(file.file_id)), + { routing: agent.deliveryRouting, consumers: agent.fileConsumers }, + ), + committedEntries: sharedFiles + .filter((file) => !rejectedIds.has(file.file_id)) + .map(measureModelBoundAttachment), + scope: 'request', + }, + flush: true, + }); + const prepared = prepare(settled, agent).sharedFiles; + fallbackFiles = prepared.filter( + (file) => rejectedIds.has(file.file_id) && file.llmDeliveryPath === 'text', + ); + assertModelBoundContent({ + filters: deps.req.config?.filters, + files: fallbackFiles, + }); + if (fallbackFiles.length > 0) { + fallbackText = await deps.extractText({ + attachments: fallbackFiles, + req: deps.req, + tokenCountFn: countTokens, + ...getTurnTextOptions(agent.deliveryRouting), + }); + } + } + const deliveredIds = new Set(fallbackFiles.map(({ file_id }) => file_id)); + const leftOut = renderLeftOutFiles( + agent, + documents.filter((file) => rejectedIds.has(file.file_id) && !deliveredIds.has(file.file_id)), + ); + const body = [text, fallbackText, leftOut].filter(Boolean).join('\n\n'); if ( - !text && + !body && imageResult.image_urls.length === 0 && documentResult.documents.length === 0 && audioResult.audios.length === 0 && @@ -208,7 +316,7 @@ export function createRunFileMessageEncoder( const formatted = formatMessage({ message: { role: 'user', - content: text ?? 'Read-only files shared for this task.', + content: body || 'Read-only files shared for this task.', image_urls: imageResult.image_urls, documents: documentResult.documents, audios: audioResult.audios, diff --git a/packages/api/src/agents/hitl/protection.spec.ts b/packages/api/src/agents/hitl/protection.spec.ts index 86f0220a26f..dbff26961d4 100644 --- a/packages/api/src/agents/hitl/protection.spec.ts +++ b/packages/api/src/agents/hitl/protection.spec.ts @@ -1,3 +1,4 @@ +import { FileContext } from 'librechat-data-provider'; import type { AssertResumeRuntimeContentAllowedInput, ResumeContentProtectionDependencies, @@ -9,6 +10,7 @@ import { isModelBoundAttachmentFile, } from '../attachments'; import { assertResumeContentAllowed, assertResumeRuntimeContentAllowed } from './protection'; +import { resolveTurnDeliveryRouting } from '../files/delivery'; const user = { id: 'user-1', @@ -146,6 +148,111 @@ describe('assertResumeRuntimeContentAllowed', () => { expect(dependencies.getFiles).toHaveBeenCalledTimes(1); }); + it('charges checkpoint files by the resuming agent automatic decisions', async () => { + /* The automatic policy withheld this file's text for Run Code, so the checkpoint never + * carried it and the resumed turn must not spend its text budget on it. */ + const dependencies = createDependencies(); + dependencies.getAgentCheckpointer.mockResolvedValue({ + getTuple: jest.fn().mockResolvedValue({ + checkpoint: { + channel_values: { + messages: [{ role: 'human', additional_kwargs: { sourceMessageId: 'source-message' } }], + }, + }, + }), + }); + dependencies.getMessages.mockResolvedValue([ + { messageId: 'source-message', files: [{ file_id: 'code-file' }] }, + ]); + const storedFile = { + file_id: 'code-file', + filename: 'sales.csv', + type: 'text/csv', + source: 'local', + context: FileContext.message_attachment, + bytes: 20, + text: 'region,total', + llmDeliveryPath: 'none', + metadata: { destinationChosen: false }, + }; + dependencies.getFiles.mockResolvedValue([storedFile]); + const routing = resolveTurnDeliveryRouting({ + agent: { provider: 'openAI' }, + config: { fileConfig: { endpoints: { openAI: { llmDeliveryPolicy: 'automatic' } } } }, + }); + const input = { ...createInput({}), isTemporary: false }; + + const [runsCode, noReader] = await Promise.all([ + assertResumeRuntimeContentAllowed( + { ...input, delivery: { routing, consumers: { executeCode: true, fileSearch: false } } }, + dependencies, + ), + assertResumeRuntimeContentAllowed( + { ...input, delivery: { routing, consumers: { executeCode: false, fileSearch: false } } }, + dependencies, + ), + ]); + + expect(runsCode.checkpointFiles).toEqual([storedFile]); + expect(noReader.checkpointFiles).toEqual([{ ...storedFile, llmDeliveryPath: 'text' }]); + expect(storedFile.llmDeliveryPath).toBe('none'); + }); + + it('charges checkpoint files by the primary agent when the host passes only its agents', async () => { + /* AgentClient lists its own agent first and passes no delivery; the resume budget must + * still follow that agent's automatic decision, not charge the withheld text. */ + const dependencies = createDependencies(); + dependencies.getAgentCheckpointer.mockResolvedValue({ + getTuple: jest.fn().mockResolvedValue({ + checkpoint: { + channel_values: { + messages: [{ role: 'human', additional_kwargs: { sourceMessageId: 'source-message' } }], + }, + }, + }), + }); + dependencies.getMessages.mockResolvedValue([ + { messageId: 'source-message', files: [{ file_id: 'workbook' }] }, + ]); + const storedFile = { + file_id: 'workbook', + filename: 'quarterly.xlsx', + type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + source: 'local', + context: FileContext.message_attachment, + bytes: 2048, + text: 'quarter,total\nQ1,42', + llmDeliveryPath: 'none', + metadata: { destinationChosen: false }, + }; + dependencies.getFiles.mockResolvedValue([storedFile]); + const deliveryRouting = resolveTurnDeliveryRouting({ + agent: { provider: 'openAI' }, + config: { fileConfig: { endpoints: { openAI: { llmDeliveryPolicy: 'automatic' } } } }, + }); + const primary = { + name: 'primary', + deliveryRouting, + fileConsumers: { executeCode: true, fileSearch: false }, + }; + const handoff = { name: 'handoff', fileConsumers: { executeCode: false, fileSearch: false } }; + + const projection = await assertResumeRuntimeContentAllowed( + { ...createInput({}), isTemporary: false, agents: [primary, handoff] }, + dependencies, + ); + + expect(projection.checkpointFiles).toEqual([storedFile]); + expect(projection.checkpointFiles.filter(isModelBoundAttachmentFile)).toEqual([]); + expect(() => + assertAgentAttachmentLimits({ + attachments: projection.checkpointFiles.filter(isModelBoundAttachmentFile), + fileConfig: { fileContextCharLimit: 3 }, + endpoint: 'openAI', + }), + ).not.toThrow(); + }); + it('hydrates checkpoint-bound files when content filters are inactive', async () => { const dependencies = createDependencies(); dependencies.getAgentCheckpointer.mockResolvedValue({ diff --git a/packages/api/src/agents/hitl/protection.ts b/packages/api/src/agents/hitl/protection.ts index 88d27200342..da436b1afc4 100644 --- a/packages/api/src/agents/hitl/protection.ts +++ b/packages/api/src/agents/hitl/protection.ts @@ -40,6 +40,8 @@ import type { import type { LocatorTraversalReporter } from '../../protection/diagnostics'; import type { TextContentFragment } from '~/protection/types'; import type { CheckAccessParams } from '~/middleware/access'; +import type { TurnDeliveryInputs } from '../files/delivery'; +import type { TurnReadingAgent } from '~/files/reading'; import { assertModelBoundContent, collectModelBoundHistoricalFileIdState, @@ -217,10 +219,26 @@ export interface AssertResumeRuntimeContentAllowedInput | 'isTemporary' | 'checkpointNamespace' > { - readonly agents: NonNullable; + /** The run's reachable agents, the primary agent first. */ + readonly agents: readonly (ResumeAgentContentInput | null | undefined)[]; readonly files: NonNullable; + /** + * The resuming agent's routing and readers, so checkpoint files are charged by its decisions. + * Defaults to the primary agent's `deliveryRouting` and `fileConsumers`. + */ + readonly delivery?: TurnDeliveryInputs; } +/** An initialized agent as the resume check reads it: its content plus how it receives files. */ +type ResumeAgentContentInput = AgentContentInput & + Pick; + +const resolveResumeDelivery = ({ + delivery, + agents, +}: Pick): TurnDeliveryInputs => + delivery ?? { routing: agents[0]?.deliveryRouting, consumers: agents[0]?.fileConsumers }; + export type ResumeRuntimeContentProtectionDependencies = Pick< ResumeContentProtectionDependencies, 'getAgentCheckpointer' | 'getMessages' | 'getFiles' | 'onTraversalFailure' @@ -1148,7 +1166,10 @@ export async function assertResumeRuntimeContentAllowed( ); throw new AttachmentObjectNotFoundError(unresolvedFileId ?? 'unknown'); } - return { resolvedFiles, checkpointFiles: applyCheckpointDelivery(checkpointFiles) }; + return { + resolvedFiles, + checkpointFiles: applyCheckpointDelivery(checkpointFiles, resolveResumeDelivery(input)), + }; } export function getUserFacingResumeError( diff --git a/packages/api/src/agents/initialize.ts b/packages/api/src/agents/initialize.ts index 4eb34948a1e..8ca944c8caa 100644 --- a/packages/api/src/agents/initialize.ts +++ b/packages/api/src/agents/initialize.ts @@ -14,6 +14,7 @@ import { hasActivePiiPatterns, replaceSpecialVars, providerEndpointMap, + resolveLLMDeliveryPolicy, } from 'librechat-data-provider'; import type { AgentToolResources, @@ -28,8 +29,8 @@ import type { TUser, TurnFileConsumers, } from 'librechat-data-provider'; +import type { AppConfig, IMongoFile, FileMethods, FileOwnerScope } from '@librechat/data-schemas'; import type { GenericTool, LCToolRegistry, ToolMap, LCTool } from '@librechat/agents'; -import type { AppConfig, IMongoFile, FileOwnerScope } from '@librechat/data-schemas'; import type { Request, Response as ServerResponse } from 'express'; import type { TFileUpdate, @@ -40,6 +41,13 @@ import type { TCheckSessionsAlive, TLoadCodeApiKey, } from './resources'; +import type { + FileTextDeriver, + PrimedSearchFile, + TurnReadingContext, + DirectContentAllocation, + TextDerivationPersister, +} from '~/files/reading'; import type { ResolvedManualSkill, ResolvedAlwaysApplySkill, @@ -75,6 +83,12 @@ import { unionPrimeAllowedTools, MAX_PRIMED_SKILLS_PER_TURN, } from './skills'; +import { + isModelBoundAttachmentFile, + assertAgentAttachmentLimits, + measureModelBoundAttachment, + resolveAgentAttachmentLimits, +} from './attachments'; import { resolveChatProjectFiles, resolveChatProjectPolicyFiles, @@ -109,6 +123,17 @@ import { resolveAttachedWorkspaceCommandTimeoutMax, resolveAttachedWorkspaceCommandTimeoutDefault, } from '~/code/command'; +import { + logTurnReading, + settleTurnFiles, + buildTurnReadingContext, + prepareAgentFileContext, +} from '~/files/reading'; +import { + applyTurnDelivery, + toClassicInspectionView, + resolveTurnDeliveryRouting, +} from './files/delivery'; import { formatChatProjectInstructions, hydrateChatProjectContextResources, @@ -116,29 +141,28 @@ import { import { assertChatProjectInstructions, ChatProjectResourcesChangedError } from '../projects/turn'; import { extractAgentContent, extractSkillContent } from '../protection/adapters/submissions'; import { createConfiguredContentInspector, inspectContent } from '../protection/runtime'; -import { assertAgentAttachmentLimits, isModelBoundAttachmentFile } from './attachments'; import { assertModelBoundContent } from '../middleware/modelBoundContent'; import { resolveAttachedWorkspaceReadFileLines } from '~/code/workspace'; +import { resolveConfiguredFileSizeLimit } from '~/files/encode/utils'; import { isValidInstructionsPromptLink } from './instructions/linked'; import { PARTIAL_RESOLVED_CONVERSATION } from './conversationSymbols'; import { registerMemoryTools, memoryToolUsageGuard } from './memory'; import { isImplicitStatefulCodeRouteAvailable } from '~/code/config'; import { applyIntentLabels, sanitizeIntentLabels } from './intent'; -import { prepareQueuedCodeFileContext } from '~/files/code/queued'; import { ContentFilterError } from '../middleware/contentFilter'; import { resolveToolRoleGrants } from '~/tools/rolePermissions'; import { createRequestAgentExecutionContext } from './runtime'; -import { resolveTurnDeliveryRouting } from './files/delivery'; import { filterFilesByEndpointRuntimeConfig } from '~/files'; import { isFatalAgentInitializationError } from './errors'; import { hasActiveFilePolicy } from '../protection/files'; import { resolveAgentCodeExecution } from '~/code/agent'; +import { resolveFileTokenLimit } from '~/files/context'; import { hasActiveFileFieldPolicy } from '~/protection'; import { applyBackgroundToolCalls } from './background'; -import { applyTurnDelivery } from './files/delivery'; import { generateArtifactsPrompt } from '~/prompts'; import { getProviderConfig } from '~/endpoints'; import { primeResources } from './resources'; +import Tokenizer from '~/utils/tokenizer'; /** * Fraction of context budget reserved as headroom when no explicit maxContextTokens is set. @@ -194,6 +218,91 @@ function sumUniqueBytes(files: Array<{ file_id?: string; bytes?: number }>): num return total; } +type TurnAttachment = IMongoFile | TFile; + +/** The request attachments the endpoint runtime policy removed from the delivered set. */ +function selectDroppedRequestFiles( + delivered: readonly T[], + kept: readonly T[], + requestFileIds: ReadonlySet, +): T[] { + const keptFiles = new Set(kept); + return delivered.filter((file) => !keptFiles.has(file) && requestFileIds.has(file.file_id)); +} + +/** The encoding the prompt's file text budget counts with (`extractFileContext`). */ +const READING_TOKEN_ENCODING = 'o200k_base'; + +interface TurnReadingSetup { + routing: TurnDeliveryRouting; + /** The provider the document encoder receives for this agent. */ + provider: string; + model?: string; + endpoint?: string; + fileConfig?: AppConfig['fileConfig']; + req?: ServerRequest; + deriveText?: FileTextDeriver; + persistDerivation?: TextDerivationPersister; + signal?: AbortSignal; +} + +/** Saves a derivation under the request's file owner, when the host can save one. */ +function bindDerivationPersister( + save: InitializeAgentDbMethods['saveFileTextDerivation'], + scope: FileOwnerScope | undefined, +): TextDerivationPersister | undefined { + if (save == null || scope == null) { + return undefined; + } + return (update) => save(update, { user: scope.userId, tenantId: scope.tenantId }); +} + +/** + * The reading evidence for one agent's turn: the automatic policy's, or under classic with a text + * deriver a context that only derives text for records the automatic policy marked. Nothing + * otherwise, so a classic turn runs none of the reading steps. The automatic policy judges text + * fit synchronously, so the tokenizer is loaded first; if it cannot load, counts fall back to + * the tokenizer's estimate. + */ +async function prepareTurnReading({ + routing, + provider, + model, + endpoint, + fileConfig, + req, + deriveText, + persistDerivation, + signal, +}: TurnReadingSetup): Promise { + const automatic = resolveLLMDeliveryPolicy(routing.endpointConfig) === 'automatic'; + if (!automatic && deriveText == null) { + return undefined; + } + if (automatic) { + await Tokenizer.initEncoding(READING_TOKEN_ENCODING).catch((error: unknown) => { + logger.warn( + '[initializeAgent] Tokenizer unavailable; judging text fit with estimated counts', + getSafeErrorMetadata(error), + ); + }); + } + return buildTurnReadingContext({ + routing, + provider, + model, + fileTokenLimit: resolveFileTokenLimit(req), + configuredFileSizeLimit: resolveConfiguredFileSizeLimit(fileConfig, { + provider: provider as Providers, + endpoint, + }), + countTokens: (text) => Tokenizer.getTokenCount(text, READING_TOKEN_ENCODING), + deriveText, + persistDerivation, + signal, + }); +} + const temporalSpecialVarRegex = /{{\s*(current_date|current_datetime|iso_datetime)\s*}}/i; const geminiModelVersionRegex = /^gemini-(\d+)(?:\.(\d+))?(?:-|$)/; const googleToolCombinationTextModels = [ @@ -869,6 +978,8 @@ export type InitializedAgent = Agent & { * call #1 the sandbox can't see the files at all. */ primedCodeFiles?: import('@librechat/agents').CodeEnvFile[]; + /** Files the File Search advert listed on this turn, as the loader primed them. */ + primedSearchFileIds?: string[]; /** * Resolved token/pricing config for this agent's endpoint (admin static * `tokenConfig` and/or fetched custom-endpoint config). Surfaced from the @@ -955,6 +1066,8 @@ export interface InitializeAgentParams { * artifacts don't reach the sandbox. */ primedCodeFiles?: import('@librechat/agents').CodeEnvFile[]; + /** Files the loader listed in the File Search advert (`tool_resources.file_search`). */ + primedSearchFiles?: PrimedSearchFile[]; /** Live workspace binding resolved by the execution-side loader. */ codeExecutionContext?: CodeExecutionContext; repositoryInstructionSource?: RepositoryInstructionSource; @@ -1041,6 +1154,18 @@ export interface InitializeAgentParams { * meta user messages before the LLM call. */ manualSkills?: string[]; + /** + * Derives an attachment's text from its retained original on this request. With it, a record + * the automatic policy marked for later extraction can still be read as text, under either + * policy; without it the turn reads only text stored at upload. + */ + deriveText?: FileTextDeriver; + /** + * The request's derivation persister (`createDerivationPersister`), shared by every agent + * so a derived file is written once per request. Without it each agent saves its own + * derivations through `saveFileTextDerivation` under the request's file owner. + */ + persistDerivation?: TextDerivationPersister; } /** @@ -1146,6 +1271,11 @@ export interface InitializeAgentDbMethods extends EndpointDbMethods { loadCodeApiKey?: TLoadCodeApiKey; /** Optional: persist file metadata updates after provisioning */ updateFile?: (data: TFileUpdate) => Promise; + /** + * Optional: saves text derived on this turn onto a record whose upload deferred extraction. + * Without it derived text stays on the turn copies and is derived again on a later turn. + */ + saveFileTextDerivation?: FileMethods['saveFileTextDerivation']; /** Resolves a role by name for the tool role-permission grants. Optional: when * absent the role half of the web-search gate is not applied. */ getRoleByName?: CheckAccessParams['getRoleByName']; @@ -1589,6 +1719,22 @@ export async function initializeAgent( }, config: appConfig, }); + const readingContext = await prepareTurnReading({ + routing: deliveryRouting, + provider: agent.provider, + model: typeof llmConfig.model === 'string' ? llmConfig.model : (agent.model ?? undefined), + endpoint: agent.endpoint ?? undefined, + fileConfig: appConfig?.fileConfig, + req: params.req, + deriveText: params.deriveText, + persistDerivation: + params.persistDerivation ?? + bindDerivationPersister(db.saveFileTextDerivation, requestFileOwnerScope), + signal: params.signal, + }); + if (readingContext != null) { + deliveryRouting.reading = readingContext; + } /** Resolve the per-agent Code API route before resource/tool priming. A * stateful agent must perform freshness checks and recovery uploads against @@ -1872,8 +2018,24 @@ export async function initializeAgent( if (!paramEndpoints.has(agent.endpoint ?? '')) { endpointFileType = EModelEndpoint.custom; } + const admissionFileIds = new Set( + (authorizedRunFiles ?? requestUsageFiles).map((file) => file.file_id), + ); + /** First-fit allocation of the request's direct content against the limits admission holds + * it to, so allocation and the admission check cannot disagree. */ + const requestAllocation = (): DirectContentAllocation => ({ + requestFileIds: [...admissionFileIds], + limits: resolveAgentAttachmentLimits({ + fileConfig: appConfig?.fileConfig, + endpoint: agent.endpoint ?? '', + endpointType: endpointFileType, + }), + measure: measureModelBoundAttachment, + agentId: agent.id, + }); if ((currentFiles && currentFiles.length) || deferredProvisionFiles.length > 0) { const endpointType = endpointFileType; + const deliveredFiles = currentFiles ?? []; currentFiles = filterFilesByEndpointRuntimeConfig(appConfig, { files: currentFiles, @@ -1881,13 +2043,27 @@ export async function initializeAgent( endpointType, skipTotalSizeLimit: true, preserveTextSources: true, + consumers: fileConsumers, }); - const requestUsageFileIds = new Set( - (authorizedRunFiles ?? requestUsageFiles).map((file) => file.file_id), - ); + /* Only filter survivors are allocated: a file the endpoint refuses never spends the + * request's direct-content allowance, and the reading records it as dropped. */ + if (readingContext != null) { + if (readingContext.policy === 'automatic') { + readingContext.recordDropped( + selectDroppedRequestFiles(deliveredFiles, currentFiles, admissionFileIds), + ); + } + currentFiles = await settleTurnFiles({ + routing: deliveryRouting, + consumers: fileConsumers, + files: currentFiles, + allocation: requestAllocation(), + signal: params.signal, + }); + } assertAgentAttachmentLimits({ attachments: currentFiles.filter( - (file) => requestUsageFileIds.has(file.file_id) && isModelBoundAttachmentFile(file), + (file) => admissionFileIds.has(file.file_id) && isModelBoundAttachmentFile(file), ), fileConfig: appConfig?.fileConfig, endpoint: agent.endpoint ?? '', @@ -1895,8 +2071,9 @@ export async function initializeAgent( }); /* The same endpoint configuration governs both paths. A file this endpoint refuses - * by size, MIME type, or a files-disabled setting must not reach the Code API or - * RAG through provisioning just because it left the delivery set. */ + * by MIME type, a files-disabled setting, or (outside the automatic reading policy, + * where size is delivery capacity) size must not reach the Code API or RAG through + * provisioning just because it left the delivery set. */ if (deferredProvisionFiles.length > 0) { /* One request, one total-size allowance. Filtering each set from zero would let a * delivery attachment and a provisioning candidate that each fit alone exceed the @@ -1915,6 +2092,7 @@ export async function initializeAgent( files: deferredProvisionFiles, endpoint: agent.endpoint ?? '', endpointType, + consumers: fileConsumers, /* The deferred pass charges its own list as it walks it, so a file in both sets * is counted there. Only what delivery spends on files the deferred pass will * not see is carried in. */ @@ -1929,7 +2107,7 @@ export async function initializeAgent( assertModelBoundContent({ filters: appConfig?.filters, - files: currentFiles, + files: currentFiles && toClassicInspectionView(currentFiles, deliveryRouting, fileConsumers), }); /* Provisioning candidates are inspected under the same policy before their bytes can @@ -2000,6 +2178,7 @@ export async function initializeAgent( * and current content policy checks. Ignore returned rows so priming cannot * observe a different post-inspection snapshot. */ + const readingSettled = readingContext?.flush(); if (requestFileOwnerId && requestUsageFiles.length > 0) { await db.updateFilesUsage(requestUsageFiles, undefined, { user: requestFileOwnerId, @@ -2012,6 +2191,7 @@ export async function initializeAgent( tenantId: user?.tenantId, }); } + await readingSettled; let runtimeToolResources = agent.tool_resources; if (attachedEnvironmentOptOut && runtimeToolResources != null) { @@ -2058,6 +2238,7 @@ export async function initializeAgent( endpoint: agent.endpoint ?? '', endpointType: endpointFileType, consumedBytes: sumUniqueBytes(committedFiles), + consumers: fileConsumers, }) as unknown as TFile[]; /* Dropped rather than fatal, matching the deferred candidates: these were not @@ -2183,6 +2364,7 @@ export async function initializeAgent( oauthActionToolNames, tools: structuredTools, primedCodeFiles, + primedSearchFiles, codeExecutionContext: loadedCodeExecutionContext, repositoryInstructionSource, } = loadToolsResult ?? { @@ -2199,6 +2381,7 @@ export async function initializeAgent( actionsEnabled: undefined, oauthActionToolNames: undefined, primedCodeFiles: undefined, + primedSearchFiles: undefined, codeExecutionContext: undefined, repositoryInstructionSource: undefined, }; @@ -2684,30 +2867,60 @@ export async function initializeAgent( fileConsumers.executeCode !== finalFileConsumers.executeCode || fileConsumers.fileSearch !== finalFileConsumers.fileSearch; Object.assign(fileConsumers, finalFileConsumers); - const finalizeAttachments = (files: Array | undefined): IMongoFile[] => { - const hydrated = toMongoFiles(files); - return consumersChanged - ? applyTurnDelivery(hydrated, { routing: deliveryRouting, consumers: fileConsumers }) - : hydrated; - }; - const finalAttachments = finalizeAttachments(primedAttachments); - const finalRequestAttachments = consumersChanged - ? applyTurnDelivery( - (primedRequestAttachments ?? []).filter((file): file is TFile => file != null), - { - routing: deliveryRouting, - consumers: fileConsumers, - }, - ) - : primedRequestAttachments; + const automaticReading = readingContext?.policy === 'automatic'; + if (automaticReading) { + const searchResource = runtimeToolResources?.[EToolResources.file_search]; + const registeredSearchFileIds = primedSearchFiles?.map((file) => file.file_id) ?? [ + ...(searchResource?.file_ids ?? []), + ...(searchResource?.files ?? []).flatMap((file) => (file == null ? [] : [file.file_id])), + ]; + if (provisionState != null) { + provisionState.searchPreparation ??= new Map(); + } + readingContext?.setSearchEvidence({ + queued: (provisionState?.vectorDBFiles ?? []).map((file) => file.file_id), + registered: registeredSearchFileIds, + preparation: provisionState?.searchPreparation, + }); + } + const primedRequestFiles = (primedRequestAttachments ?? []).filter( + (file): file is TFile => file != null, + ); + const primedFiles = toMongoFiles(primedAttachments); + const primedContextFiles = toMongoFiles(primedAgentContextAttachments); + const redeliver = (files: T[]): T[] => + consumersChanged + ? applyTurnDelivery(files, { routing: deliveryRouting, consumers: fileConsumers }) + : files; + /* The automatic reading decides the request again whether or not a reader dropped out: + * search evidence is known only now, so a file left for a search that never received it moves + * on. A derive-only classic reading settles again only when a reader dropped out, since only + * then can a marked record newly need text. Request files are the only ones the reading + * settles, so the other sets take the settled request copies and keep classic routing. */ + const settlesFinalRequest = automaticReading || (readingContext != null && consumersChanged); + const finalRequestAttachments = settlesFinalRequest + ? await settleTurnFiles({ + routing: deliveryRouting, + consumers: fileConsumers, + files: primedRequestFiles, + allocation: requestAllocation(), + signal: params.signal, + }) + : redeliver(primedRequestFiles); const requestAttachments = toMongoFiles(finalRequestAttachments); - const agentContextAttachments = finalizeAttachments(primedAgentContextAttachments); - if (consumersChanged) { + const readingChanged = settlesFinalRequest && finalRequestAttachments !== primedRequestFiles; + const settledRequestCopies = new Map( + readingChanged ? requestAttachments.map((file) => [file.file_id, file] as const) : [], + ); + const finalizeShared = (files: IMongoFile[]): IMongoFile[] => + settledRequestCopies.size === 0 + ? redeliver(files) + : redeliver(files).map((file) => settledRequestCopies.get(file.file_id) ?? file); + const finalAttachments = finalizeShared(primedFiles); + const agentContextAttachments = finalizeShared(primedContextFiles); + if (consumersChanged || readingChanged) { /* A loader may drop a reader, including a skill extra on retry. Newly model-bound text * must pass admission before it can escape initialization, just like initial fallback. */ - const admissionFileIds = new Set( - (authorizedRunFiles ?? requestUsageFiles).map((file) => file.file_id), - ); assertAgentAttachmentLimits({ attachments: requestAttachments.filter( (file) => admissionFileIds.has(file.file_id) && isModelBoundAttachmentFile(file), @@ -2718,9 +2931,14 @@ export async function initializeAgent( }); assertModelBoundContent({ filters: appConfig?.filters, - files: [...finalAttachments, ...requestAttachments, ...agentContextAttachments], + files: toClassicInspectionView( + [...finalAttachments, ...requestAttachments, ...agentContextAttachments], + deliveryRouting, + fileConsumers, + ), }); } + await readingContext?.flush(); const currentRequestFileIds = new Set(requestFileIds); const currentRequestAttachments: TFile[] = (finalRequestAttachments ?? []) .filter((file): file is TFile => file != null && currentRequestFileIds.has(file.file_id)) @@ -2807,11 +3025,14 @@ export async function initializeAgent( ? maxContextTokens : Math.max(1024, Math.round(baseContextTokens * (1 - DEFAULT_RESERVE_RATIO))), primedCodeFiles, + primedSearchFileIds: primedSearchFiles?.map((file) => file.file_id), endpointTokenConfig: options.endpointTokenConfig, instructionsPromptFacts, }; - prepareQueuedCodeFileContext(initializedAgent, [initializedAgent], user?.id); + prepareAgentFileContext(initializedAgent, [initializedAgent], user?.id, false, { + filters: appConfig?.filters, + }); const queuedFileContext = initializedAgent.dynamicToolContextMap?.queued_code_files; if (typeof queuedFileContext === 'string') { assertModelBoundContent({ @@ -2819,6 +3040,7 @@ export async function initializeAgent( files: [{ content: queuedFileContext }], }); } + logTurnReading(initializedAgent, 'init'); /** * Usage is recorded only once initialization has fully succeeded — every diff --git a/packages/api/src/agents/resources.test.ts b/packages/api/src/agents/resources.test.ts index 9e3530be7db..ea9718491f2 100644 --- a/packages/api/src/agents/resources.test.ts +++ b/packages/api/src/agents/resources.test.ts @@ -3016,6 +3016,7 @@ describe('primeResources', () => { bytes: 4096, embedded: false, usage: 0, + metadata: { destinationChosen: false }, } as TFile; const result = await primeResources({ @@ -3046,6 +3047,7 @@ describe('primeResources', () => { bytes: 2048, embedded: false, usage: 0, + metadata: { destinationChosen: false }, } as TFile; const result = await primeResources({ diff --git a/packages/api/src/agents/resources.ts b/packages/api/src/agents/resources.ts index 743b2e6a042..e9746c2c8eb 100644 --- a/packages/api/src/agents/resources.ts +++ b/packages/api/src/agents/resources.ts @@ -18,9 +18,9 @@ import type { } from 'librechat-data-provider'; import type { IMongoFile, AppConfig, IUser } from '@librechat/data-schemas'; import type { FilterQuery, QueryOptions, ProjectionType } from 'mongoose'; +import type { SearchPreparationState } from '~/files/reading/turn'; import type { CodeExecutionContext } from './execution'; import type { ServerRequest } from '~/types'; - import { TOOL_RESOURCE_KEYS } from './orphans'; /** Removes runtime-only file records before persisted Agent resources enter tool initialization. */ @@ -97,6 +97,8 @@ export type ProvisionState = { codeEnvDestinations?: Map; /** Files that need embedding into the vector DB for file_search */ vectorDBFiles: TFile[]; + /** Request-scoped indexing outcomes, shared with the turn's reading notices. */ + searchPreparation?: Map; /** Set of file_ids confirmed alive in code env (from staleness check) */ aliveFileIds: Set; /** The active agent's own resource files, which are the only ones provisioned under its diff --git a/packages/api/src/agents/run.ts b/packages/api/src/agents/run.ts index 0867a483e1d..8b8e2da3a53 100644 --- a/packages/api/src/agents/run.ts +++ b/packages/api/src/agents/run.ts @@ -64,6 +64,7 @@ import type { LangfuseTraceContext } from '~/langfuse/identity'; import type { ResolvedAlwaysApplySkill } from '~/agents/skills'; import type { CodeExecutionContext } from '~/agents/execution'; import type { SubagentCodeHostArgSpecs } from '~/code/targets'; +import type { ReadingAgent } from '~/files/reading/inventory'; import type { MCPToolAlias } from '~/tools/classification'; import type { ReviewedToolApprovals } from './hitl/modes'; import type { SubagentUsageEvent } from '~/agents/usage'; @@ -125,7 +126,7 @@ import { buildAgentInitialToolSessions } from '~/agents/codeFilesSession'; import { getDirectDispatcher, getProxyDispatcher } from '~/utils/proxy'; import { getAzureCredentials, constructAzureURL } from '~/utils/azure'; import { buildEffectiveToolApprovalPolicy } from '~/agents/hitl/allow'; -import { prepareQueuedCodeFileContext } from '~/files/code/queued'; +import { prepareAgentFileContext } from '~/files/reading/inventory'; import { getBuiltInBaseURL } from '~/endpoints/openai/initialize'; import { getProviderConfig } from '~/endpoints/config/providers'; import { buildToolApprovalHooks } from '~/agents/hitl/hooks'; @@ -435,82 +436,83 @@ export function shouldReplayReasoningContent( return isDeepSeekReasoningProvider(agent.provider, agent.model_parameters?.model ?? agent.model); } -type RunAgent = Omit & { - provisionState?: ProvisionState; - fileConsumers?: TurnFileConsumers; - azureOptions?: t.AzureOptions; - tools?: GenericTool[]; - maxContextTokens?: number; - /** Pre-ratio context budget from initializeAgent. */ - baseContextTokens?: number; - useLegacyContent?: boolean; - imageDetail?: ImageDetail; - toolContextMap?: Record; - dynamicToolContextMap?: Record; - toolRegistry?: LCToolRegistry; - /** Serializable tool definitions for event-driven execution */ - toolDefinitions?: LCTool[]; - /** Precomputed flag indicating if any tools have defer_loading enabled */ - hasDeferredTools?: boolean; - /** Both-direction identity aliases for MCP tools whose key spelling changed */ - mcpToolAliases?: MCPToolAlias[]; - /** Names of tools injected with the `run_in_background` param (excluded from eager execution). */ - backgroundToolNames?: string[]; - /** Names of tools with the host-injected `intent` param (stripped from self-spawn inputs). */ - intentToolNames?: string[]; - /** Marker-verified tool names whose intent labels are safe compaction guidance. */ - semanticIntentToolNames?: string[]; - /** - * Per-agent codeenv gate set by `initializeAgent`: admin-level - * `execute_code` capability AND the agent actually requested - * `execute_code` in its tools. Used here to enable - * `RunConfig.toolOutputReferences` only on runs where the bash tool - * is actually registered. - */ - codeEnvAvailable?: boolean; - /** - * Per-agent stateful-session gate set by `initializeAgent`: the admin - * `stateful_code_sessions` capability AND the agent's builder opt-in AND - * `codeEnvAvailable`. Carried into per-agent tool loading and prewarming. - */ - statefulCodeSessions?: boolean; - /** Per-agent stateful workspace sharing scope. */ - statefulCodeEnvironment?: Agent['stateful_code_environment']; - /** Trusted partition for transient code session ids and file references. */ - codeSessionKey?: string; - /** Trusted Code API route selected during initialization. */ - codeExecutionContext?: CodeExecutionContext; - /** Whether this initialized agent can route skills/ writes to persistent skill storage. */ - skillAuthoringAvailable?: boolean; - /** Optional per-agent summarization overrides */ - summarization?: SummarizationConfig; - /** Response field to read model reasoning from for custom OpenAI-compatible endpoints. */ - reasoningKey?: ReasoningResponseKey; - /** Whether to reconstruct `reasoning_content` from persisted history across turns. */ - includeReasoningHistory?: boolean; - /** - * Maximum characters allowed in a single tool result before truncation. - * Overrides the default computed from maxContextTokens. - */ - maxToolResultChars?: number; - /** Initialized subagent configs (loaded by initialize.js from agent.subagents.agent_ids). */ - subagentAgentConfigs?: RunAgent[]; - /** - * Inert, VIEW-checked descriptors for explicit children that are initialized - * only after the SDK selects them. These resolvers are request-scoped: they - * may use the active request's authorization and tool-loading context. - */ - lazySubagentConfigs?: LazySubagentAgent[]; - /** All-or-nothing saved-agent teams resolved by initialize.js. */ - subagentGraphConfigs?: Array<{ - definition: AgentSubagentGraph; - memberConfigs: RunAgent[]; - }>; - /** Member-scoped always-apply skills resolved during agent initialization. */ - alwaysApplySkillPrimes?: ResolvedAlwaysApplySkill[]; - /** Source subagent spawning configuration (enabled / allowSelf / agent_ids). */ - subagents?: AgentSubagentsConfig; -}; +type RunAgent = Omit & + Pick & { + provisionState?: ProvisionState; + fileConsumers?: TurnFileConsumers; + azureOptions?: t.AzureOptions; + tools?: GenericTool[]; + maxContextTokens?: number; + /** Pre-ratio context budget from initializeAgent. */ + baseContextTokens?: number; + useLegacyContent?: boolean; + imageDetail?: ImageDetail; + toolContextMap?: Record; + dynamicToolContextMap?: Record; + toolRegistry?: LCToolRegistry; + /** Serializable tool definitions for event-driven execution */ + toolDefinitions?: LCTool[]; + /** Precomputed flag indicating if any tools have defer_loading enabled */ + hasDeferredTools?: boolean; + /** Both-direction identity aliases for MCP tools whose key spelling changed */ + mcpToolAliases?: MCPToolAlias[]; + /** Names of tools injected with the `run_in_background` param (excluded from eager execution). */ + backgroundToolNames?: string[]; + /** Names of tools with the host-injected `intent` param (stripped from self-spawn inputs). */ + intentToolNames?: string[]; + /** Marker-verified tool names whose intent labels are safe compaction guidance. */ + semanticIntentToolNames?: string[]; + /** + * Per-agent codeenv gate set by `initializeAgent`: admin-level + * `execute_code` capability AND the agent actually requested + * `execute_code` in its tools. Used here to enable + * `RunConfig.toolOutputReferences` only on runs where the bash tool + * is actually registered. + */ + codeEnvAvailable?: boolean; + /** + * Per-agent stateful-session gate set by `initializeAgent`: the admin + * `stateful_code_sessions` capability AND the agent's builder opt-in AND + * `codeEnvAvailable`. Carried into per-agent tool loading and prewarming. + */ + statefulCodeSessions?: boolean; + /** Per-agent stateful workspace sharing scope. */ + statefulCodeEnvironment?: Agent['stateful_code_environment']; + /** Trusted partition for transient code session ids and file references. */ + codeSessionKey?: string; + /** Trusted Code API route selected during initialization. */ + codeExecutionContext?: CodeExecutionContext; + /** Whether this initialized agent can route skills/ writes to persistent skill storage. */ + skillAuthoringAvailable?: boolean; + /** Optional per-agent summarization overrides */ + summarization?: SummarizationConfig; + /** Response field to read model reasoning from for custom OpenAI-compatible endpoints. */ + reasoningKey?: ReasoningResponseKey; + /** Whether to reconstruct `reasoning_content` from persisted history across turns. */ + includeReasoningHistory?: boolean; + /** + * Maximum characters allowed in a single tool result before truncation. + * Overrides the default computed from maxContextTokens. + */ + maxToolResultChars?: number; + /** Initialized subagent configs (loaded by initialize.js from agent.subagents.agent_ids). */ + subagentAgentConfigs?: RunAgent[]; + /** + * Inert, VIEW-checked descriptors for explicit children that are initialized + * only after the SDK selects them. These resolvers are request-scoped: they + * may use the active request's authorization and tool-loading context. + */ + lazySubagentConfigs?: LazySubagentAgent[]; + /** All-or-nothing saved-agent teams resolved by initialize.js. */ + subagentGraphConfigs?: Array<{ + definition: AgentSubagentGraph; + memberConfigs: RunAgent[]; + }>; + /** Member-scoped always-apply skills resolved during agent initialization. */ + alwaysApplySkillPrimes?: ResolvedAlwaysApplySkill[]; + /** Source subagent spawning configuration (enabled / allowSelf / agent_ids). */ + subagents?: AgentSubagentsConfig; + }; type LazySubagentAgent = Pick< RunAgent, @@ -2390,8 +2392,13 @@ export async function createRun({ visitedCodeFileAgents.add(agent.id); enqueueSubagentChildren(agent, pendingCodeFileAgents, visitedCodeFileAgents, false, false); } + /** Only the run's own agents receive the request; a subagent child receives its task. */ + const requestAgents = new Set(agents); for (const agent of codeFileAgents.values()) { - prepareQueuedCodeFileContext(agent, codeFileAgents.values(), user?.id); + prepareAgentFileContext(agent, codeFileAgents.values(), user?.id, false, { + filters: appConfig?.filters, + receivesRequest: requestAgents.has(agent), + }); } const preparedCodeFileAgents = new WeakSet(codeFileAgents.values()); @@ -2399,7 +2406,10 @@ export async function createRun({ if (!preparedCodeFileAgents.has(agent)) { if (agent.provisionState) agent.provisionState.codeEnvDestinations = undefined; codeFileAgents.set(agent.id, agent); - prepareQueuedCodeFileContext(agent, codeFileAgents.values(), user?.id, true); + prepareAgentFileContext(agent, codeFileAgents.values(), user?.id, true, { + filters: appConfig?.filters, + receivesRequest: requestAgents.has(agent), + }); preparedCodeFileAgents.add(agent); } const isSubagent = opts.isSubagent === true; diff --git a/packages/api/src/agents/steering/__tests__/media.spec.ts b/packages/api/src/agents/steering/__tests__/media.spec.ts index 460f162aa11..7c8f455e92b 100644 --- a/packages/api/src/agents/steering/__tests__/media.spec.ts +++ b/packages/api/src/agents/steering/__tests__/media.spec.ts @@ -1,8 +1,14 @@ -import type { IMongoFile } from '@librechat/data-schemas'; +import { FileContext, resolveTurnLLMDeliveryPath } from 'librechat-data-provider'; +import type { AppConfig, IMongoFile } from '@librechat/data-schemas'; +import type { SteerMediaClient, SteerReadingAgent } from '../media'; +import type { FileTextDeriver } from '~/files/reading'; import type { SteerFileFetcher } from '../request'; -import type { SteerMediaClient } from '../media'; +import type { ServerRequest } from '~/types'; import { buildSteerMedia, collectSteerStampTargets, stampSteerPartMedia } from '../media'; +import { applyTurnDelivery, resolveTurnDeliveryRouting } from '../../files/delivery'; +import { buildTurnReadingContext, prepareTurnFiles } from '~/files/reading'; import { AttachmentObjectNotFoundError } from '~/files/encode/utils'; +import { extractFileContext } from '~/files/context'; jest.spyOn(console, 'log').mockImplementation(); @@ -37,6 +43,8 @@ function createClient({ } const user = { id: 'user-1' }; +/** A stored record carrying only the fields a test reads. */ +const asStoredFile = (fields: Partial): IMongoFile => fields as IMongoFile; const imagePart = { type: 'image_url', image_url: { url: 'data:image/png;base64,abc', detail: 'auto' }, @@ -169,6 +177,33 @@ describe('buildSteerMedia', () => { }); }); + it('checks and encodes the copies a host that derives text prepares', async () => { + const storedXlsx = { file_id: 'xlsx', type: 'text/csv', llmDeliveryPath: 'none' }; + const preparedXlsx = { ...storedXlsx, llmDeliveryPath: 'text', text: 'derived' }; + const getFiles: SteerFileFetcher = jest.fn(async () => [asStoredFile(storedXlsx)]); + const client = createClient(); + const prepareTurnAttachments = jest.fn(async () => [asStoredFile(preparedXlsx)]); + const assertFilesAllowed = jest.fn(); + + await buildSteerMedia({ + client: { ...client, prepareTurnAttachments }, + user, + item: steerItem([{ file_id: 'xlsx' }]), + getFiles, + assertFilesAllowed, + }); + + const consumers = { executeCode: false, fileSearch: false }; + expect(prepareTurnAttachments).toHaveBeenCalledWith([storedXlsx], consumers); + expect(client.resolveTurnAttachments).not.toHaveBeenCalled(); + expect(assertFilesAllowed).toHaveBeenCalledWith([preparedXlsx]); + expect(client.processAttachments).toHaveBeenCalledWith( + expect.anything(), + [preparedXlsx], + consumers, + ); + }); + it('prepends extracted file context to the steer text', async () => { const getFiles: SteerFileFetcher = jest.fn(async () => [ { file_id: 'f2', type: 'text/plain' } as unknown as IMongoFile, @@ -518,3 +553,231 @@ describe('stampSteerPartMedia', () => { expect((message.content as Array>)[1]).toBe(filesOnlyPart); }); }); + +describe('buildSteerMedia file reading', () => { + const steerText = 'use the sheet'; + const holdNote = + '"quarterly.xlsx" is attached; Run Code can open it starting with your next message.'; + /** A workbook extracted at upload, so classic routing reads it as text. */ + const workbook = { + file_id: 'xlsx', + filename: 'quarterly.xlsx', + type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + source: 'local', + context: FileContext.message_attachment, + bytes: 2048, + text: 'SENTINEL-7F3A', + llmDeliveryPath: 'text', + metadata: { destinationChosen: false }, + } as Partial as IMongoFile; + const routingFor = (fileConfig: AppConfig['fileConfig']) => + resolveTurnDeliveryRouting({ agent: { provider: 'openAI' }, config: { fileConfig } }); + const automatic = routingFor({ endpoints: { openAI: { llmDeliveryPolicy: 'automatic' } } }); + const classic = routingFor({}); + + /** + * The AgentClient surface over the real turn routing and text extraction: like BaseClient, it + * applies the running agent's routing and exposes the agent only through `options`. + */ + function createRoutedClient( + agent: SteerReadingAgent | undefined, + ): SteerMediaClient & { processAttachments: jest.Mock } { + const req = { body: {}, config: {} } as ServerRequest; + const routing = agent?.deliveryRouting; + return { + options: { agent }, + resolveTurnAttachments: (files, consumers = agent?.fileConsumers) => + applyTurnDelivery(files, { routing, consumers }), + addFileContextToMessage: async (message, files, consumers = agent?.fileConsumers) => { + const fileContext = await extractFileContext({ + attachments: files.filter((file) => { + const path = resolveTurnLLMDeliveryPath(routing, file, consumers); + return path == null || path === 'text'; + }), + req, + tokenCountFn: (text) => text.length, + }); + if (fileContext) { + message.fileContext = fileContext; + } + }, + processAttachments: jest.fn( + async (_message: Record, files: IMongoFile[]) => files, + ), + }; + } + + const steer = (client: SteerMediaClient) => + buildSteerMedia({ + client, + user, + item: steerItem([{ file_id: 'xlsx' }], steerText), + getFiles: jest.fn(async () => [workbook]), + }); + const textOf = (result: Awaited>) => + (result?.content ?? []).map((part) => part.text).join('\n'); + + it('holds a spreadsheet for Run Code under the automatic policy and says so', async () => { + /* The run's code queue cannot take a mid-run file, so the workbook waits for the next + * message's provisioning rather than reaching the prompt as text. */ + const client = createRoutedClient({ + deliveryRouting: automatic, + fileConsumers: { executeCode: true, fileSearch: true }, + }); + + const result = await steer(client); + + expect(result?.content).toEqual([{ type: 'text', text: `${steerText}\n\n${holdNote}` }]); + expect(textOf(result)).not.toContain('SENTINEL-7F3A'); + expect(client.processAttachments).toHaveBeenCalledWith( + expect.anything(), + [{ ...workbook, llmDeliveryPath: 'none' }], + { executeCode: true, fileSearch: false }, + ); + }); + + it('notes a newly held Run Code file after native validation rejects it', async () => { + const deliveryRouting = routingFor({ + endpoints: { openAI: { llmDeliveryPolicy: 'automatic' } }, + }); + const reading = buildTurnReadingContext({ + routing: deliveryRouting, + provider: 'openAI', + fileTokenLimit: 100_000, + configuredFileSizeLimit: undefined, + countTokens: (text) => text.length, + }); + deliveryRouting.reading = reading; + const client = createRoutedClient({ + deliveryRouting, + fileConsumers: { executeCode: true, fileSearch: true }, + }); + const file = asStoredFile({ + ...workbook, + file_id: 'steered-pdf', + filename: 'steered.pdf', + type: 'application/pdf', + llmDeliveryPath: 'provider', + }); + client.processMessageAttachments = async (_message, files, consumers) => { + expect(files[0].llmDeliveryPath).toBe('provider'); + reading?.recordRejections([{ file_id: file.file_id, reason: 'capacity' }]); + return applyTurnDelivery(files, { routing: deliveryRouting, consumers }); + }; + + const result = await buildSteerMedia({ + client, + user, + item: steerItem([{ file_id: file.file_id }], steerText), + getFiles: jest.fn(async () => [file]), + }); + + expect(result?.content).toEqual([ + { + type: 'text', + text: `${steerText}\n\n"steered.pdf" is attached; Run Code can open it starting with your next message.`, + }, + ]); + expect(textOf(result)).not.toContain('SENTINEL-7F3A'); + }); + + it('delivers the text of a spreadsheet when the automatic run has no Run Code', async () => { + const client = createRoutedClient({ + deliveryRouting: automatic, + fileConsumers: { executeCode: false, fileSearch: true }, + }); + + const result = await steer(client); + + expect(textOf(result)).toContain('SENTINEL-7F3A'); + expect(textOf(result)).not.toContain(holdNote); + }); + + describe('a workbook whose upload deferred extraction', () => { + const deferred = asStoredFile({ + ...workbook, + text: undefined, + llmDeliveryPath: 'none', + metadata: { destinationChosen: false, textDerivation: { outcome: 'deferred' } }, + }); + const derivedSentinel = 'DERIVED-7F3A'; + + /** An automatic run without Run Code whose reading derives text through the given stub. */ + function deferredRun(deriveText: FileTextDeriver) { + const deliveryRouting = routingFor({ + endpoints: { openAI: { llmDeliveryPolicy: 'automatic' } }, + }); + const context = buildTurnReadingContext({ + routing: deliveryRouting, + provider: 'openAI', + fileTokenLimit: 100_000, + configuredFileSizeLimit: undefined, + countTokens: (text) => text.length, + deriveText, + }); + deliveryRouting.reading = context; + return createRoutedClient({ + deliveryRouting, + fileConsumers: { executeCode: false, fileSearch: true }, + }); + } + const steerDeferred = (client: SteerMediaClient) => + buildSteerMedia({ + client, + user, + item: steerItem([{ file_id: 'xlsx' }], steerText), + getFiles: jest.fn(async () => [deferred]), + }); + const deriver = () => + jest.fn, Parameters>(async () => ({ + status: 'derived', + text: derivedSentinel, + textDerivation: { outcome: 'complete', extractor: 'document_parser', at: 1 }, + })); + + it('steers with the text the host derives for it', async () => { + const deriveText = deriver(); + const client = deferredRun(deriveText); + const routing = client.options?.agent?.deliveryRouting; + + const result = await steerDeferred({ + ...client, + prepareTurnAttachments: (files, consumers) => + prepareTurnFiles({ routing, files, consumers }), + }); + + expect(deriveText).toHaveBeenCalledTimes(1); + expect(textOf(result)).toContain(derivedSentinel); + expect(textOf(result)).not.toContain(holdNote); + expect(deferred.text).toBeUndefined(); + }); + + it('leaves it out of the steer when the host cannot prepare it', async () => { + const deriveText = deriver(); + + const result = await steerDeferred(deferredRun(deriveText)); + + expect(deriveText).not.toHaveBeenCalled(); + expect(textOf(result)).not.toContain(derivedSentinel); + expect(result?.content).toEqual([{ type: 'text', text: steerText }]); + }); + }); + + it('steers exactly as before under classic routing, even with Run Code loaded', async () => { + const withCode = createRoutedClient({ + deliveryRouting: classic, + fileConsumers: { executeCode: true, fileSearch: false }, + }); + const withoutAgent = createRoutedClient(undefined); + + const [coded, bare] = await Promise.all([steer(withCode), steer(withoutAgent)]); + + expect(coded).toEqual(bare); + expect(textOf(coded)).toContain('SENTINEL-7F3A'); + expect(textOf(coded)).not.toContain(holdNote); + expect(withCode.processAttachments).toHaveBeenCalledWith(expect.anything(), [workbook], { + executeCode: false, + fileSearch: false, + }); + }); +}); diff --git a/packages/api/src/agents/steering/media.ts b/packages/api/src/agents/steering/media.ts index 977549a59f8..a4415fcbb27 100644 --- a/packages/api/src/agents/steering/media.ts +++ b/packages/api/src/agents/steering/media.ts @@ -1,9 +1,10 @@ import { logger } from '@librechat/data-schemas'; import { formatMessage } from '@librechat/agents'; -import { ContentTypes } from 'librechat-data-provider'; +import { ContentTypes, decideFileReading, resolveLLMDeliveryPolicy } from 'librechat-data-provider'; import type { TFile, TurnFileConsumers } from 'librechat-data-provider'; import type { IMongoFile } from '@librechat/data-schemas'; import type { SteerQueueItem } from '~/stream/interfaces/IJobStore'; +import type { TurnReadingAgent } from '~/files/reading'; import type { SteerFileFetcher } from './request'; import type { SteerMediaResult } from './runtime'; import type { SteerRequestUser } from './refs'; @@ -16,6 +17,14 @@ import { prependFileContext } from '../client'; export interface SteerMediaClient { /** The turn's view of stored records, which every check and encode below must share. */ resolveTurnAttachments(files: IMongoFile[], consumers?: TurnFileConsumers): IMongoFile[]; + /** + * The same view after deriving the text its reading needs, when the host can derive it; the + * steer then checks and encodes the copies carrying that text. + */ + prepareTurnAttachments?( + files: IMongoFile[], + consumers?: TurnFileConsumers, + ): Promise; addFileContextToMessage( message: Record, files: IMongoFile[], @@ -26,8 +35,22 @@ export interface SteerMediaClient { files: IMongoFile[], consumers?: TurnFileConsumers, ): Promise; + processMessageAttachments?( + message: Record, + files: IMongoFile[], + consumers?: TurnFileConsumers, + ): Promise; + /** + * The host's running agent. `resolveTurnAttachments` applies its `deliveryRouting`, so the + * steer judges its files by the same routing, with Run Code as a reader exactly when the + * agent's final `fileConsumers` loaded it. + */ + readonly options?: { readonly agent?: SteerReadingAgent | null }; } +/** The fields of the running agent a steer reads its files by. */ +export type SteerReadingAgent = Pick; + interface PseudoMessage { messageId: string; fileContext?: string; @@ -67,6 +90,44 @@ function mergeSteerModelText(text: string, quotes?: string[] | null): string { return normalized != null ? mergeQuotedText(text, normalized) : text; } +const codeHoldNote = (filename: string): string => + `"${filename}" is attached; Run Code can open it starting with your next message.`; + +function appendCodeHoldNotes(text: string, heldFilenames: readonly string[]): string { + if (heldFilenames.length === 0) { + return text; + } + const notes = heldFilenames.map(codeHoldNote).join('\n'); + return text.length > 0 ? `${text}\n\n${notes}` : notes; +} + +/** + * How a mid-run steer reads its files. Its files arrived after resource priming, so no loaded + * tool can reach them before a new turn provisions them. Classic routing therefore judges them + * with no reader and delivers the opted-in text fallback; the automatic policy keeps a file Run + * Code reads for the next message and says so in the steer text. + */ +function resolveSteerReading( + client: SteerMediaClient, + docs: IMongoFile[], +): { fileConsumers: TurnFileConsumers; heldFilenames: string[] } { + const agent = client.options?.agent; + const routing = agent?.deliveryRouting; + if (resolveLLMDeliveryPolicy(routing?.endpointConfig) !== 'automatic') { + return { fileConsumers: { executeCode: false, fileSearch: false }, heldFilenames: [] }; + } + const fileConsumers: TurnFileConsumers = { + executeCode: agent?.fileConsumers?.executeCode === true, + fileSearch: false, + }; + const heldFilenames = docs.flatMap((file) => + decideFileReading({ routing, file, consumers: fileConsumers }).reader === 'code' + ? [file.filename] + : [], + ); + return { fileConsumers, heldFilenames }; +} + /** * Encodes authorized file docs for one steer and assembles the multimodal * content array, reusing the exact pipeline regular user turns go through: @@ -83,6 +144,7 @@ async function encodeSteerContent({ steerId, fileDocs, fileConsumers, + heldFilenames = [], }: { client: SteerMediaClient; text: string; @@ -90,11 +152,21 @@ async function encodeSteerContent({ steerId: string; fileDocs: IMongoFile[]; fileConsumers?: TurnFileConsumers; + /** Files kept for Run Code on the next message, each noted after the steer body. */ + heldFilenames?: readonly string[]; }): Promise { - const modelText = mergeSteerModelText(text, quotes); const pseudo: PseudoMessage = { messageId: `steer:${steerId}` }; - await client.addFileContextToMessage(pseudo, fileDocs, fileConsumers); - const validated = await client.processAttachments(pseudo, fileDocs, fileConsumers); + let validated: IMongoFile[] | undefined; + if (client.processMessageAttachments != null) { + validated = await client.processMessageAttachments(pseudo, fileDocs, fileConsumers); + } else { + await client.addFileContextToMessage(pseudo, fileDocs, fileConsumers); + validated = await client.processAttachments(pseudo, fileDocs, fileConsumers); + } + const settledFiles = validated ?? fileDocs; + const finalHeldFilenames = + fileConsumers == null ? heldFilenames : resolveSteerReading(client, settledFiles).heldFilenames; + const modelText = appendCodeHoldNotes(mergeSteerModelText(text, quotes), finalHeldFilenames); const formatted = formatMessage({ message: { role: 'user', @@ -148,13 +220,10 @@ export async function buildSteerMedia({ return undefined; } const docsById = new Map(rawDocs.map((file) => [file.file_id, file])); - /* These files arrived after resource priming. A loaded tool is not evidence that - * it can access them; use the opted-in text fallback until a new turn provisions them. */ - const fileConsumers: TurnFileConsumers = { executeCode: false, fileSearch: false }; - const fileDocs = client.resolveTurnAttachments( - ids.map((id) => docsById.get(id)).filter((doc): doc is IMongoFile => doc != null), - fileConsumers, - ); + const docs = ids.map((id) => docsById.get(id)).filter((doc): doc is IMongoFile => doc != null); + const { fileConsumers, heldFilenames } = resolveSteerReading(client, docs); + const fileDocs = await (client.prepareTurnAttachments?.(docs, fileConsumers) ?? + client.resolveTurnAttachments(docs, fileConsumers)); assertFilesAllowed?.(fileDocs); return encodeSteerContent({ client, @@ -163,6 +232,7 @@ export async function buildSteerMedia({ steerId: item.steerId, fileDocs, fileConsumers, + heldFilenames, }); } diff --git a/packages/api/src/files/code/priming.ts b/packages/api/src/files/code/priming.ts index 0dba3389f0f..7eecaa9d056 100644 --- a/packages/api/src/files/code/priming.ts +++ b/packages/api/src/files/code/priming.ts @@ -208,7 +208,8 @@ export type CodeFileLocation = 'sandbox' | 'programmatic'; type CodeContextFile = Pick; -const CODE_FILE_CONTEXT_ROOTS: Record = { +/** The directory primed code files mount under on each execution route. */ +export const CODE_FILE_CONTEXT_ROOTS: Readonly> = { sandbox: '/mnt/data', programmatic: '$LIBRECHAT_CODE_DATA_DIR', }; diff --git a/packages/api/src/files/context.spec.ts b/packages/api/src/files/context.spec.ts index 56d09ff5f44..2927c107db7 100644 --- a/packages/api/src/files/context.spec.ts +++ b/packages/api/src/files/context.spec.ts @@ -1,5 +1,7 @@ +import { FileSources, mergeFileConfig } from 'librechat-data-provider'; import type { TFile } from 'librechat-data-provider'; -import { getAttachmentTitleText } from './context'; +import type { ServerRequest } from '~/types'; +import { extractFileContext, getAttachmentTitleText, resolveFileTokenLimit } from './context'; const file = (filename?: string): TFile => ({ filename }) as TFile; @@ -28,3 +30,131 @@ describe('getAttachmentTitleText', () => { expect(getAttachmentTitleText([file(), file()])).toBe(''); }); }); + +type TextAttachment = { + file_id: string; + filename: string; + text: string; + source: string; + llmDeliveryPath: string; +}; + +const textFile = (file_id: string, text: string): TextAttachment => ({ + file_id, + filename: `${file_id}.txt`, + text, + source: FileSources.local, + llmDeliveryPath: 'text', +}); + +const requestWithLimit = (fileTokenLimit?: number, configLimit?: number): ServerRequest => + ({ + body: { fileTokenLimit }, + config: { fileConfig: { fileTokenLimit: configLimit } }, + }) as ServerRequest; + +describe('resolveFileTokenLimit', () => { + it('prefers the request override over the configured limit', () => { + expect(resolveFileTokenLimit(requestWithLimit(25, 50))).toBe(25); + }); + + it('falls back to the configured limit', () => { + expect(resolveFileTokenLimit(requestWithLimit(undefined, 50))).toBe(50); + }); + + it('falls back to the default limit without a request', () => { + expect(resolveFileTokenLimit()).toBe(mergeFileConfig(undefined).fileTokenLimit); + }); +}); + +describe('extractFileContext options', () => { + const tokenLimit = 10; + + it.each<{ + name: string; + text: string; + known: number | undefined; + counted: 'never' | 'only for the truncation' | 'in full'; + rendered: string; + }>([ + { + name: 'reuses a known token count instead of counting the text again', + text: 'fits', + known: 4, + counted: 'never', + rendered: '# "note.txt"\nfits\n', + }, + { + name: 'never counts the full text of a truncated file whose count is known', + text: 'x'.repeat(50), + known: 50, + counted: 'only for the truncation', + rendered: '# "note.txt"\nx', + }, + { + name: 'counts the text when the known count is unavailable', + text: 'fits', + known: undefined, + counted: 'in full', + rendered: '# "note.txt"\nfits\n', + }, + ])('$name', async ({ text, known, counted, rendered }) => { + const tokenCountFn = jest.fn((value: string): number => value.length); + const attachment = textFile('note', text); + const knownTokenCount = jest.fn((_file: TextAttachment): number | undefined => known); + + const result = await extractFileContext({ + attachments: [attachment], + req: requestWithLimit(tokenLimit), + tokenCountFn, + knownTokenCount, + }); + + expect(result).toContain(rendered); + expect(knownTokenCount).toHaveBeenCalledWith(attachment); + if (counted === 'never') { + expect(tokenCountFn).not.toHaveBeenCalled(); + } else if (counted === 'only for the truncation') { + expect(tokenCountFn).toHaveBeenCalled(); + expect(tokenCountFn).not.toHaveBeenCalledWith(text); + } else { + expect(tokenCountFn).toHaveBeenCalledWith(text); + } + }); + + it('follows truncated text with a notice only when asked to', async () => { + const tokenCountFn = (text: string): number => text.length; + const attachments = [textFile('long', 'x'.repeat(50)), textFile('short', 'fits')]; + const notice = '[Truncated: only the beginning of "long.txt" fits; the rest is omitted.]'; + + const marked = await extractFileContext({ + attachments, + req: requestWithLimit(tokenLimit), + tokenCountFn, + markTruncation: true, + }); + const unmarked = await extractFileContext({ + attachments, + req: requestWithLimit(tokenLimit), + tokenCountFn, + }); + + expect(marked).toMatch(/# "long\.txt"\nx+\n\[Truncated: [^\n]+\]\n/); + expect(marked).toContain(notice); + expect(marked?.split('[Truncated:')).toHaveLength(2); + expect(unmarked).not.toContain('[Truncated:'); + expect(marked?.replace(`\n${notice}`, '')).toBe(unmarked); + }); + + it('keeps the classic output byte-for-byte without options', async () => { + const result = await extractFileContext({ + attachments: [textFile('alpha', 'first'), textFile('beta', 'y'.repeat(20))], + req: requestWithLimit(tokenLimit), + tokenCountFn: (text: string): number => text.length, + }); + + expect(result).toBe( + 'Attached document(s):\n```md# "alpha.txt"\nfirst\n\n\n---\n\n# "beta.txt"\nyyyyyyyyy\n\n```', + ); + }); +}); diff --git a/packages/api/src/files/context.ts b/packages/api/src/files/context.ts index 211f5a07834..389d8aad745 100644 --- a/packages/api/src/files/context.ts +++ b/packages/api/src/files/context.ts @@ -29,6 +29,20 @@ export function getAttachmentTitleText(files?: TFile[] | null): string { return filenames.length > 0 ? `Attached file(s): ${filenames.join(', ')}` : ''; } +/** The per-file token limit for attachment text: the request's override, else the merged config's. */ +export function resolveFileTokenLimit(req?: ServerRequest): number | undefined { + return req?.body?.fileTokenLimit ?? mergeFileConfig(req?.config?.fileConfig).fileTokenLimit; +} + +type FileContextAttachment = Pick & { + file_id?: string; + source?: string; + llmDeliveryPath?: string; +}; + +const truncationNotice = (filename: string): string => + `[Truncated: only the beginning of "${filename}" fits; the rest is omitted.]`; + /** * Extracts text context from attachments and returns formatted text. * This handles text that was already extracted from files (OCR, transcriptions, document text, etc.) @@ -36,26 +50,28 @@ export function getAttachmentTitleText(files?: TFile[] | null): string { * @param params.attachments - Array of file attachments * @param params.req - Express request object for config access * @param params.tokenCountFn - Function to count tokens in text + * @param params.knownTokenCount - A file's already-measured text token count, so it is not recounted + * @param params.markTruncation - Follow truncated text with a line saying the rest was omitted * @returns The formatted file context text, or undefined if no text found */ -export async function extractFileContext({ +export async function extractFileContext({ attachments, req, tokenCountFn, + knownTokenCount, + markTruncation = false, }: { - attachments: readonly (Pick & { - source?: string; - llmDeliveryPath?: string; - })[]; + attachments: readonly T[]; req?: ServerRequest; tokenCountFn: TokenCountFn; + knownTokenCount?: (file: T) => number | undefined; + markTruncation?: boolean; }): Promise { if (!attachments || attachments.length === 0) { return undefined; } - const fileConfig = mergeFileConfig(req?.config?.fileConfig); - const fileTokenLimit = req?.body?.fileTokenLimit ?? fileConfig.fileTokenLimit; + const fileTokenLimit = resolveFileTokenLimit(req); if (!fileTokenLimit) { // If no token limit, return undefined (no processing) @@ -71,21 +87,23 @@ export async function extractFileContext({ } const hasTextDelivery = file.llmDeliveryPath === 'text' || source === FileSources.text; - if (hasTextDelivery && file.text) { - const { text: limitedText, wasTruncated } = await processTextWithTokenLimit({ - text: file.text, - tokenLimit: fileTokenLimit, - tokenCountFn, - }); - - if (wasTruncated) { - logger.debug( - `[extractFileContext] Text content truncated for file: ${file.filename} due to token limits`, - ); - } - - resultText += `${!resultText ? 'Attached document(s):\n```md' : '\n\n---\n\n'}# "${file.filename}"\n${limitedText}\n`; + if (!hasTextDelivery || !file.text) { + continue; } + + const { text: limitedText, wasTruncated } = await processTextWithTokenLimit({ + text: file.text, + tokenLimit: fileTokenLimit, + tokenCountFn, + knownTokenCount: knownTokenCount?.(file), + }); + + if (wasTruncated) { + logger.debug(`[extractFileContext] text truncated file_id=${file.file_id ?? 'unknown'}`); + } + + const notice = markTruncation && wasTruncated ? `\n${truncationNotice(file.filename)}` : ''; + resultText += `${!resultText ? 'Attached document(s):\n```md' : '\n\n---\n\n'}# "${file.filename}"\n${limitedText}${notice}\n`; } if (resultText) { diff --git a/packages/api/src/files/documents/crud.ts b/packages/api/src/files/documents/crud.ts index 4457a30b3d6..c2721b8adf0 100644 --- a/packages/api/src/files/documents/crud.ts +++ b/packages/api/src/files/documents/crud.ts @@ -7,7 +7,8 @@ import { assertSafeZipSize } from './zipSafety'; type FileParseFn = (file: Express.Multer.File) => Promise; -const DOCUMENT_PARSER_MAX_FILE_SIZE = 15 * megabyte; +/** The largest file the built-in document parser accepts, checked before it reads anything. */ +export const DOCUMENT_PARSER_MAX_FILE_SIZE: number = 15 * megabyte; const ODT_MAX_DECOMPRESSED_SIZE = 50 * megabyte; /** diff --git a/packages/api/src/files/encode/document.spec.ts b/packages/api/src/files/encode/document.spec.ts index f58d591ed67..fe234cdc312 100644 --- a/packages/api/src/files/encode/document.spec.ts +++ b/packages/api/src/files/encode/document.spec.ts @@ -1,59 +1,47 @@ import { Providers } from '@librechat/agents'; -import { mbToBytes } from 'librechat-data-provider'; +import { mbToBytes, fileConfig as baseFileConfig } from 'librechat-data-provider'; import type { AppConfig, IMongoFile } from '@librechat/data-schemas'; -import type { ServerRequest } from '~/types'; +import type { + NativeValidationMode, + DocumentRejection, + DocumentResult, + ServerRequest, +} from '~/types'; import { encodeAndFormatDocuments } from './document'; /** Mock the validation module */ jest.mock('~/files/validation', () => ({ + ...jest.requireActual('~/files/validation'), validatePdf: jest.fn(), validateBedrockDocument: jest.fn(), })); -/** Mock the utils module */ +/** Mock storage reads; the file size limit resolves through the real fileConfig merge */ jest.mock('./utils', () => ({ + ...jest.requireActual('./utils'), getFileStream: jest.fn(), - getConfiguredFileSizeLimit: jest.fn(), - isAttachmentObjectNotFoundError: jest.fn(() => false), })); import { validatePdf, validateBedrockDocument } from '~/files/validation'; -import { getFileStream, getConfiguredFileSizeLimit } from './utils'; +import { getFileStream, AttachmentObjectNotFoundError } from './utils'; import { Types } from 'mongoose'; +const realValidation = + jest.requireActual('~/files/validation'); + const mockedValidatePdf = validatePdf as jest.MockedFunction; const mockedValidateBedrockDocument = validateBedrockDocument as jest.MockedFunction< typeof validateBedrockDocument >; const mockedGetFileStream = getFileStream as jest.MockedFunction; -const mockedGetConfiguredFileSizeLimit = getConfiguredFileSizeLimit as jest.MockedFunction< - typeof getConfiguredFileSizeLimit ->; describe('encodeAndFormatDocuments - fileConfig integration', () => { const mockStrategyFunctions = jest.fn(); + const inheritedLimit = baseFileConfig.endpoints.default.fileSizeLimit; + beforeEach(() => { - jest.clearAllMocks(); - /** Default mock implementation for getConfiguredFileSizeLimit */ - mockedGetConfiguredFileSizeLimit.mockImplementation((req, params) => { - if (!req.config?.fileConfig) { - return undefined; - } - const { provider, endpoint } = params; - const lookupKey = endpoint ?? provider; - const fileConfig = req.config.fileConfig; - const endpoints = fileConfig.endpoints; - if (endpoints?.[lookupKey]) { - const limit = endpoints[lookupKey].fileSizeLimit; - return limit !== undefined ? mbToBytes(limit) : undefined; - } - if (endpoints?.default) { - const limit = endpoints.default.fileSizeLimit; - return limit !== undefined ? mbToBytes(limit) : undefined; - } - return undefined; - }); + jest.resetAllMocks(); }); /** Helper to create a mock request with file config */ @@ -168,7 +156,7 @@ describe('encodeAndFormatDocuments - fileConfig integration', () => { ); }); - it('should pass undefined when fileConfig.endpoints is not defined', async () => { + it('should pass the inherited default limit for an empty fileConfig', async () => { const req = { config: { fileConfig: {}, @@ -176,13 +164,11 @@ describe('encodeAndFormatDocuments - fileConfig integration', () => { } as ServerRequest; const file = createMockFile(10); - const mockContent = Buffer.from('test-pdf-content').toString('base64'); mockedGetFileStream.mockResolvedValue({ file, - content: mockContent, + content: Buffer.from('test-pdf-content').toString('base64'), metadata: file, }); - mockedValidatePdf.mockResolvedValue({ isValid: true }); await encodeAndFormatDocuments( @@ -192,12 +178,11 @@ describe('encodeAndFormatDocuments - fileConfig integration', () => { mockStrategyFunctions, ); - /** When fileConfig has no endpoints, getConfiguredFileSizeLimit returns undefined */ expect(mockedValidatePdf).toHaveBeenCalledWith( expect.any(Buffer), expect.any(Number), Providers.OPENAI, - undefined, + inheritedLimit, undefined, ); }); @@ -282,12 +267,11 @@ describe('encodeAndFormatDocuments - fileConfig integration', () => { ); }); - it('should pass undefined when provider-specific config not found and no default', async () => { + it('should pass the inherited default limit when only another provider is configured', async () => { const req = { config: { fileConfig: { endpoints: { - /** Only configure a different provider, not OpenAI */ [Providers.ANTHROPIC]: { fileSizeLimit: 25, }, @@ -297,13 +281,11 @@ describe('encodeAndFormatDocuments - fileConfig integration', () => { } as unknown as ServerRequest; const file = createMockFile(20); - const mockContent = Buffer.from('test-pdf-content').toString('base64'); mockedGetFileStream.mockResolvedValue({ file, - content: mockContent, + content: Buffer.from('test-pdf-content').toString('base64'), metadata: file, }); - mockedValidatePdf.mockResolvedValue({ isValid: true }); await encodeAndFormatDocuments( @@ -313,12 +295,11 @@ describe('encodeAndFormatDocuments - fileConfig integration', () => { mockStrategyFunctions, ); - /** When provider-specific config not found and no default, returns undefined */ expect(mockedValidatePdf).toHaveBeenCalledWith( expect.any(Buffer), expect.any(Number), Providers.OPENAI, - undefined, + inheritedLimit, undefined, ); }); @@ -1202,4 +1183,431 @@ describe('encodeAndFormatDocuments - fileConfig integration', () => { expect(result.files).toHaveLength(6); }); }); + + describe('onValidationFailure', () => { + const pdfBytes = (size: number): Buffer => { + const buffer = Buffer.alloc(size); + buffer.write('%PDF-1.4\n', 0); + return buffer; + }; + + /** Serves each file its own bytes so the real validators see what production would. */ + const serveContents = (contents: Map): void => { + mockedGetFileStream.mockImplementation(async (_req, file) => ({ + file, + content: (contents.get(file.file_id) ?? Buffer.alloc(0)).toString('base64'), + metadata: file, + })); + }; + + type ServedFile = { file: IMongoFile; bytes?: Buffer; reason?: DocumentRejection['reason'] }; + + /** Serves the bytes each row declares and returns its files in order. */ + const serveRows = (rows: ServedFile[]): IMongoFile[] => { + const contents = new Map(); + for (const { file, bytes } of rows) { + if (bytes != null) { + contents.set(file.file_id, bytes); + } + } + serveContents(contents); + return rows.map(({ file }) => file); + }; + const rejectedOf = (rows: ServedFile[]): DocumentRejection[] => + rows.flatMap(({ file, reason }) => + reason == null ? [] : [{ file_id: file.file_id, reason }], + ); + const keptOf = (rows: ServedFile[]): IMongoFile[] => + rows.filter(({ reason }) => reason == null).map(({ file }) => file); + const DOCX = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'; + + type UnreadableSet = Record<'unreadable' | 'unstored' | 'empty' | 'readable', IMongoFile>; + + /** A failed read, a record with no stored object, a zero-byte object and a readable file. */ + const serveUnreadableSet = (): UnreadableSet => { + const files: UnreadableSet = { + unreadable: createMockDocFile(0.01, 'text/plain', 'unreadable.txt'), + unstored: createMockDocFile(0.01, 'text/plain', 'unstored.txt'), + empty: createMockDocFile(0.01, 'text/plain', 'empty.txt'), + readable: createMockDocFile(0.01, 'text/plain', 'readable.txt'), + }; + mockedGetFileStream.mockImplementation(async (_req, file) => { + if (file.file_id === files.unreadable.file_id) { + throw new Error('storage unavailable'); + } + if (file.file_id === files.unstored.file_id) { + return null; + } + const content = + file.file_id === files.empty.file_id ? '' : Buffer.from('notes').toString('base64'); + return { file, content, metadata: file }; + }); + return files; + }; + + const encodeUnreadableSet = async ( + files: UnreadableSet, + onValidationFailure?: NativeValidationMode, + ): Promise => { + const consoleError = jest.spyOn(console, 'error').mockImplementation(() => undefined); + try { + const result = await encodeAndFormatDocuments( + createMockRequest(15) as ServerRequest, + [files.unreadable, files.unstored, files.empty, files.readable], + { provider: Providers.OPENAI, onValidationFailure }, + mockStrategyFunctions, + ); + expect(consoleError).toHaveBeenCalledWith( + 'Document processing failed:', + expect.objectContaining({ message: 'storage unavailable' }), + ); + return result; + } finally { + consoleError.mockRestore(); + } + }; + + /** Passes the case-insensitive provider gate but matches no case-sensitive block format. */ + const providerWithoutBlockFormat = 'OpenRouter' as Providers; + + beforeEach(() => { + mockedValidatePdf.mockImplementation(realValidation.validatePdf); + mockedValidateBedrockDocument.mockImplementation(realValidation.validateBedrockDocument); + }); + + describe('skip', () => { + it.each<{ + name: string; + request: Partial; + provider: Providers; + serve: () => ServedFile[]; + document: unknown; + }>([ + { + name: 'a PDF over the configured limit as capacity', + request: createMockRequest(1), + provider: Providers.OPENAI, + serve: () => [ + { file: createMockFile(2), bytes: pdfBytes(mbToBytes(2)), reason: 'capacity' }, + { file: createMockFile(0.01), bytes: pdfBytes(1024) }, + ], + document: expect.anything(), + }, + { + name: 'an encrypted Anthropic PDF as integrity', + request: createMockRequest(30, Providers.ANTHROPIC), + provider: Providers.ANTHROPIC, + serve: () => { + const encryptedBytes = pdfBytes(1024); + encryptedBytes.write('/Encrypt ', 100); + return [ + { file: createMockFile(0.01), bytes: encryptedBytes, reason: 'integrity' }, + { file: createMockFile(0.01), bytes: pdfBytes(1024) }, + ]; + }, + document: expect.objectContaining({ + type: 'document', + source: expect.objectContaining({ media_type: 'application/pdf' }), + }), + }, + { + name: 'Bedrock capacity and integrity failures with their own reasons', + request: createMockRequest(), + provider: Providers.BEDROCK, + serve: () => { + const brokenBytes = Buffer.alloc(1024); + brokenBytes.write('INVALID', 0); + return [ + { + file: createMockDocFile(5, 'text/csv', 'large.csv'), + bytes: Buffer.alloc(mbToBytes(5)), + reason: 'capacity', + }, + { + file: createMockDocFile(0.01, 'application/pdf', 'broken.pdf'), + bytes: brokenBytes, + reason: 'integrity', + }, + { + file: createMockDocFile(0.01, 'text/csv', 'small.csv'), + bytes: Buffer.from('a,b\n1,2'), + }, + ]; + }, + document: expect.objectContaining({ + type: 'document', + document: expect.objectContaining({ format: 'csv', name: 'small_csv' }), + }), + }, + { + name: 'a generic file over the configured limit as capacity', + request: createMockRequest(1, Providers.ANTHROPIC), + provider: Providers.ANTHROPIC, + serve: () => [ + { + file: createMockDocFile(2, 'text/plain', 'large.txt'), + bytes: Buffer.alloc(mbToBytes(2)), + reason: 'capacity', + }, + { + file: createMockDocFile(0.01, 'text/plain', 'notes.txt'), + bytes: Buffer.from('notes'), + }, + ], + document: expect.anything(), + }, + ])( + 'records $name, omitting the failures and keeping the rest', + async ({ request, provider, serve, document }) => { + const rows = serve(); + + const result = await encodeAndFormatDocuments( + request as ServerRequest, + serveRows(rows), + { provider, onValidationFailure: 'skip' }, + mockStrategyFunctions, + ); + + expect(result.rejected).toEqual(rejectedOf(rows)); + expect(result.documents).toEqual([document]); + expect(result.files).toEqual(keptOf(rows)); + }, + ); + + it.each<{ + name: string; + request: Partial; + provider: Providers; + serve: () => ServedFile[]; + reads?: number; + }>([ + { + name: 'every file when the provider has no document path', + request: createMockRequest(15), + provider: Providers.AZURE, + serve: () => [ + { file: createMockFile(1), reason: 'unsupported' }, + { file: createMockDocFile(0.01, 'text/plain', 'notes.txt'), reason: 'unsupported' }, + ], + reads: 0, + }, + { + name: 'a type with no Bedrock document format', + request: createMockRequest(), + provider: Providers.BEDROCK, + serve: () => [ + { + file: createMockDocFile(0.01, 'application/zip', 'archive.zip'), + reason: 'unsupported', + }, + { + file: createMockDocFile(0.01, 'text/csv', 'data.csv'), + bytes: Buffer.from('a,b\n1,2'), + }, + ], + reads: 1, + }, + { + name: 'a type Claude document input cannot take', + request: createMockRequest(30, Providers.ANTHROPIC), + provider: Providers.ANTHROPIC, + serve: () => [ + { file: createMockDocFile(0.01, DOCX, 'report.docx'), reason: 'unsupported' }, + { + file: createMockDocFile(0.01, 'text/markdown', 'readme.md'), + bytes: Buffer.from('# heading'), + }, + ], + reads: 1, + }, + { + name: 'a file no block format exists for', + request: createMockRequest(), + provider: providerWithoutBlockFormat, + serve: () => [ + { + file: createMockDocFile(0.01, 'text/plain', 'notes.txt'), + bytes: Buffer.from('notes'), + reason: 'unsupported', + }, + ], + }, + ])('reports $name as unsupported', async ({ request, provider, serve, reads }) => { + const rows = serve(); + + const result = await encodeAndFormatDocuments( + request as ServerRequest, + serveRows(rows), + { provider, onValidationFailure: 'skip' }, + mockStrategyFunctions, + ); + + expect(result.rejected).toEqual(rejectedOf(rows)); + expect(result.files).toEqual(keptOf(rows)); + expect(result.documents).toHaveLength(keptOf(rows).length); + if (reads != null) { + expect(mockedGetFileStream).toHaveBeenCalledTimes(reads); + } + }); + + it('returns an empty rejected list when every file encodes', async () => { + const req = createMockRequest(15) as ServerRequest; + const file = createMockFile(0.01); + serveContents(new Map([[file.file_id, pdfBytes(1024)]])); + + const result = await encodeAndFormatDocuments( + req, + [file], + { provider: Providers.OPENAI, onValidationFailure: 'skip' }, + mockStrategyFunctions, + ); + + expect(result.rejected).toEqual([]); + expect(result.files).toEqual([file]); + }); + + it('records unreadable, unstored and empty files as integrity and keeps the readable one', async () => { + const files = serveUnreadableSet(); + + const result = await encodeUnreadableSet(files, 'skip'); + + expect(result.rejected).toEqual([ + { file_id: files.unreadable.file_id, reason: 'integrity' }, + { file_id: files.unstored.file_id, reason: 'integrity' }, + { file_id: files.empty.file_id, reason: 'integrity' }, + ]); + expect(result.files).toEqual([files.readable]); + expect(result.documents).toEqual([expect.objectContaining({ type: 'file' })]); + }); + + it('still rethrows a missing storage object', async () => { + const req = createMockRequest(15) as ServerRequest; + const missing = createMockFile(0.01); + const present = createMockFile(0.01); + mockedGetFileStream.mockImplementation(async (_req, file) => { + if (file.file_id === missing.file_id) { + throw new AttachmentObjectNotFoundError(missing.file_id); + } + return { file, content: pdfBytes(1024).toString('base64'), metadata: file }; + }); + + await expect( + encodeAndFormatDocuments( + req, + [missing, present], + { provider: Providers.OPENAI, onValidationFailure: 'skip' }, + mockStrategyFunctions, + ), + ).rejects.toMatchObject({ code: 'ATTACHMENT_OBJECT_NOT_FOUND', fileId: missing.file_id }); + }); + }); + + describe('throw', () => { + it.each<{ + name: string; + request: Partial; + provider: Providers; + onValidationFailure?: NativeValidationMode; + serve: () => ServedFile; + message: string; + }>([ + { + name: 'PDF capacity error when the mode is omitted', + request: createMockRequest(1), + provider: Providers.OPENAI, + serve: () => ({ file: createMockFile(2), bytes: pdfBytes(mbToBytes(2)) }), + message: 'PDF validation failed: PDF file size (2MB) exceeds the 1MB limit', + }, + { + name: 'PDF capacity error when the mode is throw', + request: createMockRequest(1), + provider: Providers.OPENAI, + onValidationFailure: 'throw', + serve: () => ({ file: createMockFile(2), bytes: pdfBytes(mbToBytes(2)) }), + message: 'PDF validation failed: PDF file size (2MB) exceeds the 1MB limit', + }, + { + name: 'Bedrock capacity error', + request: createMockRequest(), + provider: Providers.BEDROCK, + serve: () => ({ + file: createMockDocFile(5, 'text/csv', 'large.csv'), + bytes: Buffer.alloc(mbToBytes(5)), + }), + message: + 'Document validation failed: File size (5.0MB) exceeds the 4.5MB limit for Bedrock', + }, + { + name: 'generic capacity error', + request: createMockRequest(1, Providers.ANTHROPIC), + provider: Providers.ANTHROPIC, + serve: () => ({ + file: createMockDocFile(2, 'text/plain', 'large.txt'), + bytes: Buffer.alloc(mbToBytes(2)), + }), + message: 'File size (~2.0MB) exceeds the configured limit for anthropic', + }, + ])( + 'throws the unchanged $name', + async ({ request, provider, onValidationFailure, serve, message }) => { + await expect( + encodeAndFormatDocuments( + request as ServerRequest, + serveRows([serve()]), + { provider, onValidationFailure }, + mockStrategyFunctions, + ), + ).rejects.toThrow(message); + }, + ); + + it('logs and leaves out unreadable and unstored files and keeps an empty one in files', async () => { + const files = serveUnreadableSet(); + + const result = await encodeUnreadableSet(files); + + expect(result).not.toHaveProperty('rejected'); + expect(result.files).toEqual([files.empty, files.readable]); + expect(result.documents).toEqual([expect.objectContaining({ type: 'file' })]); + }); + + it('leaves rejected off the result, leaving out files the provider cannot take', async () => { + const req = createMockRequest(30, Providers.ANTHROPIC) as ServerRequest; + const notes = createMockDocFile(0.01, 'text/plain', 'notes.txt'); + const docx = createMockDocFile(0.01, DOCX, 'report.docx'); + const readme = createMockDocFile(0.01, 'text/markdown', 'readme.md'); + serveContents( + new Map([ + [notes.file_id, Buffer.from('notes')], + [readme.file_id, Buffer.from('# heading')], + ]), + ); + + await expect( + encodeAndFormatDocuments( + createMockRequest() as ServerRequest, + [notes], + { provider: providerWithoutBlockFormat }, + mockStrategyFunctions, + ), + ).resolves.toEqual({ documents: [], files: [] }); + const anthropicResult = await encodeAndFormatDocuments( + req, + [docx, readme], + { provider: Providers.ANTHROPIC }, + mockStrategyFunctions, + ); + expect(anthropicResult).not.toHaveProperty('rejected'); + expect(anthropicResult.files).toEqual([readme]); + await expect( + encodeAndFormatDocuments( + req, + [readme], + { provider: Providers.AZURE }, + mockStrategyFunctions, + ), + ).resolves.toEqual({ documents: [], files: [] }); + }); + }); + }); }); diff --git a/packages/api/src/files/encode/document.ts b/packages/api/src/files/encode/document.ts index 57ef72db9a3..01a1303a9e3 100644 --- a/packages/api/src/files/encode/document.ts +++ b/packages/api/src/files/encode/document.ts @@ -9,18 +9,26 @@ import { } from 'librechat-data-provider'; import type { IMongoFile } from '@librechat/data-schemas'; import type { - DocumentBlock, + ValidationFailureReason, AnthropicDocumentBlock, + NativeValidationMode, + DocumentRejection, StrategyFunctions, DocumentResult, + ProcessedFile, + DocumentBlock, ServerRequest, } from '~/types'; +import { + validatePdf, + validateBedrockDocument, + getNativeDocumentSizeLimit, +} from '~/files/validation'; import { getFileStream, getConfiguredFileSizeLimit, isAttachmentObjectNotFoundError, } from './utils'; -import { validatePdf, validateBedrockDocument } from '~/files/validation'; import { runGuardedEncode } from './memoryGuard'; /** Anthropic only accepts PDFs as base64 documents; textual types must use a text source */ @@ -51,7 +59,7 @@ function getAnthropicDocumentSource( * Whether the model behind this provider is Claude, which accepts only PDFs as base64 * documents. OpenAI-compatible gateways report an OpenAI-like provider for Claude models. */ -function usesAnthropicDocumentCapabilities(provider: Providers, model?: string): boolean { +export function usesAnthropicDocumentCapabilities(provider: Providers, model?: string): boolean { return ( provider === Providers.ANTHROPIC || (isOpenAILikeProvider(provider) && (model?.toLowerCase().includes('claude') ?? false)) @@ -133,8 +141,29 @@ function formatDocumentBlock( return null; } +interface ProviderDocumentFiles { + processable: IMongoFile[]; + unsupported: IMongoFile[]; +} + +function partitionDocumentFiles( + files: IMongoFile[], + isSupported: (mimeType?: string) => boolean, +): ProviderDocumentFiles { + const processable: IMongoFile[] = []; + const unsupported: IMongoFile[] = []; + for (const file of files) { + if (isSupported(file.type)) { + processable.push(file); + } else { + unsupported.push(file); + } + } + return { processable, unsupported }; +} + /** - * Filters out files the provider's document path cannot send to the model. + * Separates out files the provider's document path cannot send to the model. * Claude rejects non-PDF binary documents with a 400 that recurs on every retry, * including when it is reached through an OpenAI-compatible gateway. Unsupported * types are skipped instead of bricking the conversation. @@ -143,32 +172,33 @@ function filterProviderDocumentFiles( provider: Providers, files: IMongoFile[], model?: string, -): IMongoFile[] { +): ProviderDocumentFiles { if (provider === Providers.BEDROCK) { - return files.filter((file) => isBedrockDocumentType(file.type)); + return partitionDocumentFiles(files, isBedrockDocumentType); } if (!usesAnthropicDocumentCapabilities(provider, model)) { - return files; - } - - const processable: IMongoFile[] = []; - const skipped: string[] = []; - for (const file of files) { - if (isAnthropicDocumentType(file.type)) { - processable.push(file); - } else { - skipped.push(`"${file.filename}" (${file.type})`); - } + return { processable: files, unsupported: [] }; } - if (skipped.length) { + const partition = partitionDocumentFiles(files, isAnthropicDocumentType); + if (partition.unsupported.length) { + const skipped = partition.unsupported.map((file) => `"${file.filename}" (${file.type})`); console.warn( `Skipping attachment(s) unsupported by Claude document input: ${skipped.join(', ')}`, ); } - return processable; + return partition; +} + +/** Records files left out in skip mode; a no-op in throw mode, where `rejected` is absent. */ +function recordRejections( + result: DocumentResult, + files: IMongoFile[], + reason: DocumentRejection['reason'], +): void { + result.rejected?.push(...files.map((file) => ({ file_id: file.file_id, reason }))); } function getBase64DecodedByteCount(content: string): number { @@ -194,29 +224,73 @@ function getBase64DecodedByteCount(content: string): number { * all others are skipped. * - **PDF**: Validated via `validatePdf` before encoding. * - **Generic types**: Encoded with a provider-specific size check. + * + * A validation failure throws by default. With `onValidationFailure: 'skip'` every file + * that yields no block is listed in `rejected` with its reason instead: a failed + * validation, a type the provider cannot take, or bytes that could not be read. A missing + * storage object still throws. */ export async function encodeAndFormatDocuments( req: ServerRequest, files: IMongoFile[], - params: { provider: Providers; endpoint?: string; useResponsesApi?: boolean; model?: string }, + params: { + provider: Providers; + endpoint?: string; + useResponsesApi?: boolean; + model?: string; + onValidationFailure?: NativeValidationMode; + }, getStrategyFunctions: (source: string) => StrategyFunctions, ): Promise { - const { provider, endpoint, useResponsesApi, model } = params; + const { provider, endpoint, useResponsesApi, model, onValidationFailure = 'throw' } = params; + const skipInvalid = onValidationFailure === 'skip'; + const result: DocumentResult = skipInvalid + ? { documents: [], files: [], rejected: [] } + : { documents: [], files: [] }; if (!files?.length) { - return { documents: [], files: [] }; + return result; } const encodingMethods: Record = {}; - const result: DocumentResult = { documents: [], files: [] }; + + const rejectInvalid = ( + file: IMongoFile, + reason: ValidationFailureReason, + message: string, + ): void => { + if (!skipInvalid) { + throw new Error(message); + } + recordRejections(result, [file], reason); + }; + + const addBlock = ( + file: IMongoFile, + block: DocumentBlock | null, + metadata: ProcessedFile['metadata'], + ): void => { + if (!block) { + recordRejections(result, [file], 'unsupported'); + return; + } + result.documents.push(block); + result.files.push(metadata); + }; const isBedrock = provider === Providers.BEDROCK; const isDocSupported = isDocumentSupportedProvider(provider); if (!isDocSupported && !isBedrock) { + recordRejections(result, files, 'unsupported'); return result; } - const processableFiles = filterProviderDocumentFiles(provider, files, model); + const { processable: processableFiles, unsupported } = filterProviderDocumentFiles( + provider, + files, + model, + ); + recordRejections(result, unsupported, 'unsupported'); if (!processableFiles.length) { return result; @@ -232,22 +306,29 @@ export async function encodeAndFormatDocuments( ), ); - for (const settledResult of results) { + for (let i = 0; i < results.length; i++) { + const settledResult = results[i]; + const sourceFile = processableFiles[i]; if (settledResult.status === 'rejected') { if (isAttachmentObjectNotFoundError(settledResult.reason)) { throw settledResult.reason; } console.error('Document processing failed:', settledResult.reason); + recordRejections(result, [sourceFile], 'integrity'); continue; } const processed = settledResult.value; - if (!processed) continue; + if (!processed) { + recordRejections(result, [sourceFile], 'integrity'); + continue; + } const { file, content, metadata } = processed; if (!content || !file) { - if (metadata) result.files.push(metadata); + if (metadata && !skipInvalid) result.files.push(metadata); + recordRejections(result, [sourceFile], 'integrity'); continue; } @@ -266,7 +347,12 @@ export async function encodeAndFormatDocuments( ); if (!validation.isValid) { - throw new Error(`Document validation failed: ${validation.error}`); + rejectInvalid( + file, + validation.reason ?? 'integrity', + `Document validation failed: ${validation.error}`, + ); + continue; } const sanitizedName = (file.filename || 'document') @@ -295,41 +381,41 @@ export async function encodeAndFormatDocuments( ); if (!validation.isValid) { - throw new Error(`PDF validation failed: ${validation.error}`); + rejectInvalid( + file, + validation.reason ?? 'integrity', + `PDF validation failed: ${validation.error}`, + ); + continue; } - const block = formatDocumentBlock( - provider, - mimeType, - content, - file.filename, - useResponsesApi, - model, + addBlock( + file, + formatDocumentBlock(provider, mimeType, content, file.filename, useResponsesApi, model), + metadata, ); - if (block) { - result.documents.push(block); - result.files.push(metadata); - } } else if (isDocSupported && !isBedrock) { const decodedByteCount = getBase64DecodedByteCount(content); - if (configuredFileSizeLimit && decodedByteCount > configuredFileSizeLimit) { - throw new Error( + const sizeLimit = getNativeDocumentSizeLimit({ + provider, + mimeType, + model, + configuredFileSizeLimit, + }); + if (sizeLimit !== undefined && decodedByteCount > sizeLimit) { + rejectInvalid( + file, + 'capacity', `File size (~${(decodedByteCount / 1024 / 1024).toFixed(1)}MB) exceeds the configured limit for ${provider}`, ); + continue; } - const block = formatDocumentBlock( - provider, - mimeType, - content, - file.filename, - useResponsesApi, - model, + addBlock( + file, + formatDocumentBlock(provider, mimeType, content, file.filename, useResponsesApi, model), + metadata, ); - if (block) { - result.documents.push(block); - result.files.push(metadata); - } } } diff --git a/packages/api/src/files/encode/utils.spec.ts b/packages/api/src/files/encode/utils.spec.ts index d4882d952dd..7ea74ece557 100644 --- a/packages/api/src/files/encode/utils.spec.ts +++ b/packages/api/src/files/encode/utils.spec.ts @@ -1,12 +1,16 @@ import { Readable } from 'node:stream'; import { Providers } from '@librechat/agents'; -import { audioMimeTypes } from 'librechat-data-provider'; +import { mbToBytes, audioMimeTypes, fileConfig as baseFileConfig } from 'librechat-data-provider'; +import type { TFileConfig } from 'librechat-data-provider'; +import type { FileSizeLimitParams } from './utils'; import type { ServerRequest } from '~/types'; import { - AttachmentObjectNotFoundError, getAudioFormat, getFileStream, + getConfiguredFileSizeLimit, + AttachmentObjectNotFoundError, isConfiguredProviderMediaType, + resolveConfiguredFileSizeLimit, } from './utils'; const file = { @@ -198,3 +202,66 @@ describe('getAudioFormat', () => { } }); }); + +describe('resolveConfiguredFileSizeLimit', () => { + const inheritedLimit = baseFileConfig.endpoints.default.fileSizeLimit; + + /** `getConfiguredFileSizeLimit` and its resolver agree for every fileConfig shape. */ + const parityCases: Array< + [string, TFileConfig | undefined, FileSizeLimitParams, number | undefined] + > = [ + ['no fileConfig', undefined, { provider: Providers.OPENAI }, undefined], + [ + 'an empty fileConfig, which inherits the 512 MB default', + {}, + { provider: Providers.ANTHROPIC }, + mbToBytes(512), + ], + [ + 'an explicit provider limit', + { endpoints: { [Providers.GOOGLE]: { fileSizeLimit: 25 } } }, + { provider: Providers.GOOGLE }, + mbToBytes(25), + ], + [ + 'an endpoint name shadowing the provider', + { + endpoints: { [Providers.OPENAI]: { fileSizeLimit: 15 }, MyGateway: { fileSizeLimit: 3 } }, + }, + { provider: Providers.OPENAI, endpoint: 'MyGateway' }, + mbToBytes(3), + ], + [ + 'an endpoint name with no entry, which does not fall back to the provider entry', + { endpoints: { [Providers.OPENAI]: { fileSizeLimit: 15 } } }, + { provider: Providers.OPENAI, endpoint: 'Unlisted' }, + inheritedLimit, + ], + [ + 'a configured default', + { endpoints: { default: { fileSizeLimit: 7 } } }, + { provider: Providers.BEDROCK }, + mbToBytes(7), + ], + [ + 'a configured default for a provider with a built-in entry', + { endpoints: { default: { fileSizeLimit: 7 } } }, + { provider: Providers.ANTHROPIC }, + inheritedLimit, + ], + [ + 'a configured 0', + { endpoints: { [Providers.OPENAI]: { fileSizeLimit: 0 } } }, + { provider: Providers.OPENAI }, + 0, + ], + ]; + + it.each(parityCases)( + 'resolves %s like getConfiguredFileSizeLimit', + (_label, config, params, expected) => { + expect(resolveConfiguredFileSizeLimit(config, params)).toBe(expected); + expect(getConfiguredFileSizeLimit(reqWith(config), params)).toBe(expected); + }, + ); +}); diff --git a/packages/api/src/files/encode/utils.ts b/packages/api/src/files/encode/utils.ts index 9c39ce2c13c..0e261731f6e 100644 --- a/packages/api/src/files/encode/utils.ts +++ b/packages/api/src/files/encode/utils.ts @@ -6,6 +6,7 @@ import { isExplicitMimeConfig, getEndpointFileConfig, } from 'librechat-data-provider'; +import type { TFileConfig } from 'librechat-data-provider'; import type { ServerRequest, StrategyFunctions, ProcessedFile } from '~/types'; import type { StoredFileRef } from '~/storage/path'; import { resolveDownloadPath } from '~/storage/path'; @@ -48,33 +49,47 @@ function isStorageNotFoundError(error: unknown): boolean { ); } +export interface FileSizeLimitParams { + provider: Providers; + endpoint?: string; +} + /** - * Extracts the configured file size limit for a specific provider from fileConfig - * @param req - The server request object containing config + * Resolves the configured file size limit for a provider or endpoint from a fileConfig. + * Any fileConfig, even an empty one, yields the merged endpoint limit, inherited default included. + * @param fileConfig - The app's fileConfig, if any * @param params - Object containing provider and optional endpoint * @param params.provider - The provider to get the limit for * @param params.endpoint - Optional endpoint name for lookup - * @returns The configured file size limit in bytes, or undefined if not configured + * @returns The configured file size limit in bytes, or undefined without a fileConfig */ -export const getConfiguredFileSizeLimit = ( - req: ServerRequest, - params: { - provider: Providers; - endpoint?: string; - }, +export const resolveConfiguredFileSizeLimit = ( + fileConfig: TFileConfig | undefined, + { provider, endpoint }: FileSizeLimitParams, ): number | undefined => { - if (!req.config?.fileConfig) { + if (!fileConfig) { return undefined; } - const { provider, endpoint } = params; - const fileConfig = mergeFileConfig(req.config.fileConfig); const endpointConfig = getEndpointFileConfig({ - fileConfig, + fileConfig: mergeFileConfig(fileConfig), endpoint: endpoint ?? provider, }); return endpointConfig?.fileSizeLimit; }; +/** + * Extracts the configured file size limit for a specific provider from fileConfig + * @param req - The server request object containing config + * @param params - Object containing provider and optional endpoint + * @param params.provider - The provider to get the limit for + * @param params.endpoint - Optional endpoint name for lookup + * @returns The configured file size limit in bytes, or undefined if not configured + */ +export const getConfiguredFileSizeLimit = ( + req: ServerRequest, + params: FileSizeLimitParams, +): number | undefined => resolveConfiguredFileSizeLimit(req.config?.fileConfig, params); + /** * Whether the admin explicitly allowed `mimeType` for this endpoint via * `fileConfig.endpoints..supportedMimeTypes`. The inherited default list does diff --git a/packages/api/src/files/filter.spec.ts b/packages/api/src/files/filter.spec.ts index f64a6017192..98dfb852374 100644 --- a/packages/api/src/files/filter.spec.ts +++ b/packages/api/src/files/filter.spec.ts @@ -1,6 +1,7 @@ import { Types } from 'mongoose'; import { Providers } from '@librechat/agents'; -import { EModelEndpoint, FileSources } from 'librechat-data-provider'; +import { EModelEndpoint, FileContext, FileSources } from 'librechat-data-provider'; +import type { TFileConfig, TurnDeliveryFile, TurnFileConsumers } from 'librechat-data-provider'; import type { IMongoFile } from '@librechat/data-schemas'; import type { AppConfig } from '@librechat/data-schemas'; import type { ServerRequest } from '~/types'; @@ -1445,3 +1446,152 @@ describe('filterFilesByEndpointConfig', () => { }); }); }); + +describe('filterFilesByEndpointRuntimeConfig under the automatic reading policy', () => { + const MB = 1024 * 1024; + + type ReadingRecord = TurnDeliveryFile & { + file_id: string; + filename: string; + type: string; + bytes: number; + }; + type EndpointConfigInput = NonNullable[string]; + + const record = (overrides: Partial = {}): ReadingRecord => ({ + file_id: 'eligible', + filename: 'report.pdf', + type: 'application/pdf', + bytes: 3 * MB, + source: FileSources.local, + context: FileContext.message_attachment, + llmDeliveryPath: 'provider', + metadata: { destinationChosen: false }, + ...overrides, + }); + + const automatic = (endpoint: EndpointConfigInput = {}): TFileConfig => ({ + endpoints: { + [EModelEndpoint.openAI]: { fileSizeLimit: 1, llmDeliveryPolicy: 'automatic', ...endpoint }, + }, + }); + + const runsCode: TurnFileConsumers = { executeCode: true, fileSearch: false }; + const searches: TurnFileConsumers = { executeCode: false, fileSearch: true }; + const noReader: TurnFileConsumers = { executeCode: false, fileSearch: false }; + + /** A file tool is loaded unless a test says otherwise: only then does the policy read at all. */ + const filter = ( + fileConfig: TFileConfig, + files: ReadingRecord[], + consumers: TurnFileConsumers | null | undefined = runsCode, + ): ReadingRecord[] => + filterFilesByEndpointRuntimeConfig({ config: {}, fileConfig } as AppConfig, { + files, + endpoint: EModelEndpoint.openAI, + consumers, + }); + + it('keeps an oversized eligible record, since its reading judges that size as capacity', () => { + const eligible = record(); + expect(filter(automatic(), [eligible])).toEqual([eligible]); + expect(filter(automatic(), [eligible], searches)).toEqual([eligible]); + }); + + it.each<[string, { consumers?: TurnFileConsumers | null }]>([ + ['no file tool is loaded', { consumers: noReader }], + ['the consumers are null', { consumers: null }], + ['the caller names no consumers', {}], + ])('drops the oversized eligible record as classic routing does when %s', (_case, consumers) => { + const classic = (files: ReadingRecord[]) => + filterFilesByEndpointRuntimeConfig({ config: {}, fileConfig: automatic() } as AppConfig, { + files, + endpoint: EModelEndpoint.openAI, + ...consumers, + }); + const withinLimit = record({ bytes: MB / 2 }); + expect(classic([record()])).toEqual([]); + expect(classic([withinLimit])).toEqual([withinLimit]); + }); + + it('keeps an eligible record whatever route it stored', () => { + const text = record({ file_id: 'text', llmDeliveryPath: 'text', text: 'extracted' }); + const none = record({ file_id: 'none', llmDeliveryPath: 'none' }); + expect(filter(automatic(), [text, none])).toEqual([text, none]); + }); + + it('applies a policy the endpoint inherits from the top-level file config', () => { + const eligible = record(); + const fileConfig: TFileConfig = { + llmDeliveryPolicy: 'automatic', + endpoints: { [EModelEndpoint.openAI]: { fileSizeLimit: 1 } }, + }; + expect(filter(fileConfig, [eligible])).toEqual([eligible]); + }); + + it.each<[string, { fileConfig?: TFileConfig; file?: Partial }]>([ + ['an image', { file: { type: 'image/png', filename: 'photo.png' } }], + ['a persistent agent file', { file: { context: FileContext.agents } }], + ['an explicit destination', { file: { metadata: { destinationChosen: true } } }], + ['an unmarked record', { file: { metadata: {} } }], + ['a legacy record', { file: { llmDeliveryPath: undefined } }], + ['a text-only record', { file: { source: FileSources.text, text: 'pasted' } }], + [ + 'a record on a classic endpoint', + { fileConfig: { endpoints: { [EModelEndpoint.openAI]: { fileSizeLimit: 1 } } } }, + ], + [ + 'a record on an endpoint set to classic', + { fileConfig: automatic({ llmDeliveryPolicy: 'classic' }) }, + ], + ['a record behind the legacy chooser', { fileConfig: automatic({ legacyFileUploadUX: true }) }], + [ + 'a type an endpoint route names', + { + fileConfig: automatic({ + defaultLLMDeliveryPath: { overrides: { 'application/pdf': 'provider' } }, + }), + }, + ], + [ + 'a type a global route names', + { fileConfig: { ...automatic(), defaultLLMDeliveryPath: { fallback: 'text' } } }, + ], + ])('still drops %s over the size limit', (_label, { fileConfig = automatic(), file = {} }) => { + const withinLimit = record({ ...file, bytes: MB / 2 }); + expect(filter(fileConfig, [record(file)])).toEqual([]); + expect(filter(fileConfig, [withinLimit])).toEqual([withinLimit]); + }); + + it('keeps the MIME allowlist, the disabled switch and the total size limit binding', () => { + expect(filter(automatic({ supportedMimeTypes: ['^text/plain$'] }), [record()])).toEqual([]); + expect(filter(automatic({ disabled: true }), [record()])).toEqual([]); + expect(filter(automatic({ totalSizeLimit: 2 }), [record()])).toEqual([]); + }); + + it('holds provider copies to the size limit when asked to bind them', () => { + const provider = record(); + const text = record({ file_id: 'text', llmDeliveryPath: 'text', text: 'extracted' }); + const bound = filterFilesByEndpointRuntimeConfig( + { config: {}, fileConfig: automatic() } as AppConfig, + { + files: [provider, text], + endpoint: EModelEndpoint.openAI, + bindProviderCopies: true, + consumers: runsCode, + }, + ); + expect(bound).toEqual([text]); + }); + + it('exempts only the eligible records in a mixed set', () => { + const eligible = record(); + const explicit = record({ file_id: 'explicit', metadata: { destinationChosen: true } }); + const small = record({ + file_id: 'small', + bytes: MB / 2, + metadata: { destinationChosen: true }, + }); + expect(filter(automatic(), [eligible, explicit, small])).toEqual([eligible, small]); + }); +}); diff --git a/packages/api/src/files/filter.ts b/packages/api/src/files/filter.ts index f6881ac1349..c85d0c2ef14 100644 --- a/packages/api/src/files/filter.ts +++ b/packages/api/src/files/filter.ts @@ -1,11 +1,18 @@ import { + fileConfig, FileSources, - getEndpointFileConfig, mergeFileConfig, - fileConfig, + getEndpointFileConfig, + hasFileToolConsumer, + isAutomaticReadingRecord, +} from 'librechat-data-provider'; +import type { + RegexLike, + TFile, + TurnDeliveryFile, + TurnFileConsumers, } from 'librechat-data-provider'; import type { AppConfig, IMongoFile } from '@librechat/data-schemas'; -import type { RegexLike, TFile } from 'librechat-data-provider'; import type { ServerRequest } from '~/types'; /** @@ -65,10 +72,13 @@ export function isLegacyFileUploadUX( return endpointFileConfig?.legacyFileUploadUX === true; } -type EndpointPolicyFile = Pick & { - source?: string; - metadata?: { routingMimeType?: string }; -}; +type EndpointPolicyFile = Pick & + Pick & { + metadata?: Pick< + NonNullable, + 'routingMimeType' | 'destinationChosen' | 'textDerivation' + > | null; + }; export function filterFilesByEndpointRuntimeConfig( appConfig: AppConfig | undefined, @@ -81,6 +91,13 @@ export function filterFilesByEndpointRuntimeConfig consumedBytes?: number; skipTotalSizeLimit?: boolean; preserveTextSources?: boolean; + /** Hold copies already routed to the provider to the size limit even when the automatic + * policy reads them: another agent's decision put their bytes on this endpoint's path. */ + bindProviderCopies?: boolean; + /** The turn's loaded file tools. The size limit is native capacity rather than admission + * only while one is loaded; otherwise the automatic policy routes nothing, so nothing + * else could take the file. */ + consumers?: TurnFileConsumers | null; }, ): T[] { const { @@ -90,6 +107,8 @@ export function filterFilesByEndpointRuntimeConfig consumedBytes = 0, skipTotalSizeLimit = false, preserveTextSources = false, + bindProviderCopies = false, + consumers, } = params; if (!files || files.length === 0) { @@ -116,11 +135,19 @@ export function filterFilesByEndpointRuntimeConfig /** Filter files based on individual file size and MIME type */ let filteredFiles = files; - /** Filter by individual file size limit */ + /** Filter by individual file size limit. For a record the automatic policy reads on a turn + * with a file tool, the limit is native-delivery capacity its reading decision already + * applies, not admission. */ if (fileSizeLimit !== undefined && fileSizeLimit > 0) { - filteredFiles = filteredFiles.filter((file) => { - return file.bytes <= fileSizeLimit; - }); + const readingConfig = { endpointConfig: endpointFileConfig, fileConfig: mergedFileConfig }; + const toolLoaded = hasFileToolConsumer(consumers); + const isCapacityOnly = (file: T): boolean => + toolLoaded && + !(bindProviderCopies && file.llmDeliveryPath === 'provider') && + isAutomaticReadingRecord(readingConfig, file); + filteredFiles = filteredFiles.filter( + (file) => file.bytes <= fileSizeLimit || isCapacityOnly(file), + ); } /** Filter by MIME type, against the type the upload was accepted as. Conversion rewrites diff --git a/packages/api/src/files/index.ts b/packages/api/src/files/index.ts index 66226ed4b4c..c66744f7d6a 100644 --- a/packages/api/src/files/index.ts +++ b/packages/api/src/files/index.ts @@ -14,6 +14,7 @@ export * from './ocr'; export * from './parse'; export * from './preflight'; export * from './provision'; +export * from './reading'; export * from './upload/routing'; export * from './upload/diagnostics'; export * from './upload/fallback'; diff --git a/packages/api/src/files/provision/callback.spec.ts b/packages/api/src/files/provision/callback.spec.ts index 28846bdb928..22479d03e70 100644 --- a/packages/api/src/files/provision/callback.spec.ts +++ b/packages/api/src/files/provision/callback.spec.ts @@ -1,14 +1,20 @@ import { Constants } from '@librechat/agents'; +import { logger } from '@librechat/data-schemas'; import { EToolResources, FileContext } from 'librechat-data-provider'; import type { TFile } from 'librechat-data-provider'; +import type { SearchPreparationState } from '../reading/turn'; import type { ProvisionState } from '~/agents/resources'; +import type { ReadingAgent } from '../reading/inventory'; import type { ProvisionToolContext } from './callback'; import type { CodeFileAgent } from '../code/queued'; import type { ServerRequest } from '~/types'; +import { collectInventoryEntries, prepareAgentFileContext } from '../reading/inventory'; +import { FileSearchPreparationError, createProvisionFilesCallback } from './callback'; +import { buildTurnReadingContext, getTurnReadingContext } from '../reading/turn'; import { mergeCodeFilesIntoContext } from '~/agents/codeFilesSession'; +import { resolveTurnDeliveryRouting } from '~/agents/files/delivery'; import { prepareQueuedCodeFileContext } from '../code/queued'; import { createSubagentCodeRouting } from '~/code/targets'; -import { createProvisionFilesCallback } from './callback'; jest.mock('@librechat/data-schemas', () => ({ logger: { error: jest.fn(), warn: jest.fn(), debug: jest.fn(), info: jest.fn() }, @@ -45,6 +51,49 @@ function state( }; } +function searchAgent( + files: TFile[], + id = 'agent-a', + scopedFileIds: string[] = [], +): CodeFileAgent & ReadingAgent { + const routing = resolveTurnDeliveryRouting({ + agent: { provider: 'openAI', endpoint: 'openAI' }, + config: { fileConfig: { endpoints: { openAI: { llmDeliveryPolicy: 'automatic' } } } }, + }); + const preparation = new Map(); + const provisionState = { ...state([], files, scopedFileIds), searchPreparation: preparation }; + const context = buildTurnReadingContext({ + routing, + provider: 'openAI', + fileTokenLimit: 1000, + configuredFileSizeLimit: 1024 * 1024, + countTokens: (text) => text.length, + }); + context?.setSearchEvidence({ + queued: files.map(({ file_id }) => file_id), + registered: [], + preparation, + }); + routing.reading = context; + return { + id, + deliveryRouting: routing, + provisionState, + fileConsumers: { executeCode: false, fileSearch: true }, + currentRequestAttachments: files, + }; +} + +const searchFile = (overrides: Partial = {}): TFile => + makeFile({ + filename: 'manual.pdf', + type: 'application/pdf', + bytes: 3_000_000, + llmDeliveryPath: 'none', + metadata: { destinationChosen: false }, + ...overrides, + }); + function buildHarness({ contexts, codeImpl, @@ -876,18 +925,24 @@ describe('createProvisionFilesCallback', () => { it('preserves cancellation raised during vector provisioning', async () => { const controller = new AbortController(); + const pending = searchFile(); + const agent = searchAgent([pending]); const vectorImpl = jest.fn(async () => { controller.abort(); controller.signal.throwIfAborted(); }); const { provisionFiles } = buildHarness({ - contexts: [['agent-a', { provisionState: state([], [makeFile()]) }]], + contexts: [[agent.id, agent]], vectorImpl, }); await expect( provisionFiles(['file_search'], 'agent-a', controller.signal), ).rejects.toMatchObject({ name: 'AbortError' }); + expect(agent.provisionState?.vectorDBFiles).toEqual([pending]); + expect(getTurnReadingContext(agent.deliveryRouting)?.searchState(pending.file_id)).toBe( + 'failed', + ); }); it('preserves a homogeneous Code API rate-limit failure from lazy provisioning', async () => { @@ -954,6 +1009,29 @@ describe('createProvisionFilesCallback', () => { }); }); + it('embeds a pending search attachment once, leaving nothing for the next batch', async () => { + const pending = makeFile({ + file_id: 'manual-pdf', + filename: 'manual.pdf', + filepath: '/uploads/manual.pdf', + type: 'application/pdf', + bytes: 3_000_000, + llmDeliveryPath: 'none', + metadata: { destinationChosen: false }, + }); + const { provisionFiles, provisionToVectorDB, agentToolContexts } = buildHarness({ + contexts: [['agent-a', { provisionState: state([], [pending]) }]], + }); + + await provisionFiles(['file_search'], 'agent-a'); + + expect(provisionToVectorDB).toHaveBeenCalledWith(expect.objectContaining({ file: pending })); + expect(pending.embedded).toBe(true); + expect(agentToolContexts.get('agent-a')?.provisionState?.vectorDBFiles).toEqual([]); + await provisionFiles(['file_search'], 'agent-a'); + expect(provisionToVectorDB).toHaveBeenCalledTimes(1); + }); + it('retries a failed upload on a later tool call instead of replaying the rejection', async () => { let attempts = 0; const codeImpl = jest.fn(async ({ file }: { file: TFile }) => { @@ -1053,7 +1131,7 @@ describe('createProvisionFilesCallback', () => { }); await expect(provisionFiles(['file_search'], 'agent-a')).rejects.toThrow( - /aborting tool execution rather than searching without them/, + FileSearchPreparationError, ); expect(agentToolContexts.get('agent-a')?.provisionState?.vectorDBFiles).toHaveLength(1); }); @@ -1068,11 +1146,150 @@ describe('createProvisionFilesCallback', () => { }); await expect(provisionFiles(['file_search'], 'agent-a')).rejects.toThrow( - /aborting tool execution rather than searching without them/, + FileSearchPreparationError, ); expect(agentToolContexts.get('agent-a')?.provisionState?.vectorDBFiles).toHaveLength(1); }); + it('retains failed search readiness until a declined embedding succeeds on retry', async () => { + const pending = searchFile({ text: 'complete cached text that was not sent' }); + const agent = searchAgent([pending]); + const context = getTurnReadingContext(agent.deliveryRouting); + let release: () => void = () => undefined; + const retry = new Promise((resolve) => { + release = resolve; + }); + const vectorImpl = jest + .fn() + .mockResolvedValueOnce({ embedded: false, fileUpdate: null }) + .mockImplementationOnce(async () => { + await retry; + return { embedded: true, fileUpdate: { file_id: pending.file_id, embedded: true } }; + }); + const { provisionFiles } = buildHarness({ contexts: [[agent.id, agent]], vectorImpl }); + const query = jest.fn(); + + await expect(provisionFiles(['file_search'], agent.id).then(query)).rejects.toMatchObject({ + code: 'file_search_preparation_failed', + }); + expect(query).not.toHaveBeenCalled(); + expect(agent.provisionState?.vectorDBFiles).toEqual([pending]); + expect(context?.searchState(pending.file_id)).toBe('failed'); + + const secondAttempt = provisionFiles(['file_search'], agent.id).then(query); + await new Promise((resolve) => setImmediate(resolve)); + expect(vectorImpl).toHaveBeenCalledTimes(2); + expect(context?.searchState(pending.file_id)).toBe('failed'); + expect(query).not.toHaveBeenCalled(); + release(); + await secondAttempt; + expect(query).toHaveBeenCalledTimes(1); + expect(agent.provisionState?.vectorDBFiles).toEqual([]); + expect(context?.searchState(pending.file_id)).toBe('ready'); + }); + + it('keeps partial indexing successes and retries only failed files', async () => { + const first = searchFile({ file_id: 'first', filename: 'first.pdf' }); + const second = searchFile({ file_id: 'second', filename: 'second.pdf' }); + const agent = searchAgent([first, second]); + let failed = false; + const vectorImpl = jest.fn(async ({ file }: { file: TFile }) => { + if (file.file_id === second.file_id && !failed) { + failed = true; + throw new Error('temporary indexing outage'); + } + return { embedded: true, fileUpdate: { file_id: file.file_id, embedded: true } }; + }); + const { provisionFiles } = buildHarness({ contexts: [[agent.id, agent]], vectorImpl }); + await expect(provisionFiles(['file_search'], agent.id)).rejects.toThrow( + FileSearchPreparationError, + ); + expect(agent.provisionState?.vectorDBFiles).toEqual([second]); + expect(getTurnReadingContext(agent.deliveryRouting)?.searchState(first.file_id)).toBe('ready'); + expect(getTurnReadingContext(agent.deliveryRouting)?.searchState(second.file_id)).toBe( + 'failed', + ); + + await provisionFiles(['file_search'], agent.id); + expect( + vectorImpl.mock.calls.filter(([args]) => args.file.file_id === first.file_id), + ).toHaveLength(1); + expect( + vectorImpl.mock.calls.filter(([args]) => args.file.file_id === second.file_id), + ).toHaveLength(2); + expect(agent.provisionState?.vectorDBFiles).toEqual([]); + expect(agent.tool_resources?.file_search?.files?.map((file) => file?.file_id)).toEqual([ + first.file_id, + second.file_id, + ]); + }); + + it('keeps failed readiness scoped to the namespace that failed', async () => { + const first = searchFile(); + const second = searchFile(); + const agentA = searchAgent([first], 'agent-a', [first.file_id]); + const agentB = searchAgent([second], 'agent-b', [second.file_id]); + const vectorImpl = jest.fn(async ({ file, entity_id }: { file: TFile; entity_id: string }) => { + if (entity_id === agentA.id) { + return { embedded: false, fileUpdate: null }; + } + return { embedded: true, fileUpdate: { file_id: file.file_id, embedded: true } }; + }); + const { provisionFiles, addEmbeddedEntity } = buildHarness({ + contexts: [ + [agentA.id, agentA], + [agentB.id, agentB], + ], + vectorImpl, + }); + await expect(provisionFiles(['file_search'], agentA.id)).rejects.toThrow( + FileSearchPreparationError, + ); + await provisionFiles(['file_search'], agentB.id); + + expect(vectorImpl).toHaveBeenCalledTimes(2); + expect(getTurnReadingContext(agentA.deliveryRouting)?.searchState(first.file_id)).toBe( + 'failed', + ); + expect(getTurnReadingContext(agentB.deliveryRouting)?.searchState(second.file_id)).toBe( + 'ready', + ); + expect(addEmbeddedEntity).toHaveBeenCalledWith({ + file_id: second.file_id, + entityId: agentB.id, + }); + expect(agentB.tool_resources?.file_search?.file_ids).toEqual([second.file_id]); + expect(agentA.provisionState?.vectorDBFiles).toEqual([first]); + }); + + it('sanitizes operational failures at the preparation boundary', async () => { + const pending = searchFile(); + const agent = searchAgent([pending]); + const secret = 'submitted-secret Bearer private-credential'; + const vectorImpl = jest.fn().mockRejectedValue( + Object.assign(new Error(secret), { + response: { status: 503, data: secret }, + config: { headers: { Authorization: secret } }, + }), + ); + const { provisionFiles } = buildHarness({ contexts: [[agent.id, agent]], vectorImpl }); + await expect(provisionFiles(['file_search'], agent.id)).rejects.toMatchObject({ + code: 'file_search_preparation_failed', + message: + 'File Search could not prepare the attached files. Try sending the message again.\n' + + '{"code":"file_search_preparation_failed"}', + }); + expect(logger.error).toHaveBeenLastCalledWith( + expect.stringContaining(`file ${pending.file_id}`), + { type: 'Error', status: 503 }, + ); + prepareAgentFileContext(agent, [agent], 'user-1', false); + expect(JSON.stringify(agent.dynamicToolContextMap)).toContain('could not prepare it'); + expect(JSON.stringify(agent.dynamicToolContextMap)).not.toContain(secret); + expect(JSON.stringify(collectInventoryEntries(agent))).not.toContain(secret); + expect(agent.provisionState?.searchPreparation?.get(pending.file_id)).toBe('failed'); + }); + it('ignores tool batches that need neither code nor search', async () => { const { provisionFiles, provisionToCodeEnv, provisionToVectorDB } = buildHarness({ contexts: [['agent-a', { provisionState: state([makeFile()], [makeFile()]) }]], diff --git a/packages/api/src/files/provision/callback.ts b/packages/api/src/files/provision/callback.ts index 7aee5546053..1278e45b973 100644 --- a/packages/api/src/files/provision/callback.ts +++ b/packages/api/src/files/provision/callback.ts @@ -7,6 +7,7 @@ import type { CodeEnvRefUpdate, ProvisionService } from './service'; import type { ProvisionToolContext } from '../code/queued'; import type { ServerRequest } from '~/types'; import { createCodeApiRateLimitBudget, isCodeApiRateLimitError } from '~/utils'; +import { getSafeErrorMetadata } from '~/utils/errors'; import { planCodeFileUploads } from '../code/queued'; import { isCodeFileToolName } from '~/agents/tools'; @@ -38,6 +39,22 @@ async function persistWithRetry( export type { ProvisionToolContext } from '../code/queued'; +const FILE_SEARCH_PREPARATION_CODE = 'file_search_preparation_failed'; + +/** A safe boundary error: indexing failed before any search query could run. */ +export class FileSearchPreparationError extends Error { + readonly code: typeof FILE_SEARCH_PREPARATION_CODE = FILE_SEARCH_PREPARATION_CODE; + + constructor() { + // Existing stream boundaries transmit error text; the client extracts this safe code to localize it. + super( + 'File Search could not prepare the attached files. Try sending the message again.\n' + + JSON.stringify({ code: FILE_SEARCH_PREPARATION_CODE }), + ); + this.name = 'FileSearchPreparationError'; + } +} + export interface ProvisionCallbackDeps { req: ServerRequest; agentToolContexts: Map; @@ -352,9 +369,16 @@ export function createProvisionFilesCallback({ if (needsSearch && provisionState.vectorDBFiles.length > 0) { const queuedVectorFiles = provisionState.vectorDBFiles; + provisionState.searchPreparation ??= new Map(); + const searchPreparation = provisionState.searchPreparation; + for (const file of queuedVectorFiles) { + if (!searchPreparation.has(file.file_id)) { + searchPreparation.set(file.file_id, 'queued'); + } + } const results = await Promise.allSettled( queuedVectorFiles.map(async (file) => { - const result = await shareProvisioning(shareKey('search', file), async () => { + await shareProvisioning(shareKey('search', file), async () => { const provisioned = await provisionToVectorDB({ req, file, @@ -362,9 +386,14 @@ export function createProvisionFilesCallback({ signal, }); signal?.throwIfAborted(); + /* A declined embedding must reject the shared work too, so a later tool + * call retries it rather than replaying a cached embedded:false result. */ + if (!provisioned.embedded) { + throw new FileSearchPreparationError(); + } /* The vectors are already stored, so a failed flag write costs a re-embed next * turn rather than this turn's results. Logged, not fatal. */ - if (provisioned.embedded && provisioned.fileUpdate) { + if (provisioned.fileUpdate) { const update = provisioned.fileUpdate; /* Vectors live under the entity that provisioned them, so the namespace is * recorded alongside the flag. Agents sharing a record, as a duplicate does @@ -379,39 +408,34 @@ export function createProvisionFilesCallback({ (error) => logger.error( `[provisionFiles] Failed to persist embedding state for file ${update.file_id}`, - error, + getSafeErrorMetadata(error), ), ); } return provisioned; }); - if (result.embedded) { - file.embedded = true; - const namespace = namespaceForFile(file); - if (namespace != null) { - const recorded = new Set(file.metadata?.embeddedEntities ?? []); - recorded.add(namespace); - file.metadata = { ...file.metadata, embeddedEntities: [...recorded] }; - } - addProvisionedFile( - file, - EToolResources.file_search, - entityIdForFile(file) !== undefined, - ); - return; + file.embedded = true; + const namespace = namespaceForFile(file); + if (namespace != null) { + const recorded = new Set(file.metadata?.embeddedEntities ?? []); + recorded.add(namespace); + file.metadata = { ...file.metadata, embeddedEntities: [...recorded] }; } - /* Resolving with embedded false is a normal outcome of the service, returned - * when the vector store declines the file, and it means the same thing as a - * throw: the vectors are not there. Treated as a failure so the file stays - * queued and search does not proceed without it. */ - throw new Error(`Vector store did not embed "${file.filename}" (${file.file_id})`); + addProvisionedFile(file, EToolResources.file_search, entityIdForFile(file) !== undefined); + searchPreparation.set(file.file_id, 'ready'); }), ); - signal?.throwIfAborted(); results.forEach((result, index) => { if (result.status === 'rejected') { - logger.error('[provisionFiles] Vector DB provisioning failed', result.reason); - failedVectorFiles.push(queuedVectorFiles[index]); + const file = queuedVectorFiles[index]; + if (!signal?.aborted) { + logger.error( + `[provisionFiles] Vector DB provisioning failed for file ${file.file_id}`, + getSafeErrorMetadata(result.reason), + ); + } + searchPreparation.set(file.file_id, 'failed'); + failedVectorFiles.push(file); } }); /* Preserved for a later retry, and fatal for this turn like a code failure. An @@ -419,6 +443,7 @@ export function createProvisionFilesCallback({ * narrows results, but a search that silently omits the file the user asked about * is a wrong answer, not a smaller one. */ provisionState.vectorDBFiles = failedVectorFiles; + signal?.throwIfAborted(); } /* Provisioning failed outright, so the sandbox or vector store does not have the @@ -436,9 +461,7 @@ export function createProvisionFilesCallback({ ); } if (failedVectorFiles.length > 0) { - throw new Error( - `Failed to provision ${failedVectorFiles.length} file(s) for search; aborting tool execution rather than searching without them`, - ); + throw new FileSearchPreparationError(); } if (needsCode) { diff --git a/packages/api/src/files/provision/service.spec.ts b/packages/api/src/files/provision/service.spec.ts index 253c97b8dd1..df92eb04d47 100644 --- a/packages/api/src/files/provision/service.spec.ts +++ b/packages/api/src/files/provision/service.spec.ts @@ -79,6 +79,50 @@ describe('createProvisionService', () => { }); }); + describe('openStoredFile', () => { + it('opens the stored original through the source strategy', async () => { + const stream = Readable.from('a,b'); + const getDownloadStream = jest.fn().mockResolvedValue(stream); + const getStrategyFunctions = jest.fn(() => ({ getDownloadStream })); + const { service } = buildService({ getStrategyFunctions }); + + await expect( + service.openStoredFile( + { file_id: 'f1', source: 's3', storageKey: 'uploads/u1/a.csv' }, + req, + ), + ).resolves.toBe(stream); + expect(getStrategyFunctions).toHaveBeenCalledWith('s3'); + expect(getDownloadStream).toHaveBeenCalledWith(req, 'uploads/u1/a.csv', { + signal: undefined, + }); + }); + + it('reads a record without a source from local storage', async () => { + const getStrategyFunctions = jest.fn(() => ({ + getDownloadStream: jest.fn().mockResolvedValue(Readable.from('a,b')), + })); + const { service } = buildService({ getStrategyFunctions }); + + await expect( + service.openStoredFile({ file_id: 'f1', filepath: '/uploads/u1/a.csv' }, req), + ).resolves.toBeInstanceOf(Readable); + expect(getStrategyFunctions).toHaveBeenCalledWith('local'); + }); + + it('opens nothing for a source without a storage download contract or a record without a path', async () => { + const { service, getDownloadStream } = buildService(); + + await expect( + service.openStoredFile({ file_id: 'f1', source: 'text', filepath: '/x/a.csv' }, req), + ).resolves.toBeNull(); + await expect( + service.openStoredFile({ file_id: 'f1', source: 'local' }, req), + ).resolves.toBeNull(); + expect(getDownloadStream).not.toHaveBeenCalled(); + }); + }); + describe('provisionToCodeEnv', () => { it('waits out Code API throttling and reopens the upload stream', async () => { const rateLimit = new AxiosError('Request failed', 'ERR_BAD_REQUEST'); @@ -182,6 +226,24 @@ describe('createProvisionService', () => { expect(uploadCodeEnvFile).not.toHaveBeenCalled(); }); + it('reads a record without a source from local storage, as the rest of the pipeline does', async () => { + const getDownloadStream = jest.fn().mockResolvedValue(Readable.from('a,b')); + const uploadCodeEnvFile = jest + .fn() + .mockResolvedValue({ storage_session_id: 's1', file_id: 'remote-1' }); + const getStrategyFunctions = jest.fn((source: string) => + source === 'execute_code' ? { handleFileUpload: uploadCodeEnvFile } : { getDownloadStream }, + ); + const { service } = buildService({ getStrategyFunctions }); + + await expect( + service.provisionToCodeEnv({ req, file: makeFile({ source: undefined }) }), + ).resolves.toMatchObject({ referenceSet: { codeEnvRef: { file_id: 'remote-1' } } }); + + expect(getStrategyFunctions).toHaveBeenCalledWith('local'); + expect(getDownloadStream).toHaveBeenCalledWith(req, '/x/data.csv', { signal: undefined }); + }); + it('prefers a durable storage key when opening the backing object', async () => { const { service, getDownloadStream } = buildService(); diff --git a/packages/api/src/files/provision/service.ts b/packages/api/src/files/provision/service.ts index e50ba1e8f77..5f9a900f9e6 100644 --- a/packages/api/src/files/provision/service.ts +++ b/packages/api/src/files/provision/service.ts @@ -9,6 +9,7 @@ import { FileSources, getCodeEnvRefs, mergeCodeEnvRef, + isOriginalBacked, resolveSandboxFilename, } from 'librechat-data-provider'; import type { CodeEnvRef, CodeEnvRefMap, TFile } from 'librechat-data-provider'; @@ -88,8 +89,26 @@ export interface CodeEnvReferenceSetResult { codeEnvRefs?: CodeEnvRefMap; } +/** What locates a record's retained original in storage. */ +export interface StoredOriginalRef { + file_id: string; + filename?: string; + source?: string | null; + filepath?: string; + storageKey?: string | null; +} + export interface ProvisionService { loadCodeApiKey: (userId: string) => Promise; + /** + * Opens a stream of the record's retained original, or null when its source has no storage + * download contract or the record names no stored object. A missing source reads as `local`. + */ + openStoredFile: ( + file: StoredOriginalRef, + req: ServerRequest, + signal?: AbortSignal, + ) => Promise; provisionToCodeEnv: (params: { req: ServerRequest; file: TFile; @@ -155,37 +174,34 @@ export function createProvisionService({ let axiosInstance; const getAxios = () => (axiosInstance ??= createAxiosInstance()); - /* Sources whose `getDownloadStream` takes `(req, filepath)` and returns a readable. - * Others diverge: `openai` takes `(file_id, client)` and `execute_code` takes - * `(fileIdentifier, identity, req)` returning an Axios response, so calling them - * through this contract fails. An allowlist keeps an unfamiliar source skipped - * rather than mis-invoked. */ - const STORAGE_STREAM_SOURCES = new Set([ - FileSources.local, - FileSources.s3, - FileSources.cloudfront, - FileSources.azure_blob, - FileSources.firebase, - ]); - - /** Resolves a storage download stream, or null when the source uses a different contract. */ + /** + * Resolves a storage download stream, or null when the source uses a different contract. + * Only original-backed sources take `(req, filepath)` and return a readable. Others diverge: + * `openai` takes `(file_id, client)` and `execute_code` takes `(fileIdentifier, identity, req)` + * returning an Axios response, so calling them through this contract fails. The allowlist + * keeps an unfamiliar source skipped rather than mis-invoked. + */ async function getStorageStream( - file: TFile, + file: StoredOriginalRef, req: ServerRequest, signal?: AbortSignal, ): Promise { - if (file.source == null || !STORAGE_STREAM_SOURCES.has(file.source)) { + if (!isOriginalBacked(file)) { logger.warn( `[provision] Cannot stream "${file.filename}" (${file.file_id}) from source "${file.source}": unsupported download contract`, ); return null; } - const { getDownloadStream } = getStrategyFunctions(file.source as string); + const downloadPath = resolveDownloadPath({ ...file, filepath: file.filepath ?? '' }); + if (!downloadPath) { + return null; + } + const { getDownloadStream } = getStrategyFunctions(file.source ?? FileSources.local); if (!getDownloadStream) { return null; } signal?.throwIfAborted(); - const stream = await getDownloadStream(req, resolveDownloadPath(file), { signal }); + const stream = await getDownloadStream(req, downloadPath, { signal }); if (signal?.aborted) { stream.destroy(signal.reason instanceof Error ? signal.reason : undefined); signal.throwIfAborted(); @@ -688,6 +704,7 @@ export function createProvisionService({ } return { loadCodeApiKey, + openStoredFile: getStorageStream, provisionToCodeEnv, provisionToVectorDB, checkCodeEnvFileAlive, diff --git a/packages/api/src/files/reading/derive.spec.ts b/packages/api/src/files/reading/derive.spec.ts new file mode 100644 index 00000000000..ed4a2a5fb08 --- /dev/null +++ b/packages/api/src/files/reading/derive.spec.ts @@ -0,0 +1,398 @@ +import fs from 'fs'; +import os from 'os'; +import path from 'path'; +import { Readable } from 'stream'; +import { FileSources } from 'librechat-data-provider'; +import type { FiltersConfig } from 'librechat-data-provider'; +import type { UploadFallbackTextExtractors } from '~/files/upload/fallback'; +import type { ProvisionService } from '~/files/provision/service'; +import type { TurnReadingFile } from './turn'; +import type { ServerRequest } from '~/types'; +import { DOCUMENT_PARSER_MAX_FILE_SIZE, parseDocument } from '~/files/documents/crud'; +import { AttachmentObjectNotFoundError } from '~/files/encode/utils'; +import { MAX_STORED_EXTRACTED_TEXT_BYTES } from '~/files/extract'; +import { ContentFilterError } from '~/middleware/contentFilter'; +import { UninspectableFileError } from '~/protection/files'; +import { createFileTextDeriver } from './derive'; +import { parseTextNative } from '~/files/text'; + +type OpenStoredFile = ProvisionService['openStoredFile']; + +const XLSX = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'; +const documentsDir = path.join(__dirname, '../documents'); + +let workDir: string; +let sequence = 0; + +beforeAll(() => { + workDir = fs.mkdtempSync(path.join(os.tmpdir(), 'reading-derive-')); +}); + +afterAll(() => { + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +const newRequest = (): ServerRequest => ({}) as ServerRequest; + +/** A deferred message attachment whose original lives at `sourcePath` in local storage. */ +function storedFile( + overrides: Partial & Pick, +): TurnReadingFile { + sequence += 1; + return { + file_id: `derive-spec-${process.pid}-${sequence}`, + source: FileSources.local, + filepath: `/uploads/u1/${overrides.filename}`, + bytes: 1024, + llmDeliveryPath: 'none', + metadata: { destinationChosen: false, textDerivation: { outcome: 'deferred' } }, + ...overrides, + }; +} + +/** Local storage: every open streams the file at `sourcePath`, observed. */ +const localStorage = (sourcePath: string) => + jest.fn, Parameters>(async () => + fs.createReadStream(sourcePath), + ); + +function writeSource(name: string, content: string | Buffer): string { + const filePath = path.join(workDir, name); + fs.writeFileSync(filePath, content); + return filePath; +} + +const tempCopies = (file: TurnReadingFile): string[] => + fs.readdirSync(os.tmpdir()).filter((name) => name.startsWith(`derive-${file.file_id}-`)); + +/** The real built-in extractors, observed. */ +function spiedExtractors() { + return { + parseDocument: jest.fn(parseDocument), + parseTextNative: jest.fn(parseTextNative), + } satisfies UploadFallbackTextExtractors; +} + +describe('createFileTextDeriver', () => { + it('derives a workbook from its stored original with the document parser', async () => { + const req = newRequest(); + const file = storedFile({ type: XLSX, filename: 'sample.xlsx' }); + const samplePath = path.join(documentsDir, 'sample.xlsx'); + const openStoredFile = localStorage(samplePath); + const extractors = spiedExtractors(); + + const result = await createFileTextDeriver({ req, openStoredFile, extractors })(file); + + expect(result).toEqual({ + status: 'derived', + text: 'Sheet One:\nData,on,first,sheet\nSecond Sheet:\nData,On\nSecond,Sheet\n', + textDerivation: { outcome: 'complete', extractor: 'document_parser', at: expect.any(Number) }, + }); + expect(openStoredFile).toHaveBeenCalledWith(file, req, undefined); + expect(extractors.parseDocument).toHaveBeenCalledWith({ + file: expect.objectContaining({ + originalname: 'sample.xlsx', + mimetype: XLSX, + size: fs.statSync(samplePath).size, + }), + }); + expect(extractors.parseTextNative).not.toHaveBeenCalled(); + expect(tempCopies(file)).toEqual([]); + }); + + it('derives a CSV with the native reader', async () => { + const content = 'region,total\nwest,4'; + const file = storedFile({ type: 'text/csv', filename: 'sales.csv', bytes: content.length }); + const extractors = spiedExtractors(); + + const result = await createFileTextDeriver({ + req: newRequest(), + openStoredFile: localStorage(writeSource('sales.csv', content)), + extractors, + })(file); + + expect(result).toEqual({ + status: 'derived', + text: content, + textDerivation: { outcome: 'complete', extractor: 'native_text', at: expect.any(Number) }, + }); + expect(extractors.parseTextNative).toHaveBeenCalledTimes(1); + expect(extractors.parseDocument).not.toHaveBeenCalled(); + }); + + it('fails too_large from the record alone, without opening the original', async () => { + const openStoredFile = localStorage(path.join(documentsDir, 'sample.xlsx')); + const derive = createFileTextDeriver({ req: newRequest(), openStoredFile }); + + await expect( + derive( + storedFile({ + type: XLSX, + filename: 'big.xlsx', + bytes: DOCUMENT_PARSER_MAX_FILE_SIZE + 1, + }), + ), + ).resolves.toEqual({ + status: 'failed', + textDerivation: { + outcome: 'failed', + extractor: 'document_parser', + reason: 'too_large', + at: expect.any(Number), + }, + persist: true, + }); + await expect( + derive( + storedFile({ + type: 'text/csv', + filename: 'big.csv', + bytes: MAX_STORED_EXTRACTED_TEXT_BYTES + 1, + }), + ), + ).resolves.toMatchObject({ status: 'failed', textDerivation: { reason: 'too_large' } }); + expect(openStoredFile).not.toHaveBeenCalled(); + }); + + const flagging: FiltersConfig = { + files: { + pii: { + fields: ['extracted_text'], + starterPatterns: [], + customPatterns: [{ id: 'private', label: 'private token', regex: 'PRIVATE-[A-Z]+' }], + }, + }, + }; + const blocking: FiltersConfig = { + files: { pii: { fields: ['extracted_text'], starterPatterns: [], uninspectable: 'block' } }, + }; + + it.each< + [string, FiltersConfig, string, typeof ContentFilterError | typeof UninspectableFileError] + >([ + ['flags', flagging, 'token,PRIVATE-SECRET', ContentFilterError], + ['cannot inspect', blocking, '', UninspectableFileError], + ])( + 'blocks text a content policy %s with its policy error', + async (_case, filters, content, error) => { + await expect( + createFileTextDeriver({ + req: newRequest(), + filters, + openStoredFile: localStorage(writeSource('blocked.csv', content)), + })(storedFile({ type: 'text/csv', filename: 'blocked.csv' })), + ).resolves.toEqual({ status: 'blocked', error: expect.any(error) }); + }, + ); + + it('fails original_missing, which persists, when storage no longer holds the original', async () => { + const missing = [ + localStorage(path.join(workDir, 'deleted.csv')), + jest.fn, Parameters>(async (file) => { + throw new AttachmentObjectNotFoundError(file.file_id); + }), + jest.fn, Parameters>(async () => { + throw Object.assign(new Error('The specified key does not exist.'), { name: 'NoSuchKey' }); + }), + ]; + for (const openStoredFile of missing) { + const file = storedFile({ type: 'text/csv', filename: 'deleted.csv' }); + await expect( + createFileTextDeriver({ req: newRequest(), openStoredFile })(file), + ).resolves.toEqual({ + status: 'failed', + textDerivation: { + outcome: 'failed', + extractor: 'native_text', + reason: 'original_missing', + at: expect.any(Number), + }, + persist: true, + }); + expect(tempCopies(file)).toEqual([]); + } + }); + + it('skips a transient storage error and a source with no download contract', async () => { + const throwing = (error: Error) => + jest.fn, Parameters>(async () => { + throw error; + }); + const unreachable = throwing(new Error('socket hang up')); + const missingBucket = throwing( + Object.assign(new Error('The specified bucket does not exist'), { + name: 'NoSuchBucket', + $metadata: { httpStatusCode: 404 }, + }), + ); + const bareNotFound = throwing(Object.assign(new Error('Not Found'), { status: 404 })); + const noContract = jest.fn, Parameters>( + async () => null, + ); + + for (const openStoredFile of [unreachable, missingBucket, bareNotFound, noContract]) { + await expect( + createFileTextDeriver({ req: newRequest(), openStoredFile })( + storedFile({ type: 'text/csv', filename: 'later.csv' }), + ), + ).resolves.toEqual({ status: 'skipped', reason: 'storage_unavailable' }); + } + }); + + it('skips, without keeping a marker, a system error the extractor hits reading the copy', async () => { + const extractors = { + parseDocument: jest.fn(parseDocument), + parseTextNative: jest.fn(async ({ path: tmpPath }: Express.Multer.File) => { + throw Object.assign(new Error('EBUSY: resource busy or locked'), { + code: 'EBUSY', + errno: -16, + path: tmpPath, + }); + }), + } satisfies UploadFallbackTextExtractors; + await expect( + createFileTextDeriver({ + req: newRequest(), + openStoredFile: localStorage(writeSource('busy.csv', 'a,b\n1,2\n')), + extractors, + })(storedFile({ type: 'text/csv', filename: 'busy.csv' })), + ).resolves.toEqual({ status: 'skipped', reason: 'storage_unavailable' }); + expect(extractors.parseTextNative).toHaveBeenCalledTimes(1); + }); + + it('keeps a parser failure the extractor raised without a system code', async () => { + const extractors = { + parseDocument: jest.fn(parseDocument), + parseTextNative: jest.fn(async () => { + throw Object.assign(new Error('Invalid structure'), { code: 'ERR_INVALID_ARG_VALUE' }); + }), + } satisfies UploadFallbackTextExtractors; + await expect( + createFileTextDeriver({ + req: newRequest(), + openStoredFile: localStorage(writeSource('broken.csv', 'a,b')), + extractors, + })(storedFile({ type: 'text/csv', filename: 'broken.csv' })), + ).resolves.toMatchObject({ + status: 'failed', + textDerivation: { outcome: 'failed', reason: 'parser' }, + persist: true, + }); + }); + + it('skips a natively readable type the text allowlist excludes without opening it', async () => { + const openStoredFile = localStorage(writeSource('notes.csv', 'a,b\n1,2\n')); + const textMimeTypes = [/^application\/json$/]; + + await expect( + createFileTextDeriver({ req: newRequest(), openStoredFile, textMimeTypes })( + storedFile({ type: 'text/csv', filename: 'notes.csv' }), + ), + ).resolves.toEqual({ status: 'skipped', reason: 'no_extractor' }); + expect(openStoredFile).not.toHaveBeenCalled(); + }); + + it('parses a document type whatever the text allowlist says, as upload extraction does', async () => { + const openStoredFile = localStorage(path.join(documentsDir, 'sample.xlsx')); + const extractors = spiedExtractors(); + + const result = await createFileTextDeriver({ + req: newRequest(), + openStoredFile, + extractors, + textMimeTypes: [/^application\/json$/], + })(storedFile({ type: XLSX, filename: 'sample.xlsx' })); + + expect(result).toMatchObject({ status: 'derived' }); + expect(extractors.parseDocument).toHaveBeenCalledTimes(1); + }); + + it('skips a type no built-in extractor reads without opening it', async () => { + const openStoredFile = localStorage(path.join(documentsDir, 'sample.xlsx')); + + await expect( + createFileTextDeriver({ req: newRequest(), openStoredFile })( + storedFile({ type: 'application/zip', filename: 'bundle.zip' }), + ), + ).resolves.toEqual({ status: 'skipped', reason: 'no_extractor' }); + expect(openStoredFile).not.toHaveBeenCalled(); + }); + + it('removes the temporary copy when the request is aborted mid-download', async () => { + const controller = new AbortController(); + const file = storedFile({ type: 'text/csv', filename: 'slow.csv' }); + const source = new Readable({ read() {} }); + source.push('region,total\n'); + let markOpened!: () => void; + const opened = new Promise((resolve) => { + markOpened = resolve; + }); + const openStoredFile = jest.fn, Parameters>( + async () => { + markOpened(); + return source; + }, + ); + + const pending = createFileTextDeriver({ req: newRequest(), openStoredFile })( + file, + controller.signal, + ); + await opened; + for (let attempt = 0; attempt < 50 && tempCopies(file).length === 0; attempt++) { + await new Promise((resolve) => setTimeout(resolve, 10)); + } + expect(tempCopies(file)).toHaveLength(1); + + controller.abort(); + + await expect(pending).resolves.toEqual({ status: 'skipped', reason: 'aborted' }); + expect(tempCopies(file)).toEqual([]); + expect(source.destroyed).toBe(true); + }); + + it('skips, without keeping anything, when the temporary copy cannot be written', async () => { + const file = storedFile({ type: 'text/csv', filename: 'nowhere.csv' }); + const tmpdir = jest.spyOn(os, 'tmpdir').mockReturnValue(path.join(workDir, 'missing-tmp')); + try { + await expect( + createFileTextDeriver({ + req: newRequest(), + openStoredFile: localStorage(writeSource('nowhere.csv', 'a,b')), + })(file), + ).resolves.toEqual({ status: 'skipped', reason: 'storage_unavailable' }); + } finally { + tmpdir.mockRestore(); + } + }); + + it('derives each file once per deriver, which the host builds once per request', async () => { + const file = storedFile({ type: 'text/csv', filename: 'once.csv' }); + const openStoredFile = localStorage(writeSource('once.csv', 'a,b')); + const derive = createFileTextDeriver({ req: newRequest(), openStoredFile }); + + const [a, b] = await Promise.all([derive(file), derive({ ...file })]); + + expect(a).toBe(b); + expect(a).toMatchObject({ status: 'derived', text: 'a,b' }); + expect(openStoredFile).toHaveBeenCalledTimes(1); + + await createFileTextDeriver({ req: newRequest(), openStoredFile })(file); + expect(openStoredFile).toHaveBeenCalledTimes(2); + }); + + it('forgets a derivation its caller aborted, so the next caller derives under its own signal', async () => { + const file = storedFile({ type: 'text/csv', filename: 'retry.csv' }); + const openStoredFile = localStorage(writeSource('retry.csv', 'a,b')); + const derive = createFileTextDeriver({ req: newRequest(), openStoredFile }); + + await expect(derive(file, AbortSignal.abort())).resolves.toEqual({ + status: 'skipped', + reason: 'aborted', + }); + await new Promise((resolve) => setImmediate(resolve)); + + await expect(derive(file)).resolves.toMatchObject({ status: 'derived', text: 'a,b' }); + expect(openStoredFile).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/api/src/files/reading/derive.ts b/packages/api/src/files/reading/derive.ts new file mode 100644 index 00000000000..ac7d440aa10 --- /dev/null +++ b/packages/api/src/files/reading/derive.ts @@ -0,0 +1,286 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { randomUUID } from 'node:crypto'; +import { pipeline } from 'node:stream/promises'; +import { logger } from '@librechat/data-schemas'; +import { fileConfig, selectBuiltInTextPlan } from 'librechat-data-provider'; +import type { + FiltersConfig, + BuiltInTextPlan, + TextDerivation, + RegexLike, +} from 'librechat-data-provider'; +import type { + DerivedText, + TurnReadingFile, + FileTextDeriver, + DerivationSkipReason, + DerivationPolicyError, +} from './turn'; +import type { UploadFallbackTextExtractors, BoundedText } from '~/files/upload/fallback'; +import type { ProvisionService } from '~/files/provision/service'; +import type { ServerRequest } from '~/types'; +import { DOCUMENT_PARSER_MAX_FILE_SIZE } from '~/files/documents/crud'; +import { isAttachmentObjectNotFoundError } from '~/files/encode/utils'; +import { MAX_STORED_EXTRACTED_TEXT_BYTES } from '~/files/extract'; +import { ContentFilterError } from '~/middleware/contentFilter'; +import { runGuardedEncode } from '~/files/encode/memoryGuard'; +import { extractBoundedText } from '~/files/upload/fallback'; +import { UninspectableFileError } from '~/protection/files'; +import { getSafeErrorMetadata } from '~/utils/errors'; + +export interface FileTextDeriverDeps { + req: ServerRequest; + /** Opens the record's retained original; the provision service's `openStoredFile`. */ + openStoredFile: ProvisionService['openStoredFile']; + filters?: FiltersConfig; + /** The built-in extractors; RAG and OCR never run at turn time. */ + extractors?: UploadFallbackTextExtractors; + /** + * The deployment's text allowlist (`fileConfig.text.supportedMimeTypes`). A natively readable + * type outside it is not parsed, as upload extraction refuses it; absent means unrestricted. + */ + textMimeTypes?: RegexLike[]; +} + +/** The largest file each built-in extractor accepts, checked before anything is downloaded. */ +const PLAN_MAX_BYTES: Readonly> = { + document_parser: DOCUMENT_PARSER_MAX_FILE_SIZE, + native_text: MAX_STORED_EXTRACTED_TEXT_BYTES, +}; + +/** Codes storage backends report for one missing object, as opposed to a missing bucket. */ +const OBJECT_NOT_FOUND_CODES: ReadonlySet = new Set([ + 'NoSuchKey', + 'BlobNotFound', + 'storage/object-not-found', +]); + +interface StorageErrorShape { + name?: string; + code?: string; + path?: string; +} + +/** + * Whether storage reports the original itself gone. A missing bucket or container, a status code + * alone, and any error on the temporary copy are not: those clear up, so a later turn retries. + * The same holds inside the extractor: a system error while it reads the temporary copy is not + * kept on the record, only a failure the parser names. + */ +function isOriginalMissing(error: unknown, tmpPath: string): boolean { + if (isAttachmentObjectNotFoundError(error)) { + return true; + } + if (typeof error !== 'object' || error == null) { + return false; + } + const storageError: StorageErrorShape = error; + if (storageError.code === 'ENOENT') { + return storageError.path != null && storageError.path !== tmpPath; + } + return ( + OBJECT_NOT_FOUND_CODES.has(storageError.name ?? '') || + OBJECT_NOT_FOUND_CODES.has(storageError.code ?? '') + ); +} + +const skipped = (reason: DerivationSkipReason): DerivedText => ({ status: 'skipped', reason }); + +const failed = ( + extractor: BuiltInTextPlan, + reason: NonNullable, +): DerivedText => ({ + status: 'failed', + textDerivation: { outcome: 'failed', extractor, reason, at: Date.now() }, + persist: true, +}); + +/** A policy refusal of the derived text, logged with the rule that refused it. */ +function blocked( + file: TurnReadingFile, + result: Exclude, +): DerivationPolicyError { + if (result.failure === 'policy') { + logger.warn( + `[readingText] file_id=${file.file_id} outcome=blocked rule=${result.finding.ruleId}`, + ); + return new ContentFilterError(result.finding); + } + logger.warn(`[readingText] file_id=${file.file_id} outcome=blocked reason=uninspectable`); + return result.error instanceof UninspectableFileError + ? result.error + : new UninspectableFileError('extracted_text'); +} + +function toDerivedText( + file: TurnReadingFile, + plan: BuiltInTextPlan, + result: BoundedText, +): DerivedText { + if ('text' in result) { + return { + status: 'derived', + text: result.text, + textDerivation: { outcome: 'complete', extractor: plan, at: Date.now() }, + }; + } + if (result.failure === 'policy' || result.failure === 'uninspectable') { + return { status: 'blocked', error: blocked(file, result) }; + } + if (result.failure === 'unavailable') { + logger.error( + `[readingText] file_id=${file.file_id} outcome=skipped reason=storage_unavailable`, + getSafeErrorMetadata(result.error), + ); + return skipped('storage_unavailable'); + } + return failed(plan, result.failure); +} + +function classifyReadError( + file: TurnReadingFile, + plan: BuiltInTextPlan, + error: unknown, + { signal, tmpPath }: { signal?: AbortSignal; tmpPath: string }, +): DerivedText { + if (signal?.aborted) { + return skipped('aborted'); + } + if (isOriginalMissing(error, tmpPath)) { + return failed(plan, 'original_missing'); + } + logger.error( + `[readingText] file_id=${file.file_id} outcome=skipped reason=storage_unavailable`, + getSafeErrorMetadata(error), + ); + return skipped('storage_unavailable'); +} + +function describeDerivation(result: DerivedText): string { + if (result.status === 'derived') { + return `outcome=complete extractor=${result.textDerivation.extractor}`; + } + if (result.status === 'failed') { + return `outcome=failed reason=${result.textDerivation.reason}`; + } + if (result.status === 'blocked') { + return 'outcome=blocked'; + } + return `outcome=skipped reason=${result.reason}`; +} + +/** An outcome a later caller should not inherit: a derivation its own caller's signal cut short. */ +const isAborted = (result: DerivedText): boolean => + result.status === 'skipped' && result.reason === 'aborted'; + +/** + * Builds a request's text deriver: reads a record's retained original back from storage and runs + * the one built-in extractor its type allows, under the same caps as upload extraction. Size caps + * are checked from the record before anything is downloaded; the download runs inside the encode + * memory guard, the parser checks the size actually downloaded, and the temporary copy is always + * removed. The deriver derives each file at most once, so the host builds one per request and + * hands it to every agent. A deterministic failure may be kept on the record and a policy refusal + * blocks the turn that needed the text. An unreachable store is not kept, and an aborted + * derivation is forgotten, so the next caller derives under its own signal. + */ +export function createFileTextDeriver({ + req, + openStoredFile, + filters, + extractors, + textMimeTypes, +}: FileTextDeriverDeps): FileTextDeriver { + const derivations = new Map>(); + + const readOriginal = async ( + file: TurnReadingFile, + plan: BuiltInTextPlan, + signal?: AbortSignal, + ): Promise => { + const tmpPath = path.join( + os.tmpdir(), + `derive-${file.file_id}-${randomUUID()}${path.extname(file.filename ?? '')}`, + ); + try { + signal?.throwIfAborted(); + const stream = await openStoredFile(file, req, signal); + if (stream == null) { + return skipped('storage_unavailable'); + } + await pipeline(stream, fs.createWriteStream(tmpPath), { signal }); + const { size } = await fs.promises.stat(tmpPath); + const upload = { + path: tmpPath, + originalname: file.filename ?? path.basename(tmpPath), + mimetype: file.type ?? '', + size, + } as Express.Multer.File; + const result = await extractBoundedText({ file: upload, filters, plan, extractors }); + if (signal?.aborted) { + return skipped('aborted'); + } + return toDerivedText(file, plan, result); + } catch (error) { + return classifyReadError(file, plan, error, { signal, tmpPath }); + } finally { + await fs.promises.rm(tmpPath, { force: true }).catch(() => undefined); + } + }; + + const deriveOnce = (file: TurnReadingFile, signal?: AbortSignal): Promise => { + const type = file.type ?? ''; + const plan = selectBuiltInTextPlan(type); + if (plan == null) { + return Promise.resolve(skipped('no_extractor')); + } + /* Upload extraction consults the allowlist only for types the native text reader takes; + * document types reach the parser regardless, so the same holds here. */ + if ( + plan === 'native_text' && + textMimeTypes != null && + !fileConfig.checkType(type, textMimeTypes) + ) { + return Promise.resolve(skipped('no_extractor')); + } + const bytes = file.bytes ?? 0; + if (bytes > PLAN_MAX_BYTES[plan]) { + return Promise.resolve(failed(plan, 'too_large')); + } + return runGuardedEncode(bytes, () => readOriginal(file, plan, signal)); + }; + + const derive = async (file: TurnReadingFile, signal?: AbortSignal): Promise => { + const startedAt = Date.now(); + const result = await deriveOnce(file, signal); + logger.debug( + `[readingText] file_id=${file.file_id} ${describeDerivation(result)} ms=${Date.now() - startedAt}`, + ); + return result; + }; + + const forget = (fileId: string, derivation: Promise): void => { + if (derivations.get(fileId) === derivation) { + derivations.delete(fileId); + } + }; + + return (file, signal) => { + const existing = derivations.get(file.file_id); + if (existing != null) { + return existing; + } + const derivation = derive(file, signal); + derivations.set(file.file_id, derivation); + derivation.then( + (result) => { + if (isAborted(result)) { + forget(file.file_id, derivation); + } + }, + () => forget(file.file_id, derivation), + ); + return derivation; + }; +} diff --git a/packages/api/src/files/reading/diagnostics.spec.ts b/packages/api/src/files/reading/diagnostics.spec.ts new file mode 100644 index 00000000000..6f5e0787d96 --- /dev/null +++ b/packages/api/src/files/reading/diagnostics.spec.ts @@ -0,0 +1,120 @@ +import { logger } from '@librechat/data-schemas'; +import { FileContext, FileSources } from 'librechat-data-provider'; +import type { TFileConfig, TurnFileConsumers } from 'librechat-data-provider'; +import type { FileTextDeriver, TurnReadingFile } from './turn'; +import { resolveTurnDeliveryRouting } from '~/agents/files/delivery'; +import { logTurnReading, logAllocation } from './diagnostics'; +import { buildTurnReadingContext } from './turn'; + +type EndpointFileConfigInput = NonNullable[string]; + +const searchOnly: TurnFileConsumers = { executeCode: false, fileSearch: true }; + +const attachment = ( + overrides: Partial & Pick, +): TurnReadingFile => ({ + type: 'application/pdf', + bytes: 1024, + filename: 'quarterly-report.pdf', + source: FileSources.local, + context: FileContext.message_attachment, + llmDeliveryPath: 'provider', + metadata: { destinationChosen: false }, + ...overrides, +}); + +const deriveText: FileTextDeriver = async () => ({ status: 'skipped', reason: 'policy' }); + +function agentFor(endpointConfig: EndpointFileConfigInput, withContext = true) { + const deliveryRouting = resolveTurnDeliveryRouting({ + agent: { provider: 'openAI', endpoint: 'openAI' }, + config: { fileConfig: { endpoints: { openAI: endpointConfig } } }, + }); + const context = withContext + ? buildTurnReadingContext({ + routing: deliveryRouting, + provider: 'openAI', + fileTokenLimit: 100_000, + configuredFileSizeLimit: 1024, + countTokens: (text) => text.length, + deriveText, + }) + : undefined; + if (context != null) { + deliveryRouting.reading = context; + } + return { + agent: { + id: 'agent_1', + deliveryRouting, + fileConsumers: searchOnly, + currentRequestAttachments: [ + attachment({ file_id: 'fits' }), + attachment({ file_id: 'large', bytes: 4096 }), + ], + }, + context, + }; +} + +describe('reading diagnostics', () => { + let info: jest.SpyInstance; + let debug: jest.SpyInstance; + + beforeEach(() => { + info = jest.spyOn(logger, 'info').mockImplementation(() => logger); + debug = jest.spyOn(logger, 'debug').mockImplementation(() => logger); + }); + + afterEach(() => { + info.mockRestore(); + debug.mockRestore(); + }); + + it('logs a summary and one line per file under the automatic policy, without filenames', () => { + const { agent, context } = agentFor({ llmDeliveryPolicy: 'automatic' }); + context?.recordDropped([attachment({ file_id: 'dropped' })]); + + logTurnReading(agent, 'init'); + + expect(info).toHaveBeenCalledTimes(1); + expect(info).toHaveBeenCalledWith( + '[fileReading] agent=agent_1 policy=automatic pass=init files=2 provider=1 text=0 search=1 code=0 unavailable=0 overflow=0 derived=0 rejected=0 dropped=1', + ); + expect(debug.mock.calls.map(([line]) => line)).toEqual([ + '[fileReading] file_id=fits category=document reader=provider path=provider classic=provider reason=native_supported skipped=none code=no_run_code', + '[fileReading] file_id=large category=document reader=search path=none classic=provider reason=native_capacity skipped=provider:native_capacity code=no_run_code', + ]); + const lines = [...info.mock.calls, ...debug.mock.calls].map(([line]) => String(line)); + expect(lines.some((line) => line.includes('quarterly-report'))).toBe(false); + }); + + it.each([ + ['without a reading context', { llmDeliveryPolicy: 'automatic' as const }, false], + ['under the classic policy with a derive-only context', {}, true], + ])('stays silent %s', (_label, endpointConfig, withContext) => { + const { agent } = agentFor(endpointConfig, withContext); + + logTurnReading(agent, 'final'); + + expect(info).not.toHaveBeenCalled(); + expect(debug).not.toHaveBeenCalled(); + }); + + it('stays silent for a turn with nothing to report', () => { + const { agent } = agentFor({ llmDeliveryPolicy: 'automatic' }); + + logTurnReading({ ...agent, currentRequestAttachments: [] }, 'run'); + + expect(info).not.toHaveBeenCalled(); + }); + + it('logs an allocation at debug', () => { + logAllocation({ agentId: 'agent_1', scope: 'history', candidates: 4, overflow: 1 }); + + expect(debug).toHaveBeenCalledWith( + '[allocation] agent=agent_1 scope=history candidates=4 overflow=1', + ); + expect(info).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/api/src/files/reading/diagnostics.ts b/packages/api/src/files/reading/diagnostics.ts new file mode 100644 index 00000000000..5818e18340c --- /dev/null +++ b/packages/api/src/files/reading/diagnostics.ts @@ -0,0 +1,102 @@ +import { logger } from '@librechat/data-schemas'; +import { decideFileReading } from 'librechat-data-provider'; +import type { + FileReading, + TurnDeliveryFile, + TurnFileConsumers, + TurnDeliveryRouting, +} from 'librechat-data-provider'; +import { getTurnReadingContext } from './turn'; + +/** Which reading the line describes: initialization's first or final decision, or a run's. */ +export type ReadingPass = 'init' | 'final' | 'run'; + +/** Whether an allocation covered the request alone or the request after its history. */ +export type AllocationScope = 'request' | 'history'; + +/** The fields of an initialized agent the turn's reading diagnostics read. */ +export interface TurnReadingAgent { + id?: string; + deliveryRouting?: TurnDeliveryRouting; + fileConsumers?: TurnFileConsumers; + requestAttachments?: readonly TurnDeliveryFile[]; + currentRequestAttachments?: readonly TurnDeliveryFile[]; +} + +const SUMMARY_READERS = ['provider', 'text', 'search', 'code', 'unavailable'] as const; + +type ReaderCounts = Record<(typeof SUMMARY_READERS)[number], number>; + +const formatSkipped = ({ skipped }: FileReading): string => + skipped.length === 0 + ? 'none' + : skipped.map(({ reader, reason }) => `${reader}:${reason}`).join(','); + +const formatFileReading = (fileId: string | undefined, reading: FileReading): string => + [ + `[fileReading] file_id=${fileId ?? 'unknown'}`, + `category=${reading.category}`, + `reader=${reading.reader}`, + `path=${reading.path ?? 'unset'}`, + `classic=${reading.classicPath ?? 'unset'}`, + `reason=${reading.reason}`, + `skipped=${formatSkipped(reading)}`, + `code=${reading.code}`, + ...(reading.needsText ? ['text=derive'] : []), + ].join(' '); + +/** + * Logs how the automatic policy reads the agent's current request attachments: one summary line + * at info and one line per file at debug. Lines carry file ids, codes and provider names only. + * Silent under the classic policy. + */ +export function logTurnReading(agent: TurnReadingAgent, pass: ReadingPass): void { + const routing = agent.deliveryRouting; + const context = getTurnReadingContext(routing); + if (routing == null || context == null || context.policy !== 'automatic') { + return; + } + const files = agent.currentRequestAttachments ?? agent.requestAttachments ?? []; + const stats = context.stats(); + if (files.length === 0 && stats.dropped === 0) { + return; + } + const counts: ReaderCounts = { provider: 0, text: 0, search: 0, code: 0, unavailable: 0 }; + const lines: string[] = []; + for (const file of files) { + const reading = decideFileReading({ routing, file, consumers: agent.fileConsumers }); + if (reading.reader !== 'unresolved') { + counts[reading.reader] += 1; + } + lines.push(formatFileReading(file.file_id, reading)); + } + logger.info( + [ + `[fileReading] agent=${agent.id ?? 'unknown'} policy=automatic pass=${pass}`, + `files=${files.length}`, + ...SUMMARY_READERS.map((reader) => `${reader}=${counts[reader]}`), + `overflow=${stats.overflow} derived=${stats.derived}`, + `rejected=${stats.rejected} dropped=${stats.dropped}`, + ].join(' '), + ); + for (const line of lines) { + logger.debug(line); + } +} + +/** Logs one first-fit allocation of direct model content. */ +export function logAllocation({ + agentId, + scope, + candidates, + overflow, +}: { + agentId?: string; + scope: AllocationScope; + candidates: number; + overflow: number; +}): void { + logger.debug( + `[allocation] agent=${agentId ?? 'unknown'} scope=${scope} candidates=${candidates} overflow=${overflow}`, + ); +} diff --git a/packages/api/src/files/reading/history.spec.ts b/packages/api/src/files/reading/history.spec.ts new file mode 100644 index 00000000000..1a5a23cdc7d --- /dev/null +++ b/packages/api/src/files/reading/history.spec.ts @@ -0,0 +1,557 @@ +import { FileContext, FileSources, decideFileReading } from 'librechat-data-provider'; +import type { TFileConfig, TurnFileConsumers, TurnDeliveryRouting } from 'librechat-data-provider'; +import type { AppConfig } from '@librechat/data-schemas'; +import type { TurnAttachmentsWithHistoryParams, HistoryAllocationFile } from './history'; +import type { FileTextDeriver, TurnReadingContext } from './turn'; +import { + isModelBoundAttachmentFile, + assertAgentAttachmentLimits, + collectHistoricalAttachmentIds, +} from '~/agents/attachments'; +import { + allocateTurnAttachmentsWithHistory, + allocateTurnAttachmentsWithRetainedContext, +} from './history'; +import { applyTurnDelivery, resolveTurnDeliveryRouting } from '~/agents/files/delivery'; +import { buildTurnReadingContext } from './turn'; + +type EndpointFileConfigInput = NonNullable[string]; +type AllocationParams = TurnAttachmentsWithHistoryParams; + +const MB = 1024 * 1024; +const PDF = 'application/pdf'; +const XLSX = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'; +const ENDPOINT = 'openAI'; + +const noReader: TurnFileConsumers = { executeCode: false, fileSearch: false }; +const searchOnly: TurnFileConsumers = { executeCode: false, fileSearch: true }; + +const attachment = ( + overrides: Partial & Pick, +): HistoryAllocationFile => ({ + type: PDF, + bytes: 3 * MB, + source: FileSources.local, + context: FileContext.message_attachment, + llmDeliveryPath: 'provider', + metadata: { destinationChosen: false }, + ...overrides, +}); + +const workbook = (file_id: string, text: string): HistoryAllocationFile => + attachment({ file_id, type: XLSX, bytes: 1024, llmDeliveryPath: 'text', text }); + +interface Turn { + agent: AllocationParams['agent']; + req: NonNullable; + routing: TurnDeliveryRouting; + context: TurnReadingContext; +} + +function setup({ + endpointConfig = { llmDeliveryPolicy: 'automatic' }, + fileConfig = {}, + consumers = searchOnly, + deriveText, +}: { + endpointConfig?: EndpointFileConfigInput; + fileConfig?: Omit; + consumers?: TurnFileConsumers; + deriveText?: FileTextDeriver; +} = {}): Turn { + const appConfig: AppConfig = { + config: {}, + fileStrategy: FileSources.local, + imageOutputType: 'png', + fileConfig: { ...fileConfig, endpoints: { [ENDPOINT]: endpointConfig } }, + }; + const routing = resolveTurnDeliveryRouting({ + agent: { provider: ENDPOINT, endpoint: ENDPOINT }, + config: appConfig, + }); + const context = buildTurnReadingContext({ + routing, + provider: ENDPOINT, + model: 'gpt-4o', + fileTokenLimit: 100_000, + configuredFileSizeLimit: undefined, + countTokens: (text) => text.length, + deriveText, + }); + if (context == null) { + throw new Error('expected a reading context'); + } + routing.reading = context; + return { + agent: { id: 'agent_1', deliveryRouting: routing, fileConsumers: consumers }, + req: { config: appConfig }, + routing, + context, + }; +} + +const allocate = ( + { agent, req }: Turn, + historical: readonly HistoryAllocationFile[], + current: HistoryAllocationFile[], +): Promise => + allocateTurnAttachmentsWithHistory({ agent, req, historical, current, endpoint: ENDPOINT }); + +const pathsById = ( + files: readonly HistoryAllocationFile[], +): Record => + Object.fromEntries(files.map((file) => [file.file_id, file.llmDeliveryPath])); + +/** The check AgentClient's `assertTurnAttachmentLimits` runs on the shared set. */ +const assertTurnLimits = ( + { req }: Turn, + historical: readonly HistoryAllocationFile[], + current: readonly HistoryAllocationFile[], +) => + assertAgentAttachmentLimits({ + attachments: [...historical, ...current.filter(isModelBoundAttachmentFile)], + req, + endpoint: ENDPOINT, + countRepeatedExtractedText: true, + enforceAttachmentCount: false, + useGlobalContextSizeLimit: true, + }); + +/** The check the topology assertion runs on the same set for the primary agent. */ +const assertAgentLimits = ( + { req }: Turn, + historical: readonly HistoryAllocationFile[], + current: readonly HistoryAllocationFile[], +) => + assertAgentAttachmentLimits({ + attachments: [...historical, ...current.filter(isModelBoundAttachmentFile)], + req, + endpoint: ENDPOINT, + countRepeatedExtractedText: true, + historicalFileIds: collectHistoricalAttachmentIds(historical, current), + }); + +describe('allocateTurnAttachmentsWithHistory', () => { + describe('outside the automatic policy', () => { + interface ClassicTurn { + agent: AllocationParams['agent']; + req?: AllocationParams['req']; + context?: TurnReadingContext; + } + + it.each<[string, () => ClassicTurn]>([ + [ + 'the routing carries no reading context', + () => { + const routing = resolveTurnDeliveryRouting({ + agent: { provider: ENDPOINT, endpoint: ENDPOINT }, + config: { + fileConfig: { + fileContextSizeLimit: 1, + endpoints: { [ENDPOINT]: { llmDeliveryPolicy: 'automatic' } }, + }, + }, + }); + return { agent: { deliveryRouting: routing, fileConsumers: searchOnly } }; + }, + ], + [ + 'the policy is classic', + () => { + const turn = setup({ + endpointConfig: {}, + fileConfig: { fileContextSizeLimit: 1 }, + deriveText: async () => ({ status: 'skipped', reason: 'no_extractor' }), + }); + expect(turn.context.policy).toBe('classic'); + return turn; + }, + ], + ])( + 'returns the current files themselves when %s, asking its context nothing', + async (_case, arrange) => { + const { agent, req, context } = arrange(); + const methods = [ + 'judge', + 'derive', + 'addOverflow', + 'markTextFailed', + 'flush', + 'knownTokenCount', + ] as const; + const spies = context == null ? [] : methods.map((method) => jest.spyOn(context, method)); + const current = [attachment({ file_id: 'a' }), attachment({ file_id: 'b' })]; + + const allocated = await allocateTurnAttachmentsWithHistory({ + agent, + req, + historical: [attachment({ file_id: 'history', bytes: 6 * MB })], + current, + endpoint: ENDPOINT, + }); + + expect(allocated).toBe(current); + for (const spy of spies) { + expect(spy).not.toHaveBeenCalled(); + } + }, + ); + }); + + describe('under the automatic policy', () => { + const history = attachment({ file_id: 'history', bytes: 6 * MB }); + const current = (): HistoryAllocationFile[] => [ + attachment({ file_id: 'a' }), + attachment({ file_id: 'b' }), + ]; + + it('charges history first, so a current PDF that no longer fits moves to File Search', async () => { + const turn = setup({ fileConfig: { fileContextSizeLimit: 10 } }); + const { routing, context } = turn; + const historyBefore = structuredClone(history); + const files = current(); + expect(() => assertTurnLimits(turn, [history], files)).toThrow( + expect.objectContaining({ limitType: 'bytes' }), + ); + + const allocated = await allocate(turn, [history], files); + + expect(pathsById(allocated)).toEqual({ a: 'provider', b: 'none' }); + expect( + decideFileReading({ routing, file: allocated[1], consumers: searchOnly }), + ).toMatchObject({ reader: 'search', reason: 'aggregate_overflow' }); + expect(allocated[0]).toBe(files[0]); + expect(context.stats().overflow).toBe(1); + expect(() => assertTurnLimits(turn, [history], allocated)).not.toThrow(); + expect(() => assertAgentLimits(turn, [history], allocated)).not.toThrow(); + expect(history).toEqual(historyBefore); + expect(context.judge(history).overflow).toBeUndefined(); + expect(decideFileReading({ routing, file: history, consumers: searchOnly })).toMatchObject({ + reader: 'provider', + }); + }); + + it('admits the same request in full without the history in front of it', async () => { + const turn = setup({ fileConfig: { fileContextSizeLimit: 10 } }); + const files = current(); + + const allocated = await allocate(turn, [], files); + + expect(allocated).toBe(files); + expect(turn.context.stats().overflow).toBe(0); + }); + + it('leaves the overflow unavailable rather than failing the turn when File Search cannot reach it', async () => { + /* File Search queued and registered nothing, as a failed provisioning leaves it, and no + * code tool is loaded, so no reader is left for the overflow. */ + const turn = setup({ fileConfig: { fileContextSizeLimit: 10 } }); + turn.context.setSearchEvidence({ queued: [], registered: [] }); + + const allocated = await allocate(turn, [history], current()); + + expect(pathsById(allocated)).toEqual({ a: 'provider', b: 'none' }); + expect( + decideFileReading({ routing: turn.routing, file: allocated[1], consumers: searchOnly }), + ).toMatchObject({ reader: 'unavailable', reason: 'aggregate_overflow' }); + expect(() => assertTurnLimits(turn, [history], allocated)).not.toThrow(); + }); + + it('keeps the current files on their classic route when no file tool is loaded, so the limit check still fails the turn', async () => { + const turn = setup({ fileConfig: { fileContextSizeLimit: 10 }, consumers: noReader }); + const files = current(); + + const allocated = await allocate(turn, [history], files); + + expect(pathsById(allocated)).toEqual({ a: 'provider', b: 'provider' }); + expect(turn.context.stats().overflow).toBe(0); + expect( + decideFileReading({ routing: turn.routing, file: allocated[1], consumers: noReader }), + ).toMatchObject({ reader: 'provider', reason: 'no_file_tools', automatic: false }); + expect(() => assertTurnLimits(turn, [history], allocated)).toThrow( + expect.objectContaining({ limitType: 'bytes' }), + ); + }); + + it('moves a text reading that no longer fits the extracted-text budget', async () => { + const turn = setup({ fileConfig: { fileContextCharLimit: 100 } }); + const replayed = workbook('replayed', 'r'.repeat(70)); + const sheet = workbook('sheet', 's'.repeat(40)); + expect( + decideFileReading({ routing: turn.routing, file: sheet, consumers: searchOnly }), + ).toMatchObject({ reader: 'text' }); + + const allocated = await allocate(turn, [replayed], [sheet]); + + expect(pathsById(allocated)).toEqual({ sheet: 'none' }); + expect( + decideFileReading({ routing: turn.routing, file: allocated[0], consumers: searchOnly }), + ).toMatchObject({ reader: 'search', reason: 'aggregate_overflow' }); + expect(() => assertTurnLimits(turn, [replayed], allocated)).not.toThrow(); + }); + + it.each([ + ['replayed', (file: HistoryAllocationFile) => ({ historical: [file], files: current() })], + [ + 'submitted', + (file: HistoryAllocationFile) => ({ historical: [], files: [file, ...current()] }), + ], + ])('does not charge a %s file this endpoint refuses', async (_label, arrange) => { + const turn = setup({ + endpointConfig: { + llmDeliveryPolicy: 'automatic', + supportedMimeTypes: ['^application/pdf$'], + }, + fileConfig: { fileContextSizeLimit: 10 }, + }); + const screenshot = attachment({ file_id: 'screenshot', type: 'image/png', bytes: 6 * MB }); + const { historical, files } = arrange(screenshot); + + const allocated = await allocate(turn, historical, files); + + expect(allocated).toBe(files); + expect(turn.context.stats().overflow).toBe(0); + }); + + it('does not charge replayed history that is no longer model-bound', async () => { + const turn = setup({ fileConfig: { fileContextSizeLimit: 10 } }); + const searched = attachment({ file_id: 'searched', bytes: 6 * MB, llmDeliveryPath: 'none' }); + const files = current(); + + expect(await allocate(turn, [searched], files)).toBe(files); + }); + + describe('count allowance', () => { + const countLimited = () => + setup({ endpointConfig: { llmDeliveryPolicy: 'automatic', fileLimit: 2 } }); + const small = (file_id: string): HistoryAllocationFile => + attachment({ file_id, bytes: 1024 }); + + it('spends no count allowance on replayed history', async () => { + const turn = countLimited(); + const historical = ['h1', 'h2', 'h3'].map(small); + const files = ['a', 'b', 'c'].map(small); + + const allocated = await allocate(turn, historical, files); + + expect(pathsById(allocated)).toEqual({ a: 'provider', b: 'provider', c: 'none' }); + expect(() => assertAgentLimits(turn, historical, allocated)).not.toThrow(); + expect(() => assertAgentLimits(turn, historical, files)).toThrow( + expect.objectContaining({ limitType: 'count' }), + ); + }); + + it('counts a replayed file submitted again once, as the current submission', async () => { + const turn = countLimited(); + const historical = ['h1', 'a'].map(small); + const files = ['a', 'b', 'c'].map(small); + + const allocated = await allocate(turn, historical, files); + + expect(pathsById(allocated)).toEqual({ a: 'provider', b: 'provider', c: 'none' }); + expect(() => assertAgentLimits(turn, historical, allocated)).not.toThrow(); + }); + + it('commits a resubmitted file with its history rather than moving it', async () => { + const turn = countLimited(); + const historical = ['h1', 'c'].map(small); + const files = ['a', 'b', 'c'].map(small); + + const allocated = await allocate(turn, historical, files); + + expect(pathsById(allocated)).toEqual({ a: 'provider', b: 'none', c: 'provider' }); + expect(turn.context.stats().overflow).toBe(1); + expect(() => assertAgentLimits(turn, historical, allocated)).not.toThrow(); + expect( + decideFileReading({ routing: turn.routing, file: historical[1], consumers: searchOnly }) + .reader, + ).toBe('provider'); + }); + }); + + it('hands the moved files to the caller, and nothing when none moved', async () => { + const turn = setup({ fileConfig: { fileContextSizeLimit: 4 } }); + const onAllocated = jest.fn(); + const files = current(); + + const allocated = await allocateTurnAttachmentsWithHistory({ + ...turn, + historical: [], + current: files, + endpoint: ENDPOINT, + onAllocated, + }); + const fitting = [attachment({ file_id: 'small', bytes: MB })]; + const unchanged = await allocateTurnAttachmentsWithHistory({ + ...setup(), + historical: [], + current: fitting, + endpoint: ENDPOINT, + onAllocated, + }); + + expect(onAllocated.mock.calls).toEqual([[allocated]]); + expect(pathsById(allocated)).toEqual({ a: 'provider', b: 'none' }); + expect(unchanged).toBe(fitting); + }); + + it('holds the shared set to a secondary agent endpoint with a tighter count', async () => { + const appConfig: AppConfig = { + config: {}, + fileStrategy: FileSources.local, + imageOutputType: 'png', + fileConfig: { + endpoints: { + [ENDPOINT]: { llmDeliveryPolicy: 'automatic', fileLimit: 10 }, + anthropic: { fileLimit: 1 }, + }, + }, + }; + const turn = setup(); + const req = { config: appConfig }; + const endpointsByAgentId = new Map([ + ['agent_1', { endpoint: ENDPOINT }], + ['agent_2', { endpoint: 'anthropic' }], + ]); + const files = ['a', 'b'].map((file_id) => attachment({ file_id, bytes: 1024 })); + /** The per-agent check the topology assertion runs for the secondary agent. */ + const assertSecondary = (shared: readonly HistoryAllocationFile[]) => + assertAgentAttachmentLimits({ + attachments: shared.filter(isModelBoundAttachmentFile), + req, + endpoint: 'anthropic', + countRepeatedExtractedText: true, + }); + + const allocated = await allocateTurnAttachmentsWithHistory({ + agent: turn.agent, + req, + historical: [], + current: files, + endpoint: ENDPOINT, + endpointsByAgentId, + }); + + expect(pathsById(allocated)).toEqual({ a: 'provider', b: 'none' }); + expect(() => assertSecondary(allocated)).not.toThrow(); + expect(() => assertSecondary(files)).toThrow(expect.objectContaining({ limitType: 'count' })); + }); + + describe('against retained file contexts', () => { + const retainedContext = (text: string): HistoryAllocationFile => ({ + file_id: 'retained-file-context:msg-1', + source: FileSources.text, + type: 'text/plain', + text, + bytes: Buffer.byteLength(text), + }); + + it('moves a text reading that no longer fits beside the retained text', async () => { + const turn = setup({ fileConfig: { fileContextCharLimit: 1000 } }); + const retained = [retainedContext('r'.repeat(500))]; + const files = [workbook('sheet', 's'.repeat(600))]; + + const allocated = await allocateTurnAttachmentsWithRetainedContext({ + ...turn, + resendFiles: false, + retained, + current: files, + endpoint: ENDPOINT, + }); + + expect(pathsById(allocated)).toEqual({ sheet: 'none' }); + expect(() => assertTurnLimits(turn, retained, allocated)).not.toThrow(); + expect(() => assertTurnLimits(turn, retained, files)).toThrow( + expect.objectContaining({ limitType: 'extracted_text' }), + ); + }); + + it('leaves the request alone when the conversation resends files', async () => { + const turn = setup({ fileConfig: { fileContextCharLimit: 1000 } }); + const files = [workbook('sheet', 's'.repeat(600))]; + + await expect( + allocateTurnAttachmentsWithRetainedContext({ + ...turn, + resendFiles: true, + retained: [retainedContext('r'.repeat(500))], + current: files, + endpoint: ENDPOINT, + }), + ).resolves.toBe(files); + expect(turn.context.stats().overflow).toBe(0); + }); + }); + + it('depends on the current order alone, not on the order of the history', async () => { + const historical = ['h1', 'h2', 'h3'].map((file_id) => + attachment({ file_id, bytes: 2 * MB }), + ); + const run = async ( + replay: readonly HistoryAllocationFile[], + files: HistoryAllocationFile[], + ) => { + const turn = setup({ fileConfig: { fileContextSizeLimit: 10 } }); + return pathsById(await allocate(turn, replay, files)); + }; + const [a, b] = current(); + + const expected = { a: 'provider', b: 'none' }; + expect(await run(historical, [a, b])).toEqual(expected); + expect(await run([...historical].reverse(), [a, b])).toEqual(expected); + expect(await run([historical[1], historical[2], historical[0]], [a, b])).toEqual(expected); + expect(await run(historical, [b, a])).toEqual({ a: 'none', b: 'provider' }); + }); + + describe('limits', () => { + it('holds the shared set to the global limits the turn check enforces', async () => { + const turn = setup({ + endpointConfig: { llmDeliveryPolicy: 'automatic', fileLimit: 4 }, + fileConfig: { fileContextSizeLimit: 10, fileContextCharLimit: 500 }, + }); + const exact = 10 * MB - 7 * MB; + const historical = [attachment({ file_id: 'history', bytes: 7 * MB })]; + const files = [ + attachment({ file_id: 'exact', bytes: exact }), + attachment({ file_id: 'byte', bytes: 1 }), + ]; + + const allocated = await allocate(turn, historical, files); + + expect(pathsById(allocated)).toEqual({ exact: 'provider', byte: 'none' }); + expect(assertTurnLimits(turn, historical, allocated).totalKnownBytes).toBe(10 * MB); + expect(() => assertTurnLimits(turn, historical, files)).toThrow( + expect.objectContaining({ limitType: 'bytes', limit: 10 * MB }), + ); + }); + + it('holds the shared set to a tighter aggregate the agent endpoint configures', async () => { + const turn = setup({ + endpointConfig: { llmDeliveryPolicy: 'automatic', totalSizeLimit: 8 }, + fileConfig: { fileContextSizeLimit: 10 }, + }); + const historical = [attachment({ file_id: 'history', bytes: 4 * MB })]; + const files = [attachment({ file_id: 'a' }), attachment({ file_id: 'b', bytes: 2 * MB })]; + + const allocated = await allocate(turn, historical, files); + + expect(pathsById(allocated)).toEqual({ a: 'provider', b: 'none' }); + expect(() => assertAgentLimits(turn, historical, allocated)).not.toThrow(); + expect(() => assertAgentLimits(turn, historical, files)).toThrow( + expect.objectContaining({ limitType: 'bytes', limit: 8 * MB }), + ); + }); + }); + + it('returns copies decided by the turn routing for files it did not move', async () => { + const turn = setup({ fileConfig: { fileContextSizeLimit: 10 } }); + const stale = attachment({ file_id: 'stale', bytes: MB, llmDeliveryPath: 'text' }); + + const allocated = await allocate(turn, [], [stale]); + + expect(allocated).toEqual( + applyTurnDelivery([stale], { routing: turn.routing, consumers: searchOnly }), + ); + expect(allocated[0].llmDeliveryPath).toBe('provider'); + }); + }); +}); diff --git a/packages/api/src/files/reading/history.ts b/packages/api/src/files/reading/history.ts new file mode 100644 index 00000000000..d3900ed676a --- /dev/null +++ b/packages/api/src/files/reading/history.ts @@ -0,0 +1,237 @@ +import { EModelEndpoint } from 'librechat-data-provider'; +import type { + TFile, + DirectContentEntry, + DirectContentLimits, + TurnFileConsumers, +} from 'librechat-data-provider'; +import type { AgentAttachmentEndpointsByAgentId } from '~/agents/attachments'; +import type { TurnReadingAgent } from './diagnostics'; +import type { TurnReadingFile } from './turn'; +import type { ServerRequest } from '~/types'; +import { + collectFileIds, + listAgentEntries, + isModelBoundAttachmentFile, + measureModelBoundAttachment, + resolveAgentAttachmentLimits, + collectHistoricalAttachmentIds, +} from '~/agents/attachments'; +import { filterFilesByEndpointRuntimeConfig } from '~/files/filter'; +import { getTurnReadingContext } from './turn'; +import { settleTurnFiles } from './settle'; + +/** A turn copy of an attachment the endpoint's file policy can judge, as AgentClient holds it. */ +export type HistoryAllocationFile = TurnReadingFile & Pick; + +/** The endpoint whose attachment policy and limits hold the turn's shared attachments. */ +interface AttachmentScope { + req?: Pick; + endpoint: string; + endpointType?: string | null; + consumers?: TurnFileConsumers | null; +} + +export interface TurnAttachmentsWithHistoryParams { + agent: Pick; + /** The replayed files the conversation's history sends again, as this turn decided them. */ + historical: readonly T[]; + /** The request's attachments in submission order; earlier files are admitted first. */ + current: T[]; + req?: Pick; + /** The endpoint the turn's attachment checks run against; the agents endpoint when absent. */ + endpoint?: string | null; + endpointType?: string | null; + /** + * Every agent the topology check holds the shared set to, with its endpoint, so the + * allocation uses the strictest limits any of those checks applies. + */ + endpointsByAgentId?: AgentAttachmentEndpointsByAgentId; + /** Receives the re-decided request attachments, only when the allocation moved one. */ + onAllocated?: (files: T[]) => void; + signal?: AbortSignal; +} + +export interface RetainedContextAllocationParams + extends Omit, 'historical'> { + /** The conversation's setting; with files resent, history replay already allocated the turn. */ + resendFiles?: boolean | null; + /** The extracted text earlier messages keep in place of their files. */ + retained: readonly T[]; +} + +/** The tighter of two limits, where a missing or zero limit leaves the dimension unlimited. */ +const tighter = (first?: number, second?: number): number | undefined => { + if (!first) { + return second; + } + if (!second) { + return first; + } + return Math.min(first, second); +}; + +const tightenLimits = ( + first: DirectContentLimits, + second: DirectContentLimits, +): DirectContentLimits => ({ + count: tighter(first.count, second.count), + bytes: tighter(first.bytes, second.bytes), + textChars: tighter(first.textChars, second.textChars), +}); + +/** + * The limits every check on the shared set enforces together: the global context limits of + * `assertTurnAttachmentLimits`, and each reachable agent's own count allowance and endpoint + * aggregate, which the topology check applies to the same set. Agent-scoped context injections + * are counted after this allocation, so they can still exceed these limits. + */ +function resolveSharedLimits( + { req, endpoint, endpointType }: AttachmentScope, + endpointsByAgentId?: AgentAttachmentEndpointsByAgentId, +): DirectContentLimits { + const turn = resolveAgentAttachmentLimits({ + req, + endpoint, + endpointType, + enforceAttachmentCount: false, + useGlobalContextSizeLimit: true, + }); + return [{ endpoint, endpointType }, ...listAgentEntries(endpointsByAgentId)] + .map((agentEndpoint) => + resolveAgentAttachmentLimits({ + req, + endpoint: agentEndpoint.endpoint, + endpointType: agentEndpoint.endpointType, + }), + ) + .reduce(tightenLimits, turn); +} + +/** The files the endpoint's runtime policy keeps, as AgentClient selects model-bound files. */ +const selectCompatible = ( + files: T[], + { req, endpoint, endpointType, consumers }: AttachmentScope, +): T[] => + filterFilesByEndpointRuntimeConfig(req?.config, { + files, + endpoint, + endpointType, + skipTotalSizeLimit: true, + preserveTextSources: true, + consumers, + }); + +const uncharged = (file: HistoryAllocationFile): DirectContentEntry => ({ + fileId: file.file_id, + counts: false, + bytes: 0, + textChars: 0, +}); + +/** + * Charges replayed content as the shared-set checks charge `[...history, ...current]`: every copy + * spends the context budgets, and a file counts once, at its first copy, unless history alone + * replays it. + */ +function chargeReplayed( + measured: ReadonlyArray<{ file: HistoryAllocationFile; entry: DirectContentEntry }>, + historyOnlyIds: ReadonlySet, +): DirectContentEntry[] { + const firstCopy = new Map(); + measured.forEach(({ file }, index) => { + if (!firstCopy.has(file.file_id)) { + firstCopy.set(file.file_id, index); + } + }); + return measured.map(({ file, entry }, index) => ({ + ...entry, + counts: + entry.counts && firstCopy.get(file.file_id) === index && !historyOnlyIds.has(file.file_id), + })); +} + +/** + * Allocates the request's direct content after the turn's replayed history, so the history + * checks AgentClient runs next cannot refuse what the automatic policy admitted. The compatible + * model-bound history is charged first, then every current file the policy cannot move; the + * files the policy reads natively or as text are admitted in submission order while they fit. + * A file that does not fit is recorded as overflow and decided again, so it moves to File Search + * or Run Code, or is left unavailable, instead of failing the turn. A current file history + * replays as well is committed with it rather than allocated, since history sends it anyway. + * Returns `current` itself unless the agent's turn reads under the automatic policy and a file + * moved. + */ +export async function allocateTurnAttachmentsWithHistory({ + agent, + historical, + current, + req, + endpoint, + endpointType, + endpointsByAgentId, + onAllocated, + signal, +}: TurnAttachmentsWithHistoryParams): Promise { + const routing = agent.deliveryRouting; + if (getTurnReadingContext(routing)?.policy !== 'automatic' || current.length === 0) { + return current; + } + const scope: AttachmentScope = { + req, + endpoint: endpoint ?? EModelEndpoint.agents, + endpointType, + consumers: agent.fileConsumers, + }; + const committedHistory = selectCompatible(historical.filter(isModelBoundAttachmentFile), scope); + const replayedIds = collectFileIds(committedHistory); + const compatibleFileIds = collectFileIds(selectCompatible(current, scope)); + const measureCurrent = (file: T): DirectContentEntry => + compatibleFileIds.has(file.file_id) ? measureModelBoundAttachment(file) : uncharged(file); + const replayed = chargeReplayed( + [ + ...committedHistory.map((file) => ({ file, entry: measureModelBoundAttachment(file) })), + ...current + .filter((file) => replayedIds.has(file.file_id)) + .map((file) => ({ file, entry: measureCurrent(file) })), + ], + collectHistoricalAttachmentIds(committedHistory, current), + ); + const allocated = await settleTurnFiles({ + routing, + consumers: agent.fileConsumers, + files: current, + allocation: { + requestFileIds: current + .filter((file) => !replayedIds.has(file.file_id)) + .map((file) => file.file_id), + limits: resolveSharedLimits(scope, endpointsByAgentId), + measure: measureCurrent, + committedEntries: replayed, + agentId: agent.id, + scope: 'history', + }, + flush: true, + signal, + }); + if (allocated !== current) { + onAllocated?.(allocated); + } + return allocated; +} + +/** + * Allocates the request against the extracted text earlier messages retain when the conversation + * does not resend files, which the turn check charges in their place. With files resent, history + * replay ran {@link allocateTurnAttachmentsWithHistory} already, so this returns `current`. + */ +export function allocateTurnAttachmentsWithRetainedContext({ + resendFiles, + retained, + ...params +}: RetainedContextAllocationParams): Promise { + if (resendFiles !== false) { + return Promise.resolve(params.current); + } + return allocateTurnAttachmentsWithHistory({ ...params, historical: retained }); +} diff --git a/packages/api/src/files/reading/index.ts b/packages/api/src/files/reading/index.ts new file mode 100644 index 00000000000..0b66db69496 --- /dev/null +++ b/packages/api/src/files/reading/index.ts @@ -0,0 +1,7 @@ +export * from './turn'; +export * from './settle'; +export * from './diagnostics'; +export * from './upload'; +export * from './derive'; +export * from './inventory'; +export * from './native'; diff --git a/packages/api/src/files/reading/inventory.spec.ts b/packages/api/src/files/reading/inventory.spec.ts new file mode 100644 index 00000000000..e76f2e8b734 --- /dev/null +++ b/packages/api/src/files/reading/inventory.spec.ts @@ -0,0 +1,709 @@ +import { Tools, FileContext, FileSources } from 'librechat-data-provider'; +import type { + TFile, + TFileConfig, + FiltersConfig, + TurnFileConsumers, + TurnDeliveryRouting, +} from 'librechat-data-provider'; +import type { BuildTurnReadingContextParams, TurnReadingContext } from './turn'; +import type { CodeExecutionContext } from '~/agents/execution'; +import type { CodeFileAgent } from '~/files/code/queued'; +import type { ReadingAgent } from './inventory'; +import { buildTurnReadingContext, getTurnReadingContext, recordNativeRejections } from './turn'; +import { prepareAgentFileContext, renderLeftOutFiles } from './inventory'; +import { resolveTurnDeliveryRouting } from '~/agents/files/delivery'; +import { ContentFilterError } from '~/middleware/contentFilter'; + +type EndpointFileConfigInput = NonNullable[string]; + +const MB = 1024 * 1024; +const PDF = 'application/pdf'; +const DOCX = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'; +const XLSX = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'; +const PARQUET = 'application/vnd.apache.parquet'; +const SENTINEL = 'SENTINEL-7F3A'; +const NO_FILES_NOTE = `- Note: Semantic search is available through the ${Tools.file_search} tool but no files are currently loaded. Request the user to upload documents to search through.`; + +const AUTOMATIC: EndpointFileConfigInput = { llmDeliveryPolicy: 'automatic' }; +const RUNS_CODE: TurnFileConsumers = { executeCode: true, fileSearch: false }; +const SEARCHES: TurnFileConsumers = { executeCode: false, fileSearch: true }; +const NO_READER: TurnFileConsumers = { executeCode: false, fileSearch: false }; + +const attachment = (overrides: Partial & Pick): TFile => ({ + user: 'user-1', + object: 'file', + filepath: `/uploads/user-1/${overrides.filename}`, + type: XLSX, + bytes: 2048, + embedded: false, + usage: 0, + source: FileSources.local, + context: FileContext.message_attachment, + llmDeliveryPath: 'none', + metadata: { destinationChosen: false }, + ...overrides, +}); + +interface AgentSetup { + provider?: string; + files?: TFile[]; + consumers?: TurnFileConsumers; + endpointConfig?: EndpointFileConfigInput; + /** Absent leaves the agent without a provision state, as a failed provisioning does. */ + codeEnvFiles?: TFile[]; + vectorDBFiles?: TFile[]; + reading?: Partial; + primedSearchFileIds?: string[]; + dynamicToolContextMap?: Record; + codeExecutionContext?: CodeExecutionContext; +} + +type TestAgent = CodeFileAgent & ReadingAgent & { deliveryRouting: TurnDeliveryRouting }; + +/** A file tool is loaded unless a test says otherwise: the inventory is written only then. */ +function agentWith({ + provider = 'openAI', + files = [], + consumers = SEARCHES, + endpointConfig = AUTOMATIC, + codeEnvFiles, + vectorDBFiles = [], + reading, + primedSearchFileIds, + dynamicToolContextMap, + codeExecutionContext, +}: AgentSetup = {}): TestAgent { + const routing = resolveTurnDeliveryRouting({ + agent: { provider, endpoint: provider }, + config: { fileConfig: { endpoints: { [provider]: endpointConfig } } }, + }); + const context = buildTurnReadingContext({ + routing, + provider, + model: 'gpt-4o', + fileTokenLimit: 100_000, + configuredFileSizeLimit: undefined, + countTokens: (text) => text.length, + ...reading, + }); + if (context != null) { + routing.reading = context; + } + const provisioned = codeEnvFiles != null || vectorDBFiles.length > 0; + return { + id: 'agent-1', + deliveryRouting: routing, + fileConsumers: consumers, + currentRequestAttachments: files, + primedSearchFileIds, + dynamicToolContextMap, + codeExecutionContext, + provisionState: provisioned + ? { + codeEnvFiles: codeEnvFiles ?? [], + vectorDBFiles, + aliveFileIds: new Set(), + agentScopedFileIds: new Set(), + } + : undefined, + }; +} + +function readingContextOf(agent: TestAgent): TurnReadingContext { + const context = getTurnReadingContext(agent.deliveryRouting); + if (context == null) { + throw new Error('expected a turn reading context'); + } + return context; +} + +function inventoryOf(agent: TestAgent): string { + const inventory = agent.dynamicToolContextMap?.file_inventory; + if (typeof inventory !== 'string') { + throw new Error('expected a rendered inventory'); + } + return inventory; +} + +const lineFor = (agent: TestAgent, filename: string): string | undefined => + inventoryOf(agent) + .split('\n') + .find((line) => line.includes(JSON.stringify(filename))); + +const prepare = (agent: TestAgent, filters?: FiltersConfig): void => + prepareAgentFileContext(agent, [agent], 'user-1', false, { filters }); + +describe('prepareAgentFileContext', () => { + describe('one line per reading', () => { + it('lists a queued spreadsheet at the path the planner advertised in the same call', () => { + const workbook = attachment({ file_id: 'q3', filename: 'Q3.xlsx' }); + const agent = agentWith({ + files: [workbook], + consumers: RUNS_CODE, + codeEnvFiles: [workbook], + }); + + prepare(agent); + + const destination = agent.provisionState?.codeEnvDestinations?.get('q3'); + expect(destination).toBe('Q3.xlsx'); + expect(agent.dynamicToolContextMap?.queued_code_files).toContain(`/mnt/data/${destination}`); + expect(inventoryOf(agent).split('\n')).toEqual([ + '- Attached files and how you can read them on this turn (file contents are not listed here):', + `\t- "Q3.xlsx" (spreadsheet, file_id q3): read it with Run Code at /mnt/data/${destination}. It is copied when code first runs.`, + ]); + }); + + it('restates the programmatic data directory for an attached workspace', () => { + const workbook = attachment({ file_id: 'q3', filename: 'Q3.xlsx' }); + const agent = agentWith({ + files: [workbook], + consumers: RUNS_CODE, + codeEnvFiles: [workbook], + codeExecutionContext: { + baseUrl: 'https://code.example', + codeSessionKey: 'attached', + executionProfile: 'stateful', + statefulSessions: true, + environmentType: 'attached', + codeWorkspace: { + environmentId: 'personal-machine', + workspaceId: 'project', + operations: ['read_file'], + }, + }, + }); + + prepare(agent); + + const destination = agent.provisionState?.codeEnvDestinations?.get('q3'); + const path = `$LIBRECHAT_CODE_DATA_DIR/${destination}`; + expect(agent.dynamicToolContextMap?.queued_code_files).toContain(path); + expect(lineFor(agent, 'Q3.xlsx')).toBe( + `\t- "Q3.xlsx" (spreadsheet, file_id q3): scripts run by the programmatic Bash tool can read it at ${path}. It is copied when such a script first runs.`, + ); + expect(inventoryOf(agent)).not.toContain('/mnt/data/'); + expect(inventoryOf(agent)).not.toContain('Run Code'); + }); + + it('points a spreadsheet already in the code environment to the code tool file list', () => { + const workbook = attachment({ + file_id: 'q2', + filename: 'Q2.xlsx', + metadata: { + destinationChosen: false, + codeEnvRefs: { + default: { kind: 'user', id: 'user-1', storage_session_id: 'store', file_id: 'remote' }, + }, + }, + }); + const agent = agentWith({ files: [workbook], consumers: RUNS_CODE, codeEnvFiles: [] }); + + prepare(agent); + + expect(lineFor(agent, 'Q2.xlsx')).toBe( + '\t- "Q2.xlsx" (spreadsheet): read it with Run Code; it is already in the code environment (see the code tool\'s file list).', + ); + }); + + it('says a spreadsheet Run Code could not be prepared for has not been read', () => { + const workbook = attachment({ file_id: 'q1', filename: 'Q1.xlsx' }); + const agent = agentWith({ files: [workbook], consumers: RUNS_CODE }); + + prepare(agent); + + expect(agent.dynamicToolContextMap?.queued_code_files).toBeUndefined(); + expect(lineFor(agent, 'Q1.xlsx')).toBe( + '\t- "Q1.xlsx" (spreadsheet): Run Code is available, but this file could not be prepared on this turn. Do not claim to have read it.', + ); + }); + + it('notes where Run Code can also open a PDF sent with the message', () => { + const pdf = attachment({ + file_id: 'brief', + filename: 'brief.pdf', + type: PDF, + llmDeliveryPath: 'provider', + }); + const agent = agentWith({ files: [pdf], consumers: RUNS_CODE, codeEnvFiles: [pdf] }); + + prepare(agent); + + expect(lineFor(agent, 'brief.pdf')).toBe( + '\t- "brief.pdf" (PDF): sent with this message. Run Code can also open it at /mnt/data/brief.pdf.', + ); + }); + + it('says complete extracted text is included without listing it', () => { + const docx = attachment({ + file_id: 'notes', + filename: 'notes.docx', + type: DOCX, + text: `Board notes ${SENTINEL}`, + llmDeliveryPath: 'text', + }); + const agent = agentWith({ files: [docx] }); + + prepare(agent); + + expect(lineFor(agent, 'notes.docx')).toBe( + '\t- "notes.docx" (document): this model cannot read this type directly. Its complete extracted text is included in this message.', + ); + expect(inventoryOf(agent)).not.toContain(SENTINEL); + }); + + it('leaves a PDF too large to send to File Search, pending its first search', () => { + const pdf = attachment({ + file_id: 'manual', + filename: 'manual.pdf', + type: PDF, + bytes: 5 * MB, + llmDeliveryPath: 'provider', + }); + const agent = agentWith({ + files: [pdf], + consumers: SEARCHES, + vectorDBFiles: [pdf], + reading: { configuredFileSizeLimit: MB }, + }); + + prepare(agent); + + expect(lineFor(agent, 'manual.pdf')).toBe( + `\t- "manual.pdf" (PDF): too large to send directly. Search it with ${Tools.file_search}; it is indexed when you first search. Results are excerpts, not the whole document.`, + ); + expect(inventoryOf(agent).match(/manual\.pdf/g)).toHaveLength(1); + }); + + it('describes failed search preparation and its retry without claiming cached text was sent', () => { + const pdf = attachment({ + file_id: 'manual', + filename: 'manual.pdf', + type: PDF, + bytes: 5 * MB, + text: SENTINEL, + llmDeliveryPath: 'none', + }); + const agent = agentWith({ + files: [pdf], + consumers: SEARCHES, + vectorDBFiles: [pdf], + reading: { configuredFileSizeLimit: MB }, + }); + const preparation = new Map([[pdf.file_id, 'failed']]); + readingContextOf(agent).setSearchEvidence({ + queued: [pdf.file_id], + registered: [], + preparation, + }); + + prepare(agent); + expect(lineFor(agent, pdf.filename)).toContain('File Search could not prepare it'); + expect(lineFor(agent, pdf.filename)).toContain('Retry file_search to try indexing again'); + expect(lineFor(agent, pdf.filename)).toContain('do not claim to have read it'); + expect(inventoryOf(agent)).not.toContain(SENTINEL); + expect(inventoryOf(agent)).not.toContain('text is included'); + + preparation.set(pdf.file_id, 'ready'); + prepare(agent); + expect(lineFor(agent, pdf.filename)).toContain('Search it with file_search.'); + expect(lineFor(agent, pdf.filename)).not.toContain('could not prepare'); + expect(lineFor(agent, pdf.filename)).not.toContain('indexed when'); + }); + + it('sends text too long to include to File Search', () => { + const log = attachment({ + file_id: 'huge', + filename: 'huge.log', + type: 'text/plain', + text: `${SENTINEL} ${'x'.repeat(64)}`, + llmDeliveryPath: 'text', + }); + const agent = agentWith({ + files: [log], + consumers: SEARCHES, + reading: { fileTokenLimit: 10 }, + }); + + prepare(agent); + + expect(lineFor(agent, 'huge.log')).toBe( + `\t- "huge.log" (text): too long to include. Search it with ${Tools.file_search}; it is indexed when you first search. Results are excerpts, not the whole document.`, + ); + expect(inventoryOf(agent)).not.toContain(SENTINEL); + }); + + it('says a file no reader can take cannot be read on this turn', () => { + const parquet = attachment({ file_id: 'data', filename: 'data.parquet', type: PARQUET }); + const agent = agentWith({ files: [parquet], consumers: SEARCHES }); + + prepare(agent); + + expect(lineFor(agent, 'data.parquet')).toBe( + '\t- "data.parquet" (spreadsheet): cannot be read on this turn because Run Code is not available. Say so if asked; do not claim to have read it.', + ); + }); + + it('says only an extracted-text copy of a text-only record was kept', () => { + const textOnly = attachment({ + file_id: 'old', + filename: 'old.xlsx', + source: FileSources.text, + text: `Q1,1200 ${SENTINEL}`, + llmDeliveryPath: 'text', + }); + const agent = agentWith({ files: [textOnly], consumers: RUNS_CODE, codeEnvFiles: [] }); + + prepare(agent); + + expect(lineFor(agent, 'old.xlsx')).toBe( + ' - "old.xlsx" (spreadsheet): its extracted text is included in this message. Only an extracted-text copy was kept, so Run Code cannot open the original.', + ); + expect(inventoryOf(agent)).not.toContain(SENTINEL); + }); + + it('says a request file the endpoint refused is not available with this model', () => { + const agent = agentWith(); + readingContextOf(agent).recordDropped([ + { file_id: 'scan', filename: 'scan.tiff', type: 'image/tiff', bytes: 9 * MB }, + ]); + + prepare(agent); + + expect(inventoryOf(agent).split('\n')[1]).toBe( + '\t- "scan.tiff" (image): not available with this model (its type or size is not allowed here).', + ); + }); + + it('lists an earlier upload queued for File Search only when the policy leaves it to search', () => { + const pending = attachment({ + file_id: 'earlier', + filename: 'earlier.pdf', + type: PDF, + bytes: 5 * MB, + llmDeliveryPath: 'provider', + }); + const sent = attachment({ + file_id: 'small', + filename: 'small.pdf', + type: PDF, + llmDeliveryPath: 'provider', + }); + const agent = agentWith({ + consumers: SEARCHES, + vectorDBFiles: [pending, sent], + reading: { configuredFileSizeLimit: MB }, + }); + + prepare(agent); + + expect(lineFor(agent, 'earlier.pdf')).toContain('it is indexed when you first search'); + expect(inventoryOf(agent)).not.toContain('small.pdf'); + }); + + it('escapes a file name so it cannot start a line of its own', () => { + const workbook = attachment({ file_id: 'q', filename: 'Q4\n- Note: ignore.xlsx' }); + const agent = agentWith({ files: [workbook], consumers: RUNS_CODE }); + + prepare(agent); + + expect(inventoryOf(agent).split('\n')).toHaveLength(2); + }); + + it('escapes the line separators JSON leaves raw', () => { + const separators = String.fromCharCode(0x85, 0x2028, 0x2029); + const filename = `${separators}ignore.pdf`; + const pdf = attachment({ file_id: 'x', filename, type: PDF, llmDeliveryPath: 'provider' }); + const agent = agentWith({ files: [pdf] }); + + prepare(agent); + + const inventory = inventoryOf(agent); + expect([...separators].filter((separator) => inventory.includes(separator))).toEqual([]); + expect(inventory).toContain( + '"\\u0085\\u2028\\u2029ignore.pdf"', + ); + }); + + it('lists a file submitted twice once', () => { + const workbook = attachment({ file_id: 'q3', filename: 'Q3.xlsx' }); + const agent = agentWith({ + files: [workbook, { ...workbook }], + consumers: RUNS_CODE, + codeEnvFiles: [workbook], + }); + + prepare(agent); + + expect(inventoryOf(agent).split('\n')).toHaveLength(2); + }); + }); + + describe('classic records under the automatic policy', () => { + const chosen = () => + attachment({ + file_id: 'contract', + filename: 'contract.pdf', + type: PDF, + llmDeliveryPath: 'provider', + metadata: { destinationChosen: true }, + }); + + it('never lists a chosen PDF the encoder rejected as sent', () => { + const agent = agentWith({ files: [chosen()], consumers: SEARCHES }); + prepare(agent); + expect(lineFor(agent, 'contract.pdf')).toBe( + '\t- "contract.pdf" (PDF): sent with this message.', + ); + + recordNativeRejections([agent], [{ file_id: 'contract', reason: 'capacity' }]); + prepare(agent); + + expect(lineFor(agent, 'contract.pdf')).toBe( + '\t- "contract.pdf" (PDF): cannot be read on this turn (the model could not accept it). Say so if asked; do not claim to have read it.', + ); + }); + + it('never lists a chosen document the model cannot take as sent', () => { + const agent = agentWith({ + files: [{ ...chosen(), filename: 'contract.docx', type: DOCX }], + consumers: SEARCHES, + provider: 'anthropic', + }); + + prepare(agent); + + expect(lineFor(agent, 'contract.docx')).toBe( + '\t- "contract.docx" (document): cannot be read on this turn (this model cannot read this type directly). Say so if asked; do not claim to have read it.', + ); + }); + }); + + describe('for an agent that does not receive the request', () => { + const childPrepare = (agent: TestAgent): void => + prepareAgentFileContext(agent, [agent], 'user-1', false, { receivesRequest: false }); + + it('never says a request file was sent or included, and keeps the Run Code paths', () => { + const pdf = attachment({ + file_id: 'brief', + filename: 'brief.pdf', + type: PDF, + llmDeliveryPath: 'provider', + }); + const docx = attachment({ + file_id: 'notes', + filename: 'notes.docx', + type: DOCX, + text: 'Board notes', + llmDeliveryPath: 'text', + }); + const workbook = attachment({ file_id: 'q3', filename: 'Q3.xlsx' }); + const agent = agentWith({ + files: [pdf, docx, workbook], + consumers: RUNS_CODE, + codeEnvFiles: [pdf, workbook], + }); + + childPrepare(agent); + + const inventory = inventoryOf(agent); + expect(inventory).not.toMatch(/sent with this message|included in this message/); + expect(lineFor(agent, 'brief.pdf')).toBe( + '\t- "brief.pdf" (PDF): not sent with your task. Do not claim to have read it unless it is shared with you. Run Code can also open it at /mnt/data/brief.pdf.', + ); + expect(lineFor(agent, 'notes.docx')).toBe( + '\t- "notes.docx" (document): not sent with your task. Do not claim to have read it unless it is shared with you.', + ); + expect(lineFor(agent, 'Q3.xlsx')).toContain('read it with Run Code at /mnt/data/Q3.xlsx.'); + }); + }); + + describe('renderLeftOutFiles', () => { + it('names a document the encoder rejected as read by File Search, in the inventory and the left-out list', () => { + const pdf = attachment({ + file_id: 'brief', + filename: 'brief.pdf', + type: PDF, + llmDeliveryPath: 'provider', + }); + const agent = agentWith({ files: [pdf], consumers: SEARCHES, vectorDBFiles: [pdf] }); + prepare(agent); + expect(lineFor(agent, 'brief.pdf')).toBe('\t- "brief.pdf" (PDF): sent with this message.'); + + recordNativeRejections([agent], [{ file_id: 'brief', reason: 'integrity' }]); + prepare(agent); + + const leftOut = `\t- "brief.pdf" (PDF): the model could not accept it. Search it with ${Tools.file_search}; it is indexed when you first search. Results are excerpts, not the whole document.`; + expect(lineFor(agent, 'brief.pdf')).toBe(leftOut); + expect(renderLeftOutFiles(agent, [pdf, pdf]).split('\n')).toEqual([ + '- Shared files left out of this message (file contents are not listed here):', + leftOut, + ]); + expect(renderLeftOutFiles(agent, [])).toBe(''); + }); + }); + + const classicAgent = (endpointConfig: EndpointFileConfigInput): TestAgent => { + const pdf = attachment({ + file_id: 'manual', + filename: 'manual.pdf', + type: PDF, + bytes: 5 * MB, + llmDeliveryPath: 'provider', + }); + const workbook = attachment({ file_id: 'q3', filename: 'Q3.xlsx' }); + return agentWith({ + files: [pdf, workbook], + consumers: { executeCode: true, fileSearch: true }, + endpointConfig, + codeEnvFiles: [workbook], + vectorDBFiles: [pdf], + primedSearchFileIds: [], + dynamicToolContextMap: { [Tools.file_search]: NO_FILES_NOTE, file_inventory: 'stale' }, + }); + }; + const untooledAgent = (): TestAgent => { + const pdf = attachment({ + file_id: 'manual', + filename: 'manual.pdf', + type: PDF, + bytes: 5 * MB, + llmDeliveryPath: 'provider', + }); + const workbook = attachment({ file_id: 'q3', filename: 'Q3.xlsx' }); + const agent = agentWith({ + files: [pdf, workbook], + consumers: NO_READER, + reading: { configuredFileSizeLimit: MB }, + primedSearchFileIds: [], + dynamicToolContextMap: { [Tools.file_search]: NO_FILES_NOTE, file_inventory: 'stale' }, + }); + readingContextOf(agent).recordDropped([ + { file_id: 'scan', filename: 'scan.tiff', type: 'image/tiff', bytes: 9 * MB }, + ]); + return agent; + }; + const queuedCodeOnly = { + [Tools.file_search]: NO_FILES_NOTE, + queued_code_files: expect.stringContaining('/mnt/data/Q3.xlsx'), + }; + + it.each<[string, () => TestAgent, Record]>([ + ['under classic routing with no policy set', () => classicAgent({}), queuedCodeOnly], + ['under classic routing', () => classicAgent({ llmDeliveryPolicy: 'classic' }), queuedCodeOnly], + ['when no file tool is loaded', untooledAgent, { [Tools.file_search]: NO_FILES_NOTE }], + ])('writes no inventory and leaves the File Search note alone %s', (_case, arrange, expected) => { + const agent = arrange(); + + prepare(agent); + + expect(agent.dynamicToolContextMap).toEqual(expected); + }); + + describe('the File Search note', () => { + const oversized = () => + attachment({ + file_id: 'manual', + filename: 'manual.pdf', + type: PDF, + bytes: 5 * MB, + llmDeliveryPath: 'provider', + }); + const agentFor = (file: TFile, primedSearchFileIds?: string[]) => + agentWith({ + files: [file], + consumers: SEARCHES, + vectorDBFiles: [file], + reading: { configuredFileSizeLimit: MB }, + primedSearchFileIds, + dynamicToolContextMap: { [Tools.file_search]: NO_FILES_NOTE }, + }); + + it('is replaced by the inventory when nothing was primed and a file awaits indexing', () => { + const agent = agentFor(oversized(), []); + + prepare(agent); + + expect(agent.dynamicToolContextMap?.[Tools.file_search]).toBeUndefined(); + expect(lineFor(agent, 'manual.pdf')).toContain('it is indexed when you first search'); + }); + + it.each([ + ['the loader primed files', ['registered'], oversized()], + ['the primed files are unknown', undefined, oversized()], + [ + 'the file is already indexed', + [], + { ...oversized(), embedded: true, metadata: { destinationChosen: false } }, + ], + ])('stays when %s', (_case, primedSearchFileIds, file) => { + const agent = agentFor(file, primedSearchFileIds); + + prepare(agent); + + expect(agent.dynamicToolContextMap?.[Tools.file_search]).toBe(NO_FILES_NOTE); + }); + + it('stays when no file is left to File Search', () => { + const workbook = attachment({ file_id: 'q3', filename: 'Q3.xlsx' }); + const agent = agentWith({ + files: [workbook], + consumers: { executeCode: true, fileSearch: true }, + codeEnvFiles: [workbook], + primedSearchFileIds: [], + dynamicToolContextMap: { [Tools.file_search]: NO_FILES_NOTE }, + }); + + prepare(agent); + + expect(agent.dynamicToolContextMap?.[Tools.file_search]).toBe(NO_FILES_NOTE); + expect(lineFor(agent, 'Q3.xlsx')).toContain('read it with Run Code'); + }); + }); + + describe('content policy', () => { + const filters: FiltersConfig = { + files: { + pii: { + fields: ['content'], + starterPatterns: [], + customPatterns: [{ id: 'private', label: 'private value', regex: 'PRIVATE-[A-Z]+' }], + }, + }, + }; + + it('inspects the rendered inventory before writing it', () => { + const workbook = attachment({ file_id: 'plan', filename: 'PRIVATE-PLAN.xlsx' }); + const agent = agentWith({ files: [workbook], consumers: RUNS_CODE }); + + expect(() => prepare(agent, filters)).toThrow(ContentFilterError); + expect(agent.dynamicToolContextMap?.file_inventory).toBeUndefined(); + }); + + it('passes an inventory whose file text alone matches, since contents are never listed', () => { + const workbook = attachment({ + file_id: 'plan', + filename: 'plan.xlsx', + source: FileSources.text, + text: 'PRIVATE-VALUE', + llmDeliveryPath: 'text', + }); + const agent = agentWith({ files: [workbook], consumers: RUNS_CODE }); + + prepare(agent, filters); + + expect(inventoryOf(agent)).not.toContain('PRIVATE-VALUE'); + }); + + it('skips the check without filters and under classic routing', () => { + const workbook = attachment({ file_id: 'plan', filename: 'PRIVATE-PLAN.xlsx' }); + const automatic = agentWith({ files: [workbook], consumers: RUNS_CODE }); + const classic = agentWith({ files: [workbook], consumers: RUNS_CODE, endpointConfig: {} }); + + expect(() => prepare(automatic)).not.toThrow(); + expect(() => prepare(classic, filters)).not.toThrow(); + expect(inventoryOf(automatic)).toContain('PRIVATE-PLAN.xlsx'); + }); + }); +}); diff --git a/packages/api/src/files/reading/inventory.ts b/packages/api/src/files/reading/inventory.ts new file mode 100644 index 00000000000..9b10317478e --- /dev/null +++ b/packages/api/src/files/reading/inventory.ts @@ -0,0 +1,467 @@ +import { + getRoutingMimeType, + hasFileToolConsumer, + Tools, + EToolResources, + isTextOnlyRecord, + decideFileReading, + categorizeForReading, + getCodeEnvRefForProfile, + resolveLLMDeliveryPolicy, + hasToolResourceProvisioning, +} from 'librechat-data-provider'; +import type { + FileReading, + FiltersConfig, + ReadingReason, + TurnDeliveryRouting, +} from 'librechat-data-provider'; +import type { CodeFileAgent, ProvisionToolContext } from '~/files/code/queued'; +import type { SearchPreparationState, TurnReadingFile } from './turn'; +import type { CodeFileLocation } from '~/files/code/priming'; +import { CODE_FILE_CONTEXT_ROOTS, getCodeFileLocation } from '~/files/code/priming'; +import { assertModelBoundContent } from '~/middleware/modelBoundContent'; +import { prepareQueuedCodeFileContext } from '~/files/code/queued'; +import { getCodeExecutionRouteKey } from '~/agents/execution'; +import { getTurnReadingContext } from './turn'; + +/** The fields of an initialized agent the file inventory reads. */ +export interface ReadingAgent + extends Pick< + CodeFileAgent, + 'id' | 'fileConsumers' | 'provisionState' | 'codeExecutionContext' | 'dynamicToolContextMap' + > { + deliveryRouting?: TurnDeliveryRouting; + currentRequestAttachments?: readonly TurnReadingFile[]; + /** Files the File Search advert listed on this turn; unknown when absent. */ + primedSearchFileIds?: readonly string[]; +} + +/** A file the File Search advert listed, as the tool loader primed it. */ +export interface PrimedSearchFile { + file_id: string; + filename: string; + fromAgent: boolean; +} + +/** The fields one file's reading and code location are decided from. */ +type DecidingAgent = Pick< + ReadingAgent, + 'deliveryRouting' | 'fileConsumers' | 'provisionState' | 'codeExecutionContext' +>; + +/** Where Run Code can open a file on this turn. */ +export type CodeFileState = + | { state: 'queued'; path: string; location: CodeFileLocation } + | { state: 'live' } + | { state: 'not_prepared' }; + +type QueuedCodeFile = Extract; + +/** Why a classic provider reading was not delivered, from the turn's encoder evidence. */ +type WithheldReason = Extract; + +/** One inventory line's inputs: a file with its reading, or a file the endpoint refused. */ +export type InventoryEntry = + | { + kind: 'read'; + file: TurnReadingFile; + reading: FileReading; + code?: CodeFileState; + search?: SearchPreparationState; + withheld?: WithheldReason; + } + | { kind: 'dropped'; file: TurnReadingFile }; + +type ReadEntry = Extract; + +export interface AgentFileContextOptions { + /** The content policy the rendered inventory must pass before it reaches the model. */ + filters?: FiltersConfig; + /** + * Whether the agent's messages carry the request and its attachments. A subagent child + * receives only its task, so nothing sent with the request is described as sent to it. + */ + receivesRequest?: boolean; +} + +/** Whose messages the rendered lines describe. */ +export interface InventoryView { + receivesRequest: boolean; +} + +const REQUEST_VIEW: InventoryView = { receivesRequest: true }; + +const INVENTORY_KEY = 'file_inventory'; + +const INVENTORY_HEADER = + '- Attached files and how you can read them on this turn (file contents are not listed here):'; + +const LEFT_OUT_HEADER = + '- Shared files left out of this message (file contents are not listed here):'; + +const REASON_PHRASES: Readonly>> = { + native_capacity: 'too large to send directly', + native_unsupported: 'this model cannot read this type directly', + native_rejected: 'the model could not accept it', + text_exceeds: 'too long to include', + aggregate_overflow: 'did not fit with the other attachments', +}; + +const NOT_READ = 'Say so if asked; do not claim to have read it.'; +const NOT_IN_TASK = + 'not sent with your task. Do not claim to have read it unless it is shared with you.'; +const TEXT_ONLY = 'only an extracted-text copy was kept, so Run Code cannot open the original.'; +const SEARCH_EXCERPTS = 'Results are excerpts, not the whole document.'; +const CODE_FILE_LIST = "(see the code tool's file list)"; + +const READABLE_CLASSIC_READERS: ReadonlySet = new Set([ + 'provider', + 'text', + 'search', + 'code', +]); + +/** Readers whose content travels inside the request's own message. */ +const MESSAGE_READERS: ReadonlySet = new Set(['provider', 'text']); + +/** Line separators JSON leaves raw, which some renderers still break a line on. */ +const RAW_LINE_BREAKS = new RegExp(`[${String.fromCharCode(0x85, 0x2028, 0x2029)}]`, 'g'); + +const NOT_PREPARED: CodeFileState = { state: 'not_prepared' }; + +const capitalize = (sentence: string): string => + sentence.charAt(0).toUpperCase() + sentence.slice(1); + +const codeRouteKey = (agent: DecidingAgent): string => + agent.codeExecutionContext == null + ? 'default' + : getCodeExecutionRouteKey(agent.codeExecutionContext); + +/** + * Queued paths come only from the destinations the planner just wrote, so the inventory and the + * lazy upload name the same path (#16578). + */ +function locateCodeFile(agent: DecidingAgent, file: TurnReadingFile): CodeFileState { + const destination = agent.provisionState?.codeEnvDestinations?.get(file.file_id); + if (destination != null) { + const location = getCodeFileLocation(agent.codeExecutionContext); + return { + state: 'queued', + path: `${CODE_FILE_CONTEXT_ROOTS[location]}/${destination}`, + location, + }; + } + if (getCodeEnvRefForProfile(file.metadata, codeRouteKey(agent)) != null) { + return { state: 'live' }; + } + return NOT_PREPARED; +} + +/** + * A classic provider reading names no other reader, so the turn's encoder evidence is what says + * whether the file went out: a recorded rejection, or a type the encoder never emits. + */ +function findWithheldReason( + agent: DecidingAgent, + file: TurnReadingFile, + reading: FileReading, +): WithheldReason | undefined { + if (reading.automatic || reading.reader !== 'provider') { + return undefined; + } + const evidence = getTurnReadingContext(agent.deliveryRouting)?.judge(file); + if (evidence?.rejected != null) { + return evidence.rejected === 'unsupported' ? 'native_unsupported' : 'native_rejected'; + } + return evidence?.native === 'unsupported' ? 'native_unsupported' : undefined; +} + +function readEntry(agent: DecidingAgent, file: TurnReadingFile): ReadEntry { + const reading = decideFileReading({ + routing: agent.deliveryRouting, + file, + consumers: agent.fileConsumers, + }); + const withheld = findWithheldReason(agent, file, reading); + const search = + reading.reader === 'search' + ? getTurnReadingContext(agent.deliveryRouting)?.searchState(file.file_id) + : undefined; + return { + kind: 'read', + file, + reading, + ...(reading.code === 'eligible' && { code: locateCodeFile(agent, file) }), + ...(search != null && { search }), + ...(withheld != null && { withheld }), + }; +} + +/** Classic routes are listed only where they name a reader; the automatic walk always does. */ +const isListed = ({ file, reading }: ReadEntry): boolean => + reading.automatic || isTextOnlyRecord(file) || READABLE_CLASSIC_READERS.has(reading.reader); + +const isAutomaticSearch = ({ reading }: ReadEntry): boolean => + reading.automatic && reading.reader === 'search'; + +/** The files not in `seen`, once each and in order. */ +function selectUnseen( + files: readonly TurnReadingFile[], + seen: ReadonlySet, +): TurnReadingFile[] { + const selected = new Map(); + for (const file of files) { + if (!seen.has(file.file_id) && !selected.has(file.file_id)) { + selected.set(file.file_id, file); + } + } + return [...selected.values()]; +} + +/** + * Decides each file the inventory lists once, from the turn's current evidence: the request's + * attachments, files queued for File Search that the automatic policy leaves to it, and request + * attachments the endpoint refused. + */ +export function collectInventoryEntries(agent: ReadingAgent): InventoryEntry[] { + const currentFiles = selectUnseen(agent.currentRequestAttachments ?? [], new Set()); + const seen = new Set(currentFiles.map(({ file_id }) => file_id)); + const pendingSearch = selectUnseen(agent.provisionState?.vectorDBFiles ?? [], seen) + .map((file) => readEntry(agent, file)) + .filter(isAutomaticSearch); + pendingSearch.forEach(({ file }) => seen.add(file.file_id)); + const dropped = selectUnseen(getTurnReadingContext(agent.deliveryRouting)?.dropped() ?? [], seen); + return [ + ...currentFiles.map((file) => readEntry(agent, file)).filter(isListed), + ...pendingSearch, + ...dropped.map((file): InventoryEntry => ({ kind: 'dropped', file })), + ]; +} + +const isPendingSearch = (entry: InventoryEntry): boolean => + entry.kind === 'read' && + entry.reading.reader === 'search' && + !hasToolResourceProvisioning(entry.file, EToolResources.file_search); + +/** + * With nothing primed, the File Search advert can only be its "no files are currently loaded" + * note, which a file awaiting indexing contradicts; the inventory lists that file instead. + */ +function reconcileSearchAdvert(agent: ReadingAgent, entries: readonly InventoryEntry[]): void { + if (agent.primedSearchFileIds?.length !== 0 || agent.dynamicToolContextMap == null) { + return; + } + if (entries.some(isPendingSearch)) { + delete agent.dynamicToolContextMap[Tools.file_search]; + } +} + +function describeType( + file: TurnReadingFile, + category: FileReading['category'] = categorizeForReading(getRoutingMimeType(file)), +): string { + const mimeType = getRoutingMimeType(file); + if (category === 'tabular') { + return 'spreadsheet'; + } + if (category === 'media') { + return mimeType.split('/', 1)[0].toLowerCase(); + } + if (mimeType === 'application/pdf') { + return 'PDF'; + } + return mimeType.startsWith('text/') ? 'text' : 'document'; +} + +/** Primed files on an attached workspace reach only scripts run by programmatic Bash. */ +const describeQueuedReader = (code: QueuedCodeFile, also = ''): string => + code.location === 'programmatic' + ? `scripts run by the programmatic Bash tool can${also} read it at ${code.path}` + : `Run Code can${also} open it at ${code.path}`; + +function describeCodeReader(code: CodeFileState): string { + if (code.state === 'queued' && code.location === 'programmatic') { + return `${describeQueuedReader(code)}. It is copied when such a script first runs.`; + } + if (code.state === 'queued') { + return `read it with Run Code at ${code.path}. It is copied when code first runs.`; + } + if (code.state === 'live') { + return `read it with Run Code; it is already in the code environment ${CODE_FILE_LIST}.`; + } + return 'Run Code is available, but this file could not be prepared on this turn. Do not claim to have read it.'; +} + +function describeAlsoInCode(code?: CodeFileState): string | undefined { + if (code?.state === 'queued') { + return `${capitalize(describeQueuedReader(code, ' also'))}.`; + } + return code?.state === 'live' ? `Run Code can also open it ${CODE_FILE_LIST}.` : undefined; +} + +const withAlsoInCode = (lead: string, code?: CodeFileState): string => { + const alsoInCode = describeAlsoInCode(code); + return alsoInCode == null ? lead : `${lead} ${alsoInCode}`; +}; + +function describeSearch(file: TurnReadingFile, state?: SearchPreparationState): string { + if (state === 'failed') { + return `File Search could not prepare it on this turn. Retry ${Tools.file_search} to try indexing again. ${NOT_READ}`; + } + const indexed = + state === 'ready' || hasToolResourceProvisioning(file, EToolResources.file_search); + const pending = indexed ? '' : '; it is indexed when you first search'; + return `search it with ${Tools.file_search}${pending}. ${SEARCH_EXCERPTS}`; +} + +function describeUnavailable(reading: FileReading): string { + if (reading.reason === 'code_unavailable') { + return `cannot be read on this turn because Run Code is not available. ${NOT_READ}`; + } + const phrase = REASON_PHRASES[reading.reason]; + return `cannot be read on this turn${phrase == null ? '' : ` (${phrase})`}. ${NOT_READ}`; +} + +function describeReader( + file: TurnReadingFile, + reading: FileReading, + code: CodeFileState = NOT_PREPARED, + search?: SearchPreparationState, +): string { + switch (reading.reader) { + case 'provider': + return 'sent with this message.'; + case 'text': + return reading.automatic + ? 'its complete extracted text is included in this message.' + : 'its extracted text is included in this message.'; + case 'search': + return describeSearch(file, search); + case 'code': + return describeCodeReader(code); + default: + return describeUnavailable(reading); + } +} + +function describeRead( + { file, reading, code, search, withheld }: ReadEntry, + { receivesRequest }: InventoryView, +): string { + if (!receivesRequest && MESSAGE_READERS.has(reading.reader)) { + return withAlsoInCode(NOT_IN_TASK, code); + } + if (withheld != null) { + return withAlsoInCode( + `cannot be read on this turn (${REASON_PHRASES[withheld]}). ${NOT_READ}`, + code, + ); + } + if (isTextOnlyRecord(file)) { + return reading.reader === 'text' + ? `its extracted text is included in this message. ${capitalize(TEXT_ONLY)}` + : TEXT_ONLY; + } + const status = describeReader(file, reading, code, search); + if (reading.reader === 'search' && search === 'failed') { + return withAlsoInCode(status, code); + } + if (reading.reader === 'unavailable' || reading.reader === 'unresolved') { + return status; + } + const phrase = REASON_PHRASES[reading.reason]; + const lead = phrase == null ? status : `${phrase}. ${capitalize(status)}`; + return reading.reader === 'code' ? lead : withAlsoInCode(lead, code); +} + +const escapeLineBreak = (separator: string): string => + `\\u${separator.charCodeAt(0).toString(16).padStart(4, '0')}`; + +/** A file name as one quoted token: JSON escapes plus the line separators JSON leaves raw. */ +const quoteFilename = (filename?: string | null): string => + JSON.stringify(filename ?? 'attachment').replace(RAW_LINE_BREAKS, escapeLineBreak); + +function renderEntry(entry: InventoryEntry, view: InventoryView): string { + const { file } = entry; + const name = quoteFilename(file.filename); + if (entry.kind === 'dropped') { + return `${name} (${describeType(file)}): not available with this model (its type or size is not allowed here).`; + } + const { reading, code } = entry; + const type = describeType(file, reading.category); + const queuedForCode = reading.reader === 'code' && code?.state === 'queued'; + const label = queuedForCode ? `${type}, file_id ${file.file_id}` : type; + return `${name} (${label}): ${describeRead(entry, view)}`; +} + +const renderLines = ( + header: string, + entries: readonly InventoryEntry[], + view: InventoryView, +): string => + entries.length === 0 + ? '' + : [header, ...entries.map((entry) => `\t- ${renderEntry(entry, view)}`)].join('\n'); + +/** + * The model's view of how each attachment is read on this turn. Pending preparation is worded + * apart from delivery, and no line carries file contents. Empty when nothing is listed. + */ +export function renderFileInventory( + entries: readonly InventoryEntry[], + view: InventoryView = REQUEST_VIEW, +): string { + return renderLines(INVENTORY_HEADER, entries, view); +} + +/** + * Names the shared files a message left out once the turn's evidence for them is recorded, and + * how each can still be read, so none disappears unnoticed. Empty without such files. + */ +export function renderLeftOutFiles( + agent: DecidingAgent, + files: readonly TurnReadingFile[], +): string { + return renderLines( + LEFT_OUT_HEADER, + selectUnseen(files, new Set()).map((file) => readEntry(agent, file)), + REQUEST_VIEW, + ); +} + +/** + * Plans the agent's queued code files, then lists how the automatic policy reads each attachment + * as `file_inventory`. The planner runs first in the same call, so the inventory restates its + * destinations and a re-plan re-renders both. Classic routing, and a turn that loads no file + * tool, write no inventory and leave the File Search advert alone. + */ +export function prepareAgentFileContext( + agent: CodeFileAgent & ReadingAgent, + contexts: Iterable, + userId?: string, + useAdvertisedNames = false, + { filters, receivesRequest = true }: AgentFileContextOptions = {}, +): void { + prepareQueuedCodeFileContext(agent, contexts, userId, useAdvertisedNames); + if (agent.dynamicToolContextMap) { + delete agent.dynamicToolContextMap[INVENTORY_KEY]; + } + if ( + resolveLLMDeliveryPolicy(agent.deliveryRouting?.endpointConfig) !== 'automatic' || + !hasFileToolConsumer(agent.fileConsumers) + ) { + return; + } + const entries = collectInventoryEntries(agent); + reconcileSearchAdvert(agent, entries); + const inventory = renderFileInventory(entries, { receivesRequest }); + if (!inventory) { + return; + } + if (filters) { + assertModelBoundContent({ filters, files: [{ content: inventory }] }); + } + agent.dynamicToolContextMap ??= {}; + agent.dynamicToolContextMap[INVENTORY_KEY] = inventory; +} diff --git a/packages/api/src/files/reading/native.spec.ts b/packages/api/src/files/reading/native.spec.ts new file mode 100644 index 00000000000..829f9fe8c2b --- /dev/null +++ b/packages/api/src/files/reading/native.spec.ts @@ -0,0 +1,678 @@ +import mongoose from 'mongoose'; +import { Readable } from 'stream'; +import { Providers } from '@librechat/agents'; +import { createModels } from '@librechat/data-schemas'; +import { megabyte, FileContext, FileSources, decideFileReading } from 'librechat-data-provider'; +import type { TFile, FiltersConfig, TurnFileConsumers } from 'librechat-data-provider'; +import type { IMongoFile } from '@librechat/data-schemas'; +import type { DocumentBlock, ServerRequest, StrategyFunctions } from '~/types'; +import { + getTurnTextOptions, + encodeNativeDocuments, + recordNativeRejections, + buildTurnReadingContext, +} from './turn'; +import { assertAgentAttachmentLimits, isModelBoundAttachmentFile } from '~/agents/attachments'; +import { resolveTurnDeliveryRouting, applyTurnDelivery } from '~/agents/files/delivery'; +import { admitNativeFallbackAttachments, prepareMessageAttachments } from './native'; +import { encodeAndFormatDocuments } from '~/files/encode/document'; +import { ContentFilterError } from '~/middleware/contentFilter'; +import { parseDocument } from '~/files/documents/crud'; +import { extractFileContext } from '~/files/context'; +import { createFileTextDeriver } from './derive'; +import { parseTextNative } from '~/files/text'; +import { prepareTurnFiles } from './settle'; +import Tokenizer from '~/utils/tokenizer'; + +/* Bridge Jest's CommonJS loader to the real PDF.js ESM implementation, including its worker. */ +jest.mock('pdfjs-dist/legacy/build/pdf.mjs', () => { + const { createRequire } = process.getBuiltinModule('node:module'); + return createRequire(__filename)('pdfjs-dist/legacy/build/pdf.mjs'); +}); + +const FileModel = createModels(mongoose).File; +const noTools: TurnFileConsumers = { executeCode: false, fileSearch: false }; +const search: TurnFileConsumers = { executeCode: false, fileSearch: true }; +const code: TurnFileConsumers = { executeCode: true, fileSearch: false }; +let countTokens: (text: string) => number; + +beforeAll(async () => { + countTokens = await Tokenizer.createExactTokenCounter('o200k_base'); +}); + +interface StoredPdf { + file: TFile; + original: Buffer; +} + +interface Message { + documents: DocumentBlock[]; + fileContext?: string; +} + +/** An uncompressed PDF with a correct xref table, text on its first page and blank later pages. */ +function pdfBytes(title: string, pages = 101): Buffer { + const content = `BT /F1 12 Tf 20 100 Td (${title}) Tj ET`; + const objects = [ + '<< /Type /Catalog /Pages 2 0 R >>', + `<< /Type /Pages /Kids [${Array.from({ length: pages }, (_, i) => `${i + 6} 0 R`).join(' ')}] /Count ${pages} >>`, + '<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>', + `<< /Length ${content.length} >>\nstream\n${content}\nendstream`, + '<< /Length 0 >>\nstream\n\nendstream', + ...Array.from( + { length: pages }, + (_, i) => + `<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Resources << /Font << /F1 3 0 R >> >> /Contents ${i === 0 ? 4 : 5} 0 R >>`, + ), + ]; + let document = '%PDF-1.4\n'; + const offsets = [0]; + for (const [index, object] of objects.entries()) { + offsets.push(Buffer.byteLength(document)); + document += `${index + 1} 0 obj\n${object}\nendobj\n`; + } + const xref = Buffer.byteLength(document); + document += `xref\n0 ${objects.length + 1}\n0000000000 65535 f \n`; + document += offsets + .slice(1) + .map((offset) => `${String(offset).padStart(10, '0')} 00000 n \n`) + .join(''); + document += `trailer\n<< /Size ${objects.length + 1} /Root 1 0 R >>\nstartxref\n${xref}\n%%EOF\n`; + return Buffer.from(document); +} + +function storedPdf( + id: string, + { text, pages = 101 }: { text?: string; pages?: number } = {}, +): StoredPdf { + const original = pdfBytes(id, pages); + return { + original, + file: { + file_id: id, + filename: `${id}.pdf`, + filepath: `/files/${id}.pdf`, + user: 'user', + object: 'file', + usage: 0, + embedded: false, + type: 'application/pdf', + bytes: original.length, + source: FileSources.local, + context: FileContext.message_attachment, + llmDeliveryPath: 'provider', + metadata: { destinationChosen: false, textDerivation: { outcome: 'deferred' } }, + ...(text != null && { text }), + }, + }; +} + +/** + * File Search is loaded unless a test says otherwise, but has queued and registered none of the + * files, as a failed provisioning leaves it: the walk passes search by, no code tool is loaded, + * and a PDF the provider rejects falls back to its text. `queuedForSearch` makes the files + * reachable, so the walk ends at File Search instead. Without any file tool the automatic + * policy reads nothing and a provider rejection fails the turn as classic routing does. + */ +function setup({ + current = [], + historical = [], + consumers = search, + queuedForSearch = false, + fileTokenLimit = 10_000, + fileContextCharLimit = 100_000, + fileLimit, + fileContextSizeLimit, + endpointTotalSizeLimit, + abortController, + filters, +}: { + current?: StoredPdf[]; + historical?: StoredPdf[]; + consumers?: TurnFileConsumers; + queuedForSearch?: boolean; + fileTokenLimit?: number; + fileContextCharLimit?: number; + fileLimit?: number; + fileContextSizeLimit?: number; + endpointTotalSizeLimit?: number; + abortController?: AbortController; + filters?: FiltersConfig; +} = {}) { + const fileConfig: NonNullable['fileConfig'] = { + fileContextCharLimit, + fileContextSizeLimit, + endpoints: { + anthropic: { + llmDeliveryPolicy: 'automatic', + fileLimit, + totalSizeLimit: endpointTotalSizeLimit, + }, + }, + }; + const req = { + body: { fileTokenLimit }, + config: { + filters, + fileConfig, + }, + } as ServerRequest; + const originals = new Map( + [...historical, ...current].map(({ file, original }) => [file.filepath, original]), + ); + const openStoredFile = jest.fn(async (file: { filepath?: string }) => { + const original = originals.get(file.filepath ?? ''); + return original == null ? null : Readable.from([original]); + }); + const storage: StrategyFunctions = { + getDownloadStream: jest.fn(async (_req, filepath) => { + const original = originals.get(filepath); + if (original == null) throw Object.assign(new Error('Missing original'), { code: 'ENOENT' }); + return Readable.from([original]); + }), + }; + const extractors = { parseDocument: jest.fn(parseDocument), parseTextNative }; + const deriveText = jest.fn(createFileTextDeriver({ req, openStoredFile, filters, extractors })); + const persistDerivation = jest.fn(async () => true); + const routing = resolveTurnDeliveryRouting({ + agent: { provider: Providers.ANTHROPIC }, + config: req.config, + }); + const context = buildTurnReadingContext({ + routing, + provider: Providers.ANTHROPIC, + fileTokenLimit, + configuredFileSizeLimit: undefined, + countTokens, + deriveText, + persistDerivation, + }); + if (context == null) throw new Error('Expected automatic reading context'); + routing.reading = context; + context.setSearchEvidence({ + queued: queuedForSearch ? [...historical, ...current].map(({ file }) => file.file_id) : [], + registered: [], + }); + const primary = { + id: 'primary', + endpoint: Providers.ANTHROPIC, + deliveryRouting: routing, + fileConsumers: consumers, + currentRequestAttachments: current.map(({ file }) => file), + requestAttachments: current.map(({ file }) => file), + attachments: current.map(({ file }) => file), + }; + const extractText = jest.fn(extractFileContext); + const addFileContextToMessage = jest.fn(async (message: Message, files: TFile[]) => { + const textFiles = applyTurnDelivery(files, { routing, consumers }).filter( + (file) => file.llmDeliveryPath === 'text', + ); + if (textFiles.length > 0) { + message.fileContext = await extractText({ + attachments: textFiles, + req, + tokenCountFn: countTokens, + ...getTurnTextOptions(routing), + }); + } + }); + const admitPreparedAttachments = (files: TFile[]): Promise => + admitNativeFallbackAttachments(client, files); + const client = { + options: { + req, + agent: primary, + abortController, + attachments: Promise.resolve(current.map(({ file }) => file)), + }, + authorizedHistoricalFiles: new Map(historical.map(({ file }) => [file.file_id, file])), + authorizedHistoricalReplayFiles: new Map(historical.map(({ file }) => [file.file_id, file])), + message_file_map: {} as Record, + modelBoundCurrentFiles: current.map(({ file }) => file), + modelBoundHistoricalSteerFiles: historical.map(({ file }) => file), + turnSharedAttachmentFiles: [...historical, ...current].map(({ file }) => file), + turnScopedAttachmentsByAgentId: new Map(), + turnAttachmentEndpointsByAgentId: new Map([ + ['primary', { endpoint: Providers.ANTHROPIC }], + ]), + getConversationAgents: () => [primary], + assertTurnAttachmentLimits: (shared: TFile[], scoped: TFile[]) => { + assertAgentAttachmentLimits({ + attachments: [...shared, ...scoped], + req, + endpoint: Providers.ANTHROPIC, + historicalFileIds: new Set(historical.map(({ file }) => file.file_id)), + countRepeatedExtractedText: true, + enforceAttachmentCount: false, + useGlobalContextSizeLimit: true, + }); + }, + processAttachments: jest.fn(async (message: Message, files: TFile[]) => { + const processed = applyTurnDelivery(files, { routing, consumers }); + const result = await encodeNativeDocuments( + processed.filter((file) => file.llmDeliveryPath === 'provider'), + primary, + (nativeFiles, onValidationFailure) => + encodeAndFormatDocuments( + req, + nativeFiles.map((file) => FileModel.hydrate(file).toObject()), + { provider: Providers.ANTHROPIC, onValidationFailure }, + () => storage, + ), + ); + recordNativeRejections([primary], result.rejected); + message.documents.push(...result.documents); + return processed; + }), + prepareTurnAttachments: jest.fn((files: TFile[]) => + prepareTurnFiles({ routing, consumers, files }), + ), + addFileContextToMessage, + admitPreparedAttachments, + }; + const prepare = async (files: TFile[]) => { + const message: Message = { documents: [] }; + const prepared = await prepareMessageAttachments({ client, message, files }); + return { message, prepared }; + }; + return { + client, + context, + primary, + prepare, + storage, + deriveText, + persistDerivation, + extractText, + extractors, + }; +} + +describe('native attachment fallback', () => { + it.each([ + ['File Search', search, true, 'search', 'none'], + ['Run Code', code, false, 'code', 'none'], + ['File Search it cannot reach', search, false, 'text', 'text'], + ] as const)( + 'recovers a real 101-page PDF initially and in a fresh history turn with %s', + async (_name, consumers, queuedForSearch, reader, path) => { + const fixture = storedPdf('CACHED-SENTINEL', { text: 'Complete cached attachment text' }); + for (const historical of [false, true]) { + const harness = setup({ + [historical ? 'historical' : 'current']: [fixture], + consumers, + queuedForSearch, + }); + expect(harness.context.judge(fixture.file).native).toBe('fits'); + + const { message, prepared } = await harness.prepare([fixture.file]); + + expect(harness.context.judge(fixture.file).rejected).toBe('capacity'); + expect(prepared?.[0].llmDeliveryPath).toBe(path); + expect( + decideFileReading({ + routing: harness.primary.deliveryRouting, + file: prepared![0], + consumers, + }).reader, + ).toBe(reader); + expect(message.documents).toEqual([]); + expect(harness.storage.getDownloadStream).toHaveBeenCalledTimes(1); + expect(harness.deriveText).not.toHaveBeenCalled(); + if (reader === 'text') { + expect(message.fileContext?.split(fixture.file.text!).length).toBe(2); + expect(message.fileContext).not.toContain('Truncated'); + if (historical) { + expect(harness.client.authorizedHistoricalFiles.get(fixture.file.file_id)).toBe( + prepared![0], + ); + expect(harness.client.modelBoundHistoricalSteerFiles[0]).toBe(prepared![0]); + } else { + expect((await harness.client.options.attachments)[0]).toBe(prepared![0]); + expect(harness.primary.currentRequestAttachments[0]).toBe(prepared![0]); + } + } else { + expect(message.fileContext).toBeUndefined(); + expect(harness.extractText).not.toHaveBeenCalled(); + } + } + expect(fixture.file.llmDeliveryPath).toBe('provider'); + }, + ); + + it.each([ + [ + 'a 101-page PDF', + () => storedPdf('NO-TOOLS', { text: 'Complete cached attachment text' }), + /100-page limit/, + ], + [ + 'a corrupt PDF', + () => { + const fixture = storedPdf('NO-TOOLS-INTEGRITY', { text: 'Cached text' }); + fixture.original = Buffer.from('not a PDF'); + fixture.file.bytes = fixture.original.length; + return fixture; + }, + /missing PDF header/, + ], + ])( + 'fails the turn on %s as classic routing does when no file tool is loaded', + async (_name, fixtureOf, error) => { + const fixture = fixtureOf(); + for (const historical of [false, true]) { + const harness = setup({ + [historical ? 'historical' : 'current']: [fixture], + consumers: noTools, + }); + + await expect(harness.prepare([fixture.file])).rejects.toThrow(error); + + expect(harness.context.judge(fixture.file).rejected).toBeUndefined(); + expect(harness.client.prepareTurnAttachments).not.toHaveBeenCalled(); + expect(harness.extractText).not.toHaveBeenCalled(); + expect(harness.deriveText).not.toHaveBeenCalled(); + } + expect(fixture.file.llmDeliveryPath).toBe('provider'); + }, + ); + + it('keeps a fitting native PDF as a real provider block without duplicating its cached text', async () => { + const fixture = storedPdf('NATIVE-SENTINEL', { pages: 1, text: 'Do not duplicate this text' }); + const harness = setup({ current: [fixture] }); + + const { message } = await harness.prepare([fixture.file]); + + expect(message.documents).toEqual([ + expect.objectContaining({ + type: 'document', + source: { + type: 'base64', + media_type: 'application/pdf', + data: fixture.original.toString('base64'), + }, + }), + ]); + expect(message.fileContext).toBeUndefined(); + expect(harness.extractText).not.toHaveBeenCalled(); + expect(harness.context.judge(fixture.file).rejected).toBeUndefined(); + }); + + it('derives complete PDF text once, then sends the same body from its restored cache', async () => { + const fixture = storedPdf('DERIVED-SENTINEL'); + const harness = setup({ current: [fixture] }); + + const first = await harness.prepare([fixture.file]); + + expect(first.prepared?.[0]).toMatchObject({ + llmDeliveryPath: 'text', + text: `DERIVED-SENTINEL${'\n'.repeat(101)}`, + }); + expect(first.message.documents).toEqual([]); + expect(first.message.fileContext?.split('DERIVED-SENTINEL').length).toBe(3); // filename and text + expect(harness.deriveText).toHaveBeenCalledTimes(1); + expect(harness.persistDerivation).toHaveBeenCalledTimes(1); + expect(fixture.file.text).toBeUndefined(); + const restored = { ...fixture, file: first.prepared![0] }; + const history = setup({ historical: [restored] }); + + const next = await history.prepare([restored.file]); + + expect(next.message.fileContext).toBe(first.message.fileContext); + expect(next.message.documents).toEqual([]); + expect(history.deriveText).not.toHaveBeenCalled(); + }); + + it('leaves excessive cached text unavailable instead of sending a truncated fallback', async () => { + const fixture = storedPdf('TOKEN-LIMIT', { + text: 'A long sentence with many distinct tokens. '.repeat(50), + }); + const harness = setup({ current: [fixture], fileTokenLimit: 5 }); + + const { message, prepared } = await harness.prepare([fixture.file]); + + expect(prepared?.[0].llmDeliveryPath).toBe('none'); + expect(message.documents).toEqual([]); + expect(message.fileContext).toBeUndefined(); + expect(harness.extractText).not.toHaveBeenCalled(); + expect(harness.deriveText).not.toHaveBeenCalled(); + }); + + it.each([ + [search, true], + [search, false], + [code, false], + ])( + 'never substitutes text for an integrity rejection (%j, queued for search: %s)', + async (consumers, queuedForSearch) => { + const fixture = storedPdf('INTEGRITY', { text: 'Cached text must not bypass corruption' }); + fixture.original = Buffer.from('not a PDF'); + fixture.file.bytes = fixture.original.length; + const harness = setup({ current: [fixture], consumers, queuedForSearch }); + + const { message, prepared } = await harness.prepare([fixture.file]); + + expect(harness.context.judge(fixture.file).rejected).toBe('integrity'); + expect(prepared?.[0].llmDeliveryPath).toBe('none'); + expect(message.documents).toEqual([]); + expect(message.fileContext).toBeUndefined(); + expect(harness.extractText).not.toHaveBeenCalled(); + expect(harness.deriveText).not.toHaveBeenCalled(); + }, + ); + + it('allocates newly derived fallback text against other admitted files before text injection', async () => { + const candidate = storedPdf('AGGREGATE-FALLBACK'); + const committed = storedPdf('COMMITTED', { pages: 1, text: 'Existing context' }); + committed.file.llmDeliveryPath = 'text'; + committed.file.metadata = { destinationChosen: true }; + const harness = setup({ current: [candidate, committed], fileContextCharLimit: 120 }); + + const { message, prepared } = await harness.prepare([candidate.file]); + + expect(harness.deriveText).toHaveBeenCalledTimes(1); + expect(prepared?.[0].llmDeliveryPath).toBe('none'); + expect(harness.context.judge(candidate.file).overflow).toBe(true); + expect(message.documents).toEqual([]); + expect(message.fileContext).toBeUndefined(); + expect(harness.extractText).not.toHaveBeenCalled(); + expect((await harness.client.options.attachments)[1]).toBe(committed.file); + }); + + it('serializes concurrent historical fallback allocation against one shared text budget', async () => { + const first = storedPdf('FIRST'); + const second = storedPdf('SECOND'); + const harness = setup({ historical: [first, second], fileContextCharLimit: 150 }); + + const outcomes = await Promise.all([ + harness.prepare([first.file]), + harness.prepare([second.file]), + ]); + + expect(outcomes.map(({ prepared }) => prepared?.[0].llmDeliveryPath).sort()).toEqual([ + 'none', + 'text', + ]); + expect(outcomes.filter(({ message }) => message.fileContext != null)).toHaveLength(1); + expect(harness.extractText).toHaveBeenCalledTimes(1); + expect( + harness.client.turnSharedAttachmentFiles.filter(isModelBoundAttachmentFile), + ).toHaveLength(1); + }); + + it.each([ + ['a current attachment beside one scoped agent', false, 1, ['primary'], 'none'], + ['a historical attachment beside one scoped agent', true, 1, ['primary'], 'text'], + ['a current attachment beside two scoped agents', false, 2, ['primary', 'parallel'], 'text'], + ] as const)( + "charges each agent's own scoped count allowance, exempting history, for %s", + async (_name, historical, fileLimit, agentIds, path) => { + const fixture = storedPdf('SHARED', { text: 'Fits the text budget' }); + const harness = setup({ [historical ? 'historical' : 'current']: [fixture], fileLimit }); + for (const id of agentIds) { + const scoped = storedPdf(`${id}-SCOPED`, { pages: 1, text: 'Scoped text' }).file; + scoped.llmDeliveryPath = 'text'; + harness.client.turnScopedAttachmentsByAgentId.set(id, [scoped]); + harness.client.turnAttachmentEndpointsByAgentId.set(id, { endpoint: Providers.ANTHROPIC }); + } + + const { message, prepared } = await harness.prepare([fixture.file]); + + expect(prepared?.[0].llmDeliveryPath).toBe(path); + if (path === 'text') { + expect(message.fileContext).toContain(fixture.file.text); + } else { + expect(message.fileContext).toBeUndefined(); + } + }, + ); + + it.each([ + ['equal endpoint budgets', false, 'text'], + ['a tighter parallel endpoint', true, 'none'], + ] as const)( + 'reserves scoped bytes separately for %s and the combined global budget', + async (_name, tighterParallel, path) => { + const fixture = storedPdf('SHARED-BYTES', { text: 'Complete fallback text' }); + const unit = fixture.file.bytes; + const harness = setup({ + current: [fixture], + fileContextSizeLimit: (3 * unit) / megabyte, + endpointTotalSizeLimit: (2 * unit) / megabyte, + }); + harness.client.options.req.config!.fileConfig!.endpoints!.parallel = { + totalSizeLimit: ((tighterParallel ? 1.5 : 2) * unit) / megabyte, + }; + for (const id of ['primary', 'parallel']) { + const scoped = storedPdf(`${id}-SCOPED`, { pages: 1, text: 'Scoped text' }).file; + scoped.bytes = unit; + scoped.llmDeliveryPath = 'text'; + harness.client.turnScopedAttachmentsByAgentId.set(id, [scoped]); + harness.client.turnAttachmentEndpointsByAgentId.set(id, { + endpoint: id === 'primary' ? Providers.ANTHROPIC : 'parallel', + }); + } + + const { message, prepared } = await harness.prepare([fixture.file]); + + expect(prepared?.[0].llmDeliveryPath).toBe(path); + expect(message.fileContext != null).toBe(path === 'text'); + expect(harness.context.judge(fixture.file).overflow).toBe(tighterParallel ? true : undefined); + }, + ); + + it('charges the full text of a current attachment replayed in history twice', async () => { + const fixture = storedPdf('REPEATED', { text: 'r'.repeat(90) }); + const harness = setup({ current: [fixture], historical: [fixture], fileContextCharLimit: 100 }); + + const { message, prepared } = await harness.prepare([fixture.file]); + + expect(prepared?.[0].llmDeliveryPath).toBe('none'); + expect(harness.context.judge(fixture.file).overflow).toBe(true); + expect(message.fileContext).toBeUndefined(); + expect(harness.extractText).not.toHaveBeenCalled(); + }); + + it('allows a later admission after the same host rejected sensitive fallback', async () => { + const blocked = storedPdf('BLOCKED', { text: 'PRIVATE-SENTINEL' }); + const safe = storedPdf('SAFE', { text: 'Safe complete text' }); + const filters: FiltersConfig = { + files: { + pii: { + fields: ['extracted_text'], + starterPatterns: [], + customPatterns: [{ id: 'private', label: 'private token', regex: 'PRIVATE-[A-Z]+' }], + }, + }, + }; + const harness = setup({ current: [blocked, safe], filters }); + harness.context.recordRejections( + [blocked, safe].map(({ file }) => ({ file_id: file.file_id, reason: 'capacity' })), + ); + const blockedFiles = await harness.client.prepareTurnAttachments([blocked.file]); + const safeFiles = await harness.client.prepareTurnAttachments([safe.file]); + + const rejected = admitNativeFallbackAttachments(harness.client, blockedFiles); + const recovered = admitNativeFallbackAttachments(harness.client, safeFiles); + + await expect(rejected).rejects.toBeInstanceOf(ContentFilterError); + await expect(recovered).resolves.toEqual(safeFiles); + expect((await harness.client.options.attachments)[1]).toBe(safeFiles[0]); + expect((await harness.client.options.attachments)[0]).toBe(blocked.file); + }); + + it('waits for admission when another message already recorded the native rejection', async () => { + const fixture = storedPdf('KNOWN-REJECTION', { text: 'Complete cached fallback' }); + const harness = setup({ current: [fixture] }); + harness.context.recordRejections([{ file_id: fixture.file.file_id, reason: 'capacity' }]); + const entered = Promise.withResolvers(); + const allowed = Promise.withResolvers(); + const admit = harness.client.admitPreparedAttachments; + harness.client.admitPreparedAttachments = async (files) => { + entered.resolve(); + await allowed.promise; + return admit(files); + }; + + const pending = harness.prepare([fixture.file]); + await entered.promise; + + expect(harness.client.addFileContextToMessage).not.toHaveBeenCalled(); + expect(harness.extractText).not.toHaveBeenCalled(); + allowed.resolve(); + const { message, prepared } = await pending; + expect(prepared?.[0].llmDeliveryPath).toBe('text'); + expect(message.fileContext).toContain(fixture.file.text); + }); + + it('stops after real validation if the request aborts before cached fallback injection', async () => { + const fixture = storedPdf('ABORTED', { text: 'Complete text never injected' }); + const controller = new AbortController(); + const aborted = new Error('Canceled request'); + const harness = setup({ current: [fixture], abortController: controller }); + const process = harness.client.processAttachments.getMockImplementation()!; + harness.client.processAttachments.mockImplementation(async (message, files) => { + const processed = await process(message, files); + controller.abort(aborted); + return processed; + }); + + await expect(harness.prepare([fixture.file])).rejects.toBe(aborted); + + expect(harness.context.judge(fixture.file).rejected).toBe('capacity'); + expect(harness.client.prepareTurnAttachments).not.toHaveBeenCalled(); + expect(harness.client.addFileContextToMessage).not.toHaveBeenCalled(); + expect(harness.extractText).not.toHaveBeenCalled(); + }); + + it.each([ + ['cached', { text: 'PRIVATE-SENTINEL' }, false], + ['derived', {}, false], + ['already rejected cached', { text: 'PRIVATE-SENTINEL' }, true], + ] as const)( + 'blocks sensitive %s fallback before injection or persistence', + async (_kind, stored, rejectedBefore) => { + const filters: FiltersConfig = { + files: { + pii: { + fields: ['extracted_text'], + starterPatterns: [], + customPatterns: [{ id: 'private', label: 'private token', regex: 'PRIVATE-[A-Z]+' }], + }, + }, + }; + const fixture = storedPdf('PRIVATE-SENTINEL', stored); + const harness = setup({ current: [fixture], filters }); + if (rejectedBefore) { + harness.context.recordRejections([{ file_id: fixture.file.file_id, reason: 'capacity' }]); + } + + await expect(harness.prepare([fixture.file])).rejects.toBeInstanceOf(ContentFilterError); + + expect(harness.client.addFileContextToMessage).not.toHaveBeenCalled(); + expect(harness.extractText).not.toHaveBeenCalled(); + expect(harness.persistDerivation).not.toHaveBeenCalled(); + expect(fixture.file.llmDeliveryPath).toBe('provider'); + }, + ); +}); diff --git a/packages/api/src/files/reading/native.ts b/packages/api/src/files/reading/native.ts new file mode 100644 index 00000000000..d2572f93fd9 --- /dev/null +++ b/packages/api/src/files/reading/native.ts @@ -0,0 +1,396 @@ +import { EModelEndpoint, resolveTurnLLMDeliveryPath } from 'librechat-data-provider'; +import type { + DirectContentEntry, + DirectContentLimits, + TurnFileConsumers, +} from 'librechat-data-provider'; +import type { AgentAttachmentEndpointsByAgentId } from '~/agents/attachments'; +import type { HistoryAllocationFile } from './history'; +import type { NativeDeliveryAgent } from './turn'; +import type { ServerRequest } from '~/types'; +import { + getAgentEntry, + listAgentIds, + isModelBoundAttachmentFile, + measureModelBoundAttachment, + resolveAgentAttachmentLimits, + collectHistoricalAttachmentIds, +} from '~/agents/attachments'; +import { getNativeValidationPolicy, getTurnReadingContext } from './turn'; +import { assertModelBoundContent } from '~/middleware/modelBoundContent'; +import { settleTurnFiles } from './settle'; + +interface MessageAttachmentClient { + options?: { agent?: NativeDeliveryAgent | null; abortController?: AbortController }; + processAttachments( + message: M, + files: T[], + consumers?: TurnFileConsumers, + ): Promise; + prepareTurnAttachments(files: T[], consumers?: TurnFileConsumers): Promise; + addFileContextToMessage(message: M, files: T[], consumers?: TurnFileConsumers): Promise; + admitPreparedAttachments?(files: T[], consumers?: TurnFileConsumers): Promise; +} + +/** + * Waits for native validation before extracting text when an automatic document may fall back. + * Other attachments keep their independent extraction and encoding in parallel. The settled + * copies are admitted before their text reaches the message and returned for history/projection. + */ +export async function prepareMessageAttachments({ + client, + message, + files, + consumers, + contextFiles = files, +}: { + client: MessageAttachmentClient; + message: M; + files: T[]; + consumers?: TurnFileConsumers; + contextFiles?: T[]; +}): Promise { + const agent = client.options?.agent; + const signal = client.options?.abortController?.signal; + const modeOf = getNativeValidationPolicy(agent); + const context = getTurnReadingContext(agent?.deliveryRouting); + const mayReject = files.some( + (file) => + modeOf(file) === 'skip' && + (context?.judge(file).rejected != null || + resolveTurnLLMDeliveryPath( + agent?.deliveryRouting ?? undefined, + file, + consumers ?? agent?.fileConsumers, + ) === 'provider'), + ); + if (!mayReject) { + const [, processed] = await Promise.all([ + consumers == null + ? client.addFileContextToMessage(message, contextFiles) + : client.addFileContextToMessage(message, contextFiles, consumers), + consumers == null + ? client.processAttachments(message, files) + : client.processAttachments(message, files, consumers), + ]); + return mergeAttachmentSurvivors(files, processed); + } + + signal?.throwIfAborted(); + const processed = await client.processAttachments(message, files, consumers); + signal?.throwIfAborted(); + let prepared = await client.prepareTurnAttachments( + mergeAttachmentSurvivors(files, processed) ?? files, + consumers, + ); + if (prepared.some((file) => context?.judge(file).rejected != null)) { + prepared = (await client.admitPreparedAttachments?.(prepared, consumers)) ?? prepared; + } + signal?.throwIfAborted(); + await client.addFileContextToMessage(message, prepared, consumers); + return prepared; +} + +/** Encoder metadata may be sparse; retain the authorized original record on surviving copies. */ +function mergeAttachmentSurvivors( + files: T[], + processed: T[] | undefined, +): T[] | undefined { + if (processed == null) { + return processed; + } + const originals = new Map(files.map((file) => [file.file_id, file])); + return processed.map((file) => { + const original = originals.get(file.file_id); + return original == null || original === file + ? file + : { ...original, ...file, metadata: file.metadata ?? original.metadata }; + }); +} + +/** The host's already loaded attachment state and limits, shared by current/history/steering. */ +interface NativeFallbackHost { + options: { + req?: ServerRequest; + endpoint?: string | null; + endpointType?: string | null; + agent?: NativeDeliveryAgent & { endpoint?: string | null }; + attachments?: T[] | Promise; + abortController?: AbortController; + }; + authorizedHistoricalFiles?: Map; + authorizedHistoricalReplayFiles?: Map; + message_file_map?: Record; + modelBoundCurrentFiles?: T[]; + modelBoundHistoricalSteerFiles?: T[]; + turnSharedAttachmentFiles?: T[]; + turnAggregateOnlyAttachmentFiles?: T[]; + turnScopedAttachmentsByAgentId?: Map; + turnAttachmentEndpointsByAgentId?: AgentAttachmentEndpointsByAgentId; + turnHistoricalAttachmentIds?: ReadonlySet; + admittedSteerAttachments?: Map; + attachmentMemoryContext?: { attachments?: T[] }; + assertTurnAttachmentLimits?(shared: T[], scoped: T[]): void; + getConversationAgents?(): Iterable< + | (NativeDeliveryAgent & { + currentRequestAttachments?: T[]; + requestAttachments?: T[]; + attachments?: T[]; + }) + | null + | undefined + >; +} + +const admissions = new WeakMap>(); + +/** Keeps the already admitted historical replay topology available before message encoding. */ +export function retainNativeAttachmentTopology( + host: NativeFallbackHost, + shared: T[], + scoped: Map, + endpoints: AgentAttachmentEndpointsByAgentId, +): void { + host.turnSharedAttachmentFiles = shared; + host.turnScopedAttachmentsByAgentId = scoped; + host.turnAttachmentEndpointsByAgentId = endpoints; +} + +/** + * Shared candidates spend the same footprint in every agent's payload. Reserve each endpoint's + * own scoped content and the global combined context separately, then use the smallest remaining + * allowance per dimension. Keeping that scope's positive limit and charged entries also handles + * an exactly exhausted dimension without treating a remaining zero as unlimited. + */ +function reserveFallbackCapacity( + host: NativeFallbackHost, + committed: T[], + scoped: Map, + historicalFileIds: ReadonlySet, +): { limits: DirectContentLimits; committedEntries: DirectContentEntry[] } { + const req = host.options.req; + const endpoint = host.options.agent?.endpoint ?? host.options.endpoint; + const endpointType = host.options.endpointType; + const measureCommitted = (files: T[]): DirectContentEntry[] => { + const seen = new Set(); + return files.map((file) => { + const measured = measureModelBoundAttachment(file); + const counts = + measured.counts && !historicalFileIds.has(file.file_id) && !seen.has(file.file_id); + seen.add(file.file_id); + return { ...measured, counts }; + }); + }; + const budgets: Array<{ limits: DirectContentLimits; entries: DirectContentEntry[] }> = [ + { + limits: resolveAgentAttachmentLimits({ + req, + endpoint, + endpointType, + enforceAttachmentCount: false, + useGlobalContextSizeLimit: true, + }), + entries: measureCommitted([ + ...committed, + ...(host.turnAggregateOnlyAttachmentFiles ?? []), + ...[...scoped.values()].flat(), + ]), + }, + ]; + const endpoints = host.turnAttachmentEndpointsByAgentId; + const agentIds = new Set([...scoped.keys(), ...listAgentIds(endpoints)]); + const appendEndpointBudget = (agentId?: string): void => { + const selectedEndpoint = agentId == null ? undefined : getAgentEntry(endpoints, agentId); + budgets.push({ + limits: resolveAgentAttachmentLimits({ + req, + endpoint: selectedEndpoint?.endpoint ?? endpoint ?? EModelEndpoint.agents, + endpointType: selectedEndpoint?.endpointType ?? endpointType, + }), + entries: measureCommitted([ + ...committed, + ...(agentId == null ? [] : (scoped.get(agentId) ?? [])), + ]), + }); + }; + if (agentIds.size === 0) { + appendEndpointBudget(); + } else { + for (const agentId of agentIds) appendEndpointBudget(agentId); + } + const limits: DirectContentLimits = {}; + const committedEntries: DirectContentEntry[] = []; + for (const dimension of ['count', 'bytes', 'textChars'] as const) { + let remaining = Infinity; + let selected: (typeof budgets)[number] | undefined; + for (const budget of budgets) { + const limit = budget.limits[dimension]; + if (!limit) continue; + const spent = budget.entries.reduce( + (sum, entry) => sum + (dimension === 'count' ? Number(entry.counts) : entry[dimension]), + 0, + ); + if (limit - spent < remaining) { + remaining = limit - spent; + selected = budget; + } + } + if (selected == null) continue; + limits[dimension] = selected.limits[dimension]; + committedEntries.push( + ...selected.entries.map((entry) => ({ + fileId: entry.fileId, + counts: dimension === 'count' && entry.counts, + bytes: dimension === 'bytes' ? entry.bytes : 0, + textChars: dimension === 'textChars' ? entry.textChars : 0, + })), + ); + } + return { limits, committedEntries }; +} + +/** + * Admits text selected after native rejection against the host's complete loaded shared/scoped + * set. Original types/bytes passed endpoint admission before encoding; this rechecks the added + * text and the aggregate before replacing the turn copies used by subsequent projection. + */ +async function admitFallback( + host: NativeFallbackHost, + files: T[], + consumers: TurnFileConsumers | undefined, +): Promise { + const signal = host.options.abortController?.signal; + signal?.throwIfAborted(); + const replacements = new Map(files.map((file) => [file.file_id, file])); + const replace = (file: T): T => replacements.get(file.file_id) ?? file; + const current = (await host.options.attachments) ?? []; + const historical = [...(host.authorizedHistoricalFiles?.values() ?? [])]; + const shared = (host.turnSharedAttachmentFiles ?? [...historical, ...current]).map(replace); + const sharedIds = new Set(shared.map((file) => file.file_id)); + for (const file of files) { + if (!sharedIds.has(file.file_id)) { + shared.push(file); + sharedIds.add(file.file_id); + } + } + const historicalFileIds = + host.turnHistoricalAttachmentIds ?? collectHistoricalAttachmentIds(historical, current); + const req = host.options.req; + const scoped = host.turnScopedAttachmentsByAgentId ?? new Map(); + const candidates = files.filter((file) => file.llmDeliveryPath === 'text'); + const candidateIds = new Set(candidates.map((file) => file.file_id)); + const copies = new Map(); + for (const file of shared) { + copies.set(file.file_id, (copies.get(file.file_id) ?? 0) + 1); + } + const { limits, committedEntries } = reserveFallbackCapacity( + host, + shared.filter((file) => !candidateIds.has(file.file_id)), + scoped, + historicalFileIds, + ); + const prepared = await settleTurnFiles({ + routing: host.options.agent?.deliveryRouting ?? undefined, + consumers: consumers ?? host.options.agent?.fileConsumers, + files, + allocation: { + requestFileIds: candidates.map((file) => file.file_id), + limits, + measure: (file) => { + const measured = measureModelBoundAttachment(file); + const multiplicity = copies.get(file.file_id) ?? 1; + return { + ...measured, + bytes: measured.bytes * multiplicity, + textChars: measured.textChars * multiplicity, + counts: measured.counts && !historicalFileIds.has(file.file_id), + }; + }, + committedEntries, + scope: 'history', + }, + flush: true, + signal, + }); + signal?.throwIfAborted(); + for (const file of prepared) { + replacements.set(file.file_id, file); + } + const admittedShared = shared.map(replace).filter(isModelBoundAttachmentFile); + host.assertTurnAttachmentLimits?.( + [...admittedShared, ...(host.turnAggregateOnlyAttachmentFiles ?? [])], + [...scoped.values()].flat(), + ); + assertModelBoundContent({ + filters: req?.config?.filters, + files: prepared.filter(isModelBoundAttachmentFile).map((file) => ({ + ...file, + source: file.source ?? undefined, + })), + }); + + const replaceMap = (map?: Map): void => { + for (const [fileId, file] of map ?? []) { + map?.set(fileId, replace(file)); + } + }; + replaceMap(host.authorizedHistoricalFiles); + replaceMap(host.authorizedHistoricalReplayFiles); + const messageFiles = host.message_file_map; + if (messageFiles != null) { + for (const [messageId, attachments] of Object.entries(messageFiles)) { + messageFiles[messageId] = attachments.map(replace).filter(isModelBoundAttachmentFile); + } + } + host.modelBoundCurrentFiles = host.modelBoundCurrentFiles?.map(replace); + host.modelBoundHistoricalSteerFiles = host.modelBoundHistoricalSteerFiles?.map(replace); + for (const [steerId, reserved] of host.admittedSteerAttachments ?? []) { + host.admittedSteerAttachments?.set(steerId, reserved.map(replace)); + } + if (host.attachmentMemoryContext?.attachments != null) { + const memoryFiles = host.attachmentMemoryContext.attachments; + for (let index = 0; index < memoryFiles.length; index++) { + memoryFiles[index] = replace(memoryFiles[index]); + } + } + if (host.turnSharedAttachmentFiles != null) { + const sharedFiles = host.turnSharedAttachmentFiles; + sharedFiles.length = shared.length; + for (let index = 0; index < shared.length; index++) { + sharedFiles[index] = replace(shared[index]); + } + } + if (host.options.attachments != null) { + host.options.attachments = current.map(replace); + } + const primary = getTurnReadingContext(host.options.agent?.deliveryRouting); + const overflowIds = prepared + .filter((file) => primary?.judge(file).overflow) + .map((file) => file.file_id); + for (const agent of host.getConversationAgents?.() ?? []) { + if (agent == null) { + continue; + } + agent.currentRequestAttachments = agent.currentRequestAttachments?.map(replace); + agent.requestAttachments = agent.requestAttachments?.map(replace); + agent.attachments = agent.attachments?.map(replace); + getTurnReadingContext(agent.deliveryRouting)?.addOverflow(overflowIds); + } + return prepared; +} + +/** Serializes allocation for parallel historical messages that spend one turn's text budget. */ +export function admitNativeFallbackAttachments( + host: NativeFallbackHost, + files: T[], + consumers?: TurnFileConsumers, +): Promise { + const previous = admissions.get(host) ?? Promise.resolve(); + const admitted = previous.then(() => admitFallback(host, files, consumers)); + const settled = admitted.then( + () => undefined, + () => undefined, + ); + admissions.set(host, settled); + return admitted; +} diff --git a/packages/api/src/files/reading/settle.spec.ts b/packages/api/src/files/reading/settle.spec.ts new file mode 100644 index 00000000000..6911376966e --- /dev/null +++ b/packages/api/src/files/reading/settle.spec.ts @@ -0,0 +1,513 @@ +import { FileContext, FileSources, decideFileReading } from 'librechat-data-provider'; +import type { TurnFileConsumers, TFileConfig, TurnDeliveryRouting } from 'librechat-data-provider'; +import type { FileTextDeriver, TurnReadingContext, TurnReadingFile, DerivedText } from './turn'; +import type { DirectContentAllocation } from './settle'; +import { applyTurnDelivery, resolveTurnDeliveryRouting } from '~/agents/files/delivery'; +import { measureAttachment, measureModelBoundAttachment } from '~/agents/attachments'; +import { settleTurnFiles, prepareTurnFiles } from './settle'; +import { buildTurnReadingContext } from './turn'; + +type EndpointFileConfigInput = NonNullable[string]; + +/** The module object settle reads through, so a spy on it sees every decision settle asks for. */ +const dataProvider = + jest.requireActual('librechat-data-provider'); + +const MB = 1024 * 1024; +const PDF = 'application/pdf'; +const XLSX = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'; + +const noReader: TurnFileConsumers = { executeCode: false, fileSearch: false }; +const searchOnly: TurnFileConsumers = { executeCode: false, fileSearch: true }; + +const derivedText: DerivedText = { + status: 'derived', + text: 'quarter,total\nQ1,42', + textDerivation: { outcome: 'complete', extractor: 'document_parser', at: 1 }, +}; + +const attachment = ( + overrides: Partial & Pick, +): TurnReadingFile => ({ + type: PDF, + bytes: 4 * MB, + source: FileSources.local, + context: FileContext.message_attachment, + llmDeliveryPath: 'provider', + metadata: { destinationChosen: false }, + ...overrides, +}); + +const deferredWorkbook = attachment({ + file_id: 'xlsx', + type: XLSX, + bytes: 64 * 1024, + llmDeliveryPath: 'none', + metadata: { destinationChosen: false, textDerivation: { outcome: 'deferred' } }, +}); + +function setup({ + endpointConfig = { llmDeliveryPolicy: 'automatic' }, + deriveText, +}: { + endpointConfig?: EndpointFileConfigInput; + deriveText?: FileTextDeriver; +} = {}): { routing: TurnDeliveryRouting; context: TurnReadingContext } { + const routing = resolveTurnDeliveryRouting({ + agent: { provider: 'openAI', endpoint: 'openAI' }, + config: { fileConfig: { endpoints: { openAI: endpointConfig } } }, + }); + const context = buildTurnReadingContext({ + routing, + provider: 'openAI', + model: 'gpt-4o', + fileTokenLimit: 100_000, + configuredFileSizeLimit: undefined, + countTokens: (text) => text.length, + deriveText, + }); + if (context == null) { + throw new Error('expected a reading context'); + } + routing.reading = context; + return { routing, context }; +} + +const byteBudget = ( + requestFileIds: readonly string[], + bytes: number, + committedExtra?: readonly TurnReadingFile[], +): DirectContentAllocation => ({ + requestFileIds, + limits: { bytes }, + measure: measureModelBoundAttachment, + committedExtra, +}); + +const pathsById = (files: readonly TurnReadingFile[]): Record => + Object.fromEntries(files.map((file) => [file.file_id, file.llmDeliveryPath])); + +describe('settleTurnFiles', () => { + describe('text derivation', () => { + it.each<[string, TurnFileConsumers, string, boolean]>([ + ['File Search', searchOnly, 'code_unavailable', true], + /* Without Run Code or File Search the policy decides nothing, but a deferred workbook's + * classic text route still has no text, so settle derives it as a no-tools turn reads it. */ + ['no file tool', noReader, 'no_file_tools', false], + ])( + 'derives the text a reading needs with %s loaded, then decides again on the derived copy', + async (_name, consumers, reason, automatic) => { + const deriveText = jest.fn, Parameters>( + async () => derivedText, + ); + const { routing } = setup({ deriveText }); + const files = [deferredWorkbook]; + + expect(decideFileReading({ routing, file: deferredWorkbook, consumers })).toMatchObject({ + reader: 'text', + reason, + needsText: true, + automatic, + }); + + const [settled] = await settleTurnFiles({ routing, consumers, files }); + + expect(settled).toMatchObject({ + file_id: 'xlsx', + llmDeliveryPath: 'text', + text: derivedText.text, + metadata: { destinationChosen: false, textDerivation: derivedText.textDerivation }, + }); + expect(decideFileReading({ routing, file: settled, consumers })).toMatchObject({ + reader: 'text', + reason, + needsText: false, + automatic, + }); + expect(deferredWorkbook.text).toBeUndefined(); + expect(deferredWorkbook.metadata?.textDerivation).toEqual({ outcome: 'deferred' }); + + await settleTurnFiles({ routing, consumers, files }); + expect(deriveText).toHaveBeenCalledTimes(1); + expect(deriveText).toHaveBeenCalledWith( + expect.objectContaining({ file_id: 'xlsx', llmDeliveryPath: 'text' }), + undefined, + ); + }, + ); + + const parserFailure: DerivedText = { + status: 'failed', + textDerivation: { outcome: 'failed', reason: 'parser' }, + persist: true, + }; + const movedToSearch = { + reader: 'search', + needsText: false, + skipped: expect.arrayContaining([{ reader: 'text', reason: 'text_unavailable' }]), + }; + + it.each<[string, TurnFileConsumers, DerivedText, Record]>([ + ['a failed derivation, with File Search loaded', searchOnly, parserFailure, movedToSearch], + [ + 'a failed derivation, with no file tool loaded', + noReader, + parserFailure, + { reader: 'unavailable', reason: 'no_file_tools', needsText: false, automatic: false }, + ], + [ + 'a skipped derivation, with File Search loaded', + searchOnly, + { status: 'skipped', reason: 'storage_unavailable' }, + movedToSearch, + ], + ])( + 'treats %s as unavailable text for the rest of the request', + async (_name, consumers, outcome, decision) => { + const { routing, context } = setup({ deriveText: async () => outcome }); + + const [settled] = await settleTurnFiles({ routing, consumers, files: [deferredWorkbook] }); + + expect(settled.llmDeliveryPath).toBe('none'); + expect(settled.text).toBeUndefined(); + expect(context.judge(deferredWorkbook).textFailed).toBe(true); + expect(decideFileReading({ routing, file: settled, consumers })).toMatchObject(decision); + }, + ); + + it('never decides or derives an unmarked record in a classic derive-only context', async () => { + const { routing, context } = setup({ + endpointConfig: {}, + deriveText: async () => derivedText, + }); + const deriveSpy = jest.spyOn(context, 'derive'); + const decideSpy = jest.spyOn(dataProvider, 'decideFileReading'); + const unmarked = [ + attachment({ file_id: 'pdf' }), + attachment({ file_id: 'plain-xlsx', type: XLSX, llmDeliveryPath: 'none' }), + attachment({ file_id: 'legacy-xlsx', type: XLSX, llmDeliveryPath: undefined }), + ]; + try { + const settled = await settleTurnFiles({ + routing, + consumers: noReader, + files: [...unmarked, deferredWorkbook], + }); + + expect(deriveSpy).toHaveBeenCalledTimes(1); + expect(deriveSpy).toHaveBeenCalledWith( + expect.objectContaining({ file_id: 'xlsx' }), + undefined, + ); + expect(decideSpy.mock.calls.map(([input]) => input.file.file_id)).toEqual(['xlsx']); + expect(settled.slice(0, unmarked.length)).toEqual( + applyTurnDelivery(unmarked, { routing, consumers: noReader }), + ); + expect(settled[unmarked.length]).toMatchObject({ file_id: 'xlsx', text: derivedText.text }); + } finally { + decideSpy.mockRestore(); + } + }); + + it('stops before deriving once the request is aborted', async () => { + const deriveText = jest.fn, Parameters>( + async () => derivedText, + ); + const { routing } = setup({ deriveText }); + const controller = new AbortController(); + controller.abort(); + + await expect( + settleTurnFiles({ + routing, + consumers: noReader, + files: [deferredWorkbook], + signal: controller.signal, + }), + ).rejects.toThrow(); + expect(deriveText).not.toHaveBeenCalled(); + }); + }); + + describe('allocation', () => { + const pdfs = ['a', 'b', 'c'].map((file_id) => attachment({ file_id })); + + it('admits direct content first-fit in request order and moves the overflow to File Search', async () => { + const { routing, context } = setup(); + + const settled = await settleTurnFiles({ + routing, + consumers: searchOnly, + files: pdfs, + allocation: byteBudget(['a', 'b', 'c'], 10 * MB), + }); + + expect(pathsById(settled)).toEqual({ a: 'provider', b: 'provider', c: 'none' }); + expect(context.judge(pdfs[2]).overflow).toBe(true); + expect(decideFileReading({ routing, file: settled[2], consumers: searchOnly })).toMatchObject( + { reader: 'search', reason: 'aggregate_overflow' }, + ); + }); + + it('leaves the overflow unavailable when File Search is loaded but cannot reach it', async () => { + /* File Search queued and registered nothing, as a failed provisioning leaves it, and no + * code tool is loaded, so the walk past the overflow finds no reader. */ + const { routing, context } = setup(); + context.setSearchEvidence({ queued: [], registered: [] }); + + const settled = await settleTurnFiles({ + routing, + consumers: searchOnly, + files: pdfs, + allocation: byteBudget(['a', 'b', 'c'], 10 * MB), + }); + + expect(settled[2].llmDeliveryPath).toBe('none'); + expect(decideFileReading({ routing, file: settled[2], consumers: searchOnly })).toMatchObject( + { reader: 'unavailable', reason: 'aggregate_overflow' }, + ); + }); + + it('allocates nothing when no file tool is loaded, keeping every file on its classic route', async () => { + const { routing, context } = setup(); + + const settled = await settleTurnFiles({ + routing, + consumers: noReader, + files: pdfs, + allocation: byteBudget(['a', 'b', 'c'], 10 * MB), + }); + + expect(pathsById(settled)).toEqual({ a: 'provider', b: 'provider', c: 'provider' }); + expect(context.stats().overflow).toBe(0); + expect(decideFileReading({ routing, file: settled[2], consumers: noReader })).toMatchObject({ + reader: 'provider', + reason: 'no_file_tools', + automatic: false, + }); + }); + + it('depends on the request order alone, not on the order of the files', async () => { + const settle = async (files: TurnReadingFile[], requestFileIds: string[]) => { + const { routing } = setup(); + const settled = await settleTurnFiles({ + routing, + consumers: searchOnly, + files, + allocation: byteBudget(requestFileIds, 10 * MB), + }); + return pathsById(settled); + }; + const [a, b, c] = pdfs; + + const expected = { a: 'provider', b: 'provider', c: 'none' }; + expect(await settle([a, b, c], ['a', 'b', 'c'])).toEqual(expected); + expect(await settle([c, a, b], ['a', 'b', 'c'])).toEqual(expected); + expect(await settle([b, c, a], ['a', 'a', 'b', 'c'])).toEqual(expected); + expect(await settle([a, b, c], ['c', 'a', 'b'])).toEqual({ + a: 'provider', + b: 'none', + c: 'provider', + }); + }); + + it('charges committed content before candidates, whatever order it arrives in', async () => { + const chosen = attachment({ + file_id: 'chosen', + bytes: 3 * MB, + metadata: { destinationChosen: true }, + }); + const history = attachment({ file_id: 'history', bytes: MB }); + const settle = async (requestFileIds: string[], extra: TurnReadingFile[]) => { + const { routing } = setup(); + const settled = await settleTurnFiles({ + routing, + consumers: searchOnly, + files: [...pdfs, chosen], + allocation: byteBudget(requestFileIds, 12 * MB, extra), + }); + return pathsById(settled); + }; + + const expected = { a: 'provider', b: 'provider', c: 'none', chosen: 'provider' }; + expect(await settle(['chosen', 'a', 'b', 'c'], [history])).toEqual(expected); + expect(await settle(['a', 'b', 'chosen', 'c'], [history])).toEqual(expected); + expect(await settle(['a', 'b', 'c', 'chosen'], [history])).toEqual(expected); + }); + + it('charges committed entries the caller measured beside the committed files', async () => { + const history = attachment({ file_id: 'history', bytes: MB }); + const settle = async ( + allocation: Pick< + DirectContentAllocation, + 'committedExtra' | 'committedEntries' + >, + ) => { + const { routing } = setup(); + const settled = await settleTurnFiles({ + routing, + consumers: searchOnly, + files: pdfs, + allocation: { + requestFileIds: ['a', 'b', 'c'], + limits: { count: 3 }, + measure: measureModelBoundAttachment, + ...allocation, + }, + }); + return pathsById(settled); + }; + const historicalFileIds = new Set(['history']); + + expect(await settle({ committedExtra: [history] })).toEqual({ + a: 'provider', + b: 'provider', + c: 'none', + }); + expect( + await settle({ committedEntries: [measureAttachment(history, { historicalFileIds })] }), + ).toEqual({ a: 'provider', b: 'provider', c: 'provider' }); + }); + + it('does not charge files left to tools', async () => { + const { routing } = setup(); + const toolFile = attachment({ + file_id: 'tool', + bytes: 50 * MB, + llmDeliveryPath: 'none', + metadata: { destinationChosen: true }, + }); + + const settled = await settleTurnFiles({ + routing, + consumers: searchOnly, + files: [toolFile, ...pdfs], + allocation: byteBudget(['tool', 'a', 'b', 'c'], 12 * MB), + }); + + expect(pathsById(settled)).toEqual({ + tool: 'none', + a: 'provider', + b: 'provider', + c: 'provider', + }); + }); + + it('charges a text-source request file as admission does, whatever its route', async () => { + const { routing } = setup(); + const pasted = attachment({ + file_id: 'pasted', + type: 'text/plain', + bytes: 3 * MB, + source: FileSources.text, + text: 'pasted notes', + llmDeliveryPath: 'none', + metadata: { destinationChosen: true }, + }); + + const settled = await settleTurnFiles({ + routing, + consumers: searchOnly, + files: [pasted, ...pdfs], + allocation: byteBudget(['pasted', 'a', 'b', 'c'], 12 * MB), + }); + + expect(pathsById(settled)).toEqual({ + pasted: 'none', + a: 'provider', + b: 'provider', + c: 'none', + }); + }); + + it('returns the same array when nothing changes', async () => { + const { routing } = setup(); + + const settled = await settleTurnFiles({ + routing, + consumers: searchOnly, + files: pdfs, + allocation: byteBudget(['a', 'b', 'c'], 100 * MB), + }); + + expect(settled).toBe(pdfs); + }); + + it('never allocates under the classic policy', async () => { + const { routing, context } = setup({ + endpointConfig: {}, + deriveText: async () => derivedText, + }); + const measureSpy = jest.fn(measureModelBoundAttachment); + + const settled = await settleTurnFiles({ + routing, + consumers: searchOnly, + files: pdfs, + allocation: { requestFileIds: ['a', 'b', 'c'], limits: { bytes: MB }, measure: measureSpy }, + }); + + expect(settled).toBe(pdfs); + expect(measureSpy).not.toHaveBeenCalled(); + expect(context.stats().overflow).toBe(0); + }); + }); + + it('is exactly applyTurnDelivery when the routing carries no reading context', async () => { + const routing = resolveTurnDeliveryRouting({ + agent: { provider: 'openAI', endpoint: 'openAI' }, + config: { fileConfig: { endpoints: { openAI: { llmDeliveryPolicy: 'automatic' } } } }, + }); + const files = [deferredWorkbook, ...['a', 'b'].map((file_id) => attachment({ file_id }))]; + + const settled = await settleTurnFiles({ + routing, + consumers: noReader, + files, + allocation: byteBudget(['a', 'b'], MB), + }); + + expect(settled).toEqual(applyTurnDelivery(files, { routing, consumers: noReader })); + expect(pathsById(settled)).toEqual({ xlsx: 'none', a: 'provider', b: 'provider' }); + expect(await settleTurnFiles({ consumers: noReader, files })).toBe(files); + }); + + describe('flush', () => { + it('defers flushing to the caller by default', async () => { + const { routing, context } = setup({ deriveText: async () => derivedText }); + const flushSpy = jest.spyOn(context, 'flush'); + + await settleTurnFiles({ routing, consumers: noReader, files: [deferredWorkbook] }); + expect(flushSpy).not.toHaveBeenCalled(); + + await settleTurnFiles({ + routing, + consumers: noReader, + files: [deferredWorkbook], + flush: true, + }); + expect(flushSpy).toHaveBeenCalledTimes(1); + }); + + it('flushes before returning when files are prepared outside initialization', async () => { + const { routing, context } = setup({ deriveText: async () => derivedText }); + const flushSpy = jest.spyOn(context, 'flush'); + + const [prepared] = await prepareTurnFiles({ + routing, + consumers: noReader, + files: [deferredWorkbook], + }); + expect(prepared.text).toBe(derivedText.text); + expect(flushSpy).toHaveBeenCalledTimes(1); + + await prepareTurnFiles({ + routing, + consumers: noReader, + files: [deferredWorkbook], + flush: false, + }); + expect(flushSpy).toHaveBeenCalledTimes(1); + }); + }); +}); diff --git a/packages/api/src/files/reading/settle.ts b/packages/api/src/files/reading/settle.ts new file mode 100644 index 00000000000..b050c2df06c --- /dev/null +++ b/packages/api/src/files/reading/settle.ts @@ -0,0 +1,188 @@ +import { decideFileReading, allocateDirectContent } from 'librechat-data-provider'; +import type { + DirectContentEntry, + TurnFileConsumers, + DirectContentLimits, + TurnDeliveryRouting, +} from 'librechat-data-provider'; +import type { TurnReadingFile, TurnReader } from './turn'; +import type { AllocationScope } from './diagnostics'; +import { + withDerivedText, + needsDerivedText, + deriveRequestedText, + getTurnReadingContext, +} from './turn'; +import { applyTurnDelivery } from '~/agents/files/delivery'; +import { logAllocation } from './diagnostics'; + +/** First-fit allocation of the request's direct model content against the turn's limits. */ +export interface DirectContentAllocation { + /** The request's attachments in submission order; earlier files are admitted first. */ + requestFileIds: readonly string[]; + limits: DirectContentLimits; + /** + * The direct content one file puts on the model path, as the matching admission check charges + * it. A file that is not model-bound measures as zero. + */ + measure: (file: T) => DirectContentEntry; + /** Content already charged against the same limits, such as replayed history. */ + committedExtra?: readonly T[]; + /** + * Content already charged that the caller measured itself, such as replayed history, which + * spends the context budgets but counts as the checks on it count repeated copies. + */ + committedEntries?: readonly DirectContentEntry[]; + agentId?: string; + scope?: AllocationScope; +} + +export interface SettleTurnFilesParams { + routing?: TurnDeliveryRouting; + consumers?: TurnFileConsumers; + files: T[]; + allocation?: DirectContentAllocation; + /** + * Save the derivations the context queued before returning. Derived text was already + * content-inspected at derivation, and a policy refusal throws instead of saving; this decides + * only whether the save waits for the caller's later checks, as initialization's does. + */ + flush?: boolean; + signal?: AbortSignal; +} + +interface SettleStep extends TurnReader { + signal?: AbortSignal; +} + +const route = (files: T[], { routing, consumers }: SettleStep): T[] => + applyTurnDelivery(files, { routing, consumers }); + +/** + * Derives text for every copy whose reading needs it. Failures are recorded on the context, so the + * next decision passes text over for the rest of the request. Returns null when no copy needed text. + * A derive-only classic context decides only the records the automatic policy marked. + */ +async function deriveNeededText( + files: T[], + step: SettleStep, +): Promise { + const { context, signal } = step; + const needed = files.filter((file) => needsDerivedText(file, step)); + if (needed.length === 0) { + return null; + } + signal?.throwIfAborted(); + const texts = await deriveRequestedText( + needed.map((file) => ({ file, contexts: [context] as const })), + signal, + ); + if (texts.size === 0) { + return files; + } + return files.map((file) => { + const derived = texts.get(file.file_id); + return derived == null ? file : withDerivedText(file, derived); + }); +} + +const isDirectReading = (file: TurnReadingFile, { routing, consumers }: SettleStep): boolean => { + const reading = decideFileReading({ routing, file, consumers }); + return reading.automatic && (reading.reader === 'provider' || reading.reader === 'text'); +}; + +/** The request's files in submission order, once each, among those still on the turn. */ +function selectRequestFiles( + files: readonly T[], + requestFileIds: readonly string[], +): T[] { + const byId = new Map(); + for (const file of files) { + if (!byId.has(file.file_id)) { + byId.set(file.file_id, file); + } + } + const selected: T[] = []; + for (const fileId of new Set(requestFileIds)) { + const file = byId.get(fileId); + if (file != null) { + selected.push(file); + } + } + return selected; +} + +/** + * Charges what the automatic policy cannot move first, measured exactly as admission measures it, + * then admits each request file it reads directly in submission order while it fits. A file that does not fit is recorded as overflow, + * so the next decision moves it to another reader or leaves it unavailable. + */ +function allocate( + files: T[], + allocation: DirectContentAllocation, + step: SettleStep, +): T[] { + const { measure, limits, committedExtra = [], committedEntries = [] } = allocation; + const candidates: DirectContentEntry[] = []; + const committed: DirectContentEntry[] = [...committedEntries, ...committedExtra.map(measure)]; + for (const file of selectRequestFiles(files, allocation.requestFileIds)) { + (isDirectReading(file, step) ? candidates : committed).push(measure(file)); + } + const overflow = allocateDirectContent(committed, candidates, limits); + logAllocation({ + agentId: allocation.agentId, + scope: allocation.scope ?? 'request', + candidates: candidates.length, + overflow: overflow.size, + }); + if (overflow.size === 0) { + return files; + } + step.context.addOverflow(overflow); + return route(files, step); +} + +/** + * Settles how a turn reads its files: applies the turn route, derives text the reading needs and + * decides again, then, with an allocation under the automatic policy, moves direct content that + * does not fit the turn's limits to another reader. Returns the input array when nothing changed. + * Without a reading context on the routing this is exactly {@link applyTurnDelivery}. + */ +export async function settleTurnFiles({ + routing, + consumers, + files, + allocation, + flush = false, + signal, +}: SettleTurnFilesParams): Promise { + const resolved = applyTurnDelivery(files, { routing, consumers }); + const context = getTurnReadingContext(routing); + if (routing == null || context == null) { + return resolved; + } + const step: SettleStep = { routing, consumers, context, signal }; + const derived = await deriveNeededText(resolved, step); + const decided = derived == null ? resolved : route(derived, step); + const settled = + allocation != null && context.policy === 'automatic' + ? allocate(decided, allocation, step) + : decided; + if (flush) { + await context.flush(); + } + return settled; +} + +/** + * {@link settleTurnFiles} without allocation, saving its derivations before returning: for readers + * outside initialization, such as history replay, steering and the run-file encoder. Those saves + * happen before the caller's endpoint filter and model-bound admission; the text they keep passed + * the `extracted_text` inspection when it was derived, and a refusal throws without saving. + */ +export function prepareTurnFiles({ + flush = true, + ...params +}: Omit, 'allocation'>): Promise { + return settleTurnFiles({ ...params, flush }); +} diff --git a/packages/api/src/files/reading/turn.spec.ts b/packages/api/src/files/reading/turn.spec.ts new file mode 100644 index 00000000000..36ad6df12a9 --- /dev/null +++ b/packages/api/src/files/reading/turn.spec.ts @@ -0,0 +1,1024 @@ +import { logger } from '@librechat/data-schemas'; +import { FileContext, FileSources, decideFileReading } from 'librechat-data-provider'; +import type { TFileConfig, TurnFileConsumers, TurnDeliveryRouting } from 'librechat-data-provider'; +import type { + DerivedText, + FileTextDeriver, + TurnReadingFile, + EncodedDocuments, + NativeDeliveryAgent, + TextDerivationPersister, + TextDerivationSave, + BuildTurnReadingContextParams, +} from './turn'; +import type { NativeValidationMode } from '~/types'; +import { + needsDerivedText, + getTurnTextOptions, + deriveRequestedText, + getTurnReadingContext, + encodeNativeDocuments, + recordNativeRejections, + buildTurnReadingContext, + getNativeValidationPolicy, + createDerivationPersister, +} from './turn'; +import { resolveTurnDeliveryRouting } from '~/agents/files/delivery'; +import { UninspectableFileError } from '~/protection/files'; + +type EndpointFileConfigInput = NonNullable[string]; + +const MB = 1024 * 1024; +const PDF = 'application/pdf'; +const DOCX = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'; +const PPTX = 'application/vnd.openxmlformats-officedocument.presentationml.presentation'; +const XLSX = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'; + +const routingFor = ( + endpoint: string, + endpointConfig: EndpointFileConfigInput = { llmDeliveryPolicy: 'automatic' }, +): TurnDeliveryRouting => + resolveTurnDeliveryRouting({ + agent: { provider: endpoint, endpoint }, + config: { fileConfig: { endpoints: { [endpoint]: endpointConfig } } }, + }); + +const attachment = ( + overrides: Partial & Pick, +): TurnReadingFile => ({ + type: PDF, + bytes: MB, + source: FileSources.local, + context: FileContext.message_attachment, + llmDeliveryPath: 'provider', + metadata: { destinationChosen: false }, + ...overrides, +}); + +const countByLength = (text: string): number => text.length; + +function contextFor(overrides: Partial = {}) { + const params: BuildTurnReadingContextParams = { + routing: routingFor('openAI'), + provider: 'openAI', + model: 'gpt-4o', + fileTokenLimit: 100_000, + configuredFileSizeLimit: undefined, + countTokens: countByLength, + ...overrides, + }; + const context = buildTurnReadingContext(params); + if (context == null) { + throw new Error('expected a reading context'); + } + return context; +} + +const nativeVerdict = ( + provider: string, + file: TurnReadingFile, + overrides: Partial = {}, +) => contextFor({ routing: routingFor(provider), provider, ...overrides }).judge(file).native; + +describe('buildTurnReadingContext', () => { + it('builds nothing under the classic policy without a text deriver', () => { + const params = { + provider: 'openAI', + fileTokenLimit: 100_000, + configuredFileSizeLimit: undefined, + countTokens: countByLength, + }; + expect(buildTurnReadingContext({ ...params, routing: routingFor('openAI', {}) })).toBe( + undefined, + ); + expect( + buildTurnReadingContext({ + ...params, + routing: routingFor('openAI', { llmDeliveryPolicy: 'classic' }), + }), + ).toBe(undefined); + expect( + buildTurnReadingContext({ + ...params, + routing: routingFor('openAI', { llmDeliveryPolicy: 'automatic', legacyFileUploadUX: true }), + }), + ).toBe(undefined); + }); + + it('builds a derive-only context under the classic policy when a deriver is wired', () => { + const deriveText: FileTextDeriver = jest.fn(); + const context = contextFor({ routing: routingFor('openAI', {}), deriveText }); + const file = attachment({ file_id: 'pdf', text: 'x'.repeat(10), bytes: 500 * MB }); + + expect(context.policy).toBe('classic'); + expect(context.canDerive).toBe(true); + expect(context.judge(file)).toEqual({}); + + context.addOverflow(['pdf']); + context.recordRejections([{ file_id: 'pdf', reason: 'capacity' }]); + context.markTextFailed('pdf'); + expect(context.judge(file)).toEqual({ textFailed: true }); + }); + + it('cannot derive under the automatic policy without a deriver', () => { + const context = contextFor(); + expect(context.policy).toBe('automatic'); + expect(context.canDerive).toBe(false); + }); + + it('is found again through the routing it is attached to, and through a copy of it', () => { + const routing = routingFor('openAI'); + const context = contextFor({ routing }); + expect(getTurnReadingContext(routing)).toBe(undefined); + + routing.reading = context; + expect(getTurnReadingContext(routing)).toBe(context); + expect(getTurnReadingContext({ ...routing })).toBe(context); + expect(getTurnReadingContext({ reading: { judge: () => ({}), canDerive: false } })).toBe( + undefined, + ); + expect(getTurnReadingContext(undefined)).toBe(undefined); + }); +}); + +describe('getTurnTextOptions', () => { + it('supplies nothing without a reading context', () => { + expect(getTurnTextOptions(undefined)).toEqual({}); + expect(getTurnTextOptions(routingFor('openAI'))).toEqual({}); + }); + + it('reuses judged counts and marks truncation under the automatic policy', () => { + const routing = routingFor('openAI'); + const context = contextFor({ routing, fileTokenLimit: 10, countTokens: () => 4 }); + routing.reading = context; + const file = attachment({ file_id: 'notes', text: 'x'.repeat(40), llmDeliveryPath: 'text' }); + expect(context.judge(file).text).toBe('fits'); + + const options = getTurnTextOptions(routing); + + expect(options.markTruncation).toBe(true); + expect(options.knownTokenCount?.(file)).toBe(4); + }); + + it('reuses counts without marking truncation in a derive-only classic context', () => { + const routing = routingFor('openAI', {}); + routing.reading = contextFor({ routing, deriveText: jest.fn() }); + + expect(getTurnTextOptions(routing)).toMatchObject({ markTruncation: false }); + }); +}); + +describe('native validation', () => { + const searchOnly: TurnFileConsumers = { executeCode: false, fileSearch: true }; + const rejectedPdf = attachment({ file_id: 'rejected-pdf' }); + const chosenPdf = attachment({ + file_id: 'chosen-pdf', + metadata: { destinationChosen: true }, + }); + const unmarkedPdf = attachment({ file_id: 'unmarked-pdf', metadata: {} }); + + const automaticAgent = (overrides: Partial = {}): NativeDeliveryAgent => { + const routing = routingFor('openAI'); + routing.reading = contextFor({ routing }); + return { + id: 'agent-1', + deliveryRouting: routing, + fileConsumers: searchOnly, + currentRequestAttachments: [rejectedPdf, chosenPdf, unmarkedPdf], + ...overrides, + }; + }; + + describe('getNativeValidationPolicy', () => { + it('leaves out only a file the automatic policy reads', () => { + const modeOf = getNativeValidationPolicy(automaticAgent()); + + expect(modeOf(rejectedPdf)).toBe('skip'); + expect(modeOf(chosenPdf)).toBe('throw'); + expect(modeOf(unmarkedPdf)).toBe('throw'); + expect(modeOf(attachment({ file_id: 'agent-file', context: FileContext.agents }))).toBe( + 'throw', + ); + }); + + it('keeps a type the operator routed to a fixed path failing as classic', () => { + const routing = routingFor('openAI', { + llmDeliveryPolicy: 'automatic', + defaultLLMDeliveryPath: { overrides: { [PDF]: 'provider' } }, + }); + routing.reading = contextFor({ routing }); + + expect( + getNativeValidationPolicy(automaticAgent({ deliveryRouting: routing }))(rejectedPdf), + ).toBe('throw'); + }); + + it('fails every file outside an automatic reading, without known consumers or without a file tool', () => { + const derivingClassic = routingFor('openAI', {}); + derivingClassic.reading = contextFor({ + routing: derivingClassic, + deriveText: async () => ({ status: 'skipped', reason: 'no_extractor' }), + }); + const agents = [ + automaticAgent({ deliveryRouting: derivingClassic }), + automaticAgent({ deliveryRouting: routingFor('openAI') }), + automaticAgent({ fileConsumers: undefined }), + automaticAgent({ fileConsumers: { executeCode: false, fileSearch: false } }), + undefined, + ]; + + expect(agents.map((agent) => getNativeValidationPolicy(agent)(rejectedPdf))).toEqual([ + 'throw', + 'throw', + 'throw', + 'throw', + 'throw', + ]); + expect( + getNativeValidationPolicy( + automaticAgent({ fileConsumers: { executeCode: true, fileSearch: false } }), + )(rejectedPdf), + ).toBe('skip'); + }); + + it('allows an automatic historical attachment to use the same fallback as a request file', () => { + const replayed = attachment({ file_id: 'replayed-pdf' }); + const agent = automaticAgent({ currentRequestAttachments: [] }); + const modeOf = getNativeValidationPolicy(agent); + + expect(modeOf(rejectedPdf)).toBe('skip'); + expect(modeOf(replayed)).toBe('skip'); + expect(modeOf({ ...replayed, metadata: { destinationChosen: true } })).toBe('throw'); + }); + }); + + describe('encodeNativeDocuments', () => { + type Encoded = EncodedDocuments; + const isInvalid = ({ file_id }: TurnReadingFile): boolean => + file_id.startsWith('rejected') || file_id.startsWith('chosen'); + /** Encodes every file but the rejected and chosen PDFs, which fail validation. */ + const encoder = () => + jest.fn(async (files: TurnReadingFile[], mode: NativeValidationMode): Promise => { + const invalid = files.filter(isInvalid); + if (mode === 'throw' && invalid.length > 0) { + throw new Error('PDF validation failed'); + } + const valid = files.filter((file) => !isInvalid(file)); + return { + documents: valid.map(({ file_id }) => `block:${file_id}`), + files: valid.map(({ file_id }) => file_id), + ...(mode === 'skip' && { + rejected: invalid.map(({ file_id }) => ({ file_id, reason: 'integrity' as const })), + }), + }; + }); + const callsOf = (encode: ReturnType) => + encode.mock.calls.map(([files, mode]) => [files.map(({ file_id }) => file_id), mode]); + + it('encodes a batch that shares one mode in a single call', async () => { + const encode = encoder(); + const fits = attachment({ file_id: 'fits-pdf' }); + + const result = await encodeNativeDocuments([rejectedPdf, fits], automaticAgent(), encode); + + expect(callsOf(encode)).toEqual([[['rejected-pdf', 'fits-pdf'], 'skip']]); + expect(result).toEqual({ + documents: ['block:fits-pdf'], + files: ['fits-pdf'], + rejected: [{ file_id: 'rejected-pdf', reason: 'integrity' }], + }); + }); + + it('still fails the turn on a classic record that fails beside an automatic one', async () => { + const encode = encoder(); + + await expect( + encodeNativeDocuments([rejectedPdf, chosenPdf], automaticAgent(), encode), + ).rejects.toThrow('PDF validation failed'); + expect(callsOf(encode)).toEqual([ + [['rejected-pdf'], 'skip'], + [['chosen-pdf'], 'throw'], + ]); + }); + + it('merges a split encode and lists only the files it left out', async () => { + const encode = encoder(); + + const result = await encodeNativeDocuments( + [rejectedPdf, unmarkedPdf], + automaticAgent(), + encode, + ); + + expect(result).toEqual({ + documents: ['block:unmarked-pdf'], + files: ['unmarked-pdf'], + rejected: [{ file_id: 'rejected-pdf', reason: 'integrity' }], + }); + }); + + it('encodes in throw mode under classic routing', async () => { + const encode = encoder(); + + await expect( + encodeNativeDocuments([rejectedPdf], { deliveryRouting: routingFor('openAI', {}) }, encode), + ).rejects.toThrow('PDF validation failed'); + expect(callsOf(encode)).toEqual([[['rejected-pdf'], 'throw']]); + }); + }); + + describe('recordNativeRejections', () => { + it('records the rejections on each agent carrying the payload, once per reading', () => { + const primary = automaticAgent(); + const handoff = automaticAgent({ id: 'agent-2' }); + const sharing = { ...primary, id: 'agent-3' }; + const levelSpy = jest.spyOn(logger, 'isDebugEnabled').mockReturnValue(true); + const debugSpy = jest.spyOn(logger, 'debug').mockImplementation(() => logger); + + recordNativeRejections( + [primary, handoff, sharing, undefined], + [{ file_id: rejectedPdf.file_id, reason: 'capacity' }], + ); + + expect( + [primary, handoff].map((agent) => { + const context = getTurnReadingContext(agent.deliveryRouting); + return [context?.judge(rejectedPdf).rejected, context?.stats().rejected]; + }), + ).toEqual([ + ['capacity', 1], + ['capacity', 1], + ]); + expect(debugSpy.mock.calls.map(([line]) => line)).toEqual([ + '[nativeDelivery] agent=agent-1 file_id=rejected-pdf provider=openAI rejected=capacity mode=skip next=search', + '[nativeDelivery] agent=agent-2 file_id=rejected-pdf provider=openAI rejected=capacity mode=skip next=search', + ]); + debugSpy.mockRestore(); + levelSpy.mockRestore(); + }); + + it('ignores rejections without a reading context or without rejections', () => { + const classic = automaticAgent({ deliveryRouting: routingFor('openAI') }); + expect(() => + recordNativeRejections([classic], [{ file_id: rejectedPdf.file_id, reason: 'integrity' }]), + ).not.toThrow(); + const agent = automaticAgent(); + + recordNativeRejections([agent], undefined); + recordNativeRejections([agent], []); + + expect(getTurnReadingContext(agent.deliveryRouting)?.stats().rejected).toBe(0); + }); + }); +}); + +describe('text verdicts', () => { + it('settles text within the limit by its byte length without counting tokens', () => { + const countTokens = jest.fn(countByLength); + const context = contextFor({ fileTokenLimit: 10, countTokens }); + + expect(context.judge(attachment({ file_id: 'short', text: 'tiny' })).text).toBe('fits'); + expect(countTokens).not.toHaveBeenCalled(); + expect(context.knownTokenCount(attachment({ file_id: 'short', text: 'tiny' }))).toBe(undefined); + }); + + it('counts tokens once when the byte length exceeds the limit', () => { + const countTokens = jest.fn((text: string) => Math.ceil(text.length / 4)); + const context = contextFor({ fileTokenLimit: 10, countTokens }); + const file = attachment({ file_id: 'long', text: 'x'.repeat(40) }); + + expect(context.judge(file).text).toBe('fits'); + expect(context.judge({ ...file }).text).toBe('fits'); + expect(countTokens).toHaveBeenCalledTimes(1); + expect(context.knownTokenCount(file)).toBe(10); + }); + + it('exceeds when the complete text has more tokens than the limit', () => { + const context = contextFor({ fileTokenLimit: 10 }); + const file = attachment({ file_id: 'long', text: 'x'.repeat(11) }); + + expect(context.judge(file).text).toBe('exceeds'); + expect(context.knownTokenCount(file)).toBe(11); + }); + + it.each([0, undefined])('exceeds when the token limit is %p', (fileTokenLimit) => { + const countTokens = jest.fn(countByLength); + const context = contextFor({ fileTokenLimit, countTokens }); + + expect(context.judge(attachment({ file_id: 'text', text: 'a' })).text).toBe('exceeds'); + expect(countTokens).not.toHaveBeenCalled(); + }); + + it('judges the text again once its length changes', () => { + const countTokens = jest.fn(countByLength); + const context = contextFor({ fileTokenLimit: 10, countTokens }); + + expect(context.judge(attachment({ file_id: 'grows', text: 'x'.repeat(12) })).text).toBe( + 'exceeds', + ); + expect(context.judge(attachment({ file_id: 'grows', text: 'x'.repeat(11) })).text).toBe( + 'exceeds', + ); + expect(countTokens).toHaveBeenCalledTimes(2); + }); + + it('never counts a stored text the reading walk gives to Run Code', () => { + const countTokens = jest.fn(countByLength); + const routing = routingFor('openAI'); + const context = contextFor({ routing, fileTokenLimit: 100_000, countTokens }); + routing.reading = context; + const csv = attachment({ + file_id: 'large-csv', + type: 'text/csv', + llmDeliveryPath: 'text', + text: 'x'.repeat(700_000), + }); + + const reading = decideFileReading({ + routing, + file: csv, + consumers: { executeCode: true, fileSearch: false }, + }); + + expect(reading.reader).toBe('code'); + expect(countTokens).not.toHaveBeenCalled(); + expect(context.knownTokenCount(csv)).toBe(undefined); + + expect(context.judge(csv).text).toBe('exceeds'); + expect(countTokens).toHaveBeenCalledTimes(1); + expect(context.knownTokenCount(csv)).toBe(700_000); + }); + + it('judges no text verdict for a record without text', () => { + const context = contextFor(); + expect(context.judge(attachment({ file_id: 'empty', text: '' }))).not.toHaveProperty('text'); + expect(context.judge(attachment({ file_id: 'none' }))).not.toHaveProperty('text'); + }); +}); + +describe('native verdicts', () => { + it.each<{ + why: string; + provider: string; + model?: string; + type: string; + bytes: number; + configuredFileSizeLimit?: number; + verdict: 'fits' | 'capacity' | 'unsupported'; + }>([ + { + why: 'a PDF on Azure OpenAI, where the encoder emits nothing', + provider: 'azureOpenAI', + type: PDF, + bytes: MB, + verdict: 'unsupported', + }, + { + why: 'a Word document on Anthropic', + provider: 'anthropic', + type: DOCX, + bytes: MB, + verdict: 'unsupported', + }, + { why: 'a PDF on Anthropic', provider: 'anthropic', type: PDF, bytes: MB, verdict: 'fits' }, + { + why: 'a Word document for a Claude model behind an OpenAI-compatible gateway', + provider: 'openAI', + model: 'claude-sonnet-4-5', + type: DOCX, + bytes: MB, + verdict: 'unsupported', + }, + { + why: 'a Word document for an OpenAI model', + provider: 'openAI', + model: 'gpt-4o', + type: DOCX, + bytes: MB, + verdict: 'fits', + }, + { + why: 'a 20 MB PDF for Claude 4 on Bedrock, exempt from the 4.5 MB document limit', + provider: 'bedrock', + model: 'us.anthropic.claude-sonnet-4-20250514-v1:0', + type: PDF, + bytes: 20 * MB, + verdict: 'fits', + }, + { + why: 'a 20 MB PDF for Claude 3.5 on Bedrock', + provider: 'bedrock', + model: 'anthropic.claude-3-5-sonnet-20240620-v1:0', + type: PDF, + bytes: 20 * MB, + verdict: 'capacity', + }, + { + why: 'a presentation on Bedrock', + provider: 'bedrock', + type: PPTX, + bytes: MB, + verdict: 'unsupported', + }, + { + why: 'a PDF above the configured limit', + provider: 'openAI', + type: PDF, + bytes: 2 * MB, + configuredFileSizeLimit: MB, + verdict: 'capacity', + }, + { + why: 'a Word document above the configured limit', + provider: 'openAI', + type: DOCX, + bytes: 2 * MB, + configuredFileSizeLimit: MB, + verdict: 'capacity', + }, + { + why: 'a PDF at the configured limit', + provider: 'openAI', + type: PDF, + bytes: MB, + configuredFileSizeLimit: MB, + verdict: 'fits', + }, + { + why: 'a PDF above the provider limit when none is configured', + provider: 'openAI', + type: PDF, + bytes: 11 * MB, + verdict: 'capacity', + }, + { + why: 'a PDF at the provider limit when none is configured', + provider: 'openAI', + type: PDF, + bytes: 10 * MB, + verdict: 'fits', + }, + ])( + 'judges $why as $verdict', + ({ provider, model, type, bytes, configuredFileSizeLimit, verdict }) => { + const overrides = { + ...(model != null && { model }), + ...(configuredFileSizeLimit != null && { configuredFileSizeLimit }), + }; + expect(nativeVerdict(provider, attachment({ file_id: 'file', type, bytes }), overrides)).toBe( + verdict, + ); + }, + ); + + it('does not judge images, audio or video', () => { + expect( + contextFor().judge(attachment({ file_id: 'image', type: 'image/png' })), + ).not.toHaveProperty('native'); + }); + + it('memoizes the verdict per file', () => { + const context = contextFor(); + expect(context.judge(attachment({ file_id: 'pdf', bytes: MB })).native).toBe('fits'); + expect(context.judge(attachment({ file_id: 'pdf', bytes: 50 * MB })).native).toBe('fits'); + expect(context.judge(attachment({ file_id: 'other', bytes: 50 * MB })).native).toBe('capacity'); + }); +}); + +describe('request evidence', () => { + it('reports overflow, encode-time rejections and derivation failures per file', () => { + const context = contextFor(); + const file = attachment({ file_id: 'pdf' }); + expect(context.judge(file)).toEqual({ native: 'fits' }); + + context.addOverflow(new Set(['pdf'])); + context.recordRejections([ + { file_id: 'pdf', reason: 'integrity' }, + { file_id: 'pdf', reason: 'capacity' }, + ]); + context.markTextFailed('pdf'); + + expect(context.judge(file)).toEqual({ + native: 'fits', + overflow: true, + rejected: 'integrity', + textFailed: true, + }); + expect(context.judge(attachment({ file_id: 'other' }))).toEqual({ native: 'fits' }); + }); + + it('judges File Search reachability only once search evidence is set', () => { + const context = contextFor(); + const queued = attachment({ file_id: 'queued' }); + const registered = attachment({ file_id: 'registered' }); + const missing = attachment({ file_id: 'missing' }); + const unsearchable = attachment({ file_id: 'zip', type: 'application/zip' }); + const chosen = attachment({ file_id: 'chosen', metadata: { destinationChosen: true } }); + + expect(context.judge(missing)).not.toHaveProperty('search'); + + context.setSearchEvidence({ queued: ['queued'], registered: ['registered'] }); + + expect(context.judge(queued).search).toBe('reachable'); + expect(context.judge(registered).search).toBe('reachable'); + expect(context.judge(missing).search).toBe('unreachable'); + expect(context.judge(unsearchable).search).toBe('reachable'); + expect(context.judge(chosen).search).toBe('reachable'); + }); + + it('reads shared preparation outcomes without rerouting already-sent attachments', () => { + const context = contextFor(); + const queued = attachment({ file_id: 'queued' }); + const preparation = new Map(); + context.setSearchEvidence({ queued: ['queued'], registered: ['indexed'], preparation }); + + expect(context.searchState('queued')).toBe('queued'); + expect(context.searchState('indexed')).toBe('ready'); + expect(context.searchState('missing')).toBeUndefined(); + + preparation.set('queued', 'failed'); + expect(context.searchState('queued')).toBe('failed'); + expect(context.judge(queued).search).toBe('reachable'); + + preparation.set('queued', 'ready'); + expect(context.searchState('queued')).toBe('ready'); + expect(context.judge(queued).search).toBe('reachable'); + expect(contextFor().searchState('queued')).toBeUndefined(); + }); + + it('keeps each dropped request file once', () => { + const context = contextFor(); + const first = attachment({ file_id: 'a', bytes: 1 }); + context.recordDropped([first, attachment({ file_id: 'b' })]); + context.recordDropped([attachment({ file_id: 'a', bytes: 2 })]); + + expect(context.dropped().map((file) => file.file_id)).toEqual(['a', 'b']); + expect(context.dropped()[0]).toBe(first); + expect(context.stats()).toEqual({ + overflow: 0, + rejected: 0, + textFailed: 0, + derived: 0, + dropped: 2, + }); + }); +}); + +describe('derive', () => { + const xlsx = attachment({ + file_id: 'xlsx', + type: XLSX, + llmDeliveryPath: 'none', + metadata: { destinationChosen: false, textDerivation: { outcome: 'deferred' } }, + }); + + it('derives each file once per request with the request signal', async () => { + const controller = new AbortController(); + const deriveText = jest.fn, Parameters>( + async () => ({ + status: 'derived', + text: 'a,b', + textDerivation: { outcome: 'complete', extractor: 'document_parser' }, + }), + ); + const context = contextFor({ deriveText, signal: controller.signal }); + + const [first, second] = await Promise.all([context.derive(xlsx), context.derive({ ...xlsx })]); + + expect(first).toBe(second); + expect(first).toMatchObject({ status: 'derived', text: 'a,b' }); + expect(deriveText).toHaveBeenCalledTimes(1); + expect(deriveText).toHaveBeenCalledWith(xlsx, controller.signal); + expect(context.stats().derived).toBe(1); + }); + + it('skips without a deriver', async () => { + await expect(contextFor().derive(xlsx)).resolves.toEqual({ + status: 'skipped', + reason: 'no_extractor', + }); + }); + + it('turns a deriver error into a skipped derivation and logs only safe metadata', async () => { + const errorSpy = jest.spyOn(logger, 'error').mockImplementation(() => logger); + try { + const context = contextFor({ + deriveText: async () => { + throw new Error('secret path /data/report.xlsx'); + }, + }); + + await expect(context.derive(xlsx)).resolves.toEqual({ + status: 'skipped', + reason: 'storage_unavailable', + }); + expect(errorSpy).toHaveBeenCalledWith( + '[readingText] file_id=xlsx outcome=skipped reason=derivation_error', + { type: 'Error' }, + ); + expect(context.stats().derived).toBe(0); + } finally { + errorSpy.mockRestore(); + } + }); + + describe('persistence', () => { + const derived: DerivedText = { + status: 'derived', + text: 'quarter,total\nQ1,42', + textDerivation: { outcome: 'complete', extractor: 'document_parser', at: 1 }, + }; + const persister = () => + jest.fn, Parameters>( + async () => true, + ); + + it.each([ + ['automatic', { llmDeliveryPolicy: 'automatic' as const }], + ['classic', {}], + ])( + 'saves text derived for a marked record once, under the %s policy', + async (_policy, endpointConfig) => { + const persistDerivation = persister(); + const context = contextFor({ + routing: routingFor('openAI', endpointConfig), + deriveText: async () => derived, + persistDerivation, + }); + + await Promise.all([context.derive(xlsx), context.derive({ ...xlsx })]); + await context.flush(); + + expect(persistDerivation).toHaveBeenCalledTimes(1); + expect(persistDerivation).toHaveBeenCalledWith({ + file_id: 'xlsx', + text: derived.text, + textDerivation: derived.textDerivation, + }); + }, + ); + + it('saves a deterministic failure as a marker alone, and nothing it may not keep', async () => { + const outcomes: Record = { + parser: { + status: 'failed', + textDerivation: { outcome: 'failed', reason: 'parser' }, + persist: true, + }, + transient: { + status: 'failed', + textDerivation: { outcome: 'failed', reason: 'extractor_unavailable' }, + persist: false, + }, + skipped: { status: 'skipped', reason: 'storage_unavailable' }, + }; + const persistDerivation = persister(); + const context = contextFor({ + deriveText: async (file) => outcomes[file.file_id], + persistDerivation, + }); + + await Promise.all( + Object.keys(outcomes).map((file_id) => context.derive({ ...xlsx, file_id })), + ); + await context.flush(); + + expect(persistDerivation).toHaveBeenCalledTimes(1); + const [[update]] = persistDerivation.mock.calls; + expect(update).toEqual({ + file_id: 'parser', + textDerivation: { outcome: 'failed', reason: 'parser' }, + }); + expect(update).not.toHaveProperty('text'); + }); + + it('caches text derived for an unmarked original and never for a settled record', async () => { + const persistDerivation = persister(); + const context = contextFor({ deriveText: async () => derived, persistDerivation }); + const unmarked = attachment({ file_id: 'pdf' }); + const settled = { + ...xlsx, + file_id: 'settled', + metadata: { textDerivation: derived.textDerivation }, + }; + + await Promise.all([context.derive(unmarked), context.derive(settled)]); + await context.flush(); + + expect(persistDerivation).toHaveBeenCalledTimes(1); + expect(persistDerivation).toHaveBeenCalledWith({ + file_id: 'pdf', + text: derived.text, + textDerivation: derived.textDerivation, + }); + }); + + it('keeps a failed marker only on a record the automatic policy deferred', async () => { + const persistDerivation = persister(); + const failure: DerivedText = { + status: 'failed', + textDerivation: { outcome: 'failed', reason: 'parser' }, + persist: true, + }; + const context = contextFor({ deriveText: async () => failure, persistDerivation }); + + await context.derive(attachment({ file_id: 'pdf' })); + await context.flush(); + + expect(persistDerivation).not.toHaveBeenCalled(); + }); + + it('forgets an aborted derivation so a later caller derives again', async () => { + const deriveText = jest + .fn(async (): Promise => derived) + .mockResolvedValueOnce({ status: 'skipped', reason: 'aborted' }); + const context = contextFor({ deriveText }); + + await expect(context.derive(xlsx)).resolves.toEqual({ status: 'skipped', reason: 'aborted' }); + await expect(context.derive(xlsx)).resolves.toBe(derived); + expect(deriveText).toHaveBeenCalledTimes(2); + }); + + it('writes nothing until the flush, so a caller saves only files that passed its checks', async () => { + const persistDerivation = persister(); + const context = contextFor({ deriveText: async () => derived, persistDerivation }); + + await context.derive(xlsx); + await new Promise((resolve) => setImmediate(resolve)); + expect(persistDerivation).not.toHaveBeenCalled(); + + await context.flush(); + expect(persistDerivation).toHaveBeenCalledTimes(1); + await context.flush(); + expect(persistDerivation).toHaveBeenCalledTimes(1); + }); + + it('settles the flush only after pending derivations and their queued writes', async () => { + let finishDerive: (() => void) | undefined; + let finishWrite: ((saved: boolean) => void) | undefined; + const context = contextFor({ + deriveText: () => + new Promise((resolve) => { + finishDerive = () => resolve(derived); + }), + persistDerivation: () => + new Promise((resolve) => { + finishWrite = resolve; + }), + }); + void context.derive(xlsx); + + let flushed = false; + const flushing = context.flush().then(() => { + flushed = true; + }); + await new Promise((resolve) => setImmediate(resolve)); + expect(finishDerive).toBeDefined(); + expect(finishWrite).toBeUndefined(); + expect(flushed).toBe(false); + + finishDerive?.(); + await new Promise((resolve) => setImmediate(resolve)); + expect(finishWrite).toBeDefined(); + expect(flushed).toBe(false); + + finishWrite?.(true); + await flushing; + expect(flushed).toBe(true); + await expect(context.flush()).resolves.toBeUndefined(); + }); + + it('logs a failed write with safe metadata and still settles the flush', async () => { + const errorSpy = jest.spyOn(logger, 'error').mockImplementation(() => logger); + try { + const context = contextFor({ + deriveText: async () => derived, + persistDerivation: async () => { + throw new Error('mongodb://user:secret@db/files'); + }, + }); + + await expect(context.derive(xlsx)).resolves.toBe(derived); + await expect(context.flush()).resolves.toBeUndefined(); + expect(errorSpy).toHaveBeenCalledWith( + '[readingText] file_id=xlsx outcome=complete persisted=failed', + { type: 'Error' }, + ); + } finally { + errorSpy.mockRestore(); + } + }); + }); + + describe('createDerivationPersister', () => { + const update = { + file_id: 'xlsx', + text: 'quarter,total', + textDerivation: { outcome: 'complete' as const, extractor: 'document_parser' as const }, + }; + const scope = { user: 'user-1', tenantId: 'tenant-1' }; + + it('writes a file once for every agent context sharing it', async () => { + const save = jest.fn, Parameters>( + async () => true, + ); + const persistDerivation = createDerivationPersister(save, scope); + const deriveText: FileTextDeriver = async () => ({ + status: 'derived', + text: update.text, + textDerivation: update.textDerivation, + }); + const primary = contextFor({ deriveText, persistDerivation }); + const handoff = contextFor({ deriveText, persistDerivation }); + + await Promise.all([primary.derive(xlsx), handoff.derive({ ...xlsx })]); + await Promise.all([primary.flush(), handoff.flush()]); + + expect(save).toHaveBeenCalledTimes(1); + expect(save).toHaveBeenCalledWith(update, scope); + }); + + it('forgets a failed write, so a later context tries again', async () => { + const save = jest + .fn, Parameters>() + .mockRejectedValueOnce(new Error('write conflict')) + .mockResolvedValueOnce(true); + const persistDerivation = createDerivationPersister(save, scope); + + await expect(persistDerivation(update)).rejects.toThrow('write conflict'); + await expect(persistDerivation(update)).resolves.toBe(true); + await expect(persistDerivation(update)).resolves.toBe(true); + expect(save).toHaveBeenCalledTimes(2); + }); + }); +}); + +describe('deriveRequestedText', () => { + const workbook = attachment({ + file_id: 'xlsx', + type: XLSX, + llmDeliveryPath: 'none', + metadata: { destinationChosen: false, textDerivation: { outcome: 'deferred' } }, + }); + + it('derives once through the first context that asked and returns the text by file id', async () => { + const firstDeriver = jest.fn, Parameters>( + async () => ({ + status: 'derived', + text: 'a,b', + textDerivation: { outcome: 'complete', extractor: 'document_parser' }, + }), + ); + const secondDeriver = jest.fn, Parameters>(); + const first = contextFor({ deriveText: firstDeriver }); + const second = contextFor({ deriveText: secondDeriver }); + + const texts = await deriveRequestedText([{ file: workbook, contexts: [first, second] }]); + + expect(texts.get('xlsx')).toMatchObject({ status: 'derived', text: 'a,b' }); + expect(firstDeriver).toHaveBeenCalledTimes(1); + expect(secondDeriver).not.toHaveBeenCalled(); + expect(second.judge(workbook).textFailed).toBeUndefined(); + }); + + it('throws the policy error of derived text a content policy refuses', async () => { + const error = new UninspectableFileError('extracted_text'); + const context = contextFor({ deriveText: async () => ({ status: 'blocked', error }) }); + + await expect(deriveRequestedText([{ file: workbook, contexts: [context] }])).rejects.toBe( + error, + ); + }); + + it('marks a file left without text failed on every context that asked', async () => { + const first = contextFor({ + deriveText: async () => ({ status: 'skipped', reason: 'storage_unavailable' }), + }); + const second = contextFor({ routing: routingFor('openAI', {}), deriveText: jest.fn() }); + + const texts = await deriveRequestedText([{ file: workbook, contexts: [first, second] }]); + + expect(texts.size).toBe(0); + expect(first.judge(workbook).textFailed).toBe(true); + expect(second.judge(workbook).textFailed).toBe(true); + }); +}); + +describe('needsDerivedText', () => { + it('decides only marked records for a derive-only classic context', () => { + const routing = routingFor('openAI', {}); + const context = contextFor({ routing, deriveText: jest.fn() }); + routing.reading = context; + const consumers = { executeCode: false, fileSearch: false }; + const marked = attachment({ + file_id: 'marked', + type: XLSX, + llmDeliveryPath: 'none', + metadata: { destinationChosen: false, textDerivation: { outcome: 'deferred' } }, + }); + const unmarked = { ...marked, file_id: 'unmarked', metadata: { destinationChosen: false } }; + + expect(needsDerivedText(marked, { routing, consumers, context })).toBe(true); + expect(needsDerivedText(unmarked, { routing, consumers, context })).toBe(false); + }); +}); diff --git a/packages/api/src/files/reading/turn.ts b/packages/api/src/files/reading/turn.ts new file mode 100644 index 00000000000..3de4275d573 --- /dev/null +++ b/packages/api/src/files/reading/turn.ts @@ -0,0 +1,767 @@ +import { Providers } from '@librechat/agents'; +import { logger } from '@librechat/data-schemas'; +import { + getRoutingMimeType, + hasFileToolConsumer, + EToolResources, + decideFileReading, + isBedrockDocumentType, + categorizeForReading, + canToolResourceConsume, + isAnthropicDocumentType, + resolveLLMDeliveryPolicy, + isAutomaticReadingRecord, + isDocumentSupportedProvider, +} from 'librechat-data-provider'; +import type { + TFile, + TextDerivation, + ReadingEvidence, + TurnDeliveryFile, + TurnFileConsumers, + TurnReadingInputs, + TLLMDeliveryPolicy, + TurnDeliveryRouting, +} from 'librechat-data-provider'; +import type { FileTextDerivationUpdate } from '@librechat/data-schemas'; +import type { DocumentRejection, NativeValidationMode } from '~/types'; +import type { ContentFilterError } from '~/middleware/contentFilter'; +import type { UninspectableFileError } from '~/protection/files'; +import { usesAnthropicDocumentCapabilities } from '~/files/encode/document'; +import { getNativeDocumentSizeLimit } from '~/files/validation'; +import { getSafeErrorMetadata } from '~/utils/errors'; + +/** Where a record's retained original is stored, for the deriver that reads it back. */ +type StoredOriginal = Partial>; + +/** A turn copy of an attachment record, identified so request-scoped evidence can name it. */ +export type TurnReadingFile = TurnDeliveryFile & StoredOriginal & { file_id: string }; + +/** Why no text was derived on this request; none of these is persisted, so a later turn retries. */ +export type DerivationSkipReason = 'policy' | 'storage_unavailable' | 'aborted' | 'no_extractor'; + +/** The content-policy error derived text raised: a finding, or text the policy cannot inspect. */ +export type DerivationPolicyError = ContentFilterError | UninspectableFileError; + +/** + * The outcome of deriving a file's text from its retained original. A `failed` outcome is + * deterministic, and `persist` says whether the record may keep it as a failed marker. A + * `blocked` outcome is a content-policy refusal of the derived text, which fails the turn that + * needed it exactly as the same text stored at upload would. + */ +export type DerivedText = + | { status: 'derived'; text: string; textDerivation: TextDerivation } + | { status: 'failed'; textDerivation: TextDerivation; persist: boolean } + | { status: 'blocked'; error: DerivationPolicyError } + | { status: 'skipped'; reason: DerivationSkipReason }; + +export type FileTextDeriver = (file: TurnReadingFile, signal?: AbortSignal) => Promise; + +/** A derivation that changed the copy's text, as {@link withDerivedText} applies it. */ +export type DerivedTextSuccess = Extract; + +/** What a derivation writes back onto a record: see {@link selectDerivationUpdate}. */ +export type TextDerivationUpdate = FileTextDerivationUpdate; + +/** Saves a derivation onto its record and resolves whether a record changed. */ +export type TextDerivationPersister = (update: TextDerivationUpdate) => Promise; + +/** The owner a derivation is saved under, as `saveFileTextDerivation` scopes its write. */ +export interface DerivationOwnerScope { + user: string; + tenantId?: string | null; +} + +/** Saves a derivation under an owner scope: the data-schemas `saveFileTextDerivation`. */ +export type TextDerivationSave = ( + update: TextDerivationUpdate, + scope: DerivationOwnerScope, +) => Promise; + +/** + * A request's derivation persister: saves under the request's file owner and writes each file at + * most once, so the reading contexts of several agents that share one deriver issue one write per + * derived file. A failed write is forgotten, so a later context may try again. The host builds + * one per request, next to the deriver, and hands it to every agent. + */ +export function createDerivationPersister( + save: TextDerivationSave, + scope: DerivationOwnerScope, +): TextDerivationPersister { + const writes = new Map>(); + return (update) => { + const existing = writes.get(update.file_id); + if (existing != null) { + return existing; + } + const write = save(update, scope); + writes.set(update.file_id, write); + write.catch(() => { + if (writes.get(update.file_id) === write) { + writes.delete(update.file_id); + } + }); + return write; + }; +} + +/** Preparation of one file in this turn's authorized File Search namespace. */ +export type SearchPreparationState = 'queued' | 'ready' | 'failed'; + +/** Which files File Search will receive on this turn, known once tools have loaded. */ +export interface SearchEvidence { + /** Files queued for embedding on this request. */ + queued: readonly string[]; + /** Files registered under the turn's `file_search` tool resource. */ + registered: readonly string[]; + /** Shared with lazy provisioning; a failed attempt stays failed until a retry succeeds. */ + preparation?: ReadonlyMap; +} + +/** Request-scoped counts for the turn's reading diagnostics. */ +export interface TurnReadingStats { + overflow: number; + rejected: number; + textFailed: number; + derived: number; + dropped: number; +} + +/** + * One agent's reading evidence for one request. Attached as `deliveryRouting.reading`, so every + * reader holding the routing sees the verdicts and request-scoped evidence the others recorded. + */ +export interface TurnReadingContext extends TurnReadingInputs { + readonly policy: TLLMDeliveryPolicy; + /** The provider the document encoder receives for this agent. */ + readonly provider: string; + /** The token count of the file's current text, when judging it required a count. */ + knownTokenCount(file: TurnDeliveryFile): number | undefined; + addOverflow(fileIds: Iterable): void; + recordRejections(rejections: ReadonlyArray): void; + markTextFailed(fileId: string): void; + setSearchEvidence(evidence: SearchEvidence): void; + /** Whether an eligible search file is indexed, queued, or failed on this turn. */ + searchState(fileId: string): SearchPreparationState | undefined; + /** Request attachments the endpoint's runtime policy removed before delivery. */ + recordDropped(files: Iterable): void; + dropped(): readonly TurnReadingFile[]; + /** Derives the file's text once per request; repeated calls share the first derivation. */ + derive(file: TurnReadingFile, signal?: AbortSignal): Promise; + /** + * Starts the record writes the request's derivations queued, once each derivation settles, and + * waits for them. Never rejects. Nothing is written before a flush, but a flush is not what + * makes the text safe to keep: derived text is content-inspected when it is derived + * (`extractBoundedText` applies the `extracted_text` policy), text a policy refuses is never + * written, and its refusal throws from {@link deriveRequestedText}. Initialization flushes only + * after endpoint admission and the full-set inspection. History replay, steering and the + * run-file encoder flush right after deriving, before their own endpoint filter and model-bound + * admission run; the scoped handoff candidates do too, but derive only for files each + * receiver's endpoint accepts. + */ + flush(): Promise; + stats(): TurnReadingStats; +} + +export interface BuildTurnReadingContextParams { + routing: TurnDeliveryRouting; + /** The provider the document encoder receives for this agent. */ + provider: string; + model?: string; + /** The request's `fileTokenLimit`; a missing or zero limit delivers no text. */ + fileTokenLimit: number | undefined; + /** The endpoint `fileSizeLimit` the document encoder applies, in bytes. */ + configuredFileSizeLimit: number | undefined; + /** A synchronous token count, ready for use. */ + countTokens: (text: string) => number; + deriveText?: FileTextDeriver; + /** + * Saves derived text, or a deterministic failure, onto the record it came from. The caller + * binds the request's file owner scope. A record marked `deferred`, or an unmarked original + * without text, takes derived text; a failed marker is kept only by a deferred record. + */ + persistDerivation?: TextDerivationPersister; + signal?: AbortSignal; +} + +type NativeVerdict = NonNullable; + +interface TextVerdict { + fit: NonNullable; + tokens?: number; +} + +const TURN_READING = Symbol('turnReading'); + +type BrandedTurnReadingContext = TurnReadingContext & { readonly [TURN_READING]: true }; + +const isTurnReadingContext = ( + reading: TurnReadingInputs | undefined, +): reading is BrandedTurnReadingContext => reading != null && TURN_READING in reading; + +/** The reading context {@link buildTurnReadingContext} built for this routing, if any. */ +export function getTurnReadingContext( + routing?: Pick | null, +): TurnReadingContext | undefined { + const reading = routing?.reading; + return isTurnReadingContext(reading) ? reading : undefined; +} + +/** The `extractFileContext` options a turn's reading supplies for its text injection. */ +export interface TurnTextOptions { + knownTokenCount?: (file: TurnDeliveryFile) => number | undefined; + markTruncation?: boolean; +} + +/** + * Reuses the token counts the reading judged text fit with, and under the automatic policy makes + * any truncation visible: the policy never sends partial text on purpose, so a truncated file + * means a reader judged it without the turn's evidence. Empty without a reading context. + */ +export function getTurnTextOptions( + routing?: Pick | null, +): TurnTextOptions { + const context = getTurnReadingContext(routing); + if (context == null) { + return {}; + } + return { + knownTokenCount: context.knownTokenCount, + markTruncation: context.policy === 'automatic', + }; +} + +/** The agent fields that decide how the document encoder treats the files it cannot send. */ +export interface NativeDeliveryAgent { + id?: string; + deliveryRouting?: TurnDeliveryRouting | null; + fileConsumers?: TurnFileConsumers; + currentRequestAttachments?: readonly TurnDeliveryFile[]; +} + +/** What a document encoder returns, as a split encode merges it. */ +export interface EncodedDocuments { + documents: D[]; + files?: M[]; + rejected?: readonly DocumentRejection[]; +} + +/** + * How the document encoder treats each file it cannot send. Only a file the automatic policy + * reads has another reader to move to, so only it is left out instead of failing the turn; + * explicit, configured, unmarked and legacy records keep failing exactly as classic. + */ +export function getNativeValidationPolicy( + agent: NativeDeliveryAgent | null | undefined, +): (file: TurnDeliveryFile) => NativeValidationMode { + const routing = agent?.deliveryRouting ?? undefined; + if ( + getTurnReadingContext(routing)?.policy !== 'automatic' || + !hasFileToolConsumer(agent?.fileConsumers) + ) { + return () => 'throw'; + } + return (file) => (isAutomaticReadingRecord(routing, file) ? 'skip' : 'throw'); +} + +const mergeEncoded = ( + skipped: EncodedDocuments, + thrown: EncodedDocuments, +): EncodedDocuments => ({ + documents: [...skipped.documents, ...thrown.documents], + files: [...(skipped.files ?? []), ...(thrown.files ?? [])], + rejected: [...(skipped.rejected ?? []), ...(thrown.rejected ?? [])], +}); + +/** + * Encodes documents with each file's own validation mode: files the automatic policy may leave + * out are encoded in skip mode, the rest in throw mode, so a classic record still fails the turn. + * A batch whose files share one mode is encoded in a single call. + */ +export async function encodeNativeDocuments( + files: F[], + agent: NativeDeliveryAgent | null | undefined, + encode: ( + files: F[], + onValidationFailure: NativeValidationMode, + ) => Promise>, +): Promise> { + const validationMode = getNativeValidationPolicy(agent); + const skippable = files.filter((file) => validationMode(file) === 'skip'); + if (skippable.length === 0) { + return encode(files, 'throw'); + } + if (skippable.length === files.length) { + return encode(files, 'skip'); + } + const skippableSet = new Set(skippable); + const [skipped, thrown] = await Promise.all([ + encode(skippable, 'skip'), + encode( + files.filter((file) => !skippableSet.has(file)), + 'throw', + ), + ]); + return mergeEncoded(skipped, thrown); +} + +/** The reader each rejected request file moves to, by file id, for the delivery log. */ +function describeNextReaders( + agent: NativeDeliveryAgent, + rejected: readonly DocumentRejection[], +): Map { + const rejectedIds = new Set(rejected.map(({ file_id }) => file_id)); + const next = new Map(); + for (const file of agent.currentRequestAttachments ?? []) { + if (file.file_id == null || !rejectedIds.has(file.file_id) || next.has(file.file_id)) { + continue; + } + const reading = decideFileReading({ + routing: agent.deliveryRouting ?? undefined, + file, + consumers: agent.fileConsumers, + }); + next.set(file.file_id, reading.reader); + } + return next; +} + +/** + * Records the documents the encoder left out on every agent whose messages carry the encoded + * payload, so each agent's later decisions and inventory move them on. Each reading context is + * recorded once, even when agents share it. + */ +export function recordNativeRejections( + agents: Iterable, + rejected?: readonly DocumentRejection[], +): void { + if (rejected == null || rejected.length === 0) { + return; + } + const recorded = new Set(); + for (const agent of agents) { + const context = getTurnReadingContext(agent?.deliveryRouting); + if (agent == null || context == null || recorded.has(context)) { + continue; + } + recorded.add(context); + context.recordRejections(rejected); + if (!logger.isDebugEnabled()) { + continue; + } + const next = describeNextReaders(agent, rejected); + for (const { file_id, reason } of rejected) { + logger.debug( + `[nativeDelivery] agent=${agent.id ?? 'unknown'} file_id=${file_id} provider=${context.provider} rejected=${reason} mode=skip next=${next.get(file_id) ?? 'unknown'}`, + ); + } + } +} + +const hasText = (text: TurnDeliveryFile['text']): text is string => + typeof text === 'string' && text.length > 0; + +/** Whether the document encoder emits any block for this type on this provider and model. */ +function encoderEmits(provider: string, mimeType: string, model?: string): boolean { + if (provider === Providers.BEDROCK) { + return isBedrockDocumentType(mimeType); + } + if (!isDocumentSupportedProvider(provider)) { + return false; + } + return ( + !usesAnthropicDocumentCapabilities(provider as Providers, model) || + isAnthropicDocumentType(mimeType) + ); +} + +/** + * Text fits when the complete text is within the token limit. Every token spans at least one + * byte, so a byte length within the limit settles it without counting. + */ +function judgeTextFit( + text: string, + limit: number | undefined, + countTokens: (text: string) => number, +): TextVerdict { + if (!limit) { + return { fit: 'exceeds' }; + } + if (Buffer.byteLength(text) <= limit) { + return { fit: 'fits' }; + } + const tokens = countTokens(text); + return { fit: tokens <= limit ? 'fits' : 'exceeds', tokens }; +} + +/** + * The write a derivation makes on its record: the text with its marker on success, the marker + * alone for a deterministic failure the record may keep. Derived text is kept by the record that + * deferred it, and cached on an unmarked original that stored none, so a document the provider + * could not take is parsed once rather than on every turn that replays it. A failed marker is + * kept only where the record was deferred; a settled or failed record is never written. + */ +function selectDerivationUpdate( + file: TurnReadingFile, + result: DerivedText, +): TextDerivationUpdate | undefined { + const outcome = file.metadata?.textDerivation?.outcome; + const hasText = typeof file.text === 'string' && file.text.length > 0; + if (result.status === 'derived' && (outcome === 'deferred' || (outcome == null && !hasText))) { + return { file_id: file.file_id, text: result.text, textDerivation: result.textDerivation }; + } + if (result.status === 'failed' && result.persist && outcome === 'deferred') { + return { file_id: file.file_id, textDerivation: result.textDerivation }; + } + return undefined; +} + +function createTurnReadingContext( + policy: TLLMDeliveryPolicy, + { + routing, + provider, + model, + fileTokenLimit, + configuredFileSizeLimit, + countTokens, + deriveText, + persistDerivation, + signal: requestSignal, + }: BuildTurnReadingContextParams, +): BrandedTurnReadingContext { + const automatic = policy === 'automatic'; + const nativeVerdicts = new Map(); + const textVerdicts = new Map(); + const overflow = new Set(); + const rejected = new Map(); + const textFailed = new Set(); + const droppedFiles = new Map(); + const derivations = new Map>(); + const queuedWrites: Array<{ file: TurnReadingFile; derivation: Promise }> = []; + let searchFileIds: ReadonlySet | undefined; + let registeredSearchFileIds: ReadonlySet | undefined; + let searchPreparation: ReadonlyMap | undefined; + let derived = 0; + + const judgeNativeUncached = (file: TurnDeliveryFile): NativeVerdict | null => { + const routingMimeType = getRoutingMimeType(file); + if (categorizeForReading(routingMimeType) === 'media') { + return null; + } + const mimeType = file.type ?? ''; + if (!encoderEmits(provider, mimeType, model)) { + return 'unsupported'; + } + const limit = getNativeDocumentSizeLimit({ + provider, + mimeType, + model, + configuredFileSizeLimit, + }); + return limit !== undefined && (file.bytes ?? 0) > limit ? 'capacity' : 'fits'; + }; + + const judgeNative = (file: TurnDeliveryFile): NativeVerdict | null => { + if (file.file_id == null) { + return judgeNativeUncached(file); + } + const cached = nativeVerdicts.get(file.file_id); + if (cached !== undefined) { + return cached; + } + const verdict = judgeNativeUncached(file); + nativeVerdicts.set(file.file_id, verdict); + return verdict; + }; + + const textKey = (file: TurnDeliveryFile, text: string): string | undefined => + file.file_id == null ? undefined : `${file.file_id}:${text.length}`; + + const judgeText = (file: TurnDeliveryFile): TextVerdict | undefined => { + if (!hasText(file.text)) { + return undefined; + } + const key = textKey(file, file.text); + const cached = key == null ? undefined : textVerdicts.get(key); + if (cached != null) { + return cached; + } + const verdict = judgeTextFit(file.text, fileTokenLimit, countTokens); + if (key != null) { + textVerdicts.set(key, verdict); + } + return verdict; + }; + + const judgeSearch = (file: TurnDeliveryFile): ReadingEvidence['search'] => { + if (searchFileIds == null) { + return undefined; + } + const unreachable = + (file.file_id == null || !searchFileIds.has(file.file_id)) && + canToolResourceConsume(EToolResources.file_search, file.type ?? '') && + isAutomaticReadingRecord(routing, file); + return unreachable ? 'unreachable' : 'reachable'; + }; + + const requestEvidence = (fileId: string | undefined): ReadingEvidence => ({ + ...(fileId != null && textFailed.has(fileId) && { textFailed: true as const }), + }); + + /** + * Exposes the text verdict as an enumerable getter judged on first read, so a reading walk that + * settles on Run Code, the provider or File Search never counts tokens in text it will not + * deliver. The verdict is memoized here and, by file and text length, for knownTokenCount. + */ + const withLazyText = (evidence: ReadingEvidence, file: TurnDeliveryFile): ReadingEvidence => { + let verdict: TextVerdict | undefined; + Object.defineProperty(evidence, 'text', { + enumerable: true, + configurable: true, + get: () => { + verdict ??= judgeText(file); + return verdict?.fit; + }, + }); + return evidence; + }; + + const judge = (file: TurnDeliveryFile): ReadingEvidence => { + if (!automatic) { + return requestEvidence(file.file_id); + } + const fileId = file.file_id; + const native = judgeNative(file); + const rejection = fileId == null ? undefined : rejected.get(fileId); + const search = judgeSearch(file); + const evidence: ReadingEvidence = { + ...(native != null && { native }), + ...(rejection != null && { rejected: rejection }), + ...(fileId != null && overflow.has(fileId) && { overflow: true as const }), + ...requestEvidence(fileId), + ...(search != null && { search }), + }; + return hasText(file.text) ? withLazyText(evidence, file) : evidence; + }; + + const knownTokenCount = (file: TurnDeliveryFile): number | undefined => { + if (!hasText(file.text)) { + return undefined; + } + const key = textKey(file, file.text); + return key == null ? undefined : textVerdicts.get(key)?.tokens; + }; + + const runDerivation = (file: TurnReadingFile, signal?: AbortSignal): Promise => { + if (deriveText == null) { + return Promise.resolve({ status: 'skipped', reason: 'no_extractor' }); + } + return deriveText(file, signal).then( + (result): DerivedText => { + if (result.status === 'derived') { + derived += 1; + } + return result; + }, + (error: unknown): DerivedText => { + logger.error( + `[readingText] file_id=${file.file_id} outcome=skipped reason=derivation_error`, + getSafeErrorMetadata(error), + ); + return { status: 'skipped', reason: signal?.aborted ? 'aborted' : 'storage_unavailable' }; + }, + ); + }; + + /** Saves the derivation onto a marked record; a failed write is logged, never thrown. */ + const persist = async (file: TurnReadingFile, result: DerivedText): Promise => { + const update = selectDerivationUpdate(file, result); + if (update == null || persistDerivation == null) { + return; + } + const line = `[readingText] file_id=${update.file_id} outcome=${update.textDerivation.outcome}`; + try { + const saved = await persistDerivation(update); + logger.debug(`${line} persisted=${saved}`); + } catch (error) { + logger.error(`${line} persisted=failed`, getSafeErrorMetadata(error)); + } + }; + + const derive = (file: TurnReadingFile, signal?: AbortSignal): Promise => { + const existing = derivations.get(file.file_id); + if (existing != null) { + return existing; + } + const derivation = runDerivation(file, signal ?? requestSignal); + derivations.set(file.file_id, derivation); + queuedWrites.push({ file, derivation }); + /* An aborted derivation answers only its own caller, like the deriver's memo. */ + const forget = () => { + if (derivations.get(file.file_id) === derivation) { + derivations.delete(file.file_id); + } + }; + derivation.then((result) => { + if (result.status === 'skipped' && result.reason === 'aborted') { + forget(); + } + }, forget); + return derivation; + }; + + const flush = async (): Promise => { + const writes = queuedWrites.splice(0); + await Promise.allSettled( + writes.map(({ file, derivation }) => derivation.then((result) => persist(file, result))), + ); + }; + + return { + [TURN_READING]: true, + policy, + provider, + canDerive: deriveText != null, + judge, + knownTokenCount, + addOverflow: (fileIds) => { + for (const fileId of fileIds) { + overflow.add(fileId); + } + }, + recordRejections: (rejections) => { + for (const { file_id, reason } of rejections) { + if (!rejected.has(file_id)) { + rejected.set(file_id, reason); + } + } + }, + markTextFailed: (fileId) => { + textFailed.add(fileId); + }, + setSearchEvidence: ({ queued, registered, preparation }) => { + searchFileIds = new Set([...queued, ...registered]); + registeredSearchFileIds = new Set(registered); + searchPreparation = preparation; + }, + searchState: (fileId) => { + const preparation = searchPreparation?.get(fileId); + if (preparation != null) { + return preparation; + } + if (registeredSearchFileIds?.has(fileId)) { + return 'ready'; + } + if (searchFileIds?.has(fileId)) { + return 'queued'; + } + return undefined; + }, + recordDropped: (files) => { + for (const file of files) { + if (!droppedFiles.has(file.file_id)) { + droppedFiles.set(file.file_id, file); + } + } + }, + dropped: () => [...droppedFiles.values()], + derive, + flush, + stats: () => ({ + overflow: overflow.size, + rejected: rejected.size, + textFailed: textFailed.size, + derived, + dropped: droppedFiles.size, + }), + }; +} + +/** + * Builds the reading evidence for one agent's turn, or nothing where the turn needs none: under + * the classic policy without a text deriver every route stays classic. Under classic with a + * deriver the context only derives text, for records the automatic policy marked, and its judge + * reports nothing but this request's derivation failures. + */ +export function buildTurnReadingContext( + params: BuildTurnReadingContextParams, +): TurnReadingContext | undefined { + const policy = resolveLLMDeliveryPolicy(params.routing.endpointConfig); + if (policy !== 'automatic' && params.deriveText == null) { + return undefined; + } + return createTurnReadingContext(policy, params); +} + +/** One agent's turn routing, the tools judged as its readers, and its reading context. */ +export interface TurnReader { + routing: TurnDeliveryRouting; + consumers?: TurnFileConsumers; + context: TurnReadingContext; +} + +/** + * Whether the file's reading needs text derived on this turn. A derive-only classic context + * serves only records the automatic policy marked, so an unmarked record costs it no decision. + */ +export function needsDerivedText( + file: TurnReadingFile, + { routing, consumers, context }: TurnReader, +): boolean { + if (context.policy !== 'automatic' && file.metadata?.textDerivation == null) { + return false; + } + return decideFileReading({ routing, file, consumers }).needsText; +} + +/** The copy carrying derived text and the marker saying how it was obtained. */ +export const withDerivedText = ( + file: T, + derived: DerivedTextSuccess, +): T => ({ + ...file, + text: derived.text, + metadata: { ...file.metadata, textDerivation: derived.textDerivation }, +}); + +/** A file whose reading needs text, with every reading context whose decision asked for it. */ +export interface TextRequest { + file: T; + /** The first context derives; the rest share its outcome. */ + contexts: readonly [TurnReadingContext, ...TurnReadingContext[]]; +} + +/** + * Derives each requested file's text once, through the first context that asked for it, and + * returns the successes by file id. A file left without text is marked failed on every context + * that asked, so their next decisions pass text over for the rest of the request. Derived text a + * content policy refuses throws its policy error: the turn needed that text, and the same text + * stored at upload would have been refused there. + */ +export async function deriveRequestedText( + requests: readonly TextRequest[], + signal?: AbortSignal, +): Promise> { + const outcomes = await Promise.all( + requests.map(async ({ file, contexts }) => ({ + file, + contexts, + result: await contexts[0].derive(file, signal), + })), + ); + const texts = new Map(); + for (const { file, contexts, result } of outcomes) { + if (result.status === 'derived') { + texts.set(file.file_id, result); + continue; + } + if (result.status === 'blocked') { + throw result.error; + } + for (const context of contexts) { + context.markTextFailed(file.file_id); + } + } + return texts; +} diff --git a/packages/api/src/files/reading/upload.spec.ts b/packages/api/src/files/reading/upload.spec.ts new file mode 100644 index 00000000000..8ce42300552 --- /dev/null +++ b/packages/api/src/files/reading/upload.spec.ts @@ -0,0 +1,513 @@ +import fs from 'fs'; +import os from 'os'; +import path from 'path'; +import { logger } from '@librechat/data-schemas'; +import type { EndpointFileConfig, FiltersConfig, TextDerivation } from 'librechat-data-provider'; +import type { ResolveUploadReadingInput, ResolvedUploadReading } from './upload'; +import { + logUploadReading, + acquireUploadText, + orderToolsForReading, + resolveUploadReading, + getUploadReadingMetadata, + ExtractorUnavailableError, + resolveUploadCodePossible, +} from './upload'; +import { UninspectableFileError, ContentFilterInputTooLargeError } from '~/protection/files'; +import { extractFileContent } from '~/protection/adapters/submissions'; +import { UnsupportedProviderAudioError } from '~/files/upload/errors'; +import { MAX_STORED_EXTRACTED_TEXT_BYTES } from '~/files/extract'; +import { ContentFilterError } from '~/middleware/contentFilter'; +import { ZipBombError } from '~/files/documents/zipSafety'; +import { parseDocument } from '~/files/documents/crud'; +import { inspectContent } from '~/protection/runtime'; + +const XLSX = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'; +const DOCX = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'; +const PPTX = 'application/vnd.openxmlformats-officedocument.presentationml.presentation'; +const PDF = 'application/pdf'; +const CSV = 'text/csv'; + +const automatic: EndpointFileConfig = { llmDeliveryPolicy: 'automatic' }; +const extractedTextBlock: FiltersConfig = { + files: { pii: { fields: ['extracted_text'], starterPatterns: [], uninspectable: 'block' } }, +}; +const contentAndTextBlock: FiltersConfig = { + files: { + pii: { fields: ['content', 'extracted_text'], starterPatterns: [], uninspectable: 'block' }, + }, +}; +const privateTokenFilters: FiltersConfig = { + files: { + pii: { + fields: ['extracted_text'], + starterPatterns: [], + customPatterns: [{ id: 'private', label: 'private token', regex: 'PRIVATE-[A-Z]+' }], + }, + }, +}; + +function decide( + overrides: Partial & Pick, + codePossible = true, +) { + const resolveCodePossible = jest.fn(async () => codePossible); + const reading = resolveUploadReading({ + endpoint: 'openAI', + endpointConfig: automatic, + isMessageAttachment: true, + resolveCodePossible, + ...overrides, + }); + return { reading, resolveCodePossible }; +} + +describe('resolveUploadReading', () => { + it('leaves a workbook to Run Code when code is possible, deferring its text', async () => { + const { reading, resolveCodePossible } = decide({ mimeType: XLSX }); + + await expect(reading).resolves.toEqual({ + path: 'none', + policy: 'automatic', + category: 'tabular', + reason: 'code_preferred', + codePreferred: true, + needsCodeAvailability: false, + keepOriginalOnExtractionFailure: false, + deferredMarker: true, + mimeType: XLSX, + }); + expect(resolveCodePossible).toHaveBeenCalledTimes(1); + }); + + it('extracts a workbook at upload when code is not possible, keeping it on failure', async () => { + const { reading, resolveCodePossible } = decide({ mimeType: XLSX }, false); + + await expect(reading).resolves.toMatchObject({ + path: 'text', + reason: 'code_unavailable', + codePreferred: false, + keepOriginalOnExtractionFailure: true, + deferredMarker: false, + }); + expect(resolveCodePossible).toHaveBeenCalledTimes(1); + }); + + it('never asks about code for documents, which keep their original on failure', async () => { + for (const mimeType of [PDF, DOCX, 'application/zip']) { + const { reading, resolveCodePossible } = decide({ mimeType }); + await expect(reading).resolves.toMatchObject({ + reason: 'automatic_default', + category: 'document', + codePreferred: false, + keepOriginalOnExtractionFailure: true, + deferredMarker: false, + }); + expect(resolveCodePossible).not.toHaveBeenCalled(); + } + }); + + it('extracts at upload, and rejects on failure, where extraction is the inspection', async () => { + for (const mimeType of [XLSX, DOCX]) { + const { reading, resolveCodePossible } = decide({ mimeType, filters: extractedTextBlock }); + await expect(reading).resolves.toMatchObject({ + path: 'text', + codePreferred: false, + keepOriginalOnExtractionFailure: false, + deferredMarker: false, + }); + expect(resolveCodePossible).not.toHaveBeenCalled(); + } + await expect( + decide({ mimeType: XLSX, filters: extractedTextBlock }).reading, + ).resolves.toMatchObject({ reason: 'inspection_requires_text' }); + }); + + it('still defers where a blocked content field, not extraction, is the inspection', async () => { + await expect( + decide({ mimeType: XLSX, filters: contentAndTextBlock }).reading, + ).resolves.toMatchObject({ reason: 'code_preferred', codePreferred: true }); + await expect( + decide({ mimeType: XLSX, filters: privateTokenFilters }).reading, + ).resolves.toMatchObject({ reason: 'code_preferred', codePreferred: true }); + }); + + it('keeps a classic row classic without asking about code', async () => { + const { reading, resolveCodePossible } = decide({ mimeType: XLSX, endpointConfig: undefined }); + + await expect(reading).resolves.toMatchObject({ + reason: 'classic_policy', + codePreferred: false, + keepOriginalOnExtractionFailure: false, + deferredMarker: false, + }); + expect(resolveCodePossible).not.toHaveBeenCalled(); + }); +}); + +describe('resolveUploadCodePossible', () => { + function resolvers({ capability = true, grant }: { capability?: boolean; grant?: boolean }) { + return { + checkCodeCapability: jest.fn(async () => capability), + getRunCodeGrant: jest.fn(async () => grant), + }; + } + + it.each([ + [['file_search', 'execute_code'], true], + [['file_search'], false], + [[], false], + ])( + 'reads saved agent tools %p as %p, with no capability or role read', + async (tools, expected) => { + const saved = resolvers({}); + await expect(resolveUploadCodePossible({ agentTools: tools, ...saved })).resolves.toBe( + expected, + ); + expect(saved.checkCodeCapability).not.toHaveBeenCalled(); + expect(saved.getRunCodeGrant).not.toHaveBeenCalled(); + }, + ); + + it.each([ + [true, undefined, true], + [true, true, true], + [true, false, false], + [false, undefined, false], + [false, true, false], + ])( + 'reads an ephemeral agent from capability %p and grant %p as %p', + async (capability, grant, expected) => { + const ephemeral = resolvers({ capability, grant }); + await expect(resolveUploadCodePossible(ephemeral)).resolves.toBe(expected); + expect(ephemeral.checkCodeCapability).toHaveBeenCalledTimes(1); + }, + ); +}); + +describe('orderToolsForReading', () => { + const tools = ['file_search', 'execute_code', 'web_search']; + + it('files a code-preferred upload under Run Code first', () => { + expect(orderToolsForReading({ codePreferred: true }, tools)).toEqual([ + 'execute_code', + 'file_search', + 'web_search', + ]); + }); + + it('leaves the order alone otherwise', () => { + expect(orderToolsForReading({ codePreferred: false }, tools)).toBe(tools); + const withoutCode = ['file_search']; + expect(orderToolsForReading({ codePreferred: true }, withoutCode)).toBe(withoutCode); + expect(orderToolsForReading({ codePreferred: true }, undefined)).toBeUndefined(); + }); +}); + +describe('acquireUploadText', () => { + let workDir: string; + + beforeAll(() => { + workDir = fs.mkdtempSync(path.join(os.tmpdir(), 'upload-reading-')); + }); + + afterAll(() => { + fs.rmSync(workDir, { recursive: true, force: true }); + }); + + const keep = (mimeType: string): ResolvedUploadReading => ({ + path: 'text', + policy: 'automatic', + category: 'document', + reason: 'automatic_default', + codePreferred: false, + needsCodeAvailability: false, + keepOriginalOnExtractionFailure: true, + deferredMarker: false, + mimeType, + }); + const classic = (mimeType: string): ResolvedUploadReading => ({ + ...keep(mimeType), + policy: 'classic', + reason: 'classic_policy', + keepOriginalOnExtractionFailure: false, + }); + + /** The real document parser over a file written to disk. */ + function parse(name: string, mimetype: string, content: string | Buffer) { + const filePath = path.join(workDir, name); + fs.writeFileSync(filePath, content); + const file = { originalname: name, path: filePath, mimetype } as Express.Multer.File; + return async () => ({ text: (await parseDocument({ file })).text }); + } + + const failedWith = (reason: TextDerivation['reason'], extractor = 'document_parser') => ({ + textDerivation: { outcome: 'failed', extractor, reason, at: expect.any(Number) }, + path: 'none', + }); + const deferredWith = (textDerivation: Omit) => ({ + textDerivation: { outcome: 'deferred', ...textDerivation, at: expect.any(Number) }, + path: 'none', + }); + const fileId = 'file-1'; + + it('passes text and errors through unchanged where failure rejects the upload', async () => { + const acquired = { text: 'a'.repeat(MAX_STORED_EXTRACTED_TEXT_BYTES + 1) }; + await expect( + acquireUploadText({ fileId, reading: classic('text/csv'), acquire: async () => acquired }), + ).resolves.toBe(acquired); + await expect( + acquireUploadText({ + fileId, + reading: classic(DOCX), + acquire: parse('broken.docx', DOCX, 'no zip'), + }), + ).rejects.toThrow(); + const empty = { text: '' }; + await expect( + acquireUploadText({ fileId, reading: classic('text/csv'), acquire: async () => empty }), + ).resolves.toBe(empty); + }); + + it('returns acquired text within the caps unchanged', async () => { + const acquired = { text: 'region,total', isTranscript: false }; + await expect( + acquireUploadText({ fileId, reading: keep('text/csv'), acquire: async () => acquired }), + ).resolves.toBe(acquired); + }); + + it('keeps the original, deferred for a later turn, when extraction fails unrecognized', async () => { + const warn = jest.spyOn(logger, 'warn'); + try { + await expect( + acquireUploadText({ + fileId, + reading: keep(DOCX), + acquire: parse('broken.docx', DOCX, 'no zip'), + }), + ).resolves.toEqual(deferredWith({ reason: 'parser' })); + await expect( + acquireUploadText({ + fileId, + reading: keep(PDF), + acquire: () => Promise.reject(new Error('connect ECONNREFUSED 10.0.0.5:8000')), + }), + ).resolves.toEqual(deferredWith({ reason: 'parser' })); + expect(warn).toHaveBeenCalledWith( + '[uploadReading] file_id=file-1 extraction=deferred failure=parser', + expect.objectContaining({ type: expect.any(String) }), + ); + expect(JSON.stringify(warn.mock.calls)).not.toContain('10.0.0.5'); + await expect( + acquireUploadText({ + fileId, + reading: keep(PPTX), + acquire: () => Promise.reject(new TypeError('cannot read properties of undefined')), + }), + ).resolves.toEqual(failedWith('parser', 'configured')); + } finally { + warn.mockRestore(); + } + }); + + it('keeps the original, marked failed, when a built-in extractor names its failure', async () => { + await expect( + acquireUploadText({ + fileId, + reading: keep(DOCX), + acquire: parse( + 'empty.docx', + DOCX, + fs.readFileSync(path.join(__dirname, '../documents/empty.docx')), + ), + }), + ).resolves.toEqual(failedWith('empty')); + await expect( + acquireUploadText({ + fileId, + reading: keep(XLSX), + acquire: () => Promise.reject(new ZipBombError('sheet: entry exceeds the cap')), + }), + ).resolves.toEqual(failedWith('expansion_limit')); + }); + + it('treats the stored-text cap and empty text as soft failures', async () => { + await expect( + acquireUploadText({ + fileId, + reading: keep('text/csv'), + acquire: async () => ({ text: 'a'.repeat(MAX_STORED_EXTRACTED_TEXT_BYTES + 1) }), + }), + ).resolves.toEqual(failedWith('too_large', 'native_text')); + await expect( + acquireUploadText({ + fileId, + reading: keep('text/csv'), + acquire: async () => ({ text: ' \n' }), + }), + ).resolves.toEqual(failedWith('empty', 'native_text')); + }); + + it('defers when only a configured step was unavailable and a built-in extractor exists', async () => { + await expect( + acquireUploadText({ + fileId, + reading: keep(PDF), + acquire: () => + Promise.reject( + new ExtractorUnavailableError( + 'OCR capability is not enabled for Agents', + 'extractor_unavailable', + ), + ), + }), + ).resolves.toEqual(deferredWith({ extractor: 'configured', reason: 'extractor_unavailable' })); + await expect( + acquireUploadText({ + fileId, + reading: keep(PPTX), + acquire: () => + Promise.reject( + new ExtractorUnavailableError( + `File type ${PPTX} is not supported for text parsing.`, + 'no_extractor', + ), + ), + }), + ).resolves.toEqual(failedWith('no_extractor', 'configured')); + /* A type the text allowlist excludes stays failed even where a built-in extractor exists. */ + await expect( + acquireUploadText({ + fileId, + reading: keep(CSV), + acquire: () => + Promise.reject( + new ExtractorUnavailableError( + `File type ${CSV} is not supported for text parsing.`, + 'no_extractor', + ), + ), + }), + ).resolves.toEqual(failedWith('no_extractor', 'configured')); + }); + + it('rejects an unavailable configured step unchanged where failure rejects the upload', async () => { + const error = new ExtractorUnavailableError( + 'OCR capability is not enabled for Agents', + 'extractor_unavailable', + ); + await expect( + acquireUploadText({ fileId, reading: classic(PDF), acquire: () => Promise.reject(error) }), + ).rejects.toBe(error); + expect(error).toBeInstanceOf(Error); + expect(error.message).toBe('OCR capability is not enabled for Agents'); + }); + + it('fails closed on text a blocking policy cannot inspect before it counts as empty', async () => { + for (const text of ['', ' \n']) { + await expect( + acquireUploadText({ + fileId, + reading: keep(DOCX), + filters: contentAndTextBlock, + acquire: async () => ({ text }), + }), + ).rejects.toBeInstanceOf(UninspectableFileError); + } + await expect( + acquireUploadText({ + fileId, + reading: keep(DOCX), + filters: privateTokenFilters, + acquire: async () => ({ text: ' ' }), + }), + ).resolves.toEqual(failedWith('empty')); + }); + + it('still rejects inspection, content policy, audio and abort errors', async () => { + const finding = inspectContent(extractFileContent({ extractedText: 'PRIVATE-SECRET' }), { + filters: privateTokenFilters, + }); + if (finding == null) { + throw new Error('expected the private token to match'); + } + const hardErrors = [ + new UninspectableFileError('extracted_text'), + new ContentFilterError(finding), + new ContentFilterInputTooLargeError('extracted_text'), + new UnsupportedProviderAudioError(), + AbortSignal.abort().reason, + ]; + for (const error of hardErrors) { + await expect( + acquireUploadText({ fileId, reading: keep(DOCX), acquire: () => Promise.reject(error) }), + ).rejects.toBe(error); + } + }); +}); + +describe('getUploadReadingMetadata', () => { + const marker: TextDerivation = { outcome: 'failed', reason: 'parser', at: 1 }; + + it('writes nothing under the classic policy', () => { + expect(getUploadReadingMetadata({ policy: 'classic', deferredMarker: false })).toEqual({}); + expect(getUploadReadingMetadata({ policy: 'classic', deferredMarker: true }, marker)).toEqual( + {}, + ); + }); + + it('writes the acquisition marker, or a deferred marker for an upload left to Run Code', () => { + expect( + getUploadReadingMetadata({ policy: 'automatic', deferredMarker: false }, marker), + ).toEqual({ textDerivation: marker }); + expect(getUploadReadingMetadata({ policy: 'automatic', deferredMarker: true })).toEqual({ + textDerivation: { outcome: 'deferred', at: expect.any(Number) }, + }); + expect(getUploadReadingMetadata({ policy: 'automatic', deferredMarker: false })).toEqual({}); + }); +}); + +describe('logUploadReading', () => { + let debug: jest.SpyInstance; + let info: jest.SpyInstance; + + beforeEach(() => { + debug = jest.spyOn(logger, 'debug'); + info = jest.spyOn(logger, 'info'); + }); + + afterEach(() => { + debug.mockRestore(); + info.mockRestore(); + }); + + it('logs a deferred upload at debug with codes only', async () => { + const reading = await decide({ mimeType: XLSX }).reading; + logUploadReading('file-1', reading); + + expect(debug).toHaveBeenCalledWith( + '[uploadReading] file_id=file-1 policy=automatic category=tabular path=none reason=code_preferred extraction=deferred', + ); + }); + + it('logs a kept original at info', async () => { + const reading = await decide({ mimeType: DOCX }).reading; + logUploadReading( + 'file-2', + { ...reading, path: 'none' }, + { outcome: 'failed', reason: 'parser' }, + ); + + expect(info).toHaveBeenCalledWith( + '[uploadReading] file_id=file-2 policy=automatic category=document path=none reason=automatic_default extraction=failed failure=parser kept=original', + ); + }); + + it('stays silent under the classic policy', async () => { + const reading = await decide({ mimeType: XLSX, endpointConfig: undefined }).reading; + logUploadReading('file-3', reading); + + expect(debug).not.toHaveBeenCalledWith(expect.stringContaining('[uploadReading]')); + expect(info).not.toHaveBeenCalledWith(expect.stringContaining('[uploadReading]')); + }); +}); diff --git a/packages/api/src/files/reading/upload.ts b/packages/api/src/files/reading/upload.ts new file mode 100644 index 00000000000..8f32e6d68f8 --- /dev/null +++ b/packages/api/src/files/reading/upload.ts @@ -0,0 +1,293 @@ +import { logger } from '@librechat/data-schemas'; +import { + EToolResources, + decideUploadReading, + selectBuiltInTextPlan, +} from 'librechat-data-provider'; +import type { + FiltersConfig, + UploadReading, + TextDerivation, + UploadReadingInput, +} from 'librechat-data-provider'; +import { + hasActiveFilePolicy, + UninspectableFileError, + ContentFilterInputTooLargeError, + getBlockedUninspectableFileField, +} from '~/protection/files'; +import { assertExtractedTextInspectable, MAX_STORED_EXTRACTED_TEXT_BYTES } from '~/files/extract'; +import { UnsupportedProviderAudioError } from '~/files/upload/errors'; +import { isAbortError, getSafeErrorMetadata } from '~/utils/errors'; +import { matchExtractionFailure } from '~/files/upload/fallback'; +import { ContentFilterError } from '~/middleware/contentFilter'; + +/** An upload's reading decision, with the type it was decided for. */ +export interface ResolvedUploadReading extends UploadReading { + mimeType: string; +} + +export type ResolveUploadReadingInput = Omit< + UploadReadingInput, + 'extractionRequiredForInspection' | 'codePossible' +> & { + filters?: FiltersConfig; + /** Asked only when the decision depends on whether Run Code can read the file. */ + resolveCodePossible: () => Promise; +}; + +/** + * Decides how an upload is read before anything is extracted. Code preference is off where the + * upload preflight defers an extracted-text fail-close to the context route, because there the + * extraction is the inspection; Run Code availability is resolved only when the decision needs it. + */ +export async function resolveUploadReading({ + filters, + resolveCodePossible, + ...input +}: ResolveUploadReadingInput): Promise { + const extractionRequiredForInspection = + hasActiveFilePolicy(filters) && + getBlockedUninspectableFileField(filters, ['content', 'extracted_text']) === 'extracted_text'; + const routing: UploadReadingInput = { ...input, extractionRequiredForInspection }; + const first = decideUploadReading(routing); + if (!first.needsCodeAvailability) { + return { ...first, mimeType: input.mimeType }; + } + const codePossible = await resolveCodePossible(); + return { ...decideUploadReading({ ...routing, codePossible }), mimeType: input.mimeType }; +} + +export interface UploadCodePossibleParams { + /** + * The saved agent's tools, already narrowed by capability and role grants: the same list the + * upload is filed against. Undefined when no agent record backs the upload. + */ + agentTools?: string[]; + checkCodeCapability: () => Promise; + getRunCodeGrant: () => Promise; +} + +/** + * Whether Run Code can read the upload: a saved agent whose narrowed tools include it, or, with no + * saved agent, the capability enabled and the role's grant not denied. + */ +export async function resolveUploadCodePossible({ + agentTools, + checkCodeCapability, + getRunCodeGrant, +}: UploadCodePossibleParams): Promise { + if (agentTools != null) { + return agentTools.includes(EToolResources.execute_code); + } + const [enabled, grant] = await Promise.all([checkCodeCapability(), getRunCodeGrant()]); + return enabled && grant !== false; +} + +/** + * Files a deferred upload under the reader that will use it: `execute_code` first, since upload + * filing takes the first compatible tool. + */ +export function orderToolsForReading( + reading: Pick, + tools?: string[], +): string[] | undefined { + if (!reading.codePreferred || tools?.includes(EToolResources.execute_code) !== true) { + return tools; + } + return [ + EToolResources.execute_code, + ...tools.filter((tool) => tool !== EToolResources.execute_code), + ]; +} + +/** Text the context route acquired, before it is inspected and stored. */ +export interface AcquiredUploadText { + text: string; + isTranscript?: boolean; +} + +/** The upload keeps its original as plain storage without text, and records why. */ +export interface UploadTextMarker { + textDerivation: TextDerivation; + /** The route the kept original takes: no reader gets text from it at upload. */ + path: 'none'; +} + +/** Why a configured extraction step could not run for an upload. */ +export type ExtractorUnavailableReason = Extract< + TextDerivation['reason'], + 'no_extractor' | 'extractor_unavailable' +>; + +/** + * A configured extraction step the upload needed is off or absent, such as OCR without its + * capability, or a type the text route does not parse. It rejects the upload where failure + * rejects it; where the original is kept, a built-in extractor may derive the text later. + */ +export class ExtractorUnavailableError extends Error { + constructor( + message: string, + readonly reason: ExtractorUnavailableReason, + ) { + super(message); + this.name = 'ExtractorUnavailableError'; + Object.setPrototypeOf(this, ExtractorUnavailableError.prototype); + } +} + +const isHardUploadFailure = (error: unknown): boolean => + error instanceof UninspectableFileError || + error instanceof ContentFilterError || + error instanceof ContentFilterInputTooLargeError || + error instanceof UnsupportedProviderAudioError || + isAbortError(error); + +/** + * The marker an extraction failure leaves. A failure a built-in step names (a parser limit, empty + * text, the zip guard) is `failed`, as is a type the deployment's text allowlist excludes: a + * later turn must not parse what the administrator kept out of text. An unavailable configured + * step, or an error nothing recognizes and that may be a passing outage, is `deferred` where a + * built-in extractor can derive the text on a later turn, and `failed` where none can. + */ +function markExtractionFailure(mimeType: string, error: unknown): TextDerivation { + const at = Date.now(); + const plan = selectBuiltInTextPlan(mimeType); + if (error instanceof ExtractorUnavailableError) { + return plan == null || error.reason === 'no_extractor' + ? { outcome: 'failed', extractor: 'configured', reason: error.reason, at } + : { outcome: 'deferred', extractor: 'configured', reason: 'extractor_unavailable', at }; + } + const recognized = matchExtractionFailure(error); + if (recognized != null) { + return { outcome: 'failed', extractor: plan ?? 'configured', reason: recognized, at }; + } + return plan == null + ? { outcome: 'failed', extractor: 'configured', reason: 'parser', at } + : { outcome: 'deferred', reason: 'parser', at }; +} + +function checkAcquiredText({ text }: AcquiredUploadText): TextDerivation['reason'] | undefined { + if (text.trim().length === 0) { + return 'empty'; + } + return Buffer.byteLength(text, 'utf8') > MAX_STORED_EXTRACTED_TEXT_BYTES + ? 'too_large' + : undefined; +} + +/** Logs the error behind a kept original as safe metadata, never the filename or text. */ +function logExtractionError(fileId: string, marker: TextDerivation, error: unknown): void { + logger.warn( + `[uploadReading] file_id=${fileId} extraction=${marker.outcome} failure=${marker.reason}`, + getSafeErrorMetadata(error), + ); +} + +/** + * Runs the context route's text acquisition. Where the reading keeps the original on failure, an + * extraction failure, empty text or text past the storage cap returns a marker and the `none` + * route instead of throwing, and logs any error behind it (see {@link markExtractionFailure}). + * Content policy, inspection, audio and abort errors still reject, and text a blocking policy + * cannot inspect fails closed before it can count as empty. Everywhere else the acquisition's + * result or error passes through unchanged. + */ +export async function acquireUploadText({ + reading, + acquire, + fileId, + filters, +}: { + reading: Pick; + acquire: () => Promise; + fileId: string; + filters?: FiltersConfig; +}): Promise { + if (!reading.keepOriginalOnExtractionFailure) { + return acquire(); + } + let acquired: T; + try { + acquired = await acquire(); + } catch (error) { + if (isHardUploadFailure(error)) { + throw error; + } + const textDerivation = markExtractionFailure(reading.mimeType, error); + logExtractionError(fileId, textDerivation, error); + return { textDerivation, path: 'none' }; + } + if (acquired.isTranscript !== true) { + assertExtractedTextInspectable({ filters, text: acquired.text }); + } + const reason = checkAcquiredText(acquired); + if (reason == null) { + return acquired; + } + const textDerivation: TextDerivation = { + outcome: 'failed', + extractor: selectBuiltInTextPlan(reading.mimeType) ?? 'configured', + reason, + at: Date.now(), + }; + return { textDerivation, path: 'none' }; +} + +/** + * The `textDerivation` an upload's record stores: the marker its text acquisition returned, or a + * deferred marker for an upload left to Run Code that a built-in extractor can read. Nothing under + * the classic policy. + */ +export function getUploadReadingMetadata( + reading: Pick, + marker?: TextDerivation, +): { textDerivation?: TextDerivation } { + if (reading.policy !== 'automatic') { + return {}; + } + if (marker != null) { + return { textDerivation: marker }; + } + return reading.deferredMarker ? { textDerivation: { outcome: 'deferred', at: Date.now() } } : {}; +} + +function describeExtraction(reading: UploadReading, marker?: TextDerivation): string { + if (marker != null) { + return [ + `extraction=${marker.outcome}`, + ...(marker.reason != null ? [`failure=${marker.reason}`] : []), + 'kept=original', + ].join(' '); + } + if (reading.codePreferred) { + return `extraction=${reading.deferredMarker ? 'deferred' : 'none'}`; + } + return `extraction=${reading.path === 'text' ? 'eager' : 'none'}`; +} + +/** + * Logs how the automatic policy read one upload. Carries the file id and codes only, never the + * filename or text. Silent under the classic policy. + */ +export function logUploadReading( + fileId: string, + reading: UploadReading, + marker?: TextDerivation, +): void { + if (reading.policy !== 'automatic') { + return; + } + const line = [ + `[uploadReading] file_id=${fileId}`, + `policy=${reading.policy}`, + `category=${reading.category}`, + `path=${reading.path}`, + `reason=${reading.reason}`, + describeExtraction(reading, marker), + ].join(' '); + if (marker?.outcome === 'failed') { + logger.info(line); + return; + } + logger.debug(line); +} diff --git a/packages/api/src/files/upload/fallback.spec.ts b/packages/api/src/files/upload/fallback.spec.ts index 1b65139d1fe..2be2e71a6c6 100644 --- a/packages/api/src/files/upload/fallback.spec.ts +++ b/packages/api/src/files/upload/fallback.spec.ts @@ -1,18 +1,25 @@ import fs from 'fs'; import os from 'os'; import path from 'path'; -import type { FiltersConfig } from 'librechat-data-provider'; +import { logger } from '@librechat/data-schemas'; +import { excelFileTypes, fullMimeTypesList, selectBuiltInTextPlan } from 'librechat-data-provider'; +import type { FiltersConfig, TextDerivation } from 'librechat-data-provider'; import type { UploadFallbackTextExtractors } from './fallback'; import { + extractBoundedText, UPLOAD_FALLBACK_TEXT_PLANS, getUploadFallbackTextPlan, + matchExtractionFailure, resolveUploadFallbackText, + classifyExtractionFailure, } from './fallback'; +import { DOCUMENT_PARSER_MAX_FILE_SIZE, parseDocument } from '~/files/documents/crud'; import { MAX_STORED_EXTRACTED_TEXT_BYTES } from '~/files/extract'; -import { parseDocument } from '~/files/documents/crud'; +import { ZipBombError } from '~/files/documents/zipSafety'; import { parseTextNative } from '~/files/text'; const XLSX_MIME = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'; +const DOCX_MIME = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'; const csvRoute = { deliveryPath: 'none' as const, destinationChosen: false, @@ -59,6 +66,49 @@ describe('getUploadFallbackTextPlan', () => { it('runs nothing for a file kept on an agent, which no turn delivers as text', () => { expect(getUploadFallbackTextPlan({ ...csvRoute, isMessageAttachment: false })).toBeNull(); }); + + it('runs nothing for an upload left to Run Code, whose text a turn derives later', () => { + const codePreferred = { reading: { codePreferred: true } }; + expect(getUploadFallbackTextPlan({ ...csvRoute, ...codePreferred })).toBeNull(); + expect( + getUploadFallbackTextPlan({ ...csvRoute, mimeType: XLSX_MIME, ...codePreferred }), + ).toBeNull(); + expect(getUploadFallbackTextPlan({ ...csvRoute, reading: { codePreferred: false } })).toBe( + UPLOAD_FALLBACK_TEXT_PLANS.nativeText, + ); + }); + + it('runs nothing for an upload whose text acquisition kept the original with a marker', () => { + const failed: TextDerivation = { outcome: 'failed', reason: 'parser', at: 1 }; + const deferred: TextDerivation = { outcome: 'deferred', reason: 'extractor_unavailable' }; + for (const textDerivation of [failed, deferred]) { + expect( + getUploadFallbackTextPlan({ + ...csvRoute, + reading: { codePreferred: false }, + textDerivation, + }), + ).toBeNull(); + } + }); + + it('picks the extractor turn-time derivation picks, for every supported type', () => { + const mimeTypes = new Set([ + ...fullMimeTypesList, + ...excelFileTypes, + 'text/csv', + 'text/tab-separated-values', + ]); + const plans = new Set(); + for (const mimeType of mimeTypes) { + const plan = getUploadFallbackTextPlan({ ...csvRoute, mimeType }); + expect({ mimeType, plan }).toEqual({ mimeType, plan: selectBuiltInTextPlan(mimeType) }); + if (plan != null) { + plans.add(plan); + } + } + expect([...plans].sort()).toEqual(Object.values(UPLOAD_FALLBACK_TEXT_PLANS).sort()); + }); }); describe('resolveUploadFallbackText', () => { @@ -146,6 +196,46 @@ describe('resolveUploadFallbackText', () => { expect(extractors.parseTextNative).not.toHaveBeenCalled(); }); + it('extracts nothing for a deferred upload', async () => { + const extractors = spiedExtractors(); + + await expect( + resolveUploadFallbackText({ + ...route, + reading: { codePreferred: true }, + file: csvUpload('deferred.csv', 'region,total'), + fileId: 'file-1', + extractors, + }), + ).resolves.toBeUndefined(); + expect(extractors.parseTextNative).not.toHaveBeenCalled(); + }); + + it('warns as before for each kind of failure, and stays silent for empty text', async () => { + const warn = jest.spyOn(logger, 'warn'); + const oversized: UploadFallbackTextExtractors = { + parseDocument, + parseTextNative: async () => ({ text: 'a'.repeat(MAX_STORED_EXTRACTED_TEXT_BYTES + 1) }), + }; + try { + await resolveUploadFallbackText({ ...route, file: csvUpload('gone.csv'), fileId: 'f1' }); + await resolveUploadFallbackText({ + ...route, + file: csvUpload('big.csv', 'a'), + fileId: 'f2', + extractors: oversized, + }); + await resolveUploadFallbackText({ ...route, file: csvUpload('none.csv', ' '), fileId: 'f3' }); + + expect(warn.mock.calls.map(([message]) => message)).toEqual([ + '[resolveUploadFallbackText] No fallback text for "gone.csv": extraction failed', + '[resolveUploadFallbackText] No fallback text for "big.csv": extracted text exceeds the storage limit', + ]); + } finally { + warn.mockRestore(); + } + }); + it('keeps the upload when extraction fails', async () => { await expect( resolveUploadFallbackText({ ...route, file: csvUpload('missing.csv'), fileId: 'file-1' }), @@ -208,3 +298,166 @@ describe('resolveUploadFallbackText', () => { ).resolves.toBeUndefined(); }); }); + +describe('extractBoundedText', () => { + const documentsDir = path.join(__dirname, '../documents'); + let workDir: string; + + beforeAll(() => { + workDir = fs.mkdtempSync(path.join(os.tmpdir(), 'bounded-text-')); + }); + + afterAll(() => { + fs.rmSync(workDir, { recursive: true, force: true }); + }); + + function upload(name: string, mimetype: string, content: string): Express.Multer.File { + const filePath = path.join(workDir, name); + fs.writeFileSync(filePath, content); + return { + originalname: name, + path: filePath, + mimetype, + size: Buffer.byteLength(content), + } as Express.Multer.File; + } + + const fixture = (name: string, mimetype: string): Express.Multer.File => + ({ originalname: name, path: path.join(documentsDir, name), mimetype }) as Express.Multer.File; + + it('returns text within every cap', async () => { + await expect( + extractBoundedText({ + file: fixture('sample.xlsx', XLSX_MIME), + plan: UPLOAD_FALLBACK_TEXT_PLANS.documentParser, + }), + ).resolves.toEqual({ + text: 'Sheet One:\nData,on,first,sheet\nSecond Sheet:\nData,On\nSecond,Sheet\n', + }); + }); + + it('fails empty for blank text and for a document the parser finds no text in', async () => { + await expect( + extractBoundedText({ + file: upload('blank.csv', 'text/csv', ' \n '), + plan: UPLOAD_FALLBACK_TEXT_PLANS.nativeText, + }), + ).resolves.toEqual({ failure: 'empty' }); + await expect( + extractBoundedText({ + file: fixture('empty.docx', DOCX_MIME), + plan: UPLOAD_FALLBACK_TEXT_PLANS.documentParser, + }), + ).resolves.toMatchObject({ failure: 'empty', error: expect.any(Error) }); + }); + + it('fails parser for a document the parser cannot read', async () => { + await expect( + extractBoundedText({ + file: upload('broken.docx', DOCX_MIME, 'not a zip archive'), + plan: UPLOAD_FALLBACK_TEXT_PLANS.documentParser, + }), + ).resolves.toMatchObject({ failure: 'parser', error: expect.any(Error) }); + }); + + it('fails too_large past the parser input cap and past the stored-text cap', async () => { + await expect( + extractBoundedText({ + file: { ...fixture('sample.docx', DOCX_MIME), size: DOCUMENT_PARSER_MAX_FILE_SIZE + 1 }, + plan: UPLOAD_FALLBACK_TEXT_PLANS.documentParser, + }), + ).resolves.toMatchObject({ failure: 'too_large' }); + await expect( + extractBoundedText({ + file: upload('huge.csv', 'text/csv', 'a'), + plan: UPLOAD_FALLBACK_TEXT_PLANS.nativeText, + extractors: { + parseDocument, + parseTextNative: async () => ({ text: 'a'.repeat(MAX_STORED_EXTRACTED_TEXT_BYTES + 1) }), + }, + }), + ).resolves.toEqual({ failure: 'too_large' }); + }); + + it('fails expansion_limit when the zip guard refuses the archive', async () => { + const bomb = new ZipBombError('sheet: entry exceeds the 25MB per-entry decompressed cap'); + + await expect( + extractBoundedText({ + file: upload('bomb.xlsx', XLSX_MIME, 'x'), + plan: UPLOAD_FALLBACK_TEXT_PLANS.documentParser, + extractors: { parseDocument: () => Promise.reject(bomb), parseTextNative }, + }), + ).resolves.toEqual({ failure: 'expansion_limit', error: bomb }); + }); + + it('refuses text a content policy flags, and text a blocking policy cannot inspect', async () => { + const flagging: FiltersConfig = { + files: { + pii: { + fields: ['extracted_text'], + starterPatterns: [], + customPatterns: [{ id: 'private', label: 'private token', regex: 'PRIVATE-[A-Z]+' }], + }, + }, + }; + const blocking: FiltersConfig = { + files: { pii: { fields: ['extracted_text'], starterPatterns: [], uninspectable: 'block' } }, + }; + + await expect( + extractBoundedText({ + file: upload('flagged.csv', 'text/csv', 'token,PRIVATE-SECRET'), + filters: flagging, + plan: UPLOAD_FALLBACK_TEXT_PLANS.nativeText, + }), + ).resolves.toEqual({ + failure: 'policy', + finding: expect.objectContaining({ ruleId: expect.any(String), field: 'extracted_text' }), + }); + await expect( + extractBoundedText({ + file: upload('blank-blocked.csv', 'text/csv', ''), + filters: blocking, + plan: UPLOAD_FALLBACK_TEXT_PLANS.nativeText, + }), + ).resolves.toMatchObject({ failure: 'uninspectable' }); + }); +}); + +describe('classifyExtractionFailure', () => { + it.each([ + [new ZipBombError('archive: total decompressed size exceeds the 100MB cap'), 'expansion_limit'], + [new Error('ODT content.xml exceeds the 50MB decompressed limit'), 'expansion_limit'], + [new Error('File "a.pdf" exceeds the 15MB document parser limit (16MB).'), 'too_large'], + [new Error('No text found in document'), 'empty'], + [new Error('Unable to extract text from "a.pdf". It may be image-based.'), 'empty'], + [new Error('Invalid PDF structure'), 'parser'], + [Object.assign(new Error('Invalid PDF structure'), { code: 'ERR_OUT_OF_RANGE' }), 'parser'], + ['not an error', 'parser'], + [Object.assign(new Error('no space left on device'), { code: 'ENOSPC' }), 'unavailable'], + [Object.assign(new Error('i/o error'), { code: 'EIO' }), 'unavailable'], + [Object.assign(new Error('resource busy or locked'), { code: 'EBUSY' }), 'unavailable'], + [Object.assign(new Error('too many open files'), { code: 'EMFILE' }), 'unavailable'], + [Object.assign(new Error('try again'), { code: 'EAGAIN' }), 'unavailable'], + [Object.assign(new Error('read failed'), { errno: -5, syscall: 'read' }), 'unavailable'], + [ + Object.assign(new Error('incorrect header check'), { errno: -3, code: 'Z_DATA_ERROR' }), + 'parser', + ], + [ + Object.assign(new ZipBombError('archive: total decompressed size exceeds the 100MB cap'), { + code: 'EIO', + }), + 'expansion_limit', + ], + ])('classifies %p as %s', (error, reason) => { + expect(classifyExtractionFailure(error)).toBe(reason); + }); + + it('recognizes only the failures it can name, leaving the rest unclassified', () => { + expect(matchExtractionFailure(new Error('No text found in document'))).toBe('empty'); + expect(matchExtractionFailure(new Error('Invalid PDF structure'))).toBeUndefined(); + expect(matchExtractionFailure('not an error')).toBeUndefined(); + }); +}); diff --git a/packages/api/src/files/upload/fallback.ts b/packages/api/src/files/upload/fallback.ts index 96b8bce20ac..b08a4e61b05 100644 --- a/packages/api/src/files/upload/fallback.ts +++ b/packages/api/src/files/upload/fallback.ts @@ -1,17 +1,22 @@ import { logger } from '@librechat/data-schemas'; -import { isNativelyReadableText, documentParserMimeTypes } from 'librechat-data-provider'; +import { selectBuiltInTextPlan } from 'librechat-data-provider'; import type { FiltersConfig, + UploadReading, + TextDerivation, + BuiltInTextPlan, EndpointFileConfig, TDefaultLLMDeliveryPath, } from 'librechat-data-provider'; +import type { ProtectionFinding } from '~/protection/types'; import { extractInspectableFileText, getFileExtractionLogDetails, MAX_STORED_EXTRACTED_TEXT_BYTES, } from '~/files/extract'; +import { hasActiveFileFieldPolicy, UninspectableFileError } from '~/protection/files'; import { extractFileContent } from '~/protection/adapters/submissions'; -import { hasActiveFileFieldPolicy } from '~/protection/files'; +import { ZipBombError } from '~/files/documents/zipSafety'; import { parseDocument } from '~/files/documents/crud'; import { inspectContent } from '~/protection/runtime'; import { parseTextNative } from '~/files/text'; @@ -19,10 +24,9 @@ import { parseTextNative } from '~/files/text'; export const UPLOAD_FALLBACK_TEXT_PLANS = { documentParser: 'document_parser', nativeText: 'native_text', -} as const; +} as const satisfies Record; -export type UploadFallbackTextPlan = - (typeof UPLOAD_FALLBACK_TEXT_PLANS)[keyof typeof UPLOAD_FALLBACK_TEXT_PLANS]; +export type UploadFallbackTextPlan = BuiltInTextPlan; export interface UploadFallbackTextRoute { /** The route the upload resolved to. */ @@ -34,6 +38,10 @@ export interface UploadFallbackTextRoute { mimeType: string; /** The upload endpoint's file config, which opts in through `textFallbackWithoutTools`. */ endpointConfig?: Pick; + /** The upload's reading decision; an upload left to Run Code reads no text now. */ + reading?: Pick; + /** The marker the upload's text acquisition returned, when it kept the original without text. */ + textDerivation?: TextDerivation; } interface ExtractedText { @@ -46,8 +54,92 @@ export interface UploadFallbackTextExtractors { parseTextNative: (file: Express.Multer.File) => Promise; } +/** Why a built-in extractor produced no text a file can keep. */ +export type ExtractionFailureReason = Extract< + NonNullable, + 'empty' | 'parser' | 'too_large' | 'expansion_limit' +>; + +/** + * Why bounded extraction kept no text: the extractor failed, a file policy refused its text, or + * the system could not serve the read (`unavailable`), which says nothing about the file. + */ +export type BoundedTextFailure = + | ExtractionFailureReason + | 'policy' + | 'uninspectable' + | 'unavailable'; + +/** + * Text within every cap, or why there is none: `error` is set when extraction threw, and + * `finding` is the content policy's finding when it refused the text. + */ +export type BoundedText = + | { text: string } + | { failure: 'policy'; finding: ProtectionFinding } + | { failure: Exclude; error?: unknown }; + const builtInExtractors: UploadFallbackTextExtractors = { parseDocument, parseTextNative }; +/** Fragments of the built-in extractors' plain error messages, mapped to stable reasons. */ +const EXTRACTION_FAILURE_MESSAGES: ReadonlyArray = [ + ['No text found in document', 'empty'], + ['Unable to extract text from', 'empty'], + ['MB document parser limit', 'too_large'], + ['MB storage limit', 'too_large'], + ['MB decompressed limit', 'expansion_limit'], +]; + +/** The stable reason a built-in extractor failed with, or undefined for an unrecognized error. */ +export function matchExtractionFailure(error: unknown): ExtractionFailureReason | undefined { + if (error instanceof ZipBombError) { + return 'expansion_limit'; + } + const message = error instanceof Error ? error.message : ''; + return EXTRACTION_FAILURE_MESSAGES.find(([fragment]) => message.includes(fragment))?.[1]; +} + +interface SystemErrorShape { + code?: string; + errno?: number; + syscall?: string; +} + +/** POSIX error names (`ENOSPC`, `EIO`, `EBUSY`, `EMFILE`, `EAGAIN`...), never Node's `ERR_*` codes. */ +const SYSTEM_ERROR_CODE = /^E[A-Z0-9]+$/; + +/** + * Whether the error is the operating system refusing a read, such as a full disk, a busy or + * locked file, or descriptor exhaustion, rather than anything the file's content caused. These + * clear up, so a later read may succeed. An `errno` alone is not enough: zlib reports a corrupt + * archive with one (`Z_DATA_ERROR`), so it counts only with the system call that failed. + */ +export function isSystemError(error: unknown): boolean { + if (typeof error !== 'object' || error == null) { + return false; + } + const { code, errno, syscall }: SystemErrorShape = error; + return ( + (typeof code === 'string' && SYSTEM_ERROR_CODE.test(code)) || + (typeof errno === 'number' && typeof syscall === 'string') + ); +} + +/** + * The stable reason a built-in extractor failed with. A failure the extractor names keeps its + * reason, a system error is `unavailable`, and anything else unrecognized is a parser error. + */ +export function classifyExtractionFailure(error: unknown): ExtractionFailureReason | 'unavailable' { + return matchExtractionFailure(error) ?? (isSystemError(error) ? 'unavailable' : 'parser'); +} + +/** Whether the upload's text is left to a turn, which derives it from the stored original. */ +const isUploadTextDeferred = ({ + reading, + textDerivation, +}: Pick): boolean => + reading?.codePreferred === true || textDerivation != null; + /** * Which built-in extractor stores text for an upload left to tools, or `null` when none runs. * @@ -57,7 +149,7 @@ const builtInExtractors: UploadFallbackTextExtractors = { parseDocument, parseTe * and files kept on an agent's tool resources never reach a prompt. A file filed * under a tool that reads it still gets text, because a later turn may run without that tool: a * handoff agent, or the same agent after its tools or grants change. Only built-in extractors - * run, so a file meant for a tool never costs a RAG or OCR call. + * run, so a file meant for a tool never costs a RAG or OCR call. A deferred upload runs none. */ export function getUploadFallbackTextPlan( route: UploadFallbackTextRoute, @@ -66,16 +158,65 @@ export function getUploadFallbackTextPlan( route.endpointConfig?.textFallbackWithoutTools !== true || route.deliveryPath !== 'none' || route.destinationChosen || - !route.isMessageAttachment + !route.isMessageAttachment || + isUploadTextDeferred(route) ) { return null; } - if (documentParserMimeTypes.some((pattern) => pattern.test(route.mimeType))) { - return UPLOAD_FALLBACK_TEXT_PLANS.documentParser; + return selectBuiltInTextPlan(route.mimeType); +} + +/** + * Reads a file with one built-in extractor and keeps its text only within every cap: non-empty, + * within the stored-text limit, and clear of any `extracted_text` content policy. Extraction runs + * under the policy's fail-close, so text a blocking policy cannot inspect is refused. Never + * throws: upload fallback and turn-time derivation share these caps and each decides what a + * failure means for its file. + */ +export async function extractBoundedText({ + file, + filters, + plan, + extractors = builtInExtractors, +}: { + file: Express.Multer.File; + filters?: FiltersConfig; + plan: UploadFallbackTextPlan; + extractors?: UploadFallbackTextExtractors; +}): Promise { + try { + const result = await extractInspectableFileText({ + filters, + extract: () => + plan === UPLOAD_FALLBACK_TEXT_PLANS.documentParser + ? extractors.parseDocument({ file }) + : extractors.parseTextNative(file), + }); + const text = result?.text; + if (typeof text !== 'string' || text.trim().length === 0) { + return { failure: 'empty' }; + } + if (Buffer.byteLength(text, 'utf8') > MAX_STORED_EXTRACTED_TEXT_BYTES) { + return { failure: 'too_large' }; + } + if (filters == null || !hasActiveFileFieldPolicy(filters, ['extracted_text'])) { + return { text }; + } + const finding = inspectContent(extractFileContent({ extractedText: text }), { filters }); + return finding == null ? { text } : { failure: 'policy', finding }; + } catch (error) { + const failure = + error instanceof UninspectableFileError ? 'uninspectable' : classifyExtractionFailure(error); + return { failure, error }; } - return isNativelyReadableText(route.mimeType) ? UPLOAD_FALLBACK_TEXT_PLANS.nativeText : null; } +/** The warning a failure without a thrown error logs; an empty result logs nothing. */ +const FALLBACK_SKIP_REASONS: Partial> = { + too_large: 'extracted text exceeds the storage limit', + policy: 'extracted text matched a content policy', +}; + /** * Text a turn without a reading tool can fall back to for this upload, when one applies. * @@ -114,30 +255,13 @@ export async function resolveUploadFallbackText({ ); return undefined; }; - try { - const result = await extractInspectableFileText({ - filters, - extract: () => - plan === UPLOAD_FALLBACK_TEXT_PLANS.documentParser - ? extractors.parseDocument({ file }) - : extractors.parseTextNative(file), - }); - const text = result?.text; - if (typeof text !== 'string' || text.trim().length === 0) { - return undefined; - } - if (Buffer.byteLength(text, 'utf8') > MAX_STORED_EXTRACTED_TEXT_BYTES) { - return skip('extracted text exceeds the storage limit'); - } - if ( - filters != null && - hasActiveFileFieldPolicy(filters, ['extracted_text']) && - inspectContent(extractFileContent({ extractedText: text }), { filters }) != null - ) { - return skip('extracted text matched a content policy'); - } - return text; - } catch (error) { - return skip('extraction failed', error); + const result = await extractBoundedText({ file, filters, plan, extractors }); + if ('text' in result) { + return result.text; + } + if ('error' in result) { + return skip('extraction failed', result.error); } + const reason = FALLBACK_SKIP_REASONS[result.failure]; + return reason == null ? undefined : skip(reason); } diff --git a/packages/api/src/files/validation.spec.ts b/packages/api/src/files/validation.spec.ts index 97c8ee56979..c07a682ec23 100644 --- a/packages/api/src/files/validation.spec.ts +++ b/packages/api/src/files/validation.spec.ts @@ -1,6 +1,14 @@ import { Providers } from '@librechat/agents'; import { mbToBytes } from 'librechat-data-provider'; -import { validatePdf, validateBedrockDocument, validateVideo, validateAudio } from './validation'; +import type { NativeDocumentSizeLimitParams } from './validation'; +import { + validatePdf, + validateAudio, + validateImage, + validateVideo, + validateBedrockDocument, + getNativeDocumentSizeLimit, +} from './validation'; describe('PDF Validation with fileConfig.endpoints.*.fileSizeLimit', () => { /** Helper to create a PDF buffer with valid header */ @@ -882,3 +890,357 @@ describe('PDF Validation with fileConfig.endpoints.*.fileSizeLimit', () => { }); }); }); + +describe('getNativeDocumentSizeLimit', () => { + const pdf = 'application/pdf'; + const docx = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'; + const claude4 = 'anthropic.claude-sonnet-4-20250514-v1:0'; + const nova = 'amazon.nova-pro-v1:0'; + + it.each<[string, NativeDocumentSizeLimitParams, number | undefined]>([ + [ + 'Anthropic PDF, unconfigured', + { provider: Providers.ANTHROPIC, mimeType: pdf }, + mbToBytes(32), + ], + [ + 'Anthropic PDF, configured lower', + { provider: Providers.ANTHROPIC, mimeType: pdf, configuredFileSizeLimit: mbToBytes(5) }, + mbToBytes(5), + ], + [ + 'Anthropic PDF, configured higher', + { provider: Providers.ANTHROPIC, mimeType: pdf, configuredFileSizeLimit: mbToBytes(50) }, + mbToBytes(50), + ], + [ + 'Anthropic PDF, configured 0', + { provider: Providers.ANTHROPIC, mimeType: pdf, configuredFileSizeLimit: 0 }, + 0, + ], + ['OpenAI PDF, unconfigured', { provider: Providers.OPENAI, mimeType: pdf }, mbToBytes(10)], + ['Azure PDF, unconfigured', { provider: Providers.AZURE, mimeType: pdf }, mbToBytes(10)], + [ + 'OpenRouter PDF, unconfigured', + { provider: Providers.OPENROUTER, mimeType: pdf }, + mbToBytes(10), + ], + [ + 'OpenAI PDF, configured 0', + { provider: Providers.OPENAI, mimeType: pdf, configuredFileSizeLimit: 0 }, + 0, + ], + ['Google PDF, unconfigured', { provider: Providers.GOOGLE, mimeType: pdf }, mbToBytes(20)], + ['VertexAI PDF, unconfigured', { provider: Providers.VERTEXAI, mimeType: pdf }, mbToBytes(20)], + [ + 'Google PDF, configured', + { provider: Providers.GOOGLE, mimeType: pdf, configuredFileSizeLimit: mbToBytes(25) }, + mbToBytes(25), + ], + [ + 'PDF on a provider without a PDF limit', + { provider: 'unsupported', mimeType: pdf }, + undefined, + ], + [ + 'PDF on a provider without a PDF limit, configured', + { provider: 'unsupported', mimeType: pdf, configuredFileSizeLimit: mbToBytes(5) }, + undefined, + ], + ['Bedrock PDF, default model', { provider: Providers.BEDROCK, mimeType: pdf }, mbToBytes(4.5)], + [ + 'Bedrock PDF, Claude 4', + { provider: Providers.BEDROCK, mimeType: pdf, model: claude4 }, + mbToBytes(32), + ], + [ + 'Bedrock PDF, Nova', + { provider: Providers.BEDROCK, mimeType: pdf, model: nova }, + mbToBytes(32), + ], + [ + 'Bedrock DOCX, Nova', + { provider: Providers.BEDROCK, mimeType: docx, model: nova }, + mbToBytes(32), + ], + [ + 'Bedrock DOCX, Claude 4', + { provider: Providers.BEDROCK, mimeType: docx, model: claude4 }, + mbToBytes(4.5), + ], + [ + 'Bedrock CSV, Nova', + { provider: Providers.BEDROCK, mimeType: 'text/csv', model: nova }, + mbToBytes(4.5), + ], + [ + 'Bedrock PDF, Claude 4, configured lower than the exemption', + { + provider: Providers.BEDROCK, + mimeType: pdf, + model: claude4, + configuredFileSizeLimit: mbToBytes(10), + }, + mbToBytes(10), + ], + [ + 'Bedrock CSV, configured 0', + { provider: Providers.BEDROCK, mimeType: 'text/csv', configuredFileSizeLimit: 0 }, + 0, + ], + [ + 'generic text, unconfigured', + { provider: Providers.ANTHROPIC, mimeType: 'text/plain' }, + undefined, + ], + [ + 'generic text, configured', + { + provider: Providers.ANTHROPIC, + mimeType: 'text/plain', + configuredFileSizeLimit: mbToBytes(1), + }, + mbToBytes(1), + ], + [ + 'generic text, configured 0', + { provider: Providers.OPENAI, mimeType: 'text/plain', configuredFileSizeLimit: 0 }, + undefined, + ], + [ + 'generic spreadsheet on Google, configured', + { + provider: Providers.GOOGLE, + mimeType: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + configuredFileSizeLimit: mbToBytes(25), + }, + mbToBytes(25), + ], + ])('%s', (_label, params, expected) => { + expect(getNativeDocumentSizeLimit(params)).toBe(expected); + }); +}); + +describe('validator parity with getNativeDocumentSizeLimit', () => { + const pdfBuffer = Buffer.alloc(1024); + pdfBuffer.write('%PDF-1.4\n', 0); + + const pdfCases: Array<{ + name: string; + provider: Providers; + model?: string; + configured?: number; + limit: number; + }> = [ + { name: 'Anthropic, unconfigured', provider: Providers.ANTHROPIC, limit: mbToBytes(32) }, + { + name: 'Anthropic, configured', + provider: Providers.ANTHROPIC, + configured: mbToBytes(3), + limit: mbToBytes(3), + }, + { name: 'OpenAI, unconfigured', provider: Providers.OPENAI, limit: mbToBytes(10) }, + { + name: 'OpenAI, configured', + provider: Providers.OPENAI, + configured: mbToBytes(15), + limit: mbToBytes(15), + }, + { name: 'Google, unconfigured', provider: Providers.GOOGLE, limit: mbToBytes(20) }, + { + name: 'VertexAI, configured', + provider: Providers.VERTEXAI, + configured: mbToBytes(8), + limit: mbToBytes(8), + }, + { name: 'Bedrock, default model', provider: Providers.BEDROCK, limit: mbToBytes(4.5) }, + { + name: 'Bedrock, Claude 4', + provider: Providers.BEDROCK, + model: 'anthropic.claude-sonnet-4-20250514-v1:0', + limit: mbToBytes(32), + }, + { + name: 'Bedrock, Nova, configured', + provider: Providers.BEDROCK, + model: 'amazon.nova-pro-v1:0', + configured: mbToBytes(8), + limit: mbToBytes(8), + }, + ]; + + describe.each(pdfCases)('validatePdf on $name', ({ provider, model, configured, limit }) => { + it('accepts a PDF exactly at the limit', async () => { + const result = await validatePdf(pdfBuffer, limit, provider, configured, model); + expect(result).toEqual({ isValid: true }); + }); + + it('rejects a PDF one byte above the limit as a capacity failure', async () => { + const result = await validatePdf(pdfBuffer, limit + 1, provider, configured, model); + expect(result).toMatchObject({ isValid: false, reason: 'capacity' }); + }); + }); + + const bedrockCases: Array<{ name: string; mimeType: string; model?: string; limit: number }> = [ + { name: 'CSV', mimeType: 'text/csv', limit: mbToBytes(4.5) }, + { + name: 'Nova DOCX', + mimeType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + model: 'amazon.nova-pro-v1:0', + limit: mbToBytes(32), + }, + { + name: 'Claude 4 DOCX', + mimeType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + model: 'anthropic.claude-sonnet-4-20250514-v1:0', + limit: mbToBytes(4.5), + }, + ]; + + describe.each(bedrockCases)('validateBedrockDocument on $name', ({ mimeType, model, limit }) => { + it('accepts a document exactly at the limit', async () => { + const result = await validateBedrockDocument(limit, mimeType, undefined, undefined, model); + expect(result).toEqual({ isValid: true }); + }); + + it('rejects a document one byte above the limit as a capacity failure', async () => { + const result = await validateBedrockDocument( + limit + 1, + mimeType, + undefined, + undefined, + model, + ); + expect(result).toMatchObject({ isValid: false, reason: 'capacity' }); + }); + }); + + it.each([ + Providers.ANTHROPIC, + Providers.OPENAI, + Providers.GOOGLE, + Providers.VERTEXAI, + Providers.BEDROCK, + ])('rejects a 1-byte PDF on %s when the configured limit is 0', async (provider) => { + const result = await validatePdf(pdfBuffer, 1, provider, 0); + expect(result).toMatchObject({ isValid: false, reason: 'capacity' }); + }); + + it('accepts any PDF size on a provider without a PDF limit, configured or not', async () => { + const provider = 'unsupported' as Providers; + await expect(validatePdf(pdfBuffer, mbToBytes(100), provider)).resolves.toEqual({ + isValid: true, + }); + await expect(validatePdf(pdfBuffer, mbToBytes(100), provider, 0)).resolves.toEqual({ + isValid: true, + }); + }); +}); + +describe('validation failure reasons', () => { + const validPdf = (): Buffer => { + const buffer = Buffer.alloc(1024); + buffer.write('%PDF-1.4\n', 0); + return buffer; + }; + + it.each([ + [Providers.ANTHROPIC, 35, 'PDF file size (35MB) exceeds the 32MB limit'], + [Providers.OPENAI, 12, 'PDF file size (12MB) exceeds the 10MB limit'], + [Providers.GOOGLE, 25, 'PDF file size (25MB) exceeds the 20MB limit'], + ])( + 'reports an oversized %s PDF as capacity with the unchanged message', + async (provider, sizeMB, error) => { + const result = await validatePdf(validPdf(), mbToBytes(sizeMB), provider); + expect(result).toEqual({ isValid: false, reason: 'capacity', error }); + }, + ); + + it('reports an Anthropic page estimate over 100 as capacity', async () => { + const pages = '/Type /Page\n'.repeat(101); + const result = await validatePdf(Buffer.from(`%PDF-1.4\n${pages}`), 1024, Providers.ANTHROPIC); + expect(result).toEqual({ + isValid: false, + reason: 'capacity', + error: "PDF has approximately 101 pages, exceeding Anthropic's 100-page limit", + }); + }); + + const tooSmall = Buffer.alloc(3); + const headerless = Buffer.alloc(1024); + headerless.write('INVALID', 0); + + it.each([ + [ + 'a too-small Anthropic PDF', + () => validatePdf(tooSmall, tooSmall.length, Providers.ANTHROPIC), + 'Invalid PDF file: too small or corrupted', + ], + [ + 'a missing Anthropic PDF header', + () => validatePdf(headerless, headerless.length, Providers.ANTHROPIC), + 'Invalid PDF file: missing PDF header', + ], + [ + 'a too-small Bedrock PDF', + () => validateBedrockDocument(tooSmall.length, 'application/pdf', tooSmall), + 'Invalid PDF file: too small or corrupted', + ], + [ + 'a missing Bedrock PDF header', + () => validateBedrockDocument(headerless.length, 'application/pdf', headerless), + 'Invalid PDF file: missing PDF header', + ], + ])('reports %s as integrity with the unchanged message', async (_label, validate, error) => { + await expect(validate()).resolves.toEqual({ isValid: false, reason: 'integrity', error }); + }); + + it('reports an encrypted Anthropic PDF as integrity', async () => { + const buffer = validPdf(); + buffer.write('/Encrypt ', 100); + const result = await validatePdf(buffer, buffer.length, Providers.ANTHROPIC); + expect(result).toEqual({ + isValid: false, + reason: 'integrity', + error: 'PDF is password-protected or encrypted. Anthropic requires unencrypted PDFs.', + }); + }); + + it('reports an oversized Bedrock document as capacity with the unchanged message', async () => { + const result = await validateBedrockDocument(mbToBytes(5), 'text/csv'); + expect(result).toEqual({ + isValid: false, + reason: 'capacity', + error: 'File size (5.0MB) exceeds the 4.5MB limit for Bedrock', + }); + }); + + it('reports oversized media as capacity and truncated media as integrity', async () => { + const media = Buffer.alloc(1024); + const tiny = Buffer.alloc(5); + await expect(validateVideo(media, mbToBytes(25), Providers.GOOGLE)).resolves.toMatchObject({ + isValid: false, + reason: 'capacity', + }); + await expect(validateAudio(media, mbToBytes(25), Providers.GOOGLE)).resolves.toMatchObject({ + isValid: false, + reason: 'capacity', + }); + await expect(validateImage(media, mbToBytes(6), Providers.ANTHROPIC)).resolves.toMatchObject({ + isValid: false, + reason: 'capacity', + }); + await expect(validateVideo(tiny, tiny.length, Providers.GOOGLE)).resolves.toMatchObject({ + isValid: false, + reason: 'integrity', + }); + await expect(validateAudio(tiny, tiny.length, Providers.GOOGLE)).resolves.toMatchObject({ + isValid: false, + reason: 'integrity', + }); + await expect(validateImage(tiny, tiny.length, Providers.OPENAI)).resolves.toMatchObject({ + isValid: false, + reason: 'integrity', + }); + }); +}); diff --git a/packages/api/src/files/validation.ts b/packages/api/src/files/validation.ts index 0bde847b876..26ab3facd86 100644 --- a/packages/api/src/files/validation.ts +++ b/packages/api/src/files/validation.ts @@ -1,31 +1,32 @@ import { Providers } from '@librechat/agents'; import { mbToBytes, isOpenAILikeProvider } from 'librechat-data-provider'; +import type { ValidationFailureReason } from '~/types'; export interface ValidationResult { isValid: boolean; error?: string; + reason?: ValidationFailureReason; } -export interface PDFValidationResult { - isValid: boolean; - error?: string; -} +export type PDFValidationResult = ValidationResult; -export interface VideoValidationResult { - isValid: boolean; - error?: string; -} +export type VideoValidationResult = ValidationResult; -export interface AudioValidationResult { - isValid: boolean; - error?: string; -} +export type AudioValidationResult = ValidationResult; -export interface ImageValidationResult { - isValid: boolean; - error?: string; +export type ImageValidationResult = ValidationResult; + +export interface NativeDocumentSizeLimitParams { + provider: Providers | string; + mimeType: string; + model?: string; + /** Configured endpoint `fileSizeLimit` in bytes */ + configuredFileSizeLimit?: number; } +const pdfMimeType = 'application/pdf'; +const docxMimeType = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'; + export async function validatePdf( pdfBuffer: Buffer, fileSize: number, @@ -40,19 +41,38 @@ export async function validatePdf( if (provider === Providers.BEDROCK) { return validateBedrockDocument( fileSize, - 'application/pdf', + pdfMimeType, pdfBuffer, configuredFileSizeLimit, model, ); } - if (isOpenAILikeProvider(provider)) { - return validateOpenAIPdf(fileSize, configuredFileSizeLimit); - } + return validatePdfSize(fileSize, provider, configuredFileSizeLimit); +} - if (provider === Providers.GOOGLE || provider === Providers.VERTEXAI) { - return validateGooglePdf(fileSize, configuredFileSizeLimit); +/** + * Validates a PDF's size against the limit the provider's native document path enforces. + * Providers without a PDF limit (anything but Anthropic, OpenAI-like and Google/Vertex) pass. + */ +function validatePdfSize( + fileSize: number, + provider: Providers, + configuredFileSizeLimit?: number, +): PDFValidationResult { + const effectiveLimit = getNativeDocumentSizeLimit({ + provider, + mimeType: pdfMimeType, + configuredFileSizeLimit, + }); + + if (effectiveLimit !== undefined && fileSize > effectiveLimit) { + const limitMB = Math.round(effectiveLimit / (1024 * 1024)); + return { + isValid: false, + reason: 'capacity', + error: `PDF file size (${Math.round(fileSize / (1024 * 1024))}MB) exceeds the ${limitMB}MB limit`, + }; } return { isValid: true }; @@ -71,20 +91,15 @@ async function validateAnthropicPdf( configuredFileSizeLimit?: number, ): Promise { try { - const providerLimit = mbToBytes(32); - const effectiveLimit = configuredFileSizeLimit ?? providerLimit; - - if (fileSize > effectiveLimit) { - const limitMB = Math.round(effectiveLimit / (1024 * 1024)); - return { - isValid: false, - error: `PDF file size (${Math.round(fileSize / (1024 * 1024))}MB) exceeds the ${limitMB}MB limit`, - }; + const sizeValidation = validatePdfSize(fileSize, Providers.ANTHROPIC, configuredFileSizeLimit); + if (!sizeValidation.isValid) { + return sizeValidation; } if (!pdfBuffer || pdfBuffer.length < 5) { return { isValid: false, + reason: 'integrity', error: 'Invalid PDF file: too small or corrupted', }; } @@ -93,6 +108,7 @@ async function validateAnthropicPdf( if (!pdfHeader.startsWith('%PDF-')) { return { isValid: false, + reason: 'integrity', error: 'Invalid PDF file: missing PDF header', }; } @@ -105,6 +121,7 @@ async function validateAnthropicPdf( ) { return { isValid: false, + reason: 'integrity', error: 'PDF is password-protected or encrypted. Anthropic requires unencrypted PDFs.', }; } @@ -115,6 +132,7 @@ async function validateAnthropicPdf( if (estimatedPages > 100) { return { isValid: false, + reason: 'capacity', error: `PDF has approximately ${estimatedPages} pages, exceeding Anthropic's 100-page limit`, }; } @@ -124,6 +142,7 @@ async function validateAnthropicPdf( console.error('PDF validation error:', error); return { isValid: false, + reason: 'integrity', error: 'Failed to validate PDF file', }; } @@ -160,9 +179,6 @@ const isBedrockClaude4Plus = (model?: string): boolean => const isBedrockNova = (model?: string): boolean => model != null && /(?:^|\.)amazon\.nova-/.test(model); -const pdfMimeType = 'application/pdf'; -const docxMimeType = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'; - /** * Returns true when the given model + MIME type combination is exempt from * Bedrock's default 4.5 MB per-document limit. @@ -181,6 +197,51 @@ const isExemptFromBedrockDocLimit = (model?: string, mimeType?: string): boolean return false; }; +/** Provider fallback for a PDF sent as a native document, or undefined when it has none. */ +const getPdfProviderLimit = (provider: Providers | string): number | undefined => { + if (provider === Providers.ANTHROPIC) { + return mbToBytes(32); + } + if (isOpenAILikeProvider(provider)) { + return mbToBytes(10); + } + if (provider === Providers.GOOGLE || provider === Providers.VERTEXAI) { + return mbToBytes(20); + } + return undefined; +}; + +/** + * The per-file byte limit the document encoder enforces before sending a file natively, + * resolved from metadata alone. A configured limit replaces the provider fallback rather + * than capping it, so a configured 0 rejects every Bedrock document and every PDF with a + * provider limit. Outside those, only a truthy configured limit applies. + * @returns The limit in bytes, or undefined when no size limit applies + */ +export function getNativeDocumentSizeLimit({ + provider, + mimeType, + model, + configuredFileSizeLimit, +}: NativeDocumentSizeLimitParams): number | undefined { + if (provider === Providers.BEDROCK) { + const providerLimit = isExemptFromBedrockDocLimit(model, mimeType) + ? mbToBytes(32) + : mbToBytes(4.5); + return configuredFileSizeLimit ?? providerLimit; + } + + if (mimeType !== pdfMimeType) { + return configuredFileSizeLimit || undefined; + } + + const providerLimit = getPdfProviderLimit(provider); + if (providerLimit === undefined) { + return undefined; + } + return configuredFileSizeLimit ?? providerLimit; +} + /** * Validates a document against Bedrock size limits. The default limit is 4.5 MB, * but Claude 4+ (PDF) and Nova (PDF/DOCX) models are exempt per AWS docs. @@ -200,15 +261,18 @@ export async function validateBedrockDocument( model?: string, ): Promise { try { - const exempt = isExemptFromBedrockDocLimit(model, mimeType); - /** Default 4.5 MB; exempt models (Claude 4+ PDF, Nova PDF/DOCX) default to 32 MB when unconfigured */ - const providerLimit = exempt ? mbToBytes(32) : mbToBytes(4.5); - const effectiveLimit = configuredFileSizeLimit ?? providerLimit; + const effectiveLimit = getNativeDocumentSizeLimit({ + provider: Providers.BEDROCK, + mimeType, + model, + configuredFileSizeLimit, + }); - if (fileSize > effectiveLimit) { + if (effectiveLimit !== undefined && fileSize > effectiveLimit) { const limitMB = (effectiveLimit / (1024 * 1024)).toFixed(1); return { isValid: false, + reason: 'capacity', error: `File size (${(fileSize / (1024 * 1024)).toFixed(1)}MB) exceeds the ${limitMB}MB limit for Bedrock`, }; } @@ -217,6 +281,7 @@ export async function validateBedrockDocument( if (fileBuffer.length < 5) { return { isValid: false, + reason: 'integrity', error: 'Invalid PDF file: too small or corrupted', }; } @@ -225,6 +290,7 @@ export async function validateBedrockDocument( if (!pdfHeader.startsWith('%PDF-')) { return { isValid: false, + reason: 'integrity', error: 'Invalid PDF file: missing PDF header', }; } @@ -235,59 +301,12 @@ export async function validateBedrockDocument( console.error('Bedrock document validation error:', error); return { isValid: false, + reason: 'integrity', error: 'Failed to validate document file', }; } } -/** - * Validates if a PDF meets OpenAI's requirements - * @param fileSize - The file size in bytes - * @param configuredFileSizeLimit - Optional configured file size limit from fileConfig (in bytes) - * @returns Promise that resolves to validation result - */ -async function validateOpenAIPdf( - fileSize: number, - configuredFileSizeLimit?: number, -): Promise { - const providerLimit = mbToBytes(10); - const effectiveLimit = configuredFileSizeLimit ?? providerLimit; - - if (fileSize > effectiveLimit) { - const limitMB = Math.round(effectiveLimit / (1024 * 1024)); - return { - isValid: false, - error: `PDF file size (${Math.round(fileSize / (1024 * 1024))}MB) exceeds the ${limitMB}MB limit`, - }; - } - - return { isValid: true }; -} - -/** - * Validates if a PDF meets Google's requirements - * @param fileSize - The file size in bytes - * @param configuredFileSizeLimit - Optional configured file size limit from fileConfig (in bytes) - * @returns Promise that resolves to validation result - */ -async function validateGooglePdf( - fileSize: number, - configuredFileSizeLimit?: number, -): Promise { - const providerLimit = mbToBytes(20); - const effectiveLimit = configuredFileSizeLimit ?? providerLimit; - - if (fileSize > effectiveLimit) { - const limitMB = Math.round(effectiveLimit / (1024 * 1024)); - return { - isValid: false, - error: `PDF file size (${Math.round(fileSize / (1024 * 1024))}MB) exceeds the ${limitMB}MB limit`, - }; - } - - return { isValid: true }; -} - /** * Validates video files for different providers * @param videoBuffer - The video file as a buffer @@ -310,6 +329,7 @@ export async function validateVideo( const limitMB = Math.round(effectiveLimit / (1024 * 1024)); return { isValid: false, + reason: 'capacity', error: `Video file size (${Math.round(fileSize / (1024 * 1024))}MB) exceeds the ${limitMB}MB limit`, }; } @@ -318,6 +338,7 @@ export async function validateVideo( if (!videoBuffer || videoBuffer.length < 10) { return { isValid: false, + reason: 'integrity', error: 'Invalid video file: too small or corrupted', }; } @@ -347,6 +368,7 @@ export async function validateAudio( const limitMB = Math.round(effectiveLimit / (1024 * 1024)); return { isValid: false, + reason: 'capacity', error: `Audio file size (${Math.round(fileSize / (1024 * 1024))}MB) exceeds the ${limitMB}MB limit`, }; } @@ -355,6 +377,7 @@ export async function validateAudio( if (!audioBuffer || audioBuffer.length < 10) { return { isValid: false, + reason: 'integrity', error: 'Invalid audio file: too small or corrupted', }; } @@ -384,6 +407,7 @@ export async function validateImage( const limitMB = Math.round(effectiveLimit / (1024 * 1024)); return { isValid: false, + reason: 'capacity', error: `Image file size (${Math.round(fileSize / (1024 * 1024))}MB) exceeds the ${limitMB}MB limit`, }; } @@ -397,6 +421,7 @@ export async function validateImage( const limitMB = Math.round(effectiveLimit / (1024 * 1024)); return { isValid: false, + reason: 'capacity', error: `Image file size (${Math.round(fileSize / (1024 * 1024))}MB) exceeds the ${limitMB}MB limit`, }; } @@ -405,6 +430,7 @@ export async function validateImage( if (!imageBuffer || imageBuffer.length < 10) { return { isValid: false, + reason: 'integrity', error: 'Invalid image file: too small or corrupted', }; } diff --git a/packages/api/src/index.ts b/packages/api/src/index.ts index 54ba5cd1852..7bdb47a8b09 100644 --- a/packages/api/src/index.ts +++ b/packages/api/src/index.ts @@ -101,6 +101,7 @@ export * from './plugins'; export * from './endpoints'; /* Files */ export * from './files'; +export * from './files/reading/history'; /* Images */ export * from './images'; /* Storage */ diff --git a/packages/api/src/types/files.ts b/packages/api/src/types/files.ts index 83c3bfc1230..7490befcc65 100644 --- a/packages/api/src/types/files.ts +++ b/packages/api/src/types/files.ts @@ -1,4 +1,4 @@ -import type { BedrockDocumentFormat } from 'librechat-data-provider'; +import type { BedrockDocumentFormat, ReadingEvidence } from 'librechat-data-provider'; import type { IMongoFile } from '@librechat/data-schemas'; import type { Readable } from 'stream'; import type { DownloadURLParams } from '~/storage/types'; @@ -124,6 +124,21 @@ export type DocumentBlock = | OpenAIInputFileBlock | BedrockDocumentBlock; +/** + * `capacity`: larger than the provider accepts. + * `integrity`: the stored bytes could not be read or are not a usable file. + */ +export type ValidationFailureReason = Exclude; + +/** Whether the document encoder fails the turn on an invalid file or leaves the file out. */ +export type NativeValidationMode = 'throw' | 'skip'; + +/** A document the encoder left out in skip mode instead of failing the turn, and why. */ +export interface DocumentRejection { + file_id: string; + reason: NonNullable; +} + export interface DocumentResult { documents: DocumentBlock[]; files: Array<{ @@ -134,6 +149,8 @@ export interface DocumentResult { filename: string; type: string; }>; + /** Present only when encoding with `onValidationFailure: 'skip'` */ + rejected?: DocumentRejection[]; } /** Google audio block format */ diff --git a/packages/api/src/utils/message.ts b/packages/api/src/utils/message.ts index 968aad51130..bf3006b3095 100644 --- a/packages/api/src/utils/message.ts +++ b/packages/api/src/utils/message.ts @@ -1,8 +1,8 @@ import { Constants } from 'librechat-data-provider'; -import type { TFile, TMessage } from 'librechat-data-provider'; +import type { TMessage } from 'librechat-data-provider'; /** Minimal shape for request file entries (from `req.body.files`) */ -type RequestFile = { file_id?: string }; +export type RequestFile = { file_id?: string }; type GetMessagesByParentId = ( filter: { user: string; messageId: string; conversationId?: string }, @@ -12,6 +12,9 @@ type GetMessagesByParentId = ( /** Fields to strip from files before client transmission */ const FILE_STRIP_FIELDS = ['text', '_id', '__v'] as const; +/** A file as sent to the client: without its extracted text or database ids. */ +export type TransmittedFile = Omit; + /** Fields to strip from messages before client transmission. * Both are prompt-building inputs: `fileContext` is text extracted from * attachments and `image_urls` holds base64 image inputs. Terminal events are @@ -35,9 +38,7 @@ const MESSAGE_STRIP_FIELDS = ['fileContext', 'image_urls'] as const; * } * } */ -export function sanitizeFileForTransmit>( - file: T, -): Omit { +export function sanitizeFileForTransmit(file: T): TransmittedFile { const sanitized = { ...file }; for (const field of FILE_STRIP_FIELDS) { delete sanitized[field as keyof typeof sanitized]; @@ -46,10 +47,10 @@ export function sanitizeFileForTransmit>( } /** Filters attachments to those whose `file_id` appears in `requestFiles`, then sanitizes each. */ -export function buildMessageFiles>( +export function buildMessageFiles( requestFiles: RequestFile[], attachments: T[], -): Omit[] { +): TransmittedFile[] { const requestFileIds = new Set(); for (const f of requestFiles) { if (f.file_id) { @@ -57,7 +58,7 @@ export function buildMessageFiles>( } } - const files: Omit[] = []; + const files: TransmittedFile[] = []; for (const attachment of attachments) { if (attachment.file_id != null && requestFileIds.has(attachment.file_id)) { files.push(sanitizeFileForTransmit(attachment)); diff --git a/packages/api/src/utils/text.spec.ts b/packages/api/src/utils/text.spec.ts index 5a0c21a018c..2a8d625775b 100644 --- a/packages/api/src/utils/text.spec.ts +++ b/packages/api/src/utils/text.spec.ts @@ -405,6 +405,51 @@ describe('processTextWithTokenLimit', () => { }); }); + describe('with a known token count', () => { + it('uses the known count instead of counting text under the limit', async () => { + const tokenCountFn = jest.fn((text: string): number => Math.ceil(text.length / 4)); + const text = createTextOfLength(40); + + const result = await processTextWithTokenLimit({ + text, + tokenLimit: 100, + tokenCountFn, + knownTokenCount: 10, + }); + + expect(result).toEqual({ text, tokenCount: 10, wasTruncated: false }); + expect(tokenCountFn).not.toHaveBeenCalled(); + }); + + it('truncates from the known count without counting the full text again', async () => { + const tokenCountFn = jest.fn((text: string): number => Math.ceil(text.length / 4)); + const text = createTextOfLength(4000); + const counted = await processTextWithTokenLimit({ text, tokenLimit: 100, tokenCountFn }); + tokenCountFn.mockClear(); + + const result = await processTextWithTokenLimit({ + text, + tokenLimit: 100, + tokenCountFn, + knownTokenCount: 1000, + }); + + expect(result).toEqual(counted); + expect(result.wasTruncated).toBe(true); + expect(tokenCountFn).not.toHaveBeenCalledWith(text); + }); + + it('counts the text when no known count is supplied', async () => { + const tokenCountFn = jest.fn((text: string): number => Math.ceil(text.length / 4)); + const text = createTextOfLength(40); + + await processTextWithTokenLimit({ text, tokenLimit: 100, tokenCountFn }); + + expect(tokenCountFn).toHaveBeenCalledTimes(1); + expect(tokenCountFn).toHaveBeenCalledWith(text); + }); + }); + describe('edge cases', () => { it('should handle empty text', async () => { const { tokenCountFn } = createMockTokenCounter(); diff --git a/packages/api/src/utils/text.ts b/packages/api/src/utils/text.ts index 2d3977465ad..2c590d8107f 100644 --- a/packages/api/src/utils/text.ts +++ b/packages/api/src/utils/text.ts @@ -32,6 +32,8 @@ const TRUNCATION_SAFETY_BUFFER = 0.98; * @param text - The text content to process * @param tokenLimit - The maximum number of tokens allowed * @param tokenCountFn - Function to count tokens (can be sync or async) + * @param knownTokenCount - The text's token count when the caller already measured it; used as + * the original count instead of counting the text again * @returns Promise resolving to object with processed text, token count, and truncation status * * @remarks @@ -44,13 +46,15 @@ export async function processTextWithTokenLimit({ tokenLimit, tokenCountFn, preserve = 'start', + knownTokenCount, }: { text: string; tokenLimit: number; tokenCountFn: TokenCountFn; preserve?: 'start' | 'end'; + knownTokenCount?: number; }): Promise<{ text: string; tokenCount: number; wasTruncated: boolean }> { - const originalTokenCount = await tokenCountFn(text); + const originalTokenCount = knownTokenCount ?? (await tokenCountFn(text)); if (originalTokenCount <= tokenLimit) { return { diff --git a/packages/data-provider/src/file-config.spec.ts b/packages/data-provider/src/file-config.spec.ts index 3acee7e7d82..a6480f80e06 100644 --- a/packages/data-provider/src/file-config.spec.ts +++ b/packages/data-provider/src/file-config.spec.ts @@ -10,6 +10,7 @@ import { getConfiguredMimeAccept, getDocumentFileExtension, bedrockDocumentMimeTypes, + resolveLLMDeliveryPolicy, isAnthropicDocumentType, isPermissiveMimeConfig, isExplicitMimeConfig, @@ -2077,6 +2078,137 @@ describe('textFallbackWithoutTools config merging', () => { }); }); +describe('llmDeliveryPolicy config merging', () => { + type DynamicConfig = Parameters[0]; + const endpointConfigFor = (dynamic: DynamicConfig, endpoint: string) => + getEndpointFileConfig({ fileConfig: mergeFileConfig(dynamic), endpoint }); + const policyFor = (dynamic: DynamicConfig, endpoint: string) => + resolveLLMDeliveryPolicy(endpointConfigFor(dynamic, endpoint)); + + it('resolves to classic unless configured, leaving the merged config unchanged', () => { + const merged = mergeFileConfig({ + fileTokenLimit: 5, + endpoints: { [EModelEndpoint.openAI]: { fileLimit: 3 }, default: { fileLimit: 4 } }, + }); + + expect(resolveLLMDeliveryPolicy(undefined)).toBe('classic'); + expect(resolveLLMDeliveryPolicy(null)).toBe('classic'); + expect(resolveLLMDeliveryPolicy({})).toBe('classic'); + expect(endpointConfigFor(undefined, EModelEndpoint.openAI).llmDeliveryPolicy).toBeUndefined(); + expect(policyFor(undefined, EModelEndpoint.openAI)).toBe('classic'); + expect(policyFor({ endpoints: { default: {} } }, EModelEndpoint.openAI)).toBe('classic'); + expect(baseFileConfig.endpoints.default).not.toHaveProperty('llmDeliveryPolicy'); + expect(mergeFileConfig(undefined)).not.toHaveProperty('llmDeliveryPolicy'); + expect(merged).not.toHaveProperty('llmDeliveryPolicy'); + expect(merged.endpoints[EModelEndpoint.openAI]).not.toHaveProperty('llmDeliveryPolicy'); + expect(merged.endpoints.default).not.toHaveProperty('llmDeliveryPolicy'); + expect(merged.endpoints[EModelEndpoint.agents]).not.toHaveProperty('llmDeliveryPolicy'); + }); + + it('accepts the setting at the top level and on an endpoint', () => { + expect( + fileConfigSchema.safeParse({ + llmDeliveryPolicy: 'automatic', + endpoints: { openAI: { llmDeliveryPolicy: 'classic' } }, + }).success, + ).toBe(true); + expect(fileConfigSchema.safeParse({ llmDeliveryPolicy: 'auto' }).success).toBe(false); + expect( + fileConfigSchema.safeParse({ endpoints: { openAI: { llmDeliveryPolicy: true } } }).success, + ).toBe(false); + }); + + const endpointConfigured: DynamicConfig = { + endpoints: { [EModelEndpoint.openAI]: { llmDeliveryPolicy: 'automatic' } }, + }; + const defaultConfigured: DynamicConfig = { + endpoints: { default: { llmDeliveryPolicy: 'automatic' } }, + }; + const overriding: DynamicConfig = { + llmDeliveryPolicy: 'automatic', + endpoints: { + [EModelEndpoint.openAI]: { llmDeliveryPolicy: 'classic' }, + default: { llmDeliveryPolicy: 'classic' }, + MyGateway: { llmDeliveryPolicy: 'automatic' }, + }, + }; + + it.each<[string, DynamicConfig, string, 'automatic' | 'classic']>([ + [ + 'reaches an endpoint configured for it', + endpointConfigured, + EModelEndpoint.openAI, + 'automatic', + ], + [ + 'stays classic on an endpoint beside the configured one', + endpointConfigured, + EModelEndpoint.anthropic, + 'classic', + ], + [ + 'is inherited from the top level', + { llmDeliveryPolicy: 'automatic' }, + EModelEndpoint.anthropic, + 'automatic', + ], + [ + 'is inherited from the default endpoint by a custom endpoint', + defaultConfigured, + 'MyGateway', + 'automatic', + ], + [ + 'is inherited from the default endpoint by a known endpoint', + defaultConfigured, + EModelEndpoint.bedrock, + 'automatic', + ], + [ + 'lets an endpoint turn off what the top level sets', + overriding, + EModelEndpoint.openAI, + 'classic', + ], + [ + 'lets the default endpoint turn off what the top level sets', + overriding, + EModelEndpoint.anthropic, + 'classic', + ], + [ + 'lets an endpoint turn on what the default endpoint turns off', + overriding, + 'MyGateway', + 'automatic', + ], + ])('%s', (_case, dynamic, endpoint, expected) => { + expect(policyFor(dynamic, endpoint)).toBe(expected); + }); + + it('keeps classic wherever the legacy chooser is on', () => { + expect( + resolveLLMDeliveryPolicy({ llmDeliveryPolicy: 'automatic', legacyFileUploadUX: true }), + ).toBe('classic'); + expect( + resolveLLMDeliveryPolicy({ llmDeliveryPolicy: 'automatic', legacyFileUploadUX: false }), + ).toBe('automatic'); + expect( + policyFor( + { llmDeliveryPolicy: 'automatic', legacyFileUploadUX: true }, + EModelEndpoint.openAI, + ), + ).toBe('classic'); + + const dynamic: DynamicConfig = { + llmDeliveryPolicy: 'automatic', + endpoints: { [EModelEndpoint.openAI]: { legacyFileUploadUX: true } }, + }; + expect(policyFor(dynamic, EModelEndpoint.openAI)).toBe('classic'); + expect(policyFor(dynamic, EModelEndpoint.anthropic)).toBe('automatic'); + }); +}); + describe('agent attachment context limits', () => { it('keeps the turn-memory ceiling separate from agent upload storage', () => { expect(baseFileConfig.fileContextSizeLimit).toBe(128 * 1024 * 1024); diff --git a/packages/data-provider/src/file-config.ts b/packages/data-provider/src/file-config.ts index fadeb3352a5..b9c961bafef 100644 --- a/packages/data-provider/src/file-config.ts +++ b/packages/data-provider/src/file-config.ts @@ -638,6 +638,22 @@ export const defaultLLMDeliveryPathSchema = z.object({ export type TDefaultLLMDeliveryPathConfig = z.infer; type TDeliveryPathOverrides = NonNullable; +export const LLMDeliveryPolicy = z.enum(['classic', 'automatic']); +export type TLLMDeliveryPolicy = z.infer; + +/** + * How unified attachments are read for an endpoint. The compatibility default lives only here, + * because zod defaults never reach the merged runtime config; `legacyFileUploadUX` keeps its + * chooser semantics by forcing `classic`. + */ +export function resolveLLMDeliveryPolicy( + config?: Pick | null, +): TLLMDeliveryPolicy { + return config?.llmDeliveryPolicy === 'automatic' && config.legacyFileUploadUX !== true + ? 'automatic' + : 'classic'; +} + export const endpointFileConfigSchema = z.object({ disabled: z.boolean().optional(), fileLimit: z.number().min(0).optional(), @@ -647,6 +663,7 @@ export const endpointFileConfigSchema = z.object({ defaultLLMDeliveryPath: defaultLLMDeliveryPathSchema.optional(), legacyFileUploadUX: z.boolean().optional(), textFallbackWithoutTools: z.boolean().optional(), + llmDeliveryPolicy: LLMDeliveryPolicy.optional(), }); const skillFileConfigSchema = z.object({ @@ -690,6 +707,7 @@ export const fileConfigSchema = z.object({ defaultLLMDeliveryPath: defaultLLMDeliveryPathSchema.optional(), legacyFileUploadUX: z.boolean().optional(), textFallbackWithoutTools: z.boolean().optional(), + llmDeliveryPolicy: LLMDeliveryPolicy.optional(), }); export type TFileConfig = z.infer; @@ -1046,6 +1064,7 @@ function mergeWithDefault( legacyFileUploadUX: endpointConfig.legacyFileUploadUX ?? defaultConfig.legacyFileUploadUX, textFallbackWithoutTools: endpointConfig.textFallbackWithoutTools ?? defaultConfig.textFallbackWithoutTools, + llmDeliveryPolicy: endpointConfig.llmDeliveryPolicy ?? defaultConfig.llmDeliveryPolicy, }; } @@ -1138,6 +1157,7 @@ export function getEndpointFileConfig(params: { legacyFileUploadUX: mergedFileConfig.legacyFileUploadUX ?? baseDefaultConfig.legacyFileUploadUX, textFallbackWithoutTools: mergedFileConfig.textFallbackWithoutTools ?? baseDefaultConfig.textFallbackWithoutTools, + llmDeliveryPolicy: mergedFileConfig.llmDeliveryPolicy ?? baseDefaultConfig.llmDeliveryPolicy, }; const userDefaultConfig = mergedFileConfig.endpoints.default; const defaultConfig = userDefaultConfig @@ -1278,6 +1298,10 @@ function buildMergedFileConfig(dynamic: DynamicFileConfig | undefined): FileConf mergedConfig.textFallbackWithoutTools = dynamic.textFallbackWithoutTools; } + if (dynamic.llmDeliveryPolicy !== undefined) { + mergedConfig.llmDeliveryPolicy = dynamic.llmDeliveryPolicy; + } + if (dynamic.serverFileSizeLimit !== undefined) { mergedConfig.serverFileSizeLimit = mbToBytes(dynamic.serverFileSizeLimit); } @@ -1404,6 +1428,10 @@ function buildMergedFileConfig(dynamic: DynamicFileConfig | undefined): FileConf if (dynamicEndpoint.textFallbackWithoutTools !== undefined) { mergedEndpoint.textFallbackWithoutTools = dynamicEndpoint.textFallbackWithoutTools; } + + if (dynamicEndpoint.llmDeliveryPolicy !== undefined) { + mergedEndpoint.llmDeliveryPolicy = dynamicEndpoint.llmDeliveryPolicy; + } } return mergedConfig; diff --git a/packages/data-provider/src/index.ts b/packages/data-provider/src/index.ts index 3238b8ddab6..5dbae5bb8b4 100644 --- a/packages/data-provider/src/index.ts +++ b/packages/data-provider/src/index.ts @@ -9,6 +9,7 @@ export * from './langchain'; export * from './filters'; export * from './file-config'; export * from './resolve-llm-delivery-path'; +export * from './reading'; /* messages */ export * from './messages'; export * from './previews'; diff --git a/packages/data-provider/src/messages.spec.ts b/packages/data-provider/src/messages.spec.ts index d453f6b8562..7fcb1009fe3 100644 --- a/packages/data-provider/src/messages.spec.ts +++ b/packages/data-provider/src/messages.spec.ts @@ -2,7 +2,13 @@ import type { SummaryContentPart } from './types/content'; import type { ParentMessage } from './messages'; import type { TFile } from './types/files'; import type { TMessage } from './types'; -import { buildTree, findMessageById, isCompactedLeaf, isUserInitiatedCompaction } from './messages'; +import { + buildTree, + findMessageById, + hydrateMessageFile, + isCompactedLeaf, + isUserInitiatedCompaction, +} from './messages'; import { ContentTypes } from './types/runs'; const msg = (messageId: string, parentMessageId: string, over: Partial = {}): TMessage => @@ -147,6 +153,28 @@ describe('buildTree', () => { expect(tree?.[0].files?.[0]).toBe(file); }); + it('restores separate code and text delivery paths when the same workbook is reused', () => { + const refreshed = { + file_id: 'workbook', + filename: 'sales.xlsx', + llmDeliveryPath: 'text', + preview: 'fresh-preview', + } as TFile; + const codeFile = { file_id: refreshed.file_id, llmDeliveryPath: 'none' as const }; + const textFile = { file_id: refreshed.file_id, llmDeliveryPath: 'text' as const }; + const messages = [ + msg('code-turn', '', { files: [codeFile] }), + msg('text-turn', 'code-turn', { files: [textFile] }), + ]; + + const restored = buildTree({ messages, fileMap: { workbook: refreshed } }); + + expect(restored?.[0].files?.[0]).toEqual({ ...refreshed, ...codeFile }); + expect(asParent(restored?.[0]).children[0].files?.[0]).toBe(refreshed); + expect(refreshed.llmDeliveryPath).toBe('text'); + expect(messages[0].files?.[0]).toBe(codeFile); + }); + describe('memoization', () => { const chain = () => [ msg('u1', '00000000-0000-0000-0000-000000000000', { isCreatedByUser: true }), @@ -199,6 +227,38 @@ describe('buildTree', () => { }); }); +describe('hydrateMessageFile', () => { + const stored = { file_id: 'file', filename: 'report.pdf', llmDeliveryPath: 'provider' } as TFile; + + it('returns the stored record when the message copy names no path or the same path', () => { + expect(hydrateMessageFile({ file_id: stored.file_id }, { file: stored })).toBe(stored); + expect( + hydrateMessageFile( + { file_id: stored.file_id, llmDeliveryPath: 'provider' }, + { file: stored }, + ), + ).toBe(stored); + }); + + it("carries the message copy's differing delivery path onto the stored record", () => { + const copy = { + file_id: stored.file_id, + filename: 'stub.pdf', + llmDeliveryPath: 'text' as const, + }; + expect(hydrateMessageFile(copy, { file: stored })).toEqual({ + ...stored, + llmDeliveryPath: 'text', + }); + expect(stored.llmDeliveryPath).toBe('provider'); + }); + + it('returns the message copy itself when no stored record backs it', () => { + const copy = { file_id: 'missing', llmDeliveryPath: 'text' as const }; + expect(hydrateMessageFile(copy, { file: stored })).toBe(copy); + }); +}); + describe('isCompactedLeaf', () => { const summary = (overrides: Record = {}) => ({ type: ContentTypes.SUMMARY, diff --git a/packages/data-provider/src/messages.ts b/packages/data-provider/src/messages.ts index b424f648907..c348bef90bd 100644 --- a/packages/data-provider/src/messages.ts +++ b/packages/data-provider/src/messages.ts @@ -24,6 +24,24 @@ export function stripReasoningLabelMetadata(part: TMessageContentParts): TMessag export type ParentMessage = TMessage & { children: TMessage[]; depth: number }; +type MessageFile = NonNullable[number]; + +/** + * The stored record stands in for a message's file, but the delivery path that message's turn + * used lives only on the message's copy (a workbook kept for Run Code may have been read as + * text), so it is carried over for the preview. + */ +export function hydrateMessageFile(file: MessageFile, fileMap: Record): MessageFile { + const stored = fileMap[file.file_id ?? '']; + if (stored == null) { + return file; + } + if (file.llmDeliveryPath == null || file.llmDeliveryPath === stored.llmDeliveryPath) { + return stored; + } + return { ...stored, llmDeliveryPath: file.llmDeliveryPath }; +} + /** * Memoizes built trees per messages-array identity. The same query data feeds * several independent `select`s (ChatView plus the branch-tail helpers), which @@ -93,7 +111,7 @@ export function buildTree({ }; if (message.files && fileMap) { - extendedMessage.files = message.files.map((file) => fileMap[file.file_id ?? ''] ?? file); + extendedMessage.files = message.files.map((file) => hydrateMessageFile(file, fileMap)); } messageMap[message.messageId] = extendedMessage; diff --git a/packages/data-provider/src/reading.spec.ts b/packages/data-provider/src/reading.spec.ts new file mode 100644 index 00000000000..b4f67b088d8 --- /dev/null +++ b/packages/data-provider/src/reading.spec.ts @@ -0,0 +1,179 @@ +import type { DirectContentEntry } from './reading'; +import { + excelFileTypes, + fullMimeTypesList, + retrievalMimeTypesList, + fileConfig as baseFileConfig, + codeInterpreterMimeTypesList, +} from './file-config'; +import { + AFTER_FAILURE, + READING_ORDER, + isOriginalBacked, + isTextOnlyRecord, + categorizeForReading, + allocateDirectContent, +} from './reading'; +import { FileSources } from './types/files'; + +const TABULAR_TYPES = [ + ...excelFileTypes, + 'application/vnd.oasis.opendocument.spreadsheet', + 'text/csv', + 'application/csv', + 'text/tab-separated-values', + 'application/x-parquet', + 'application/vnd.apache.parquet', +]; + +const DOCUMENT_TYPES = [ + 'application/pdf', + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + 'application/vnd.openxmlformats-officedocument.presentationml.presentation', + 'application/msword', + 'application/zip', + 'application/epub+zip', + 'application/json', + 'text/markdown', + 'text/plain', + 'message/rfc822', + 'application/vnd.oasis.opendocument.text', +]; + +const MEDIA_TYPES = ['image/png', 'image/jpeg', 'audio/mpeg', 'video/mp4']; + +const entry = ( + fileId: string, + bytes: number, + textChars = 0, + counts = true, +): DirectContentEntry => ({ fileId, bytes, textChars, counts }); + +const sorted = (ids: ReadonlySet): string[] => Array.from(ids).sort(); + +describe('allocateDirectContent', () => { + it('admits candidates first-fit in the order given', () => { + const overflow = allocateDirectContent([], [entry('a', 60), entry('b', 50), entry('c', 30)], { + bytes: 100, + }); + expect(sorted(overflow)).toEqual(['b']); + }); + + it('charges committed content before any candidate', () => { + const overflow = allocateDirectContent( + [entry('history', 70)], + [entry('a', 60), entry('b', 50), entry('c', 30)], + { bytes: 100 }, + ); + expect(sorted(overflow)).toEqual(['a', 'b']); + }); + + it('admits a candidate that lands exactly on the limit', () => { + expect(allocateDirectContent([entry('h', 40)], [entry('a', 60)], { bytes: 100 }).size).toBe(0); + }); + + it('judges each limit dimension on its own', () => { + const candidates = [entry('a', 10, 500), entry('b', 10, 600), entry('c', 10, 100)]; + expect(sorted(allocateDirectContent([], candidates, { textChars: 1000 }))).toEqual(['b']); + expect(sorted(allocateDirectContent([], candidates, { count: 2 }))).toEqual(['c']); + expect(sorted(allocateDirectContent([], candidates, { bytes: 25 }))).toEqual(['c']); + }); + + it('does not charge the count for a file that does not count', () => { + const candidates = [entry('a', 1), entry('output', 1, 0, false), entry('b', 1)]; + expect(sorted(allocateDirectContent([entry('h', 1)], candidates, { count: 3 }))).toEqual([]); + expect(sorted(allocateDirectContent([entry('h', 1)], candidates, { count: 2 }))).toEqual(['b']); + }); + + it('treats a missing or zero limit as unlimited', () => { + const candidates = [entry('a', 10_000, 10_000), entry('b', 10_000, 10_000)]; + expect(allocateDirectContent([], candidates, {}).size).toBe(0); + expect(allocateDirectContent([], candidates, { count: 0, bytes: 0, textChars: 0 }).size).toBe( + 0, + ); + expect(sorted(allocateDirectContent([], candidates, { bytes: 0, textChars: 15_000 }))).toEqual([ + 'b', + ]); + }); +}); + +describe('categorizeForReading', () => { + it.each(TABULAR_TYPES)('classifies %s as tabular', (mimeType) => { + expect(categorizeForReading(mimeType)).toBe('tabular'); + }); + + it.each(DOCUMENT_TYPES)('classifies %s as a document', (mimeType) => { + expect(categorizeForReading(mimeType)).toBe('document'); + }); + + it.each(MEDIA_TYPES)('classifies %s as media', (mimeType) => { + expect(categorizeForReading(mimeType)).toBe('media'); + }); + + it('ignores parameters and case, as browsers send both', () => { + expect(categorizeForReading('Text/CSV; charset=utf-8')).toBe('tabular'); + expect(categorizeForReading(' APPLICATION/VND.MS-EXCEL ')).toBe('tabular'); + expect(categorizeForReading('IMAGE/PNG')).toBe('media'); + expect(categorizeForReading('text/plain; charset=utf-8')).toBe('document'); + }); + + it('treats an unknown or missing type as a document', () => { + expect(categorizeForReading('')).toBe('document'); + expect(categorizeForReading('application/octet-stream')).toBe('document'); + }); +}); + +describe('tabular classification stays inside admission', () => { + it('admits every known type it classifies as tabular', () => { + const known = [ + ...fullMimeTypesList, + ...codeInterpreterMimeTypesList, + ...retrievalMimeTypesList, + ...TABULAR_TYPES, + ]; + const unadmitted = known.filter( + (mimeType) => + categorizeForReading(mimeType) === 'tabular' && + !baseFileConfig.checkType(mimeType, baseFileConfig.endpoints.default.supportedMimeTypes), + ); + expect(unadmitted).toEqual([]); + }); +}); + +describe('record sources', () => { + it('treats a record without a source as holding the original', () => { + expect(isOriginalBacked({})).toBe(true); + expect(isOriginalBacked({ source: null })).toBe(true); + }); + + it('marks only a text source as text-only', () => { + expect(isTextOnlyRecord({ source: FileSources.text })).toBe(true); + expect(isTextOnlyRecord({ source: FileSources.local })).toBe(false); + expect(isTextOnlyRecord({})).toBe(false); + }); +}); + +describe('reading rule data', () => { + it('orders each category over all four readers exactly once', () => { + Object.values(READING_ORDER).forEach((order) => { + expect(order.slice().sort()).toEqual(['code', 'provider', 'search', 'text']); + }); + }); + + it('never retries a direct reader after a capacity or encode-time failure', () => { + const direct = ['provider', 'text']; + Object.values(AFTER_FAILURE).forEach((map) => { + expect(map.native_rejected?.some((reader) => direct.includes(reader))).toBe(false); + expect(map.aggregate_overflow?.some((reader) => direct.includes(reader))).toBe(false); + expect(map.native_capacity?.includes('provider') ?? false).toBe(false); + }); + }); + + it('tries File Search before text after a native capacity failure', () => { + Object.values(AFTER_FAILURE).forEach((map) => { + const order = map.native_capacity ?? []; + expect(order.indexOf('search')).toBe(0); + expect(order.indexOf('text')).toBe(order.length - 1); + }); + }); +}); diff --git a/packages/data-provider/src/reading.ts b/packages/data-provider/src/reading.ts new file mode 100644 index 00000000000..8514430dc93 --- /dev/null +++ b/packages/data-provider/src/reading.ts @@ -0,0 +1,248 @@ +import type { TurnDeliveryFile } from './resolve-llm-delivery-path'; +import type { TDefaultLLMDeliveryPath } from './file-config'; +import { excelMimeTypes } from './file-config'; +import { FileSources } from './types/files'; + +/** A reader the automatic policy can give an attachment to on a turn. */ +export type ReaderKind = 'provider' | 'text' | 'search' | 'code'; + +/** The automatic reading categories; images, audio and video stay on their classic route. */ +export type ReadingCategory = 'tabular' | 'document'; + +/** Why a reader was passed over. Stable log and notice codes. */ +export type SkipReason = + | 'code_unavailable' + | 'native_unsupported' + | 'native_capacity' + | 'native_rejected' + | 'aggregate_overflow' + | 'text_exceeds' + | 'text_unavailable' + | 'search_unavailable'; + +/** Why a file kept its classic route, named after the eligibility gate that kept it. */ +export type ClassicReason = + | 'classic_policy' + | 'legacy_record' + | 'explicit_destination' + | 'unmarked_record' + | 'not_message_attachment' + | 'text_only_record' + | 'configured_route' + | 'consumers_unknown' + | 'no_file_tools' + | 'media_category'; + +export type ReadingReason = + | ClassicReason + | 'code_preferred' + | 'code_selected' + | 'native_supported' + | 'text_fits' + | 'search_selected' + | SkipReason + | 'no_reader'; + +export type CodeEligibility = + | 'eligible' + | 'no_run_code' + | 'incompatible' + | 'text_only_record' + | 'unstreamable' + | 'declined'; + +export interface ReaderSkip { + reader: ReaderKind; + reason: SkipReason; +} + +/** One attachment's reading on one turn. Recomputed from its inputs and never stored. */ +export interface FileReading { + /** Turn-copy value; always inside the stored vocabulary. */ + path: TDefaultLLMDeliveryPath | undefined; + /** What classic routing answers for the same inputs (inspection view, diagnostics). */ + classicPath: TDefaultLLMDeliveryPath | undefined; + automatic: boolean; + reader: ReaderKind | 'unavailable' | 'unresolved'; + /** + * The skip that best explains the reading, or the selected reader's own reason when nothing was + * skipped. A limit the file hit outranks a reader that is merely missing; among missing readers + * the category's preferred one names the reason. + */ + reason: ReadingReason; + skipped: ReadonlyArray; + code: CodeEligibility; + category: ReadingCategory | 'media'; + /** The reader is text, but the text must first be derived from the retained original. */ + needsText: boolean; +} + +/** Plain-data verdicts the host supplies for one file on this request. */ +export interface ReadingEvidence { + /** Whether the encoder would emit this type for the provider, and within its size limit. */ + native?: 'fits' | 'capacity' | 'unsupported'; + /** The encoder rejected the file in skip mode on this request. */ + rejected?: 'capacity' | 'integrity' | 'unsupported'; + /** Judged only when the record carries text. */ + text?: 'fits' | 'exceeds'; + /** Direct content did not fit the request's first-fit allocation. */ + overflow?: true; + /** Text derivation failed on this request. */ + textFailed?: true; + /** Final pass only: whether File Search will actually receive the file. */ + search?: 'reachable' | 'unreachable'; +} + +/** The turn's evidence seam: synchronous, memoized, and side-effect free for the decision. */ +export interface TurnReadingInputs { + judge(file: TurnDeliveryFile): ReadingEvidence; + canDerive: boolean; +} + +/** The built-in extractors text can be derived with, without a RAG or OCR call. */ +export type BuiltInTextPlan = 'document_parser' | 'native_text'; + +/** How the text a reader may need was obtained; written only under the automatic policy. */ +export interface TextDerivation { + outcome: 'deferred' | 'complete' | 'failed'; + extractor?: BuiltInTextPlan | 'configured'; + reason?: + | 'too_large' + | 'expansion_limit' + | 'parser' + | 'empty' + | 'no_extractor' + | 'extractor_unavailable' + | 'original_missing'; + at?: number; +} + +/** Each category's reader order: the tabular row, and the document rows judged by capability. */ +export const READING_ORDER: Readonly> = { + tabular: ['code', 'provider', 'text', 'search'], + document: ['provider', 'text', 'search', 'code'], +}; + +/** + * Where a walk continues after a reader fails for this reason. Reasons not listed continue the + * current queue: the category order, or the list a previous failure switched to. A direct reader + * is never retried after a capacity or encode-time failure, and text follows a native capacity + * failure only after File Search, and only when it genuinely fits. + */ +export const AFTER_FAILURE: Readonly< + Record>> +> = { + tabular: { + native_capacity: ['search', 'text'], + aggregate_overflow: ['search'], + text_exceeds: ['search'], + native_rejected: ['search'], + }, + document: { + native_capacity: ['search', 'code', 'text'], + aggregate_overflow: ['search', 'code'], + text_exceeds: ['search', 'code'], + native_rejected: ['search', 'code'], + }, +}; + +export const READER_PATH: Readonly> = { + provider: 'provider', + text: 'text', + search: 'none', + code: 'none', +}; + +/** Sources whose original can be streamed back from storage, so a tool or extractor can read it. */ +export const ORIGINAL_BACKED_SOURCES: ReadonlySet = new Set([ + FileSources.local, + FileSources.s3, + FileSources.cloudfront, + FileSources.azure_blob, + FileSources.firebase, +]); + +/** + * A missing source reads as the schema default, `local`, as the encoders and agent file binding + * read it. The provision service refuses a missing source today, so it must coerce the same way + * when it adopts this set. + */ +export const isOriginalBacked = (file: Pick): boolean => + ORIGINAL_BACKED_SOURCES.has(file.source ?? FileSources.local); + +/** A record that holds only extracted text, with no original behind it. */ +export const isTextOnlyRecord = (file: Pick): boolean => + file.source === FileSources.text; + +/** Classification only; admission stays with `supportedMimeTypes`. */ +const TABULAR_PATTERNS: readonly RegExp[] = [ + excelMimeTypes, + /^application\/vnd\.oasis\.opendocument\.spreadsheet$/, + /^(text|application)\/csv$/, + /^text\/tab-separated-values$/, + /^application\/(x-parquet|vnd\.apache\.parquet)$/, +]; + +const MEDIA_MIME_PATTERN = /^(image|audio|video)\//; + +export function categorizeForReading(mimeType: string): ReadingCategory | 'media' { + const type = mimeType.split(';', 1)[0].trim().toLowerCase(); + if (MEDIA_MIME_PATTERN.test(type)) { + return 'media'; + } + return TABULAR_PATTERNS.some((pattern) => pattern.test(type)) ? 'tabular' : 'document'; +} + +export interface DirectContentEntry { + fileId: string; + /** Whether the file counts toward the attachment count limit. */ + counts: boolean; + bytes: number; + textChars: number; +} + +/** A missing or zero limit leaves that dimension unlimited. */ +export interface DirectContentLimits { + count?: number; + bytes?: number; + textChars?: number; +} + +type DirectContentUsage = Required; + +const chargeDirectContent = ( + usage: DirectContentUsage, + entry: DirectContentEntry, +): DirectContentUsage => ({ + count: usage.count + (entry.counts ? 1 : 0), + bytes: usage.bytes + entry.bytes, + textChars: usage.textChars + entry.textChars, +}); + +const withinLimit = (value: number, limit?: number): boolean => !limit || value <= limit; + +/** + * First-fit allocation of direct model content in the given (stable) candidate order, after + * charging `committed`. Returns the ids that do not fit, which must take another reader. + */ +export function allocateDirectContent( + committed: readonly DirectContentEntry[], + candidates: readonly DirectContentEntry[], + limits: DirectContentLimits, +): ReadonlySet { + const fits = (usage: DirectContentUsage): boolean => + withinLimit(usage.count, limits.count) && + withinLimit(usage.bytes, limits.bytes) && + withinLimit(usage.textChars, limits.textChars); + const overflow = new Set(); + let used = committed.reduce(chargeDirectContent, { count: 0, bytes: 0, textChars: 0 }); + for (const entry of candidates) { + const next = chargeDirectContent(used, entry); + if (fits(next)) { + used = next; + continue; + } + overflow.add(entry.fileId); + } + return overflow; +} diff --git a/packages/data-provider/src/resolve-llm-delivery-path.spec.ts b/packages/data-provider/src/resolve-llm-delivery-path.spec.ts index ab3728cedd8..49178731b95 100644 --- a/packages/data-provider/src/resolve-llm-delivery-path.spec.ts +++ b/packages/data-provider/src/resolve-llm-delivery-path.spec.ts @@ -1,8 +1,18 @@ import type { + UploadReading, TurnFileConsumers, TurnDeliveryFile, + UploadReadingInput, TurnDeliveryRouting, } from './resolve-llm-delivery-path'; +import type { + ReaderKind, + FileReading, + ClassicReason, + BuiltInTextPlan, + ReadingEvidence, + TurnReadingInputs, +} from './reading'; import type { TDefaultLLMDeliveryPathConfig } from './file-config'; import type { EndpointFileConfig } from './types/files'; import type { TEndpoint } from './config'; @@ -18,8 +28,19 @@ import { resolveDefaultLLMDeliveryPath, resolveUploadLLMDeliveryPath, SYSTEM_LLM_DELIVERY_DEFAULTS, + decideFileReading, + decideUploadReading, + judgeCodeEligibility, + selectBuiltInTextPlan, + isAutomaticReadingRecord, + resolveClassicTurnLLMDeliveryPath, } from './resolve-llm-delivery-path'; -import { mergeFileConfig, supportedMimeTypes, getEndpointFileConfig } from './file-config'; +import { + excelFileTypes, + mergeFileConfig, + supportedMimeTypes, + getEndpointFileConfig, +} from './file-config'; import { EToolResources } from './types/tools'; function resolveTurnLLMDeliveryPath({ @@ -1081,3 +1102,1230 @@ describe('resolveTurnLLMDeliveryPath', () => { ).toBe('none'); }); }); + +const XLSX = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'; +const DOCX = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'; +const PPTX = 'application/vnd.openxmlformats-officedocument.presentationml.presentation'; +const ODS = 'application/vnd.oasis.opendocument.spreadsheet'; +const TSV = 'text/tab-separated-values'; +const PARQUET = 'application/vnd.apache.parquet'; +const MSWORD = 'application/msword'; +const PDF = 'application/pdf'; +const ZIP = 'application/zip'; + +const NO_TOOLS: TurnFileConsumers = { executeCode: false, fileSearch: false }; +const CODE_ONLY: TurnFileConsumers = { executeCode: true, fileSearch: false }; +const SEARCH_ONLY: TurnFileConsumers = { executeCode: false, fileSearch: true }; +const BOTH_TOOLS: TurnFileConsumers = { executeCode: true, fileSearch: true }; + +const automaticConfig: EndpointFileConfig = { llmDeliveryPolicy: 'automatic' }; + +function automaticRouting( + endpoint: string, + overrides: Partial = {}, +): Partial { + return { + endpoint, + endpointConfig: automaticConfig, + fileConfig: mergeFileConfig(undefined), + sttConfigured: true, + ...overrides, + }; +} + +const withEvidence = (evidence: ReadingEvidence = {}, canDerive = false): TurnReadingInputs => ({ + judge: () => evidence, + canDerive, +}); + +function attachment(type: string, overrides: Partial = {}): TurnDeliveryFile { + return { + file_id: 'file_1', + type, + bytes: 2048, + source: 'local', + context: 'message_attachment', + llmDeliveryPath: 'none', + metadata: { destinationChosen: false }, + ...overrides, + }; +} + +const decide = ( + file: TurnDeliveryFile, + consumers: TurnFileConsumers | undefined, + routing: Partial | undefined = automaticRouting('openAI'), +): FileReading => decideFileReading({ routing, file, consumers }); + +const skippedReaders = (reading: FileReading): ReaderKind[] => + reading.skipped.map(({ reader }) => reader); + +describe('judgeCodeEligibility', () => { + const file = attachment(XLSX); + + it('names the first reason Run Code cannot read the record', () => { + expect(judgeCodeEligibility(file, undefined)).toBe('no_run_code'); + expect(judgeCodeEligibility(file, SEARCH_ONLY)).toBe('no_run_code'); + expect(judgeCodeEligibility({ ...file, source: 'text' }, CODE_ONLY)).toBe('text_only_record'); + expect(judgeCodeEligibility({ ...file, type: 'audio/mpeg' }, CODE_ONLY)).toBe('incompatible'); + expect(judgeCodeEligibility({ ...file, type: eml }, CODE_ONLY)).toBe('incompatible'); + expect(judgeCodeEligibility({ ...file, source: 'openai' }, CODE_ONLY)).toBe('unstreamable'); + expect( + judgeCodeEligibility({ ...file, metadata: { destinationChosen: true } }, CODE_ONLY), + ).toBe('declined'); + }); + + it('counts an enabled Run Code before the sandbox holds a copy', () => { + expect(judgeCodeEligibility(file, CODE_ONLY)).toBe('eligible'); + expect(judgeCodeEligibility({ ...file, source: undefined }, BOTH_TOOLS)).toBe('eligible'); + expect( + judgeCodeEligibility( + { ...file, metadata: { destinationChosen: true, codeEnvRef: codeRef } }, + CODE_ONLY, + ), + ).toBe('eligible'); + }); + + it('judges the stored type the sandbox receives', () => { + const converted = { ...file, type: 'audio/mpeg', metadata: { routingMimeType: XLSX } }; + expect(judgeCodeEligibility(converted, CODE_ONLY)).toBe('incompatible'); + }); +}); + +describe('decideFileReading', () => { + describe('eligibility gate', () => { + const eligible = attachment(XLSX, { text: 'region,total' }); + const withConfig = (endpointConfig: EndpointFileConfig) => + automaticRouting('openAI', { endpointConfig: { ...automaticConfig, ...endpointConfig } }); + const cases: Array< + [ + ClassicReason, + Partial | undefined, + TurnDeliveryFile, + TurnFileConsumers | undefined, + ] + > = [ + ['legacy_record', undefined, eligible, CODE_ONLY], + [ + 'legacy_record', + automaticRouting('openAI'), + { ...eligible, llmDeliveryPath: null }, + CODE_ONLY, + ], + ['classic_policy', automaticRouting('openAI', { endpointConfig: {} }), eligible, CODE_ONLY], + ['classic_policy', withConfig({ llmDeliveryPolicy: 'classic' }), eligible, CODE_ONLY], + ['classic_policy', withConfig({ legacyFileUploadUX: true }), eligible, CODE_ONLY], + [ + 'explicit_destination', + automaticRouting('openAI'), + { ...eligible, metadata: { destinationChosen: true } }, + CODE_ONLY, + ], + ['unmarked_record', automaticRouting('openAI'), { ...eligible, metadata: {} }, CODE_ONLY], + ['unmarked_record', automaticRouting('openAI'), { ...eligible, metadata: null }, CODE_ONLY], + [ + 'not_message_attachment', + automaticRouting('openAI'), + { ...eligible, context: 'agents' }, + CODE_ONLY, + ], + [ + 'not_message_attachment', + automaticRouting('openAI'), + { ...eligible, context: 'execute_code' }, + CODE_ONLY, + ], + [ + 'not_message_attachment', + automaticRouting('openAI'), + { ...eligible, context: undefined }, + CODE_ONLY, + ], + ['text_only_record', automaticRouting('openAI'), { ...eligible, source: 'text' }, CODE_ONLY], + [ + 'configured_route', + withConfig({ defaultLLMDeliveryPath: { overrides: { [XLSX]: 'text' } } }), + eligible, + CODE_ONLY, + ], + [ + 'configured_route', + withConfig({ defaultLLMDeliveryPath: { overrides: { 'application/*': 'none' } } }), + eligible, + NO_TOOLS, + ], + [ + 'configured_route', + automaticRouting('openAI', { + fileConfig: { endpoints: {}, defaultLLMDeliveryPath: { fallback: 'text' } }, + }), + eligible, + CODE_ONLY, + ], + ['consumers_unknown', automaticRouting('openAI'), eligible, undefined], + ['no_file_tools', automaticRouting('openAI'), eligible, NO_TOOLS], + ['no_file_tools', automaticRouting('bedrock'), attachment(XLSX), NO_TOOLS], + ['no_file_tools', automaticRouting('openAI'), attachment(PPTX, { text: 'slides' }), NO_TOOLS], + ['no_file_tools', automaticRouting('openAI'), attachment('audio/mpeg'), NO_TOOLS], + [ + 'media_category', + automaticRouting('openAI'), + attachment('image/png', { llmDeliveryPath: 'provider' }), + CODE_ONLY, + ], + ['media_category', automaticRouting('openAI'), attachment('audio/mpeg'), SEARCH_ONLY], + ['media_category', automaticRouting('google'), attachment('video/mp4'), BOTH_TOOLS], + ]; + + it.each(cases)('keeps the classic route: %s', (reason, routing, file, consumers) => { + const reading = decideFileReading({ routing, file, consumers }); + + expect(reading).toMatchObject({ reason, automatic: false, skipped: [], needsText: false }); + expect(reading.path).toBe(resolveClassicTurnLLMDeliveryPath(routing, file, consumers)); + expect(reading.classicPath).toBe(reading.path); + }); + + it('checks the gates in order', () => { + const configured = automaticRouting('openAI', { + endpointConfig: { + ...automaticConfig, + defaultLLMDeliveryPath: { overrides: { [XLSX]: 'text' } }, + }, + }); + let file = attachment(XLSX, { + source: 'text', + context: 'agents', + metadata: { destinationChosen: true }, + }); + const reasonOf = ( + routing: Partial, + consumers?: TurnFileConsumers, + ): FileReading['reason'] => decideFileReading({ routing, file, consumers }).reason; + + expect(reasonOf(configured)).toBe('explicit_destination'); + file = { ...file, metadata: {} }; + expect(reasonOf(configured)).toBe('unmarked_record'); + file = { ...file, metadata: { destinationChosen: false } }; + expect(reasonOf(configured)).toBe('not_message_attachment'); + file = { ...file, context: 'message_attachment' }; + expect(reasonOf(configured)).toBe('text_only_record'); + file = { ...file, source: 'local' }; + expect(reasonOf(configured)).toBe('configured_route'); + expect(reasonOf(automaticRouting('openAI'))).toBe('consumers_unknown'); + expect(reasonOf(automaticRouting('openAI'), NO_TOOLS)).toBe('no_file_tools'); + expect(reasonOf(automaticRouting('openAI'), CODE_ONLY)).toBe('code_preferred'); + file = { ...file, type: 'image/png' }; + expect(reasonOf(automaticRouting('openAI'), NO_TOOLS)).toBe('no_file_tools'); + expect(reasonOf(automaticRouting('openAI'), CODE_ONLY)).toBe('media_category'); + }); + }); + + describe('tabular order: code, provider, text, search', () => { + it('prefers Run Code whenever it can read the file', () => { + expect(decide(attachment(XLSX, { text: 'region,total' }), BOTH_TOOLS)).toMatchObject({ + path: 'none', + reader: 'code', + reason: 'code_preferred', + automatic: true, + skipped: [], + code: 'eligible', + category: 'tabular', + classicPath: 'text', + }); + }); + + it.each([...excelFileTypes, ODS, 'text/csv', 'application/csv', TSV])( + 'gives %s to Run Code', + (mimeType) => { + expect(decide(attachment(mimeType), CODE_ONLY)).toMatchObject({ + category: 'tabular', + reader: 'code', + reason: 'code_preferred', + }); + }, + ); + + it('falls back to native delivery where the provider takes the type', () => { + const reading = decide(attachment(XLSX), SEARCH_ONLY, automaticRouting('bedrock')); + expect(reading).toMatchObject({ + reader: 'provider', + path: 'provider', + reason: 'code_unavailable', + }); + expect(skippedReaders(reading)).toEqual(['code']); + }); + + it('then to complete stored text', () => { + const routing = automaticRouting('openAI', { reading: withEvidence({ text: 'fits' }) }); + const reading = decide(attachment(XLSX, { text: 'region,total' }), SEARCH_ONLY, routing); + expect(reading).toMatchObject({ reader: 'text', path: 'text', needsText: false }); + expect(reading.skipped).toEqual([ + { reader: 'code', reason: 'code_unavailable' }, + { reader: 'provider', reason: 'native_unsupported' }, + ]); + }); + + it('then to File Search', () => { + const reading = decide(attachment(XLSX), SEARCH_ONLY); + expect(reading).toMatchObject({ reader: 'search', path: 'none' }); + expect(skippedReaders(reading)).toEqual(['code', 'provider', 'text']); + }); + + it('reports the file unavailable when no reader can take it', () => { + const routing = automaticRouting('openAI', { + reading: withEvidence({ search: 'unreachable' }), + }); + const reading = decide(attachment(XLSX), SEARCH_ONLY, routing); + expect(reading).toMatchObject({ + reader: 'unavailable', + path: 'none', + reason: 'code_unavailable', + }); + expect(skippedReaders(reading)).toEqual(['code', 'provider', 'text', 'search']); + }); + + it('keeps the classic route when no file tool is loaded', () => { + const reading = decide(attachment(XLSX, { text: 'region,total' }), NO_TOOLS); + expect(reading).toMatchObject({ + reader: 'text', + path: 'text', + classicPath: 'text', + reason: 'no_file_tools', + automatic: false, + skipped: [], + }); + }); + }); + + describe('document order: provider, text, search, code', () => { + it('sends a document natively where the provider takes it', () => { + const routing = automaticRouting('openAI', { reading: withEvidence({ native: 'fits' }) }); + expect( + decide(attachment(PDF, { llmDeliveryPath: 'provider' }), BOTH_TOOLS, routing), + ).toMatchObject({ + reader: 'provider', + path: 'provider', + reason: 'native_supported', + skipped: [], + category: 'document', + }); + }); + + it('then complete stored text', () => { + const reading = decide(attachment(DOCX, { text: 'parsed' }), BOTH_TOOLS); + expect(reading).toMatchObject({ reader: 'text', path: 'text', reason: 'native_unsupported' }); + expect(decide(attachment('text/markdown', { text: '# notes' }), CODE_ONLY)).toMatchObject({ + reader: 'text', + reason: 'native_unsupported', + }); + }); + + it('then File Search, before Run Code', () => { + const reading = decide(attachment(DOCX), BOTH_TOOLS); + expect(reading).toMatchObject({ reader: 'search', path: 'none' }); + expect(skippedReaders(reading)).toEqual(['provider', 'text']); + }); + + it('then Run Code', () => { + const reading = decide(attachment(DOCX), CODE_ONLY); + expect(reading).toMatchObject({ reader: 'code', path: 'none', reason: 'text_unavailable' }); + expect(skippedReaders(reading)).toEqual(['provider', 'text', 'search']); + }); + + it('keeps an email export with stored text on its text route', () => { + const mail = attachment(eml, { llmDeliveryPath: 'text', text: 'mail body' }); + const reading = decide(mail, BOTH_TOOLS); + expect(reading).toMatchObject({ + reader: 'text', + path: 'text', + code: 'incompatible', + category: 'document', + automatic: true, + }); + expect(reading.path).toBe(reading.classicPath); + }); + + it('sends a Word 97 document natively on Bedrock', () => { + const doc = attachment(MSWORD); + expect(decide(doc, CODE_ONLY, automaticRouting('bedrock'))).toMatchObject({ + category: 'document', + reader: 'provider', + path: 'provider', + reason: 'native_supported', + }); + expect(decide(doc, CODE_ONLY, automaticRouting('openAI'))).toMatchObject({ + reader: 'code', + path: 'none', + }); + }); + + it('tries File Search before Run Code for a presentation', () => { + const searched = decide(attachment(PPTX), BOTH_TOOLS); + expect(searched).toMatchObject({ reader: 'search', path: 'none' }); + expect(skippedReaders(searched)).toEqual(['provider', 'text']); + + const coded = decide(attachment(PPTX), CODE_ONLY); + expect(coded).toMatchObject({ reader: 'code', path: 'none' }); + expect(skippedReaders(coded)).toEqual(['provider', 'text', 'search']); + }); + + it('leaves archives and columnar data to Run Code or nothing', () => { + expect(decide(attachment(ZIP), CODE_ONLY)).toMatchObject({ reader: 'code', path: 'none' }); + expect(decide(attachment(ZIP), SEARCH_ONLY)).toMatchObject({ reader: 'unavailable' }); + expect(decide(attachment(ZIP), NO_TOOLS)).toMatchObject({ + reader: 'unavailable', + path: 'none', + reason: 'no_file_tools', + automatic: false, + }); + expect(decide(attachment(PARQUET), CODE_ONLY)).toMatchObject({ + reader: 'code', + reason: 'code_preferred', + }); + expect(decide(attachment(PARQUET), SEARCH_ONLY)).toMatchObject({ + reader: 'unavailable', + path: 'none', + }); + expect(decide(attachment('application/x-parquet'), NO_TOOLS)).toMatchObject({ + reader: 'unavailable', + path: 'none', + reason: 'no_file_tools', + automatic: false, + }); + }); + }); + + describe('evidence', () => { + /* The walk runs only with a file tool loaded, so "no tool can take it" is a loaded File + * Search that will not receive the file: the row is then walked to its end. */ + const unreachable = { search: 'unreachable' } as const; + + it.each<[string, string, ReadingEvidence]>([ + ['native capacity', 'openAI', { native: 'capacity' }], + ['capacity discovered by encoding', 'anthropic', { native: 'fits', rejected: 'capacity' }], + ])( + 'sends a document over %s to search, then code, then complete fitting text last', + (_name, endpoint, evidence) => { + const pdf = attachment(PDF, { llmDeliveryPath: 'provider', text: 'extracted' }); + const routing = automaticRouting(endpoint, { + reading: withEvidence({ ...evidence, text: 'fits' }), + }); + + expect(decide(pdf, BOTH_TOOLS, routing)).toMatchObject({ + reader: 'search', + reason: 'native_capacity', + }); + expect(decide(pdf, CODE_ONLY, routing)).toMatchObject({ + reader: 'code', + reason: 'native_capacity', + }); + const unreached = automaticRouting(endpoint, { + reading: withEvidence({ ...evidence, text: 'fits', ...unreachable }), + }); + const text = decide(pdf, SEARCH_ONLY, unreached); + expect(text).toMatchObject({ + reader: 'text', + path: 'text', + reason: 'native_capacity', + needsText: false, + }); + expect(skippedReaders(text)).toEqual(['provider', 'search', 'code']); + + const exceeding = automaticRouting(endpoint, { + reading: withEvidence({ ...evidence, text: 'exceeds', ...unreachable }), + }); + expect(decide(pdf, SEARCH_ONLY, exceeding)).toMatchObject({ + reader: 'unavailable', + path: 'none', + reason: 'native_capacity', + }); + }, + ); + + it('sends a spreadsheet over native capacity to File Search before its text', () => { + const sheet = attachment(XLSX, { text: 'region,total' }); + const routing = automaticRouting('bedrock', { + reading: withEvidence({ native: 'capacity', text: 'fits' }), + }); + + const searched = decide(sheet, SEARCH_ONLY, routing); + expect(searched).toMatchObject({ reader: 'search', reason: 'native_capacity' }); + expect(skippedReaders(searched)).toEqual(['code', 'provider']); + + const unreached = automaticRouting('bedrock', { + reading: withEvidence({ native: 'capacity', text: 'fits', ...unreachable }), + }); + const text = decide(sheet, SEARCH_ONLY, unreached); + expect(text).toMatchObject({ reader: 'text', path: 'text', reason: 'native_capacity' }); + expect(skippedReaders(text)).toEqual(['code', 'provider', 'search']); + expect(decide(sheet, CODE_ONLY, routing)).toMatchObject({ reader: 'code', skipped: [] }); + }); + + it('derives a capacity fallback while keeping integrity failures away from text', () => { + const pdf = attachment(PDF, { llmDeliveryPath: 'provider' }); + const routing = automaticRouting('anthropic', { + reading: withEvidence({ native: 'fits', rejected: 'capacity', ...unreachable }, true), + }); + expect(decide(pdf, SEARCH_ONLY, routing)).toMatchObject({ + reader: 'text', + reason: 'native_capacity', + needsText: true, + }); + + const invalid = automaticRouting('anthropic', { + reading: withEvidence({ native: 'fits', rejected: 'integrity', ...unreachable }, true), + }); + expect(decide(pdf, SEARCH_ONLY, invalid)).toMatchObject({ + reader: 'unavailable', + reason: 'native_rejected', + needsText: false, + }); + }); + + it('preserves unsupported encoding as a capability failure with a fitting-text fallback', () => { + const routing = automaticRouting('anthropic', { + reading: withEvidence({ native: 'fits', rejected: 'unsupported', text: 'fits' }), + }); + expect(decide(attachment(PDF, { text: 'extracted' }), BOTH_TOOLS, routing)).toMatchObject({ + reader: 'text', + reason: 'native_unsupported', + }); + }); + + it('moves past a provider the encoder would skip', () => { + const routing = automaticRouting('anthropic', { + reading: withEvidence({ native: 'unsupported' }), + }); + const reading = decide(attachment(PDF, { text: 'extracted' }), BOTH_TOOLS, routing); + expect(reading).toMatchObject({ reader: 'text', reason: 'native_unsupported' }); + }); + + it('never offers a direct reader after an encode-time rejection', () => { + const routing = automaticRouting('bedrock', { + reading: withEvidence({ rejected: 'integrity', text: 'fits' }), + }); + const unreached = automaticRouting('bedrock', { + reading: withEvidence({ rejected: 'integrity', text: 'fits', ...unreachable }), + }); + const pdf = attachment(PDF, { text: 'extracted' }); + + const unavailable = decide(pdf, SEARCH_ONLY, unreached); + expect(unavailable).toMatchObject({ reader: 'unavailable', reason: 'native_rejected' }); + expect(skippedReaders(unavailable)).toEqual(['provider', 'search', 'code']); + expect(decide(pdf, SEARCH_ONLY, routing)).toMatchObject({ + reader: 'search', + reason: 'native_rejected', + }); + + const sheet = decide(attachment(XLSX, { text: 'region,total' }), SEARCH_ONLY, unreached); + expect(sheet.reader).toBe('unavailable'); + expect(skippedReaders(sheet)).toEqual(['code', 'provider', 'search']); + }); + + it('sends text over the limit to search and code instead of truncating it', () => { + const routing = automaticRouting('openAI', { reading: withEvidence({ text: 'exceeds' }) }); + const docx = attachment(DOCX, { text: 'long' }); + + const coded = decide(docx, CODE_ONLY, routing); + expect(coded).toMatchObject({ reader: 'code', reason: 'text_exceeds' }); + expect(coded.skipped).toContainEqual({ reader: 'text', reason: 'text_exceeds' }); + const unreached = automaticRouting('openAI', { + reading: withEvidence({ text: 'exceeds', ...unreachable }), + }); + expect(decide(docx, SEARCH_ONLY, unreached)).toMatchObject({ + reader: 'unavailable', + path: 'none', + }); + + const csv = decide(attachment('text/csv', { text: 'long' }), SEARCH_ONLY, routing); + expect(csv).toMatchObject({ reader: 'search' }); + expect(skippedReaders(csv)).toEqual(['code', 'provider', 'text']); + }); + + it('gives overflowed direct content to search and code only', () => { + const routing = automaticRouting('openAI', { + reading: withEvidence({ overflow: true, text: 'fits' }), + }); + const unreached = automaticRouting('openAI', { + reading: withEvidence({ overflow: true, text: 'fits', ...unreachable }), + }); + const pdf = attachment(PDF, { llmDeliveryPath: 'provider', text: 'extracted' }); + + const unavailable = decide(pdf, SEARCH_ONLY, unreached); + expect(unavailable).toMatchObject({ reader: 'unavailable', path: 'none' }); + expect(unavailable.skipped[0]).toEqual({ reader: 'provider', reason: 'aggregate_overflow' }); + expect(skippedReaders(unavailable)).toEqual(['provider', 'search', 'code']); + expect(decide(pdf, SEARCH_ONLY, routing)).toMatchObject({ + reader: 'search', + reason: 'aggregate_overflow', + }); + + const docx = decide(attachment(DOCX, { text: 'parsed' }), CODE_ONLY, routing); + expect(docx).toMatchObject({ reader: 'code' }); + expect(docx.skipped).toContainEqual({ reader: 'text', reason: 'aggregate_overflow' }); + }); + + it('names the limit the file hit rather than a reader that was missing', () => { + const exceeds = automaticRouting('openAI', { reading: withEvidence({ text: 'exceeds' }) }); + const unreached = automaticRouting('openAI', { + reading: withEvidence({ text: 'exceeds', ...unreachable }), + }); + + const largeCsv = decide(attachment('text/csv', { text: 'long' }), SEARCH_ONLY, unreached); + expect(largeCsv).toMatchObject({ reader: 'unavailable', reason: 'text_exceeds' }); + expect(skippedReaders(largeCsv)).toEqual(['code', 'provider', 'text', 'search']); + + const hugeLog = decide(attachment('text/plain', { text: 'long' }), SEARCH_ONLY, exceeds); + expect(hugeLog).toMatchObject({ reader: 'search', reason: 'text_exceeds' }); + expect(hugeLog.skipped[0]).toEqual({ reader: 'provider', reason: 'native_unsupported' }); + }); + + it('names the preferred reader that was missing when no limit was hit', () => { + const unreached = automaticRouting('openAI', { reading: withEvidence(unreachable) }); + expect(decide(attachment(PARQUET), SEARCH_ONLY)).toMatchObject({ + reader: 'unavailable', + reason: 'code_unavailable', + }); + expect(decide(attachment(PPTX), SEARCH_ONLY, unreached)).toMatchObject({ + reader: 'unavailable', + reason: 'text_unavailable', + }); + expect(decide(attachment(DOCX, { text: 'parsed' }), BOTH_TOOLS)).toMatchObject({ + reader: 'text', + reason: 'native_unsupported', + }); + }); + + it('continues the row when File Search will not receive the file', () => { + const routing = automaticRouting('openAI', { + reading: withEvidence({ search: 'unreachable' }), + }); + const reading = decide(attachment(PPTX), BOTH_TOOLS, routing); + expect(reading).toMatchObject({ reader: 'code', path: 'none' }); + expect(reading.skipped).toContainEqual({ reader: 'search', reason: 'search_unavailable' }); + expect(decide(attachment(XLSX), SEARCH_ONLY, routing)).toMatchObject({ + reader: 'unavailable', + }); + }); + + it('asks the turn for evidence once per decision', () => { + const judge = jest.fn((): ReadingEvidence => ({ native: 'capacity' })); + decide( + attachment(PDF), + BOTH_TOOLS, + automaticRouting('openAI', { reading: { judge, canDerive: false } }), + ); + expect(judge).toHaveBeenCalledTimes(1); + }); + }); + + describe('text derived from the original', () => { + const deferred = attachment(XLSX, { + metadata: { destinationChosen: false, textDerivation: { outcome: 'deferred' } }, + }); + const deriving = (evidence: ReadingEvidence = {}) => + automaticRouting('openAI', { reading: withEvidence(evidence, true) }); + + const unreachable = { search: 'unreachable' } as const; + + it('asks for text when a deriver is wired and code cannot read the file', () => { + expect(decide(deferred, SEARCH_ONLY, deriving(unreachable))).toMatchObject({ + reader: 'text', + path: 'text', + needsText: true, + automatic: true, + }); + expect(decide(deferred, CODE_ONLY, deriving())).toMatchObject({ + reader: 'code', + needsText: false, + }); + }); + + it('derives the text its classic route names when no file tool is loaded', () => { + expect(decide(deferred, NO_TOOLS, deriving())).toMatchObject({ + reader: 'text', + path: 'text', + classicPath: 'text', + needsText: true, + automatic: false, + reason: 'no_file_tools', + }); + expect(decide(deferred, NO_TOOLS, automaticRouting('openAI'))).toMatchObject({ + path: 'none', + classicPath: 'text', + reader: 'unavailable', + needsText: false, + reason: 'no_file_tools', + }); + }); + + it('does not ask without a deriver, an original, an extractor, or after a failure', () => { + const failed = { + ...deferred, + metadata: { destinationChosen: false, textDerivation: { outcome: 'failed' as const } }, + }; + const unreadable = [ + decide( + deferred, + SEARCH_ONLY, + automaticRouting('openAI', { reading: withEvidence(unreachable) }), + ), + decide({ ...deferred, source: 'openai' }, SEARCH_ONLY, deriving(unreachable)), + decide({ ...deferred, type: PPTX }, SEARCH_ONLY, deriving(unreachable)), + decide(failed, SEARCH_ONLY, deriving(unreachable)), + decide(deferred, SEARCH_ONLY, deriving({ textFailed: true, ...unreachable })), + ]; + unreadable.forEach((reading) => { + expect(reading).toMatchObject({ reader: 'unavailable', needsText: false }); + expect(reading.skipped).toContainEqual({ reader: 'text', reason: 'text_unavailable' }); + }); + + /* With no reading inputs at all, File Search is reachable and takes the file; text is + * still passed over rather than asked for. */ + const bare = decide(deferred, SEARCH_ONLY, automaticRouting('openAI')); + expect(bare).toMatchObject({ reader: 'search', needsText: false }); + expect(bare.skipped).toContainEqual({ reader: 'text', reason: 'text_unavailable' }); + expect(decide(deferred, SEARCH_ONLY, deriving({ textFailed: true }))).toMatchObject({ + reader: 'search', + }); + }); + + it('uses stored text rather than deriving it again', () => { + expect(decide({ ...deferred, text: 'region,total' }, SEARCH_ONLY, deriving())).toMatchObject({ + reader: 'text', + needsText: false, + automatic: true, + }); + expect(decide({ ...deferred, text: 'region,total' }, NO_TOOLS, deriving())).toMatchObject({ + reader: 'text', + needsText: false, + automatic: false, + }); + }); + }); + + describe('marked records under classic routing', () => { + const marked = attachment(XLSX, { + metadata: { destinationChosen: false, textDerivation: { outcome: 'deferred' } }, + }); + const classicRouting = (reading?: TurnReadingInputs) => + automaticRouting('openAI', { endpointConfig: {}, reading }); + + it('derives the text a classic text route needs', () => { + expect(decide(marked, NO_TOOLS, classicRouting(withEvidence({}, true)))).toMatchObject({ + path: 'text', + classicPath: 'text', + reader: 'text', + needsText: true, + automatic: false, + reason: 'classic_policy', + }); + }); + + it('delivers nothing rather than empty text when it cannot', () => { + const failed = { + ...marked, + metadata: { destinationChosen: false, textDerivation: { outcome: 'failed' as const } }, + }; + const readings = [ + decide(marked, NO_TOOLS, classicRouting()), + decide(marked, NO_TOOLS, classicRouting(withEvidence({}, false))), + decide(failed, NO_TOOLS, classicRouting(withEvidence({}, true))), + decide(marked, NO_TOOLS, classicRouting(withEvidence({ textFailed: true }, true))), + decide({ ...marked, source: 'openai' }, NO_TOOLS, classicRouting(withEvidence({}, true))), + ]; + readings.forEach((reading) => { + expect(reading).toMatchObject({ path: 'none', classicPath: 'text', needsText: false }); + }); + expect(decide(marked, CODE_ONLY, classicRouting()).reader).toBe('code'); + }); + + it('leaves every record classic routing wrote as it was', () => { + const unmarked = attachment(XLSX); + expect(decide(unmarked, NO_TOOLS, classicRouting()).path).toBe('text'); + expect(decide({ ...marked, text: 'region,total' }, NO_TOOLS, classicRouting())).toMatchObject( + { + path: 'text', + needsText: false, + }, + ); + expect(decide({ ...marked, type: ZIP }, NO_TOOLS, classicRouting()).path).toBe('none'); + }); + }); + + describe('automatic counterparts of classic pins', () => { + it('reads a classic-era spreadsheet with Run Code when it runs, and as text when not', () => { + const sheet = attachment(XLSX, { llmDeliveryPath: 'text', text: 'region,total' }); + expect(decide(sheet, CODE_ONLY)).toMatchObject({ path: 'none', classicPath: 'text' }); + expect(decide(sheet, SEARCH_ONLY)).toMatchObject({ + path: 'text', + classicPath: 'text', + automatic: true, + }); + expect(decide(sheet, NO_TOOLS)).toMatchObject({ + path: 'text', + classicPath: 'text', + automatic: false, + reason: 'no_file_tools', + }); + }); + + it('reads stored text as a primary reader without the fallback flag', () => { + const slides = attachment(PPTX, { text: 'slide text' }); + expect(decide(slides, BOTH_TOOLS)).toMatchObject({ path: 'text', classicPath: 'none' }); + expect(decide(slides, NO_TOOLS)).toMatchObject({ + path: 'none', + classicPath: 'none', + reason: 'no_file_tools', + }); + }); + + it('keeps an explicit none override with the flag off', () => { + const routing = automaticRouting('openAI', { + endpointConfig: { + ...automaticConfig, + defaultLLMDeliveryPath: { overrides: { 'text/csv': 'none' } }, + }, + }); + expect(decide(attachment('text/csv', { text: 'a,b' }), NO_TOOLS, routing)).toMatchObject({ + path: 'none', + reason: 'configured_route', + }); + }); + }); +}); + +describe('isAutomaticReadingRecord', () => { + const config = automaticRouting('openAI'); + const record = attachment(XLSX); + + it('accepts an inferred message attachment under the automatic policy', () => { + expect(isAutomaticReadingRecord(config, record)).toBe(true); + expect(isAutomaticReadingRecord(config, { ...record, source: undefined })).toBe(true); + expect(isAutomaticReadingRecord(config, { ...record, source: 'openai' })).toBe(true); + expect(isAutomaticReadingRecord(config, { ...record, llmDeliveryPath: 'text' })).toBe(true); + }); + + it.each<[string, Partial | undefined, TurnDeliveryFile]>([ + ['no endpoint config', undefined, record], + ['the classic policy', { endpointConfig: {} }, record], + [ + 'the legacy chooser', + { endpointConfig: { ...automaticConfig, legacyFileUploadUX: true } }, + record, + ], + ['a record predating routing', config, { ...record, llmDeliveryPath: null }], + ['an explicit destination', config, { ...record, metadata: { destinationChosen: true } }], + ['a missing marker', config, { ...record, metadata: {} }], + ['no metadata', config, { ...record, metadata: null }], + ['an agent resource', config, { ...record, context: 'agents' }], + ['a run artifact', config, { ...record, context: 'run_artifact' }], + ['no context', config, { ...record, context: undefined }], + ['a text-only record', config, { ...record, source: 'text' }], + ['media', config, { ...record, type: 'image/png' }], + ])('rejects %s', (_name, gateConfig, file) => { + expect(isAutomaticReadingRecord(gateConfig, file)).toBe(false); + }); + + it('rejects a type a configured route names, which the decision keeps classic', () => { + const pdf = attachment(PDF, { llmDeliveryPath: 'provider', bytes: 20 * 1024 * 1024 }); + const configured = automaticRouting('openAI', { + endpointConfig: { + ...automaticConfig, + fileSizeLimit: 5 * 1024 * 1024, + defaultLLMDeliveryPath: { overrides: { [PDF]: 'provider' } }, + }, + }); + const globallyConfigured = automaticRouting('openAI', { + fileConfig: { endpoints: {}, defaultLLMDeliveryPath: { fallback: 'text' } }, + }); + + expect(isAutomaticReadingRecord(configured, pdf)).toBe(false); + expect(decide(pdf, BOTH_TOOLS, configured)).toMatchObject({ + automatic: false, + reason: 'configured_route', + path: 'provider', + }); + expect(isAutomaticReadingRecord(globallyConfigured, pdf)).toBe(false); + expect(isAutomaticReadingRecord(config, pdf)).toBe(true); + }); + + it('judges media by the type routing saw before conversion', () => { + const converted = { + ...record, + type: 'image/png', + metadata: { destinationChosen: false, routingMimeType: 'text/csv' }, + }; + const heic = { + ...record, + type: 'text/csv', + metadata: { destinationChosen: false, routingMimeType: 'image/heic' }, + }; + expect(isAutomaticReadingRecord(config, converted)).toBe(true); + expect(isAutomaticReadingRecord(config, heic)).toBe(false); + }); +}); + +describe('decideUploadReading', () => { + const base: UploadReadingInput = { + mimeType: XLSX, + endpoint: 'openAI', + endpointConfig: automaticConfig, + sttConfigured: true, + isMessageAttachment: true, + extractionRequiredForInspection: false, + }; + const classicFields = { + codePreferred: false, + needsCodeAvailability: false, + keepOriginalOnExtractionFailure: false, + deferredMarker: false, + }; + + it.each<[string, Partial, Partial]>([ + [ + 'a spreadsheet Run Code can read', + { codePossible: true }, + { + path: 'none', + policy: 'automatic', + category: 'tabular', + reason: 'code_preferred', + codePreferred: true, + needsCodeAvailability: false, + keepOriginalOnExtractionFailure: false, + deferredMarker: true, + }, + ], + [ + 'a spreadsheet before code availability is known', + {}, + { + path: 'text', + reason: 'automatic_default', + codePreferred: false, + needsCodeAvailability: true, + keepOriginalOnExtractionFailure: true, + }, + ], + [ + 'a spreadsheet where Run Code is not possible', + { codePossible: false }, + { + path: 'text', + reason: 'code_unavailable', + codePreferred: false, + keepOriginalOnExtractionFailure: true, + }, + ], + [ + 'csv', + { mimeType: 'text/csv', codePossible: true }, + { path: 'none', codePreferred: true, deferredMarker: true }, + ], + [ + 'tsv', + { mimeType: TSV, codePossible: true }, + { path: 'none', codePreferred: true, deferredMarker: true }, + ], + [ + 'ods', + { mimeType: ODS, codePossible: true }, + { path: 'none', codePreferred: true, deferredMarker: true }, + ], + [ + 'parquet, which no built-in extractor reads', + { mimeType: PARQUET, codePossible: true }, + { path: 'none', codePreferred: true, deferredMarker: false }, + ], + [ + 'a spreadsheet an inspection policy must extract', + { codePossible: true, extractionRequiredForInspection: true }, + { path: 'text', reason: 'inspection_requires_text', ...classicFields }, + ], + [ + 'a spreadsheet an inspection policy must extract, before code availability is known', + { extractionRequiredForInspection: true }, + { path: 'text', reason: 'inspection_requires_text', ...classicFields }, + ], + [ + 'a PDF an inspection policy must extract', + { mimeType: PDF, endpoint: 'azureOpenAI', extractionRequiredForInspection: true }, + { path: 'text', reason: 'automatic_default', ...classicFields }, + ], + [ + 'a natively sent PDF under an inspection policy', + { mimeType: PDF, extractionRequiredForInspection: true }, + { path: 'provider', reason: 'automatic_default', ...classicFields }, + ], + [ + 'a Word document an inspection policy must extract', + { mimeType: DOCX, extractionRequiredForInspection: true }, + { path: 'text', reason: 'automatic_default', ...classicFields }, + ], + [ + 'a PDF on a document-capable endpoint', + { mimeType: PDF, codePossible: true }, + { + path: 'provider', + category: 'document', + reason: 'automatic_default', + keepOriginalOnExtractionFailure: true, + }, + ], + [ + 'a PDF the endpoint cannot take natively', + { mimeType: PDF, endpoint: 'azureOpenAI' }, + { path: 'text', reason: 'automatic_default', keepOriginalOnExtractionFailure: true }, + ], + [ + 'a Word document', + { mimeType: DOCX }, + { path: 'text', reason: 'automatic_default', keepOriginalOnExtractionFailure: true }, + ], + [ + 'a Word document on Bedrock', + { mimeType: DOCX, endpoint: 'bedrock' }, + { path: 'provider', reason: 'automatic_default', keepOriginalOnExtractionFailure: true }, + ], + [ + 'an archive', + { mimeType: ZIP, codePossible: true }, + { + path: 'none', + reason: 'automatic_default', + codePreferred: false, + keepOriginalOnExtractionFailure: true, + deferredMarker: false, + }, + ], + [ + 'an image', + { mimeType: 'image/png', codePossible: true }, + { path: 'provider', category: 'media', reason: 'automatic_default', ...classicFields }, + ], + [ + 'an explicit Run Code upload', + { toolResource: 'execute_code', codePossible: true }, + { path: 'none', reason: 'explicit_destination', ...classicFields }, + ], + [ + 'an explicit context upload', + { toolResource: 'context' }, + { path: 'text', reason: 'explicit_destination', ...classicFields }, + ], + [ + 'a permanent agent upload', + { isMessageAttachment: false, codePossible: true }, + { path: 'text', reason: 'agent_resource', ...classicFields }, + ], + [ + 'an upload on a classic endpoint', + { endpointConfig: {}, codePossible: true }, + { path: 'text', policy: 'classic', reason: 'classic_policy', ...classicFields }, + ], + [ + 'an upload through the legacy chooser', + { endpointConfig: { ...automaticConfig, legacyFileUploadUX: true }, codePossible: true }, + { path: 'provider', policy: 'classic', reason: 'classic_policy', ...classicFields }, + ], + [ + 'a type an endpoint route names', + { + endpointConfig: { + ...automaticConfig, + defaultLLMDeliveryPath: { overrides: { [XLSX]: 'text' } }, + }, + codePossible: true, + }, + { path: 'text', reason: 'configured_route', ...classicFields }, + ], + [ + 'a type a global route names', + { + fileConfig: { endpoints: {}, defaultLLMDeliveryPath: { fallback: 'provider' } }, + codePossible: true, + }, + { path: 'provider', reason: 'configured_route', ...classicFields }, + ], + ])('decides %s', (_name, overrides, expected) => { + expect(decideUploadReading({ ...base, ...overrides })).toMatchObject(expected); + }); + + it('never changes an upload route under the classic policy', () => { + const mimeTypes = [ + XLSX, + ODS, + 'text/csv', + TSV, + PARQUET, + PDF, + DOCX, + PPTX, + ZIP, + 'image/png', + 'audio/mpeg', + ]; + const configs: EndpointFileConfig[] = [ + {}, + { llmDeliveryPolicy: 'classic' }, + { ...automaticConfig, legacyFileUploadUX: true }, + ]; + const departures: string[] = []; + mimeTypes.forEach((mimeType) => + ['openAI', 'bedrock', 'azureOpenAI', 'agents'].forEach((endpoint) => + configs.forEach((endpointConfig) => + [undefined, true, false].forEach((codePossible) => + [true, false].forEach((isMessageAttachment) => + [true, false].forEach((extractionRequiredForInspection) => { + const input = { + ...base, + mimeType, + endpoint, + endpointConfig, + codePossible, + isMessageAttachment, + extractionRequiredForInspection, + }; + const reading = decideUploadReading(input); + const classicPath = resolveUploadLLMDeliveryPath(input); + const unchanged = + reading.policy === 'classic' && + reading.path === classicPath && + !reading.codePreferred && + !reading.needsCodeAvailability && + !reading.keepOriginalOnExtractionFailure && + !reading.deferredMarker; + if (!unchanged) { + departures.push(`${mimeType} on ${endpoint}: ${reading.path} vs ${classicPath}`); + } + }), + ), + ), + ), + ), + ); + expect(departures).toEqual([]); + }); +}); + +describe('selectBuiltInTextPlan', () => { + it.each<[string, BuiltInTextPlan | null]>([ + [XLSX, 'document_parser'], + ['application/vnd.ms-excel', 'document_parser'], + ['application/x-dos_ms_excel', 'document_parser'], + [ODS, 'document_parser'], + [PDF, 'document_parser'], + [DOCX, 'document_parser'], + ['application/vnd.oasis.opendocument.text', 'document_parser'], + ['text/csv', 'native_text'], + [TSV, 'native_text'], + ['application/csv', 'native_text'], + ['application/json', 'native_text'], + ['text/markdown', 'native_text'], + [eml, 'native_text'], + [PPTX, null], + [MSWORD, null], + [ZIP, null], + [PARQUET, null], + ['application/epub+zip', null], + ['image/png', null], + ['audio/mpeg', null], + ])('plans %s as %s', (mimeType, plan) => { + expect(selectBuiltInTextPlan(mimeType)).toBe(plan); + }); +}); + +describe('classic equivalence', () => { + const mimeTypes = [ + ...excelFileTypes, + ODS, + 'text/csv', + 'application/csv', + TSV, + 'application/x-parquet', + PARQUET, + PDF, + DOCX, + PPTX, + MSWORD, + ZIP, + 'application/epub+zip', + 'application/json', + 'text/markdown', + eml, + 'image/png', + 'audio/mpeg', + 'video/mp4', + ]; + const endpoints = [ + 'openAI', + 'anthropic', + 'bedrock', + 'google', + 'azureOpenAI', + 'agents', + 'MyGateway', + ]; + const consumerSets: Array = [ + undefined, + NO_TOOLS, + CODE_ONLY, + SEARCH_ONLY, + BOTH_TOOLS, + ]; + const shapes: Array> = [ + { text: 'stored text' }, + { embedded: true }, + { llmDeliveryPath: 'text', text: 'stored text' }, + { llmDeliveryPath: 'provider' }, + { text: 'stored text', metadata: { destinationChosen: true } }, + { text: 'stored text', metadata: null }, + { llmDeliveryPath: undefined, text: 'stored text' }, + ]; + const overrides: EndpointFileConfig = { + textFallbackWithoutTools: true, + defaultLLMDeliveryPath: { + overrides: { 'text/csv': 'none', 'application/pdf': 'text', 'image/*': 'none' }, + }, + }; + + it.each<[string, EndpointFileConfig]>([ + ['unset', {}], + ['classic', { llmDeliveryPolicy: 'classic' }], + ])('keeps every turn route identical with the policy %s', (_name, policy) => { + const mismatches: string[] = []; + let combinations = 0; + const configs: EndpointFileConfig[] = [{}, overrides]; + configs.forEach((config) => + endpoints.forEach((endpoint) => { + const routing: Partial = { + endpoint, + endpointConfig: { ...config, ...policy }, + fileConfig: mergeFileConfig(undefined), + sttConfigured: true, + reading: withEvidence({ native: 'capacity', text: 'exceeds', overflow: true }, true), + }; + mimeTypes.forEach((mimeType) => + shapes.forEach((shape) => { + const file = attachment(mimeType, shape); + consumerSets.forEach((consumers) => { + combinations += 1; + const classicPath = resolveClassicTurnLLMDeliveryPath(routing, file, consumers); + const reading = decideFileReading({ routing, file, consumers }); + const turnPath = resolveStoredTurnPath(routing, file, consumers); + if (reading.path !== classicPath || turnPath !== classicPath || reading.automatic) { + mismatches.push( + `${mimeType} on ${endpoint}: ${String(reading.path)} vs ${String(classicPath)}`, + ); + } + }); + }), + ); + }), + ); + + expect(combinations).toBe(2 * endpoints.length * mimeTypes.length * shapes.length * 5); + expect(mismatches).toEqual([]); + }); +}); diff --git a/packages/data-provider/src/resolve-llm-delivery-path.ts b/packages/data-provider/src/resolve-llm-delivery-path.ts index 6b41d378fc8..dce6b8f1c08 100644 --- a/packages/data-provider/src/resolve-llm-delivery-path.ts +++ b/packages/data-provider/src/resolve-llm-delivery-path.ts @@ -1,4 +1,22 @@ -import type { TDefaultLLMDeliveryPath, TDefaultLLMDeliveryPathConfig } from './file-config'; +import type { + ReaderKind, + ReaderSkip, + SkipReason, + FileReading, + ClassicReason, + ReadingReason, + TextDerivation, + BuiltInTextPlan, + CodeEligibility, + ReadingCategory, + ReadingEvidence, + TurnReadingInputs, +} from './reading'; +import type { + TLLMDeliveryPolicy, + TDefaultLLMDeliveryPath, + TDefaultLLMDeliveryPathConfig, +} from './file-config'; import type { EndpointFileConfig, FileConfig, RegexLike } from './types/files'; import type { CodeEnvReferenceSet } from './codeEnvRef'; import type { TEndpoint } from './config'; @@ -6,7 +24,9 @@ import { retrievalMimeTypes, isExplicitMimeConfig, isBedrockDocumentType, + documentParserMimeTypes, codeInterpreterMimeTypes, + resolveLLMDeliveryPolicy, fileConfig as baseFileConfig, } from './file-config'; import { @@ -16,9 +36,18 @@ import { isMediaSupportedProvider, isDocumentSupportedProvider, } from './schemas'; +import { + READER_PATH, + AFTER_FAILURE, + READING_ORDER, + isTextOnlyRecord, + isOriginalBacked, + categorizeForReading, +} from './reading'; import { normalizeEndpointName } from './utils'; import { EToolResources } from './types/tools'; import { getCodeEnvRefs } from './codeEnvRef'; +import { FileContext } from './types/files'; /** * The native provider a custom endpoint declares, when it declares one. A custom endpoint @@ -144,18 +173,26 @@ export function hasTextExtractionPath(mimeType: string): boolean { } /** - * Resolves the default file path destination for a given mime type. - * Resolution chain: endpoint overrides -> endpoint fallback -> global overrides -> global fallback -> system defaults. + * The built-in extractor that can turn a stored type into text, or null when none can. Only + * built-in readers count, so text for a file meant for a tool never costs a RAG or OCR call. */ -export function resolveDefaultLLMDeliveryPath( +export function selectBuiltInTextPlan(mimeType: string): BuiltInTextPlan | null { + if (documentParserMimeTypes.some((pattern) => pattern.test(mimeType))) { + return 'document_parser'; + } + return isNativelyReadableText(mimeType) ? 'native_text' : null; +} + +/** + * The route an explicit endpoint or global `defaultLLMDeliveryPath` assigns a type, or undefined + * when neither names it: endpoint overrides, endpoint fallback, global overrides, then global + * fallback, with an exact type ahead of its wildcard at each level. + */ +export function matchConfiguredDeliveryPath( mimeType: string, endpointConfig?: TDefaultLLMDeliveryPathConfig, globalConfig?: TDefaultLLMDeliveryPathConfig, - endpoint?: string, - useResponsesApi?: boolean, - sttConfigured?: boolean, - supportedMimeTypes?: RegexLike[], -): TDefaultLLMDeliveryPath { +): TDefaultLLMDeliveryPath | undefined { const wildcard = mimeType.split('/')[0] + '/*'; if (endpointConfig?.overrides) { @@ -184,6 +221,28 @@ export function resolveDefaultLLMDeliveryPath( return globalConfig.fallback; } + return undefined; +} + +/** + * Resolves the default file path destination for a given mime type. + * Resolution chain: endpoint overrides -> endpoint fallback -> global overrides -> global fallback -> system defaults. + */ +export function resolveDefaultLLMDeliveryPath( + mimeType: string, + endpointConfig?: TDefaultLLMDeliveryPathConfig, + globalConfig?: TDefaultLLMDeliveryPathConfig, + endpoint?: string, + useResponsesApi?: boolean, + sttConfigured?: boolean, + supportedMimeTypes?: RegexLike[], +): TDefaultLLMDeliveryPath { + const configured = matchConfiguredDeliveryPath(mimeType, endpointConfig, globalConfig); + if (configured != null) { + return configured; + } + + const wildcard = mimeType.split('/')[0] + '/*'; const systemDefault = (SYSTEM_LLM_DELIVERY_DEFAULTS.overrides[mimeType] ?? SYSTEM_LLM_DELIVERY_DEFAULTS.overrides[wildcard] ?? SYSTEM_LLM_DELIVERY_DEFAULTS.fallback) as TDefaultLLMDeliveryPath; @@ -249,20 +308,8 @@ export function resolveDefaultLLMDeliveryPath( return systemDefault; } -/** - * Delivery path for an upload that named no tool resource. The legacy chooser makes the - * destination explicit, so nothing is inferred there. - */ -export function resolveDefaultUploadLLMDeliveryPath({ - mimeType, - endpointConfig, - fileConfig, - endpoint, - endpointProvider, - useResponsesApi, - sttConfigured, -}: { - mimeType: string; +/** The endpoint a delivery route is resolved against, at upload or on a turn. */ +export interface DeliveryRouteInputs { endpointConfig?: EndpointFileConfig; fileConfig?: FileConfig; endpoint?: string; @@ -271,10 +318,24 @@ export function resolveDefaultUploadLLMDeliveryPath({ endpointProvider?: string | null; useResponsesApi?: boolean; sttConfigured?: boolean; -}): TDefaultLLMDeliveryPath { - if (endpointConfig?.legacyFileUploadUX === true) { - return 'provider'; - } +} + +/** + * The configured or capability-gated route for a type on an endpoint, without the legacy + * chooser. Where no configured route names the type, this is the system default the endpoint's + * encoders can actually deliver. + */ +export function resolveSystemDeliveryPath( + mimeType: string, + { + endpointConfig, + fileConfig, + endpoint, + endpointProvider, + useResponsesApi, + sttConfigured, + }: DeliveryRouteInputs, +): TDefaultLLMDeliveryPath { /* The media opt-in exists for OpenAI-format parts, so an endpoint known to run as * something else — a custom endpoint declaring `provider: anthropic` — keeps the * capability gate it had, where audio still reaches transcription. */ @@ -290,6 +351,20 @@ export function resolveDefaultUploadLLMDeliveryPath({ ); } +/** + * Delivery path for an upload that named no tool resource. The legacy chooser makes the + * destination explicit, so nothing is inferred there. + */ +export function resolveDefaultUploadLLMDeliveryPath({ + mimeType, + ...routing +}: DeliveryRouteInputs & { mimeType: string }): TDefaultLLMDeliveryPath { + if (routing.endpointConfig?.legacyFileUploadUX === true) { + return 'provider'; + } + return resolveSystemDeliveryPath(mimeType, routing); +} + /** Delivery path for an upload, honoring an explicitly chosen tool resource. */ export function resolveUploadLLMDeliveryPath({ toolResource, @@ -300,15 +375,9 @@ export function resolveUploadLLMDeliveryPath({ useResponsesApi, endpointProvider, sttConfigured, -}: { +}: DeliveryRouteInputs & { toolResource?: string | null; mimeType: string; - endpointConfig?: EndpointFileConfig; - fileConfig?: FileConfig; - endpoint?: string; - endpointProvider?: string | null; - useResponsesApi?: boolean; - sttConfigured?: boolean; }): TDefaultLLMDeliveryPath { if (toolResource === EToolResources.context || toolResource === EToolResources.ocr) { return 'text'; @@ -364,6 +433,15 @@ export interface TurnFileConsumers { fileSearch: boolean; } +/** + * Whether a file tool is loaded for the turn. The automatic policy decides readers only then; + * without Run Code or File Search every attachment keeps its classic route, so a deployment + * that enables the policy sees no change in conversations that attach no tool. + */ +export function hasFileToolConsumer(consumers?: TurnFileConsumers | null): boolean { + return consumers?.executeCode === true || consumers?.fileSearch === true; +} + /** * Whether the record shows this file reached a tool's own store: vectors for file search, a * sandbox pointer for code execution. Nothing else writes either, so their presence is proof @@ -419,11 +497,17 @@ export interface TurnDeliveryRouting { endpointProvider?: string; useResponsesApi?: boolean; sttConfigured: boolean; + /** This turn's reading evidence for the automatic policy; absent where the host supplies none. */ + reading?: TurnReadingInputs; } /** The fields of an attachment record that decide its delivery on a turn. */ export interface TurnDeliveryFile { + file_id?: string; type?: string; + bytes?: number; + source?: string | null; + context?: string | null; text?: string | null; /** Stored as an upload-time inference, so any string may be read back. */ llmDeliveryPath?: string | null; @@ -436,6 +520,7 @@ export interface TurnDeliveryFile { destinationChosen?: boolean; /** Vector namespaces holding this file, written as each embedding succeeds. */ embeddedEntities?: string[]; + textDerivation?: TextDerivation; } & CodeEnvReferenceSet) | null; } @@ -449,7 +534,7 @@ export function hasInferredLLMDeliveryPath(file: TurnDeliveryFile): boolean { } /** - * Delivery path for one attachment on one agent's turn. + * Classic delivery path for one attachment on one agent's turn. * * A record predating routing and a destination the user chose keep what they stored. An * inferred route re-resolves against the endpoint handling the turn. A `none` route leaves @@ -464,7 +549,7 @@ export function hasInferredLLMDeliveryPath(file: TurnDeliveryFile): boolean { * that never received the file. Run Code is judged by the tool set: see * {@link hasTurnFileConsumer}. */ -export function resolveTurnLLMDeliveryPath( +export function resolveClassicTurnLLMDeliveryPath( routing: Partial | undefined, file: TurnDeliveryFile, consumers?: TurnFileConsumers, @@ -473,8 +558,7 @@ export function resolveTurnLLMDeliveryPath( return isLLMDeliveryPath(file.llmDeliveryPath) ? file.llmDeliveryPath : undefined; } const { endpointConfig } = routing; - /* Conversion changes the stored type, so use the type routing originally saw. */ - const mimeType = file.metadata?.routingMimeType ?? file.type ?? ''; + const mimeType = getRoutingMimeType(file); const path = resolveUploadLLMDeliveryPath({ mimeType, ...routing }); const hasFallbackText = typeof file.text === 'string' && file.text.length > 0; if ( @@ -489,6 +573,498 @@ export function resolveTurnLLMDeliveryPath( return path; } +/** + * Delivery path for one attachment on one agent's turn: the name every turn reader resolves. + * + * Returns the path of {@link decideFileReading}. Under classic routing a record the automatic + * policy never marked gets exactly the classic path from that decision, so it is returned from + * {@link resolveClassicTurnLLMDeliveryPath} directly, without classifying the file. + */ +export function resolveTurnLLMDeliveryPath( + routing: Partial | undefined, + file: TurnDeliveryFile, + consumers?: TurnFileConsumers, +): TDefaultLLMDeliveryPath | undefined { + if ( + resolveLLMDeliveryPolicy(routing?.endpointConfig) !== 'automatic' && + file.metadata?.textDerivation == null + ) { + return resolveClassicTurnLLMDeliveryPath(routing, file, consumers); + } + return decideFileReading({ routing, file, consumers }).path; +} + +/** + * Whether Run Code can read this record on this turn. Depends only on consumers and the record: + * an enabled Run Code with no sandbox copy yet is eligible, since its first call uploads it. + */ +export function judgeCodeEligibility( + file: TurnDeliveryFile, + consumers?: TurnFileConsumers, +): CodeEligibility { + if (consumers?.executeCode !== true) { + return 'no_run_code'; + } + if (isTextOnlyRecord(file)) { + return 'text_only_record'; + } + /* The sandbox receives the stored bytes, so compatibility is judged on the stored type. */ + if (!canToolResourceConsume(EToolResources.execute_code, file.type ?? '')) { + return 'incompatible'; + } + if (!isOriginalBacked(file)) { + return 'unstreamable'; + } + if ( + file.metadata?.destinationChosen === true && + !hasToolResourceProvisioning(file, EToolResources.execute_code) + ) { + return 'declined'; + } + return 'eligible'; +} + +export interface FileReadingInput { + routing: Partial | undefined; + file: TurnDeliveryFile; + consumers?: TurnFileConsumers; +} + +type ReaderVerdict = 'ok' | 'needs_text' | SkipReason; + +type ReadingBase = Pick; + +interface ReadingWalk { + routing: Partial; + file: TurnDeliveryFile; + consumers: TurnFileConsumers; + mimeType: string; + category: ReadingCategory; + code: CodeEligibility; + evidence: ReadingEvidence; +} + +const SEARCH_ONLY: TurnFileConsumers = { executeCode: false, fileSearch: true }; + +const SELECTED_REASON: Readonly> = { + code: 'code_selected', + provider: 'native_supported', + text: 'text_fits', + search: 'search_selected', +}; + +const hasStoredText = (file: TurnDeliveryFile): boolean => + typeof file.text === 'string' && file.text.length > 0; + +/** Whether text can be derived from the retained original on this request. */ +function canDeriveText( + routing: Partial | undefined, + file: TurnDeliveryFile, + evidence: ReadingEvidence, +): boolean { + return ( + routing?.reading?.canDerive === true && + isOriginalBacked(file) && + file.metadata?.textDerivation?.outcome !== 'failed' && + evidence.textFailed !== true && + selectBuiltInTextPlan(file.type ?? '') != null + ); +} + +function judgeProvider({ mimeType, routing, evidence }: ReadingWalk): ReaderVerdict { + if ( + resolveSystemDeliveryPath(mimeType, routing) !== 'provider' || + evidence.native === 'unsupported' + ) { + return 'native_unsupported'; + } + if (evidence.rejected === 'capacity') { + return 'native_capacity'; + } + if (evidence.rejected === 'unsupported') { + return 'native_unsupported'; + } + if (evidence.rejected === 'integrity') { + return 'native_rejected'; + } + if (evidence.native === 'capacity') { + return 'native_capacity'; + } + return evidence.overflow === true ? 'aggregate_overflow' : 'ok'; +} + +function judgeText({ routing, file, evidence }: ReadingWalk): ReaderVerdict { + if (!hasStoredText(file)) { + return canDeriveText(routing, file, evidence) ? 'needs_text' : 'text_unavailable'; + } + if (evidence.text === 'exceeds') { + return 'text_exceeds'; + } + return evidence.overflow === true ? 'aggregate_overflow' : 'ok'; +} + +function judgeSearch({ consumers, file, evidence }: ReadingWalk): ReaderVerdict { + const reachable = + consumers.fileSearch && + canToolResourceConsume(EToolResources.file_search, file.type ?? '') && + isOriginalBacked(file) && + evidence.search !== 'unreachable'; + return reachable ? 'ok' : 'search_unavailable'; +} + +const READER_JUDGES: Readonly ReaderVerdict>> = { + code: ({ code }) => (code === 'eligible' ? 'ok' : 'code_unavailable'), + provider: judgeProvider, + text: judgeText, + search: judgeSearch, +}; + +/** Skip reasons naming a limit the file hit, most significant first. */ +const LIMIT_SKIPS: readonly SkipReason[] = [ + 'native_capacity', + 'native_rejected', + 'aggregate_overflow', + 'text_exceeds', +]; + +/** + * The skip that best explains a reading. A limit the file hit outranks a reader that is merely + * missing, and among missing readers the first the walk judged wins, so the category's preferred + * reader names the reason. The provider not taking the type, true of most documents, explains a + * reading only when nothing else does. + */ +function significantSkip(skipped: readonly ReaderSkip[]): SkipReason | undefined { + const limit = LIMIT_SKIPS.find((reason) => skipped.some((skip) => skip.reason === reason)); + if (limit != null) { + return limit; + } + return (skipped.find(({ reason }) => reason !== 'native_unsupported') ?? skipped[0])?.reason; +} + +function selectedReason( + reader: ReaderKind, + category: ReadingCategory, + skipped: readonly ReaderSkip[], +): ReadingReason { + const skip = significantSkip(skipped); + if (skip != null) { + return skip; + } + return reader === 'code' && category === 'tabular' ? 'code_preferred' : SELECTED_REASON[reader]; +} + +/** + * Judges each reader of the category in order and returns the first that can read the file. A + * failure may redirect the rest of the walk, and every reader is judged at most once, so the walk + * ends after at most one judgment per reader. + */ +function walkReaders(walk: ReadingWalk, base: ReadingBase): FileReading { + const skipped: ReaderSkip[] = []; + const tried = new Set(); + let queue = READING_ORDER[walk.category]; + for (let i = 0; i < queue.length; i++) { + const reader = queue[i]; + if (tried.has(reader)) { + continue; + } + tried.add(reader); + const verdict = READER_JUDGES[reader](walk); + if (verdict === 'ok' || verdict === 'needs_text') { + return { + ...base, + skipped, + path: READER_PATH[reader], + reader, + reason: selectedReason(reader, walk.category, skipped), + needsText: verdict === 'needs_text', + }; + } + skipped.push({ reader, reason: verdict }); + const next = AFTER_FAILURE[walk.category][verdict]; + if (next != null) { + queue = next; + i = -1; + } + } + return { + ...base, + skipped, + path: 'none', + reader: 'unavailable', + reason: significantSkip(skipped) ?? 'no_reader', + needsText: false, + }; +} + +/** The reader a classic route amounts to, for diagnostics only. */ +function classicReader( + path: TDefaultLLMDeliveryPath | undefined, + code: CodeEligibility, + mimeType: string, + file: TurnDeliveryFile, + consumers?: TurnFileConsumers, +): FileReading['reader'] { + if (path == null) { + return 'unresolved'; + } + if (path !== 'none') { + return path; + } + if (code === 'eligible') { + return 'code'; + } + return consumers?.fileSearch === true && hasTurnFileConsumer(mimeType, SEARCH_ONLY, file) + ? 'search' + : 'unavailable'; +} + +/** + * The classic route, except for a record the automatic policy marked and left without text: its + * classic `text` route either derives that text first or, when it cannot, delivers nothing rather + * than an empty text part. + */ +function classicReading( + base: ReadingBase, + { routing, file, consumers }: FileReadingInput, + mimeType: string, + reason: ClassicReason, +): FileReading { + const reading = { ...base, skipped: [], reason, needsText: false }; + const derivation = file.metadata?.textDerivation; + if (base.classicPath !== 'text' || derivation == null || hasStoredText(file)) { + const reader = classicReader(base.classicPath, base.code, mimeType, file, consumers); + return { ...reading, path: base.classicPath, reader }; + } + const evidence = routing?.reading?.judge(file) ?? {}; + if (derivation.outcome === 'deferred' && canDeriveText(routing, file, evidence)) { + return { ...reading, path: 'text', reader: 'text', needsText: true }; + } + return { + ...reading, + path: 'none', + reader: classicReader('none', base.code, mimeType, file, consumers), + }; +} + +/** The endpoint file configuration the eligibility gate reads. */ +type ReadingConfig = Pick; + +/** The type routing saw at upload; conversion rewrites the stored type. */ +export function getRoutingMimeType(file: TurnDeliveryFile): string { + return file.metadata?.routingMimeType ?? file.type ?? ''; +} + +/** + * Eligibility gates the record and the endpoint's file configuration decide alone: the first that + * keeps the file on its classic route, or null when none does. + */ +function recordGateReason( + config: ReadingConfig, + file: TurnDeliveryFile, + mimeType: string, +): ClassicReason | null { + if (file.llmDeliveryPath == null) { + return 'legacy_record'; + } + if (resolveLLMDeliveryPolicy(config.endpointConfig) !== 'automatic') { + return 'classic_policy'; + } + if (file.metadata?.destinationChosen === true) { + return 'explicit_destination'; + } + if (file.metadata?.destinationChosen !== false) { + return 'unmarked_record'; + } + if (file.context !== FileContext.message_attachment) { + return 'not_message_attachment'; + } + if (isTextOnlyRecord(file)) { + return 'text_only_record'; + } + const configured = matchConfiguredDeliveryPath( + mimeType, + config.endpointConfig?.defaultLLMDeliveryPath, + config.fileConfig?.defaultLLMDeliveryPath, + ); + return configured != null ? 'configured_route' : null; +} + +/** + * Whether the automatic policy reads this record once the turn's consumers are known: every + * eligibility gate of {@link decideFileReading} except the consumer one. The endpoint filter asks + * this before consumers exist, so it agrees with the decision on every file the decision judges. + */ +export function isAutomaticReadingRecord( + config: ReadingConfig | undefined, + file: TurnDeliveryFile, +): boolean { + const mimeType = getRoutingMimeType(file); + return ( + config != null && + recordGateReason(config, file, mimeType) == null && + categorizeForReading(mimeType) !== 'media' + ); +} + +type EligibleReading = Pick; + +/** The eligibility gate in precedence order: why a file keeps its classic route, or what the walk reads. */ +function gateFileReading( + { routing, file, consumers }: FileReadingInput, + mimeType: string, + category: ReadingCategory | 'media', +): ClassicReason | EligibleReading { + if (routing == null) { + return 'legacy_record'; + } + const recordReason = recordGateReason(routing, file, mimeType); + if (recordReason != null) { + return recordReason; + } + if (consumers == null) { + return 'consumers_unknown'; + } + if (!hasFileToolConsumer(consumers)) { + return 'no_file_tools'; + } + if (category === 'media') { + return 'media_category'; + } + return { routing, consumers, category }; +} + +/** + * How one attachment is read on one agent's turn. Classic routing answers unless every + * eligibility gate passes; then the category's readers are judged in order against the turn's + * consumers and evidence. The returned path always stays inside the stored vocabulary. + */ +export function decideFileReading(input: FileReadingInput): FileReading { + const { routing, file, consumers } = input; + const mimeType = getRoutingMimeType(file); + const category = categorizeForReading(mimeType); + const base: ReadingBase = { + classicPath: resolveClassicTurnLLMDeliveryPath(routing, file, consumers), + category, + code: judgeCodeEligibility(file, consumers), + automatic: false, + }; + const gate = gateFileReading(input, mimeType, category); + if (typeof gate === 'string') { + return classicReading(base, input, mimeType, gate); + } + const evidence = gate.routing.reading?.judge(file) ?? {}; + return walkReaders( + { ...gate, file, mimeType, code: base.code, evidence }, + { ...base, automatic: true }, + ); +} + +export interface UploadReadingInput extends DeliveryRouteInputs { + toolResource?: string | null; + mimeType: string; + isMessageAttachment: boolean; + /** + * The upload preflight would defer an extracted-text fail-close only for the context route, so + * that route's extraction is the inspection and a failed extraction must reject the upload. + */ + extractionRequiredForInspection: boolean; + /** Undefined until the caller resolves it; asked for only when the decision needs it. */ + codePossible?: boolean; +} + +export type UploadReadingReason = + | 'explicit_destination' + | 'classic_policy' + | 'agent_resource' + | 'configured_route' + | 'automatic_default' + | 'inspection_requires_text' + | 'code_unavailable' + | 'code_preferred'; + +export interface UploadReading { + path: TDefaultLLMDeliveryPath; + policy: TLLMDeliveryPolicy; + category: ReadingCategory | 'media'; + reason: UploadReadingReason; + /** Run Code will read the file, so nothing is extracted at upload. */ + codePreferred: boolean; + /** Resolve `codePossible`, then decide again. */ + needsCodeAvailability: boolean; + /** + * An extraction failure keeps the original, marked failed, instead of rejecting the upload. + * Never set where an inspection policy relies on that extraction. + */ + keepOriginalOnExtractionFailure: boolean; + /** Write `textDerivation: { outcome: 'deferred' }`, since a built-in extractor can derive text later. */ + deferredMarker: boolean; +} + +/** + * How an upload is read, decided before any extraction. Under the automatic policy a tabular + * message attachment is left to Run Code when code is possible at upload, and other message + * attachments outside the media rows keep their original when extraction fails, unless an + * inspection policy relies on that extraction; everything else stays classic. + */ +export function decideUploadReading(input: UploadReadingInput): UploadReading { + const path = resolveUploadLLMDeliveryPath(input); + const policy = resolveLLMDeliveryPolicy(input.endpointConfig); + const category = categorizeForReading(input.mimeType); + const classic = ( + reason: UploadReadingReason, + keepOriginalOnExtractionFailure = false, + ): UploadReading => ({ + path, + policy, + category, + reason, + codePreferred: false, + needsCodeAvailability: false, + keepOriginalOnExtractionFailure, + deferredMarker: false, + }); + + if (input.toolResource != null) { + return classic('explicit_destination'); + } + if (policy !== 'automatic') { + return classic('classic_policy'); + } + if (!input.isMessageAttachment) { + return classic('agent_resource'); + } + const configured = matchConfiguredDeliveryPath( + input.mimeType, + input.endpointConfig?.defaultLLMDeliveryPath, + input.fileConfig?.defaultLLMDeliveryPath, + ); + if (configured != null) { + return classic('configured_route'); + } + const keepOriginal = category !== 'media' && !input.extractionRequiredForInspection; + if ( + category !== 'tabular' || + !canToolResourceConsume(EToolResources.execute_code, input.mimeType) + ) { + return classic('automatic_default', keepOriginal); + } + if (input.extractionRequiredForInspection) { + return classic('inspection_requires_text'); + } + if (input.codePossible === undefined) { + return { ...classic('automatic_default', keepOriginal), needsCodeAvailability: true }; + } + if (!input.codePossible) { + return classic('code_unavailable', keepOriginal); + } + return { + ...classic('code_preferred'), + path: 'none', + codePreferred: true, + deferredMarker: selectBuiltInTextPlan(input.mimeType) != null, + }; +} + /** Why an upload cannot be accepted, when nothing would be able to read it. */ export type UploadRejection = 'no-agent-resource' | 'context-disabled' | 'no-consumer'; diff --git a/packages/data-provider/src/types/files.ts b/packages/data-provider/src/types/files.ts index 50c6e4dbe92..a3a72b763a5 100644 --- a/packages/data-provider/src/types/files.ts +++ b/packages/data-provider/src/types/files.ts @@ -1,5 +1,6 @@ -import type { TDefaultLLMDeliveryPathConfig } from '../file-config'; +import type { TDefaultLLMDeliveryPathConfig, TLLMDeliveryPolicy } from '../file-config'; import type { CodeEnvRef, CodeEnvRefMap } from '../codeEnvRef'; +import type { TextDerivation } from '../reading'; import { EToolResources } from './tools'; export enum FileSources { @@ -55,6 +56,9 @@ export type EndpointFileConfig = { /** Delivers the text extracted at upload for a file routed to tools (`none`) on a turn that * runs no tool able to read it. Off by default, which leaves such a file out of the prompt. */ textFallbackWithoutTools?: boolean; + /** How unified attachments are read. Absent means `classic`; read it through + * `resolveLLMDeliveryPolicy`, which also lets `legacyFileUploadUX` force `classic`. */ + llmDeliveryPolicy?: TLLMDeliveryPolicy; }; export type FileConfig = { @@ -96,6 +100,9 @@ export type FileConfig = { /** Delivers the text extracted at upload for a file routed to tools (`none`) on a turn that * runs no tool able to read it. Off by default, which leaves such a file out of the prompt. */ textFallbackWithoutTools?: boolean; + /** How unified attachments are read. Absent means `classic`; read it through + * `resolveLLMDeliveryPolicy`, which also lets `legacyFileUploadUX` force `classic`. */ + llmDeliveryPolicy?: TLLMDeliveryPolicy; }; export type FileConfigInput = { @@ -132,6 +139,9 @@ export type FileConfigInput = { /** Delivers the text extracted at upload for a file routed to tools (`none`) on a turn that * runs no tool able to read it. Off by default, which leaves such a file out of the prompt. */ textFallbackWithoutTools?: boolean; + /** How unified attachments are read. Absent means `classic`; read it through + * `resolveLLMDeliveryPolicy`, which also lets `legacyFileUploadUX` force `classic`. */ + llmDeliveryPolicy?: TLLMDeliveryPolicy; }; /** The immutable origin of a file explicitly published from an agent execution. */ @@ -215,6 +225,8 @@ export type TFile = { destinationChosen?: boolean; /** The type the delivery route was resolved against, when conversion changed it. */ routingMimeType?: string; + /** How the text a reader may need was obtained; written only under the automatic policy. */ + textDerivation?: TextDerivation; }; llmDeliveryPath?: 'provider' | 'text' | 'none'; createdAt?: string | Date; diff --git a/packages/data-schemas/src/methods/file.spec.ts b/packages/data-schemas/src/methods/file.spec.ts index 4a867c5dcc9..72d0c1a339c 100644 --- a/packages/data-schemas/src/methods/file.spec.ts +++ b/packages/data-schemas/src/methods/file.spec.ts @@ -2,6 +2,7 @@ import mongoose from 'mongoose'; import { v4 as uuidv4 } from 'uuid'; import { MongoMemoryServer } from 'mongodb-memory-server'; import { EToolResources, FileContext } from 'librechat-data-provider'; +import type { TextDerivation } from 'librechat-data-provider'; import type { IChatProject, IMongoFile } from '~/types'; import { _resetStrictCache } from '~/models/plugins/tenantIsolation'; import { tenantStorage, runAsSystem } from '~/config/tenantContext'; @@ -1926,6 +1927,249 @@ describe('File Methods', () => { }); }); + describe('saveFileTextDerivation', () => { + const deferred: TextDerivation = { outcome: 'deferred', at: 1_000 }; + const complete: TextDerivation = { + outcome: 'complete', + extractor: 'document_parser', + at: 2_000, + }; + const failed: TextDerivation = { + outcome: 'failed', + extractor: 'document_parser', + reason: 'parser', + at: 3_000, + }; + const codeEnvRef = { + kind: 'user' as const, + id: 'user-1', + storage_session_id: 'session-1', + file_id: 'code-file-1', + executionProfile: 'default' as const, + }; + + const seedFile = async ( + overrides: Partial = {}, + ): Promise<{ file_id: string; user: string }> => { + const file_id = uuidv4(); + const user = overrides.user ?? new mongoose.Types.ObjectId(); + await fileMethods.createFile({ + file_id, + filename: 'workbook.xlsx', + filepath: '/uploads/workbook.xlsx', + type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + bytes: 2048, + context: FileContext.message_attachment, + llmDeliveryPath: 'none', + metadata: { + destinationChosen: false, + routingMimeType: 'application/vnd.ms-excel', + codeEnvRefs: { default: codeEnvRef }, + textDerivation: deferred, + }, + ...overrides, + user, + }); + return { file_id, user: user.toString() }; + }; + + it('round-trips the marker through createFile on the strict metadata path', async () => { + const { file_id } = await seedFile(); + + const stored = await fileMethods.findFileById(file_id); + + expect(stored?.metadata?.textDerivation).toEqual(deferred); + expect(stored?.metadata?.destinationChosen).toBe(false); + expect(stored?.metadata?.routingMimeType).toBe('application/vnd.ms-excel'); + }); + + it('saves derived text and the marker on a deferred record without text', async () => { + const { file_id, user } = await seedFile(); + + const saved = await fileMethods.saveFileTextDerivation( + { file_id, text: 'Region,Total\nNorth,42', textDerivation: complete }, + { user }, + ); + + expect(saved).toBe(true); + const stored = await fileMethods.findFileById(file_id); + expect(stored?.text).toBe('Region,Total\nNorth,42'); + expect(stored?.llmDeliveryPath).toBe('none'); + expect(stored?.metadata).toEqual({ + destinationChosen: false, + routingMimeType: 'application/vnd.ms-excel', + codeEnvRefs: { default: codeEnvRef }, + textDerivation: complete, + }); + }); + + it.each([ + ['empty', ''], + ['null', null], + ])('applies when the deferred record holds %s text', async (_label, text) => { + const { file_id, user } = await seedFile(); + await File.collection.updateOne({ file_id }, { $set: { text } }); + + const saved = await fileMethods.saveFileTextDerivation( + { file_id, text: 'derived', textDerivation: complete }, + { user }, + ); + + expect(saved).toBe(true); + const stored = await fileMethods.findFileById(file_id); + expect(stored?.text).toBe('derived'); + expect(stored?.metadata?.textDerivation).toEqual(complete); + }); + + it('caches derived text on an unmarked record without text', async () => { + const { file_id, user } = await seedFile({ metadata: { destinationChosen: false } }); + + const saved = await fileMethods.saveFileTextDerivation( + { file_id, text: 'derived', textDerivation: complete }, + { user }, + ); + + expect(saved).toBe(true); + const stored = await fileMethods.findFileById(file_id); + expect(stored?.text).toBe('derived'); + expect(stored?.metadata?.textDerivation).toEqual(complete); + }); + + it.each([ + ['complete', complete], + ['failed', failed], + ])('leaves a %s record untouched', async (_label, textDerivation) => { + const { file_id, user } = await seedFile({ + metadata: { destinationChosen: false, textDerivation }, + }); + + const saved = await fileMethods.saveFileTextDerivation( + { file_id, text: 'derived', textDerivation: complete }, + { user }, + ); + + expect(saved).toBe(false); + const stored = await fileMethods.findFileById(file_id); + expect(stored?.text).toBeUndefined(); + expect(stored?.metadata?.textDerivation).toEqual(textDerivation); + }); + + it('leaves a deferred record that already holds text untouched', async () => { + const { file_id, user } = await seedFile({ text: 'extracted at upload' }); + + const saved = await fileMethods.saveFileTextDerivation( + { file_id, text: 'derived', textDerivation: complete }, + { user }, + ); + + expect(saved).toBe(false); + const stored = await fileMethods.findFileById(file_id); + expect(stored?.text).toBe('extracted at upload'); + expect(stored?.metadata?.textDerivation).toEqual(deferred); + }); + + it('enforces the owner scope', async () => { + const { file_id, user } = await seedFile(); + const otherUser = new mongoose.Types.ObjectId().toString(); + + const denied = await fileMethods.saveFileTextDerivation( + { file_id, text: 'derived', textDerivation: complete }, + { user: otherUser }, + ); + const unscoped = await fileMethods.saveFileTextDerivation( + { file_id, text: 'derived', textDerivation: complete }, + { user: '' }, + ); + + expect(denied).toBe(false); + expect(unscoped).toBe(false); + expect((await fileMethods.findFileById(file_id))?.text).toBeUndefined(); + + const allowed = await fileMethods.saveFileTextDerivation( + { file_id, text: 'derived', textDerivation: complete }, + { user }, + ); + expect(allowed).toBe(true); + }); + + it('enforces the tenant scope', async () => { + const { file_id, user } = await tenantStorage.run({ tenantId: 'tenant-a' }, () => + seedFile({ tenantId: 'tenant-a' }), + ); + + const denied = await fileMethods.saveFileTextDerivation( + { file_id, text: 'derived', textDerivation: complete }, + { user, tenantId: 'tenant-b' }, + ); + + expect(denied).toBe(false); + expect((await fileMethods.findFileById(file_id))?.text).toBeUndefined(); + + const allowed = await tenantStorage.run({ tenantId: 'tenant-a' }, () => + fileMethods.saveFileTextDerivation( + { file_id, text: 'derived', textDerivation: complete }, + { user, tenantId: 'tenant-a' }, + ), + ); + expect(allowed).toBe(true); + expect((await fileMethods.findFileById(file_id))?.text).toBe('derived'); + }); + + it('keeps the upload TTL, the retention deadline and the content timestamp', async () => { + const { file_id, user } = await seedFile(); + const expiresAt = new Date(Date.now() + 30 * 60_000); + const expiredAt = new Date(Date.now() + 30 * 24 * 3_600_000); + const updatedAt = new Date('2020-01-01T00:00:00.000Z'); + await File.collection.updateOne({ file_id }, { $set: { expiresAt, expiredAt, updatedAt } }); + + const saved = await fileMethods.saveFileTextDerivation( + { file_id, text: 'derived', textDerivation: complete }, + { user }, + ); + + expect(saved).toBe(true); + const stored = await fileMethods.findFileById(file_id); + expect(stored?.expiresAt).toEqual(expiresAt); + expect(stored?.expiredAt).toEqual(expiredAt); + expect(stored?.updatedAt).toEqual(updatedAt); + }); + + it('saves a failed marker without text, after which the record takes no text', async () => { + const { file_id, user } = await seedFile(); + + const saved = await fileMethods.saveFileTextDerivation( + { file_id, textDerivation: failed }, + { user }, + ); + const later = await fileMethods.saveFileTextDerivation( + { file_id, text: 'derived', textDerivation: complete }, + { user }, + ); + + expect(saved).toBe(true); + expect(later).toBe(false); + const stored = await fileMethods.findFileById(file_id); + expect(stored?.text).toBeUndefined(); + expect(stored?.metadata?.textDerivation).toEqual(failed); + expect(stored?.metadata?.destinationChosen).toBe(false); + }); + + it('rejects a marker outside the outcome vocabulary', async () => { + const { file_id, user } = await seedFile(); + const outcome: string = 'abandoned'; + + await expect( + fileMethods.saveFileTextDerivation( + { file_id, textDerivation: { outcome: outcome as TextDerivation['outcome'] } }, + { user }, + ), + ).rejects.toThrow(/not a valid enum value/); + + const stored = await fileMethods.findFileById(file_id); + expect(stored?.metadata?.textDerivation).toEqual(deferred); + }); + }); + describe('updateFilesUsage', () => { it('should update usage for multiple files', async () => { const userId = new mongoose.Types.ObjectId(); diff --git a/packages/data-schemas/src/methods/file.ts b/packages/data-schemas/src/methods/file.ts index 2c74b1c9764..3539ebbf3d0 100644 --- a/packages/data-schemas/src/methods/file.ts +++ b/packages/data-schemas/src/methods/file.ts @@ -11,6 +11,7 @@ import type { CodeFileCommitData, RunArtifactRunScope, PublishRunArtifactInput, + FileTextDerivationUpdate, } from '~/types/file'; import type { IChatProjectDocument } from '~/types'; import { tenantSafeBulkWrite } from '~/utils/tenantBulkWrite'; @@ -354,6 +355,10 @@ export function createFileMethods(mongoose: typeof import('mongoose')): { user?: string; tenantId?: string | null; }) => Promise; + saveFileTextDerivation: ( + update: FileTextDerivationUpdate, + scope: { user: string; tenantId?: string | null }, + ) => Promise; deleteFile: (file_id: string) => Promise; deleteFiles: (file_ids: string[], user?: string) => Promise<{ deletedCount?: number }>; deleteFileByFilter: (filter: FilterQuery) => Promise; @@ -1250,6 +1255,50 @@ export function createFileMethods(mongoose: typeof import('mongoose')): { }).lean(); } + /** + * Saves text derived at turn time onto a record whose upload deferred extraction. It applies + * only while the record is still marked `deferred` and holds no text, so a record that was + * never deferred, or one another derivation already settled, is left as it is. The text comes + * from the unchanged original, so the upload TTL, the retention deadline and the content + * timestamp all stay where they are. + * + * @param update - The file, the derived text when there is any, and the marker to record + * @param scope - Owner scope; a mismatch leaves the record unchanged + * @returns Whether a record was updated + */ + async function saveFileTextDerivation( + update: FileTextDerivationUpdate, + scope: { user: string; tenantId?: string | null }, + ): Promise { + if (!scope.user) { + return false; + } + const File = mongoose.models.File as Model; + const { file_id, text, textDerivation } = update; + /* A deferred record, or an unmarked one, takes a derivation while it stores no text. */ + const filter = withOwnerScope( + { + file_id, + $and: [ + { + $or: [ + { 'metadata.textDerivation.outcome': 'deferred' }, + { 'metadata.textDerivation': { $exists: false } }, + ], + }, + { $or: [{ text: { $exists: false } }, { text: null }, { text: '' }] }, + ], + }, + { userId: scope.user, tenantId: scope.tenantId }, + ); + const result = await File.updateOne( + filter, + { $set: { ...(text != null && { text }), 'metadata.textDerivation': textDerivation } }, + { timestamps: false, runValidators: true }, + ); + return result.modifiedCount > 0; + } + /** * Removes deleted file ids from the owner's Chat Projects so a project never * keeps a dangling reference, bumping the revision of every project it changes. @@ -1570,6 +1619,7 @@ export function createFileMethods(mongoose: typeof import('mongoose')): { updateFileCodeEnvRef, addFileEmbeddedEntity, updateFileUsage, + saveFileTextDerivation, deleteFile, deleteFiles, deleteFileByFilter, diff --git a/packages/data-schemas/src/schema/file.ts b/packages/data-schemas/src/schema/file.ts index f8445ad9dcc..276b31966ad 100644 --- a/packages/data-schemas/src/schema/file.ts +++ b/packages/data-schemas/src/schema/file.ts @@ -1,6 +1,6 @@ import mongoose, { Schema } from 'mongoose'; import { FileContext, FileSources } from 'librechat-data-provider'; -import type { RunFileProvenance } from 'librechat-data-provider'; +import type { RunFileProvenance, TextDerivation } from 'librechat-data-provider'; import type { IMongoFile } from '~/types'; import { codeEnvRefMapSchema, codeEnvRefSchema } from './codeEnvRef'; @@ -19,6 +19,16 @@ const runFileProvenanceSchema = new Schema( { _id: false }, ); +const textDerivationSchema = new Schema( + { + outcome: { type: String, enum: ['deferred', 'complete', 'failed'], required: true }, + extractor: { type: String }, + reason: { type: String }, + at: { type: Number }, + }, + { _id: false }, +); + const file: Schema = new Schema( { user: { @@ -177,6 +187,13 @@ const file: Schema = new Schema( type: String, default: undefined, }, + /** How the text a reader may need was obtained, under the automatic reading policy. + * A `deferred` or unmarked record without text accepts text derived at turn + * time; a failed marker is kept only by a deferred record. */ + textDerivation: { + type: textDerivationSchema, + default: undefined, + }, }, llmDeliveryPath: { /* What upload time inferred about delivery, from the endpoint and MIME type it saw. diff --git a/packages/data-schemas/src/types/file.ts b/packages/data-schemas/src/types/file.ts index 012f9bbb072..95e012cdd24 100644 --- a/packages/data-schemas/src/types/file.ts +++ b/packages/data-schemas/src/types/file.ts @@ -4,6 +4,7 @@ import type { CodeEnvRefMap, FileContext, RunFileProvenance, + TextDerivation, TFile, } from 'librechat-data-provider'; @@ -82,6 +83,14 @@ export type RunArtifactClaim = { file?: RunArtifactFile; }; +/** What a turn-time text derivation writes onto the record whose upload deferred extraction. */ +export interface FileTextDerivationUpdate { + file_id: string; + /** The derived text; absent when only a failed marker is recorded. */ + text?: string; + textDerivation: TextDerivation; +} + export interface IMongoFile extends Omit { user: Types.ObjectId; conversationId?: string; @@ -161,6 +170,8 @@ export interface IMongoFile extends Omit { destinationChosen?: boolean; /** The type the delivery route was resolved against, when conversion changed it. */ routingMimeType?: string; + /** How the text a reader may need was obtained, under the automatic reading policy. */ + textDerivation?: TextDerivation; }; /** Upload-time inference, not a durable contract. See the schema field for why. */ llmDeliveryPath?: string; From 20bb97d3fdf1e06f6c2245130ec9ecebba995588 Mon Sep 17 00:00:00 2001 From: Joe Licata Date: Tue, 6 Oct 2026 17:07:01 -0400 Subject: [PATCH 2/2] =?UTF-8?q?=F0=9F=A7=AE=20test:=20Classify=20the=20Upl?= =?UTF-8?q?oad-Time=20Run=20Code=20Capability=20Read?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The capability-gate invariant pins every read of the Run Code and File Search capabilities per file. Upload processing gained one: the read that decides whether a spreadsheet is left to Run Code, paired with the Run Code role grant inside resolveUploadCodePossible. The count for process.js moves from five to six with that pairing recorded. --- api/server/services/__tests__/toolCapabilityGates.spec.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/api/server/services/__tests__/toolCapabilityGates.spec.js b/api/server/services/__tests__/toolCapabilityGates.spec.js index 1b8693ddc28..c947ab6d7b2 100644 --- a/api/server/services/__tests__/toolCapabilityGates.spec.js +++ b/api/server/services/__tests__/toolCapabilityGates.spec.js @@ -38,9 +38,11 @@ const CLASSIFIED = { /** `codeEnvAvailable` for the memory-agent initializer. */ 'api/server/controllers/agents/client.js': 1, /** Upload processing: two consumer-map entries, the role-gated consumer checks, - * and the code-environment/file-search branches. All consumer selection shares - * the role grant resolved before routing. */ - 'api/server/services/Files/process.js': 5, + * the code-environment/file-search branches, and the upload-time Run Code read + * that decides whether a spreadsheet is left to code, paired with the Run Code + * grant inside `resolveUploadCodePossible`. All consumer selection shares the + * role grant resolved before routing. */ + 'api/server/services/Files/process.js': 6, /** Agent-management upload purposes. */ 'api/server/routes/agents/management.js': 2, /** Two gates, each paired when the embedder wires `getRoleByName`; one in a