diff --git a/.env.example b/.env.example index 4146dc3e5a7..78a3fc446be 100644 --- a/.env.example +++ b/.env.example @@ -1198,6 +1198,17 @@ EMAIL_FROM=noreply@librechat.ai # # Optional: For EU region # MAILGUN_HOST=https://api.eu.mailgun.net +#========================# +# Stored File Links # +#========================# + +# Serve files kept in S3 or Azure Blob Storage through LibreChat (/api/stored-files) +# instead of presigned or public storage URLs. Links never expire, and browsers never +# reach the bucket or container, so it can stay private (AZURE_STORAGE_PUBLIC_ACCESS=false). +# The route admits a file's owner, any signed-in user for avatars, and anyone the file's +# own access rules allow (files shared through agents), within their tenant. +# STORAGE_PROXY_FILES=false + #========================# # Firebase CDN # #========================# diff --git a/api/server/index.js b/api/server/index.js index 9d28fc7a17a..2314d4cf12e 100644 --- a/api/server/index.js +++ b/api/server/index.js @@ -15,6 +15,8 @@ const mongoSanitize = require('express-mongo-sanitize'); const { logger, runAsSystem } = require('@librechat/data-schemas'); const { isEnabled, + STORED_FILES_ROUTE, + isStoredFileProxyEnabled, issueCsp, apiNotFound, createMetrics, @@ -432,6 +434,9 @@ const startServer = async () => { app.use('/api/models', routes.models); app.use('/api/config', preAuthTenantMiddleware, optionalJwtAuth, routes.config); app.use('/api/assistants', routes.assistants); + if (isStoredFileProxyEnabled()) { + app.use(STORED_FILES_ROUTE, routes.storedFiles); + } app.use('/api/files', await routes.files.initialize()); app.use( '/images/', diff --git a/api/server/middleware/accessResources/fileAccess.js b/api/server/middleware/accessResources/fileAccess.js index 1864db4c459..e68cb4cc7eb 100644 --- a/api/server/middleware/accessResources/fileAccess.js +++ b/api/server/middleware/accessResources/fileAccess.js @@ -76,6 +76,32 @@ const denyFileAccess = (res) => message: 'Insufficient permissions to access this file', }); +/** + * Whether a user may access a file: same tenant (for tenant-scoped files), then + * ownership, then agent-based access through the agents it is attached to. + * @param {{ id: string, role?: string, tenantId?: unknown }} user + * @param {{ file_id: string, user?: unknown, tenantId?: unknown }} file + * @returns {Promise} + */ +const canAccessFile = async (user, file) => { + const fileTenantId = getTenantId(file.tenantId); + // Tenant-scoped files are restricted to their tenant. Legacy files without + // tenantId remain governed by owner/agent ACLs for non-tenant migrations. + if (fileTenantId && fileTenantId !== getTenantId(user.tenantId)) { + logger.warn(`[fileAccess] User ${user.id} denied cross-tenant access to file ${file.file_id}`); + return false; + } + if (file.user && file.user.toString() === user.id) { + return true; + } + return checkAgentBasedFileAccess({ + userId: user.id, + role: user.role, + fileId: file.file_id, + fileOwner: file.user, + }); +}; + /** * Middleware to check if user can access a file * Checks: 1) File ownership, 2) Agent-based access through attached agents @@ -107,30 +133,7 @@ const fileAccess = async (req, res, next) => { }); } - const fileTenantId = getTenantId(file.tenantId); - const userTenantId = getTenantId(req.user?.tenantId); - // Tenant-scoped files are restricted to their tenant. Legacy files without - // tenantId remain governed by owner/agent ACLs for non-tenant migrations. - if (fileTenantId && fileTenantId !== userTenantId) { - logger.warn( - `[fileAccess] User ${userId} denied cross-tenant access to file ${fileId} (route ${req.originalUrl})`, - ); - return denyFileAccess(res); - } - - if (file.user && file.user.toString() === userId) { - req.fileAccess = { file }; - return next(); - } - - /** Agent-based access (file inherits agent permissions) */ - const hasAgentAccess = await checkAgentBasedFileAccess({ - userId, - role: userRole, - fileId, - fileOwner: file.user, - }); - if (hasAgentAccess) { + if (await canAccessFile({ id: userId, role: userRole, tenantId: req.user?.tenantId }, file)) { req.fileAccess = { file }; return next(); } @@ -150,4 +153,5 @@ const fileAccess = async (req, res, next) => { module.exports = { fileAccess, + canAccessFile, }; diff --git a/api/server/middleware/accessResources/fileAccess.spec.js b/api/server/middleware/accessResources/fileAccess.spec.js index 68aea1157d8..03c44da7a56 100644 --- a/api/server/middleware/accessResources/fileAccess.spec.js +++ b/api/server/middleware/accessResources/fileAccess.spec.js @@ -2,7 +2,7 @@ const mongoose = require('mongoose'); const { tenantStorage } = require('@librechat/data-schemas'); const { ResourceType, PrincipalType, PrincipalModel } = require('librechat-data-provider'); const { MongoMemoryServer } = require('mongodb-memory-server'); -const { fileAccess } = require('./fileAccess'); +const { fileAccess, canAccessFile } = require('./fileAccess'); const { User, Role, AclEntry } = require('~/db/models'); const { createAgent, createFile } = require('~/models'); @@ -646,4 +646,69 @@ describe('fileAccess middleware', () => { expect(res.status).toHaveBeenCalledWith(403); }); }); + describe('canAccessFile', () => { + const viewer = () => ({ id: testUser._id.toString(), role: 'USER' }); + + test('admits the owner of a file record', async () => { + const file = await createFile({ + user: testUser._id.toString(), + file_id: 'owned_file', + filepath: '/test/owned.txt', + filename: 'owned.txt', + type: 'text/plain', + size: 100, + }); + + await expect(canAccessFile(viewer(), file)).resolves.toBe(true); + }); + + test('admits a viewer of an agent the file is attached to, and refuses others', async () => { + const file = await createFile({ + user: otherUser._id.toString(), + file_id: 'agent_file', + filepath: '/test/agent.txt', + filename: 'agent.txt', + type: 'text/plain', + size: 100, + }); + await expect(canAccessFile(viewer(), file)).resolves.toBe(false); + + const agent = await createAgent({ + id: `agent_${Date.now()}`, + name: 'Shared Agent', + provider: 'openai', + model: 'gpt-4', + author: otherUser._id, + tool_resources: { file_search: { file_ids: ['agent_file'] } }, + }); + await AclEntry.create({ + principalType: PrincipalType.USER, + principalId: testUser._id, + principalModel: PrincipalModel.USER, + resourceType: ResourceType.AGENT, + resourceId: agent._id, + permBits: 1, + grantedBy: otherUser._id, + }); + + await expect(canAccessFile(viewer(), file)).resolves.toBe(true); + }); + + test("refuses a tenant-scoped file to another tenant's user, even its owner", async () => { + const file = await tenantStorage.run({ tenantId: 'tenant-a' }, async () => + createFile({ + user: testUser._id.toString(), + file_id: 'tenant_file', + filepath: '/test/tenant.txt', + filename: 'tenant.txt', + type: 'text/plain', + size: 100, + tenantId: 'tenant-a', + }), + ); + + await expect(canAccessFile({ ...viewer(), tenantId: 'tenant-b' }, file)).resolves.toBe(false); + await expect(canAccessFile({ ...viewer(), tenantId: 'tenant-a' }, file)).resolves.toBe(true); + }); + }); }); diff --git a/api/server/routes/__tests__/storedFiles.spec.js b/api/server/routes/__tests__/storedFiles.spec.js new file mode 100644 index 00000000000..1bdcca6f9d6 --- /dev/null +++ b/api/server/routes/__tests__/storedFiles.spec.js @@ -0,0 +1,126 @@ +const express = require('express'); +const jwt = require('jsonwebtoken'); +const request = require('supertest'); +const { Readable } = require('stream'); +const { mockClient } = require('aws-sdk-client-mock'); +const { sdkStreamMixin } = require('@smithy/util-stream'); +const { S3Client, GetObjectCommand } = require('@aws-sdk/client-s3'); + +jest.mock('~/models', () => ({ + findSession: jest.fn(), + getUserById: jest.fn(), + getFiles: jest.fn(), + getAgents: jest.fn(), +})); + +jest.mock('~/server/services/PermissionService', () => ({ + getEffectivePermissions: jest.fn(), +})); + +const OWNER_ID = '65cfb246f7ecadb8b1e8036b'; +const OTHER_ID = '65cfb246f7ecadb8b1e8036c'; +const KEY_PATH = `/api/stored-files/s3/images/${OWNER_ID}/file-1__photo.png`; + +describe('stored file route', () => { + const originalEnv = { ...process.env }; + const s3Mock = mockClient(S3Client); + let app; + let models; + + beforeAll(() => { + process.env.AWS_REGION = 'eu-west-2'; + process.env.AWS_BUCKET_NAME = 'test-bucket'; + process.env.JWT_REFRESH_SECRET = 'stored-files-secret'; + process.env.ENFORCE_TWO_FACTOR_AUTHENTICATION = 'false'; + const { STORED_FILES_ROUTE } = require('@librechat/api'); + models = require('~/models'); + app = express(); + app.use(STORED_FILES_ROUTE, require('../storedFiles')); + }); + + afterAll(() => { + process.env = originalEnv; + s3Mock.restore(); + }); + + beforeEach(() => { + jest.clearAllMocks(); + s3Mock.reset(); + s3Mock + .on(GetObjectCommand) + .callsFake(() => ({ Body: sdkStreamMixin(Readable.from([Buffer.from('png')])) })); + models.findSession.mockResolvedValue({ _id: 'session' }); + models.getUserById.mockResolvedValue({ provider: 'local', role: 'USER' }); + models.getFiles.mockResolvedValue([]); + models.getAgents.mockResolvedValue([]); + }); + + const signedCookie = (userId) => + `refreshToken=${jwt.sign({ id: userId }, process.env.JWT_REFRESH_SECRET, { expiresIn: '1h' })}`; + + it('streams the object to the owner named in the key', async () => { + const res = await request(app).get(KEY_PATH).set('Cookie', signedCookie(OWNER_ID)); + + expect(res.status).toBe(200); + expect(res.headers['content-type']).toBe('image/png'); + expect(res.body.toString()).toBe('png'); + expect(s3Mock.commandCalls(GetObjectCommand)[0].args[0].input).toEqual({ + Bucket: 'test-bucket', + Key: `images/${OWNER_ID}/file-1__photo.png`, + }); + }); + + it('asks for a session cookie', async () => { + expect((await request(app).get(KEY_PATH)).status).toBe(401); + expect(s3Mock.commandCalls(GetObjectCommand)).toHaveLength(0); + }); + + it('refuses a cookie whose session has ended', async () => { + models.findSession.mockResolvedValue(null); + + const res = await request(app).get(KEY_PATH).set('Cookie', signedCookie(OWNER_ID)); + + expect(res.status).toBe(403); + expect(s3Mock.commandCalls(GetObjectCommand)).toHaveLength(0); + }); + + it("does not reveal another user's file", async () => { + const res = await request(app).get(KEY_PATH).set('Cookie', signedCookie(OTHER_ID)); + + expect(res.status).toBe(404); + expect(s3Mock.commandCalls(GetObjectCommand)).toHaveLength(0); + expect(models.getFiles).toHaveBeenCalledWith( + { + user: OWNER_ID, + source: 's3', + $or: [{ storageKey: `images/${OWNER_ID}/file-1__photo.png` }, { filepath: KEY_PATH }], + }, + null, + { text: 0 }, + 1, + ); + }); + + it('serves a file attached to an agent the viewer can view', async () => { + const { getEffectivePermissions } = require('~/server/services/PermissionService'); + models.getFiles.mockResolvedValue([{ file_id: 'file-1', user: OWNER_ID }]); + models.getAgents.mockResolvedValue([{ _id: 'agent-db-id', id: 'agent_1', author: OWNER_ID }]); + getEffectivePermissions.mockResolvedValue(1); + + const res = await request(app).get(KEY_PATH).set('Cookie', signedCookie(OTHER_ID)); + + expect(res.status).toBe(200); + expect(getEffectivePermissions).toHaveBeenCalledWith( + expect.objectContaining({ userId: OTHER_ID, role: 'USER', resourceId: 'agent-db-id' }), + ); + }); + + it('serves avatars to any signed-in user', async () => { + const res = await request(app) + .get(`/api/stored-files/s3/avatars/${OWNER_ID}/avatar-123.png`) + .set('Cookie', signedCookie(OTHER_ID)); + + expect(res.status).toBe(200); + expect(models.getFiles).not.toHaveBeenCalled(); + }); +}); diff --git a/api/server/routes/index.js b/api/server/routes/index.js index 14afa6f2691..4c604dae806 100644 --- a/api/server/routes/index.js +++ b/api/server/routes/index.js @@ -14,6 +14,7 @@ const adminUsers = require('./admin/users'); const adminAuditLog = require('./admin/audit'); const endpoints = require('./endpoints'); const staticRoute = require('./static'); +const storedFiles = require('./storedFiles'); const messages = require('./messages'); const memories = require('./memories'); const presets = require('./presets'); @@ -89,5 +90,6 @@ module.exports = { assistants, categories, staticRoute, + storedFiles, accessPermissions, }; diff --git a/api/server/routes/storedFiles.js b/api/server/routes/storedFiles.js new file mode 100644 index 00000000000..52dfdd5e09b --- /dev/null +++ b/api/server/routes/storedFiles.js @@ -0,0 +1,43 @@ +const cookie = require('cookie'); +const { FileSources } = require('librechat-data-provider'); +const { + isEnabled, + s3FileSource, + azureFileSource, + getStoredFileURL, + authenticateViewer, + createStoredFileHandler, +} = require('@librechat/api'); +const { canAccessFile } = require('~/server/middleware/accessResources/fileAccess'); +const { findSession, getUserById, getFiles } = require('~/models'); + +const cookieAuth = { + parseCookies: cookie.parse, + isOpenIdReuseEnabled: () => isEnabled(process.env.OPENID_REUSE_TOKENS), + findSession, + getUserById, +}; + +/** The stored file's own access rules, as for `/api/files/download`. */ +const canViewFile = async (viewer, { source, key, ownerId }) => { + const [file] = await getFiles( + { + user: ownerId, + source, + $or: [{ storageKey: key }, { filepath: getStoredFileURL(source, key) }], + }, + null, + { text: 0 }, + 1, + ); + if (!file) { + return false; + } + return canAccessFile({ id: viewer.userId, role: viewer.role, tenantId: viewer.tenantId }, file); +}; + +module.exports = createStoredFileHandler({ + authenticate: (req) => authenticateViewer(req, cookieAuth), + sources: { [FileSources.s3]: s3FileSource, [FileSources.azure_blob]: azureFileSource }, + canViewFile, +}); diff --git a/api/server/services/Files/Azure/crud.js b/api/server/services/Files/Azure/crud.js index 1fea4db7d5c..bd1cfed90d1 100644 --- a/api/server/services/Files/Azure/crud.js +++ b/api/server/services/Files/Azure/crud.js @@ -5,6 +5,8 @@ const fetch = require('node-fetch'); const { logger } = require('@librechat/data-schemas'); const { deleteRagFile, + getAzureBlobPath, + getAzureFileLink, assertRemoteFileURL, getSafeErrorMetadata, getAzureContainerClient, @@ -44,7 +46,7 @@ async function saveBufferToAzure({ const blobPath = `${basePath}/${userId}/${fileName}`; const blockBlobClient = containerClient.getBlockBlobClient(blobPath); await blockBlobClient.uploadData(buffer); - return blockBlobClient.url; + return getAzureFileLink(blockBlobClient.url, blobPath, containerName); } catch (error) { logger.error('[saveBufferToAzure] Error uploading buffer:', error); throw error; @@ -106,7 +108,7 @@ async function getAzureURL({ fileName, basePath = defaultBasePath, userId, conta const containerClient = await getAzureContainerClient(containerName); const blobPath = userId ? `${basePath}/${userId}/${fileName}` : `${basePath}/${fileName}`; const blockBlobClient = containerClient.getBlockBlobClient(blobPath); - return blockBlobClient.url; + return getAzureFileLink(blockBlobClient.url, blobPath, containerName); } catch (error) { logger.error('[getAzureURL] Error retrieving blob URL:', error); throw error; @@ -125,7 +127,8 @@ async function deleteFileFromAzure(req, file) { try { const containerClient = await getAzureContainerClient(AZURE_CONTAINER_NAME); - const blobPath = file.filepath.split(`${AZURE_CONTAINER_NAME}/`)[1]; + const blobPath = + getAzureBlobPath(file.filepath) ?? file.filepath.split(`${AZURE_CONTAINER_NAME}/`)[1]; if (!blobPath.includes(req.user.id)) { throw new Error('User ID not found in blob path'); } @@ -193,7 +196,7 @@ async function streamFileToAzure({ }, ); - return blockBlobClient.url; + return getAzureFileLink(blockBlobClient.url, blobPath, containerName); } catch (error) { logger.error('[streamFileToAzure] Error streaming file:', error); throw error; @@ -243,6 +246,16 @@ async function uploadFileToAzure({ } } +async function downloadBlob(containerClient, blobPath, signal) { + const response = await containerClient.getBlockBlobClient(blobPath).download(0, undefined, { + abortSignal: signal, + }); + if (!response.readableStreamBody) { + throw new Error('Azure Blob download returned no readable stream'); + } + return response.readableStreamBody; +} + /** * Retrieves a readable stream for a blob from Azure Blob Storage. * @@ -252,6 +265,11 @@ async function uploadFileToAzure({ */ async function getAzureFileStream(_req, fileURL, { signal } = {}) { try { + /** LibreChat's own links hold the blob path in the configured container. */ + const storedBlobPath = getAzureBlobPath(fileURL); + if (storedBlobPath != null) { + return await downloadBlob(await getAzureContainerClient(), storedBlobPath, signal); + } const url = new URL(fileURL); const configuredClient = await getAzureContainerClient(); const configuredURL = configuredClient.url ? new URL(configuredClient.url) : undefined; @@ -280,13 +298,7 @@ async function getAzureFileStream(_req, fileURL, { signal } = {}) { if (!blobPath) { throw new Error('Invalid Azure Blob URL'); } - const response = await containerClient.getBlockBlobClient(blobPath).download(0, undefined, { - abortSignal: signal, - }); - if (!response.readableStreamBody) { - throw new Error('Azure Blob download returned no readable stream'); - } - return response.readableStreamBody; + return await downloadBlob(containerClient, blobPath, signal); } catch (error) { logger.error('[getAzureFileStream] Error getting blob stream:', getSafeErrorMetadata(error)); throw error; diff --git a/api/server/services/Files/Azure/crud.spec.js b/api/server/services/Files/Azure/crud.spec.js index bafbb98430e..64ca53237af 100644 --- a/api/server/services/Files/Azure/crud.spec.js +++ b/api/server/services/Files/Azure/crud.spec.js @@ -1,17 +1,27 @@ const mockDownload = jest.fn(); -const mockGetBlockBlobClient = jest.fn(() => ({ download: mockDownload })); +const mockUploadData = jest.fn(); +const mockDelete = jest.fn(); +const mockGetBlockBlobClient = jest.fn((blobPath) => ({ + url: `https://account.blob.core.windows.net/files/${blobPath}`, + download: mockDownload, + uploadData: mockUploadData, + delete: mockDelete, +})); const mockGetAzureContainerClient = jest.fn(async () => ({ url: 'https://account.blob.core.windows.net/files', + createIfNotExists: jest.fn(), getBlockBlobClient: mockGetBlockBlobClient, })); const mockGetSafeErrorMetadata = jest.fn(() => ({ type: 'Error', status: 403 })); jest.mock('@librechat/data-schemas', () => ({ - logger: { error: jest.fn() }, + logger: { error: jest.fn(), debug: jest.fn() }, })); jest.mock('@librechat/api', () => ({ deleteRagFile: jest.fn(), + getAzureBlobPath: jest.requireActual('@librechat/api').getAzureBlobPath, + getAzureFileLink: jest.requireActual('@librechat/api').getAzureFileLink, assertRemoteFileURL: jest.fn((url) => url), getSafeErrorMetadata: (...args) => mockGetSafeErrorMetadata(...args), getAzureContainerClient: (...args) => mockGetAzureContainerClient(...args), @@ -21,7 +31,8 @@ jest.mock('@librechat/api', () => ({ })); const { logger } = require('@librechat/data-schemas'); -const { getAzureFileStream } = require('./crud'); +const { getStoredFileURL } = jest.requireActual('@librechat/api'); +const { getAzureFileStream, saveBufferToAzure, deleteFileFromAzure } = require('./crud'); describe('getAzureFileStream', () => { it('downloads private blobs through the authenticated Azure client', async () => { @@ -76,3 +87,50 @@ describe('getAzureFileStream', () => { expect(JSON.stringify(logger.error.mock.calls)).not.toContain(signedUrl); }); }); + +describe('stored file links', () => { + const storedLink = getStoredFileURL('azure_blob', 'images/user-1/photo one.png'); + + beforeEach(() => { + jest.clearAllMocks(); + process.env.STORAGE_PROXY_FILES = 'true'; + }); + + afterAll(() => { + delete process.env.STORAGE_PROXY_FILES; + }); + + it('stores LibreChat links for uploads when the proxy is on, and blob URLs when off', async () => { + const upload = { userId: 'user-1', buffer: Buffer.from('x'), fileName: 'photo one.png' }; + await expect(saveBufferToAzure(upload)).resolves.toBe(storedLink); + + process.env.STORAGE_PROXY_FILES = 'false'; + await expect(saveBufferToAzure(upload)).resolves.toBe( + 'https://account.blob.core.windows.net/files/images/user-1/photo one.png', + ); + }); + + it('keeps blob URLs for containers other than the configured one', async () => { + await expect( + saveBufferToAzure({ + userId: 'user-1', + buffer: Buffer.from('x'), + fileName: 'photo.png', + containerName: 'other', + }), + ).resolves.toMatch(/^https:/); + }); + + it('streams and deletes the blob behind a stored link from the configured container', async () => { + const stream = { pipe: jest.fn() }; + mockDownload.mockResolvedValue({ readableStreamBody: stream }); + + await expect(getAzureFileStream({}, storedLink)).resolves.toBe(stream); + expect(mockGetAzureContainerClient).toHaveBeenCalledWith(); + expect(mockGetBlockBlobClient).toHaveBeenCalledWith('images/user-1/photo one.png'); + + await deleteFileFromAzure({ user: { id: 'user-1' } }, { filepath: storedLink }); + expect(mockGetBlockBlobClient).toHaveBeenLastCalledWith('images/user-1/photo one.png'); + expect(mockDelete).toHaveBeenCalled(); + }); +}); diff --git a/packages/api/src/images/authorization.spec.ts b/packages/api/src/images/authorization.spec.ts index 2c24a445acc..3986e74aa8a 100644 --- a/packages/api/src/images/authorization.spec.ts +++ b/packages/api/src/images/authorization.spec.ts @@ -2,7 +2,7 @@ import jwt from 'jsonwebtoken'; import { createHash } from 'node:crypto'; import type { NextFunction, Request, Response } from 'express'; import type { ImageAuthorizationDeps } from './authorization'; -import { createImageAuthorizationMiddleware } from './authorization'; +import { authenticateViewer, createImageAuthorizationMiddleware } from './authorization'; const VIEWER_ID = '65cfb246f7ecadb8b1e8036b'; const OWNER_ID = '65cfb246f7ecadb8b1e8036c'; @@ -664,3 +664,91 @@ describe('createImageAuthorizationMiddleware', () => { expect(deps.getAssistant).not.toHaveBeenCalled(); }); }); + +describe('authenticateViewer', () => { + const originalEnforcement = process.env.ENFORCE_TWO_FACTOR_AUTHENTICATION; + let deps: ImageAuthorizationDeps; + + beforeEach(() => { + process.env.ENFORCE_TWO_FACTOR_AUTHENTICATION = 'false'; + process.env.JWT_REFRESH_SECRET = 'image-authorization-secret'; + deps = createDeps(); + }); + + afterAll(() => { + if (originalEnforcement === undefined) { + delete process.env.ENFORCE_TWO_FACTOR_AUTHENTICATION; + } else { + process.env.ENFORCE_TWO_FACTOR_AUTHENTICATION = originalEnforcement; + } + }); + + const viewerRequest = (token?: string) => + createRequest( + '/api/stored-files/s3/images/x/y.png', + token ? `refreshToken=${token}` : undefined, + ); + + it('reports a request without a session cookie as missing', async () => { + await expect(authenticateViewer(viewerRequest(), deps)).resolves.toEqual({ + status: 'missing', + }); + expect(deps.findSession).not.toHaveBeenCalled(); + expect(deps.getUserById).not.toHaveBeenCalled(); + }); + + it('authenticates an active session of a current account', async () => { + const token = signUser(VIEWER_ID); + + const auth = await authenticateViewer(viewerRequest(token), deps); + + expect(auth).toEqual( + expect.objectContaining({ + status: 'authenticated', + userId: VIEWER_ID, + role: 'USER', + tenantId: 'tenant-a', + }), + ); + expect(deps.findSession).toHaveBeenCalledWith({ userId: VIEWER_ID, refreshToken: token }); + expect(deps.getUserById).toHaveBeenCalledWith(VIEWER_ID, expect.any(String)); + }); + + it('refuses a token without a session', async () => { + (deps.findSession as jest.Mock).mockResolvedValue(null); + + await expect(authenticateViewer(viewerRequest(signUser(VIEWER_ID)), deps)).resolves.toEqual({ + status: 'invalid', + }); + }); + + it('refuses a token issued before the credentials changed', async () => { + (deps.getUserById as jest.Mock).mockResolvedValue({ + credentialsChangedAt: new Date(Date.now() + 60_000), + }); + + await expect(authenticateViewer(viewerRequest(signUser(VIEWER_ID)), deps)).resolves.toEqual({ + status: 'invalid', + }); + }); + + it('refuses an account that must still enroll in two-factor authentication', async () => { + process.env.ENFORCE_TWO_FACTOR_AUTHENTICATION = 'true'; + (deps.getUserById as jest.Mock).mockResolvedValue({ + provider: 'local', + twoFactorEnabled: false, + }); + + await expect(authenticateViewer(viewerRequest(signUser(VIEWER_ID)), deps)).resolves.toEqual({ + status: 'invalid', + }); + }); + + it('refuses a token whose account no longer exists', async () => { + (deps.getUserById as jest.Mock).mockResolvedValue(null); + + await expect(authenticateViewer(viewerRequest(signUser(VIEWER_ID)), deps)).resolves.toEqual({ + status: 'invalid', + }); + }); +}); diff --git a/packages/api/src/images/authorization.ts b/packages/api/src/images/authorization.ts index 78170b9a816..9b4abd95be0 100644 --- a/packages/api/src/images/authorization.ts +++ b/packages/api/src/images/authorization.ts @@ -61,11 +61,21 @@ type ResolvedImageConfig = { assistantEndpoints: AssistantConfig[]; }; -type CookieAuthResult = +export type CookieAuthResult = | { status: 'missing' } | { status: 'invalid' } | ({ status: 'authenticated' } & TwoFactorTokenCredential); +/** A cookie-authenticated viewer, with the account fields routes authorize against. */ +export type ViewerAuthResult = + | { status: 'missing' } + | { status: 'invalid' } + | ({ + status: 'authenticated'; + role?: string | null; + tenantId?: string; + } & TwoFactorTokenCredential); + type OpenIdCookieAuthResult = | { status: 'invalid' } | ({ status: 'legacy'; userId: string } & TokenIssuance) @@ -227,7 +237,7 @@ function getStoredPathCandidates(canonicalPath: string): string[] { async function authenticateRequest( req: ImageRequest, - deps: ImageAuthorizationDeps, + deps: Pick, ): Promise { const cookieHeader = req.headers.cookie; if (!cookieHeader) { @@ -272,6 +282,30 @@ async function authenticateRequest( return session ? { status: 'authenticated', ...credential } : { status: 'invalid' }; } +/** + * Identifies the signed-in user from the session cookie, for routes an image tag loads + * (it sends no bearer token), with the account checks the image route makes before it + * trusts a viewer: a token issued before the user's credentials changed or two-factor + * enrollment, or an account still required to enroll, is invalid. + */ +export async function authenticateViewer( + req: ImageRequest, + deps: Pick< + ImageAuthorizationDeps, + 'parseCookies' | 'isOpenIdReuseEnabled' | 'findSession' | 'getUserById' + >, +): Promise { + const auth = await authenticateRequest(req, deps); + if (auth.status !== 'authenticated') { + return auth; + } + const viewer = await runAsSystem(() => deps.getUserById(auth.userId, IMAGE_USER_PROJECTION)); + if (!viewer || isTokenRetired(auth, viewer) || isTwoFactorEnrollmentRequired(viewer)) { + return { status: 'invalid' }; + } + return { ...auth, role: viewer.role, tenantId: viewer.tenantId }; +} + function isSharedAssistant(assistant: AssistantImageRecord, configs: AssistantConfig[]): boolean { let matchingConfigs: AssistantConfig[] = []; if (assistant.endpoint) { diff --git a/packages/api/src/storage/azure/__tests__/source.test.ts b/packages/api/src/storage/azure/__tests__/source.test.ts new file mode 100644 index 00000000000..2313112ee4f --- /dev/null +++ b/packages/api/src/storage/azure/__tests__/source.test.ts @@ -0,0 +1,82 @@ +import { FileSources } from 'librechat-data-provider'; +import { azureFileSource, getAzureBlobPath, getAzureFileLink } from '../source'; +import { getStoredFileURL } from '~/storage/proxy/link'; + +const mockDownload = jest.fn(); +const mockGetProperties = jest.fn(); +const mockGetBlockBlobClient = jest.fn((_blobPath: string) => ({ + download: mockDownload, + getProperties: mockGetProperties, +})); + +jest.mock('~/cdn/azure', () => ({ + getAzureContainerClient: jest.fn(async () => ({ getBlockBlobClient: mockGetBlockBlobClient })), +})); + +const blobPath = 'images/user-1/photo one.png'; +const blobURL = `https://account.blob.core.windows.net/files/${blobPath}`; + +describe('Azure stored file links', () => { + const originalEnv = { ...process.env }; + + beforeEach(() => { + jest.clearAllMocks(); + process.env.STORAGE_PROXY_FILES = 'true'; + delete process.env.AZURE_CONTAINER_NAME; + }); + + afterAll(() => { + process.env = originalEnv; + }); + + it('links blobs of the configured container to LibreChat when the proxy is on', () => { + const link = getAzureFileLink(blobURL, blobPath); + expect(link).toBe(getStoredFileURL(FileSources.azure_blob, blobPath)); + expect(getAzureBlobPath(link)).toBe(blobPath); + expect(getAzureBlobPath(blobURL)).toBeNull(); + }); + + it('keeps blob URLs when the proxy is off or the container is not the configured one', () => { + expect(getAzureFileLink(blobURL, blobPath, 'other')).toBe(blobURL); + expect(getAzureFileLink(blobURL, blobPath, 'files')).not.toBe(blobURL); + process.env.STORAGE_PROXY_FILES = 'false'; + expect(getAzureFileLink(blobURL, blobPath)).toBe(blobURL); + }); + + it('reads owners from blob paths and recognises Azure avatar names', () => { + expect(azureFileSource.parseKey(blobPath)).toEqual({ + userId: 'user-1', + fileName: 'photo one.png', + avatar: false, + }); + expect(azureFileSource.parseKey('images/user-1/avatar-1700000000000.png')?.avatar).toBe(true); + expect( + azureFileSource.parseKey('images/user-1/agent-agent_abc-avatar-1700000000000.png')?.avatar, + ).toBe(true); + expect(azureFileSource.parseKey('images/user-1')).toBeNull(); + }); + + it('downloads blobs and answers HEAD from blob properties', async () => { + const signal = new AbortController().signal; + const body = { pipe: jest.fn() }; + mockDownload.mockResolvedValue({ readableStreamBody: body, contentLength: 5 }); + mockGetProperties.mockResolvedValue({ contentLength: 5 }); + + await expect(azureFileSource.read(blobPath, { head: false, signal })).resolves.toEqual({ + body, + contentLength: 5, + }); + expect(mockGetBlockBlobClient).toHaveBeenCalledWith(blobPath); + expect(mockDownload).toHaveBeenCalledWith(0, undefined, { abortSignal: signal }); + + await expect(azureFileSource.read(blobPath, { head: true, signal })).resolves.toEqual({ + contentLength: 5, + }); + }); + + it('recognises missing blobs', () => { + expect(azureFileSource.isNotFound({ statusCode: 404 })).toBe(true); + expect(azureFileSource.isNotFound({ code: 'BlobNotFound' })).toBe(true); + expect(azureFileSource.isNotFound({ statusCode: 403 })).toBe(false); + }); +}); diff --git a/packages/api/src/storage/azure/index.ts b/packages/api/src/storage/azure/index.ts new file mode 100644 index 00000000000..6ffed2ee350 --- /dev/null +++ b/packages/api/src/storage/azure/index.ts @@ -0,0 +1 @@ +export * from './source'; diff --git a/packages/api/src/storage/azure/source.ts b/packages/api/src/storage/azure/source.ts new file mode 100644 index 00000000000..0a0058fd51c --- /dev/null +++ b/packages/api/src/storage/azure/source.ts @@ -0,0 +1,67 @@ +import { FileSources } from 'librechat-data-provider'; +import type { Readable } from 'stream'; +import type { StoredFileSource } from '~/storage/proxy'; +import { + getStoredFileURL, + getKeyFromStoredFileURL, + isStoredFileProxyEnabled, +} from '~/storage/proxy/link'; +import { getAzureContainerClient } from '~/cdn/azure'; + +/** Azure keeps user and agent avatars beside other images, named by these patterns. */ +const AVATAR_FILE_PATTERN = /^(?:agent-.+-)?avatar-\d+\.[^/]+$/; + +const defaultContainer = () => process.env.AZURE_CONTAINER_NAME || 'files'; + +/** + * The link to store for a blob: LibreChat's own when `STORAGE_PROXY_FILES` is on, + * otherwise the blob's URL. Stored links name no container, so a blob in any + * container but the configured one keeps its URL. + */ +export function getAzureFileLink( + blobURL: string, + blobPath: string, + containerName?: string | null, +): string { + if (!isStoredFileProxyEnabled() || (containerName && containerName !== defaultContainer())) { + return blobURL; + } + return getStoredFileURL(FileSources.azure_blob, blobPath); +} + +/** The blob path behind one of LibreChat's links to Azure, or `null` for a blob URL. */ +export function getAzureBlobPath(link: string): string | null { + return getKeyFromStoredFileURL(link, FileSources.azure_blob); +} + +/** Serves blobs of the configured container through LibreChat's stored-file route. */ +export const azureFileSource: StoredFileSource = { + parseKey(key) { + const [, userId, ...rest] = key.split('/'); + const fileName = rest.join('/'); + if (!userId || !fileName) { + return null; + } + return { userId, fileName, avatar: AVATAR_FILE_PATTERN.test(fileName) }; + }, + async read(key, { head, signal }) { + const containerClient = await getAzureContainerClient(); + if (!containerClient) { + throw new Error('[azureFileSource] Azure Blob Storage not initialized'); + } + const blob = containerClient.getBlockBlobClient(key); + if (head) { + const properties = await blob.getProperties({ abortSignal: signal }); + return { contentLength: properties.contentLength }; + } + const response = await blob.download(0, undefined, { abortSignal: signal }); + return { + body: response.readableStreamBody as Readable | undefined, + contentLength: response.contentLength, + }; + }, + isNotFound(error) { + const { statusCode, code } = (error ?? {}) as { statusCode?: number; code?: string }; + return statusCode === 404 || code === 'BlobNotFound'; + }, +}; diff --git a/packages/api/src/storage/index.ts b/packages/api/src/storage/index.ts index 3639c5fa652..9105528165c 100644 --- a/packages/api/src/storage/index.ts +++ b/packages/api/src/storage/index.ts @@ -1,5 +1,7 @@ export * from './cloudfront'; export * from './s3'; +export * from './proxy'; +export * from './azure'; export * from './types'; export * from './images'; export * from './avatar'; diff --git a/packages/api/src/storage/proxy/__tests__/handler.test.ts b/packages/api/src/storage/proxy/__tests__/handler.test.ts new file mode 100644 index 00000000000..8251f8ad334 --- /dev/null +++ b/packages/api/src/storage/proxy/__tests__/handler.test.ts @@ -0,0 +1,163 @@ +import express from 'express'; +import request from 'supertest'; +import { Readable } from 'stream'; +import { tenantStorage } from '@librechat/data-schemas'; +import type { StoredFileSource, StoredFileViewer, StoredObject } from '../handler'; +import type { ViewerAuthResult } from '~/images/authorization'; +import { getStoredFileURL, STORED_FILES_ROUTE } from '../link'; +import { createStoredFileHandler } from '../handler'; + +jest.mock('@librechat/data-schemas', () => ({ + logger: { debug: jest.fn(), info: jest.fn(), warn: jest.fn(), error: jest.fn() }, + tenantStorage: { run: jest.fn((_context: unknown, fn: () => unknown) => fn()) }, +})); + +const owner = 'a1b2c3d4e5f6a1b2c3d4e5f6'; +const viewer = 'ffffffffffffffffffffffff'; +const key = `images/${owner}/file-1__photo.png`; + +/** A storage source whose keys are `[t//]//`, avatars under `avatars/`. */ +function createSource(): StoredFileSource & { + read: jest.Mock, [string, { head: boolean; signal: AbortSignal }]>; +} { + return { + parseKey: (value) => { + const parts = value.split('/'); + const tenantId = parts[0] === 't' ? parts[1] : undefined; + const [basePath, userId, ...rest] = tenantId ? parts.slice(2) : parts; + return basePath && userId && rest.length + ? { userId, fileName: rest.join('/'), tenantId, avatar: basePath === 'avatars' } + : null; + }, + read: jest.fn(async (_key: string, { head }: { head: boolean; signal: AbortSignal }) => ({ + contentLength: 5, + body: head ? undefined : Readable.from([Buffer.from('image')]), + })), + isNotFound: (error) => (error as { name?: string })?.name === 'NotFound', + }; +} + +describe('createStoredFileHandler', () => { + let auth: ViewerAuthResult; + let source: ReturnType; + let canViewFile: jest.Mock< + Promise, + [StoredFileViewer, { source: string; key: string; ownerId: string }] + >; + + const app = () => { + const server = express(); + server.use( + STORED_FILES_ROUTE, + createStoredFileHandler({ + authenticate: async () => auth, + sources: { s3: source }, + canViewFile, + }), + ); + return server; + }; + const link = (value = key, from = 's3') => getStoredFileURL(from, value); + + beforeEach(() => { + jest.clearAllMocks(); + auth = { status: 'authenticated', userId: owner }; + source = createSource(); + canViewFile = jest.fn( + async (_viewer: StoredFileViewer, _file: { source: string; key: string; ownerId: string }) => + false, + ); + }); + + it('serves an image inline to its owner', async () => { + const res = await request(app()).get(link()); + expect(res.status).toBe(200); + expect(res.body.toString()).toBe('image'); + expect(res.headers['content-type']).toBe('image/png'); + expect(res.headers['content-disposition']).toBe('inline'); + expect(res.headers['content-security-policy']).toContain('sandbox'); + expect(res.headers['x-content-type-options']).toBe('nosniff'); + expect(res.headers['cache-control']).toBe('private, max-age=3600'); + expect(source.read).toHaveBeenCalledWith(key, expect.objectContaining({ head: false })); + }); + + it('answers HEAD with the size and no body', async () => { + const res = await request(app()).head(link()); + expect(res.status).toBe(200); + expect(res.headers['content-length']).toBe('5'); + expect(source.read).toHaveBeenCalledWith(key, expect.objectContaining({ head: true })); + }); + + it('sends documents as attachments', async () => { + const res = await request(app()).get(link(`uploads/${owner}/file-2__notes.html`)); + expect(res.headers['content-type']).toBe('application/octet-stream'); + expect(res.headers['content-disposition']).toBe('attachment; filename="file-2__notes.html"'); + }); + + it('refuses requests without a valid session', async () => { + auth = { status: 'missing' }; + expect((await request(app()).get(link())).status).toBe(401); + auth = { status: 'invalid' }; + expect((await request(app()).get(link())).status).toBe(403); + expect(source.read).not.toHaveBeenCalled(); + }); + + it("hides other users' files unless the file's own access rules admit the viewer", async () => { + auth = { status: 'authenticated', userId: viewer, role: 'USER' }; + expect((await request(app()).get(link())).status).toBe(404); + expect(source.read).not.toHaveBeenCalled(); + expect(canViewFile).toHaveBeenCalledWith( + { userId: viewer, role: 'USER', tenantId: undefined }, + { source: 's3', key, ownerId: owner }, + ); + + canViewFile.mockResolvedValueOnce(true); + expect((await request(app()).get(link())).status).toBe(200); + }); + + it('shows avatars to any signed-in user without a file lookup', async () => { + auth = { status: 'authenticated', userId: viewer }; + expect((await request(app()).get(link(`avatars/${owner}/avatar-1.png`))).status).toBe(200); + expect(canViewFile).not.toHaveBeenCalled(); + }); + + it("keeps other tenants' files and avatars out of reach", async () => { + canViewFile.mockResolvedValue(true); + const avatar = link(`t/tenant-a/avatars/${owner}/avatar-1.png`); + const image = link(`t/tenant-a/images/${owner}/file-1__a.png`); + + auth = { status: 'authenticated', userId: viewer, tenantId: 'tenant-b' }; + expect((await request(app()).get(avatar)).status).toBe(404); + expect((await request(app()).get(image)).status).toBe(404); + expect(canViewFile).not.toHaveBeenCalled(); + + auth = { status: 'authenticated', userId: viewer, tenantId: 'tenant-a' }; + expect((await request(app()).get(avatar)).status).toBe(200); + expect((await request(app()).get(image)).status).toBe(200); + expect(tenantStorage.run).toHaveBeenCalledWith( + { tenantId: 'tenant-a', userId: viewer }, + expect.any(Function), + ); + }); + + it('serves only configured sources and keys that name an owner', async () => { + expect((await request(app()).get(link(key, 'azure_blob'))).status).toBe(404); + expect((await request(app()).get(`${STORED_FILES_ROUTE}/s3/${owner}`)).status).toBe(404); + expect((await request(app()).get(`${STORED_FILES_ROUTE}/s3`)).status).toBe(404); + expect(source.read).not.toHaveBeenCalled(); + }); + + it('maps a missing object to 404 and other failures to a bare 500', async () => { + source.read.mockRejectedValueOnce(Object.assign(new Error('gone'), { name: 'NotFound' })); + expect((await request(app()).get(link())).status).toBe(404); + + source.read.mockRejectedValueOnce(new Error('secret detail')); + const res = await request(app()).get(link()); + expect(res.status).toBe(500); + expect(res.text).not.toContain('secret'); + }); + + it('allows only reads', async () => { + expect((await request(app()).delete(link())).status).toBe(405); + }); +}); diff --git a/packages/api/src/storage/proxy/__tests__/link.test.ts b/packages/api/src/storage/proxy/__tests__/link.test.ts new file mode 100644 index 00000000000..2413e0ddce3 --- /dev/null +++ b/packages/api/src/storage/proxy/__tests__/link.test.ts @@ -0,0 +1,57 @@ +import { + getStoredFileURL, + parseStoredFileURL, + STORED_FILES_ROUTE, + getKeyFromStoredFileURL, + isStoredFileProxyEnabled, +} from '../link'; + +const key = 'images/a1b2c3d4e5f6a1b2c3d4e5f6/file-1__photo one.png'; + +describe('stored file links', () => { + const originalEnv = { ...process.env }; + + beforeEach(() => { + process.env.DOMAIN_CLIENT = 'https://chat.example.com'; + }); + + afterAll(() => { + process.env = originalEnv; + }); + + it('round-trips a source and key through an encoded link', () => { + const url = getStoredFileURL('s3', key); + expect(url).toBe( + `${STORED_FILES_ROUTE}/s3/images/a1b2c3d4e5f6a1b2c3d4e5f6/file-1__photo%20one.png`, + ); + expect(parseStoredFileURL(url)).toEqual({ source: 's3', key }); + expect(parseStoredFileURL(`${url}?v=123`)).toEqual({ source: 's3', key }); + }); + + it('reads a key only for the source it was written for', () => { + const url = getStoredFileURL('s3', key); + expect(getKeyFromStoredFileURL(url, 's3')).toBe(key); + expect(getKeyFromStoredFileURL(url, 'azure_blob')).toBeNull(); + }); + + it('rejects values that are not stored file links', () => { + expect(parseStoredFileURL('https://bucket.s3.amazonaws.com/images/x.png')).toBeNull(); + expect(parseStoredFileURL('/images/user/x.png')).toBeNull(); + expect(parseStoredFileURL(`${STORED_FILES_ROUTE}/s3`)).toBeNull(); + expect(parseStoredFileURL(`${STORED_FILES_ROUTE}/s3/%E0%A4%A`)).toBeNull(); + }); + + it('prefixes the base path of a subdirectory deployment', () => { + process.env.DOMAIN_CLIENT = 'https://example.com/chat/'; + const url = getStoredFileURL('s3', key); + expect(url.startsWith(`/chat${STORED_FILES_ROUTE}/s3/images/`)).toBe(true); + expect(parseStoredFileURL(url)).toEqual({ source: 's3', key }); + }); + + it('reads STORAGE_PROXY_FILES on each call', () => { + delete process.env.STORAGE_PROXY_FILES; + expect(isStoredFileProxyEnabled()).toBe(false); + process.env.STORAGE_PROXY_FILES = 'true'; + expect(isStoredFileProxyEnabled()).toBe(true); + }); +}); diff --git a/packages/api/src/storage/proxy/handler.ts b/packages/api/src/storage/proxy/handler.ts new file mode 100644 index 00000000000..1c8384f1d05 --- /dev/null +++ b/packages/api/src/storage/proxy/handler.ts @@ -0,0 +1,198 @@ +import { pipeline } from 'stream/promises'; +import { logger, tenantStorage } from '@librechat/data-schemas'; +import type { Request, Response } from 'express'; +import type { Readable } from 'stream'; +import type { ViewerAuthResult } from '~/images/authorization'; +import { sanitizeContentDispositionFilename } from '~/storage/validation'; +import { getSafeErrorMetadata } from '~/utils/errors'; + +/** Who a stored object belongs to, read from its key. */ +export interface StoredFileKey { + userId: string; + fileName: string; + tenantId?: string; + /** An avatar, which any signed-in user in the tenant may see. */ + avatar?: boolean; +} + +export interface StoredObject { + body?: Readable; + contentLength?: number; +} + +/** What the route needs from a storage strategy to serve its objects. */ +export interface StoredFileSource { + /** Reads a key's owner and file name; `null` for a key the strategy never writes. */ + parseKey: (key: string) => StoredFileKey | null; + /** Reads an object, or only its size when `head` is set. */ + read: (key: string, options: { head: boolean; signal: AbortSignal }) => Promise; + /** Whether a `read` failure means the object does not exist. */ + isNotFound: (error: unknown) => boolean; +} + +export interface StoredFileViewer { + userId: string; + role?: string | null; + tenantId?: string; +} + +export interface StoredFileDeps { + /** Identifies the signed-in user from the request's session cookie, as `/images` does. */ + authenticate: (req: Request) => Promise; + /** The storage strategies served, by file source (`s3`, ...). Other sources get a 404. */ + sources: Partial>; + /** + * Whether a viewer who does not own the object may read it: the access rules of the + * file record `ownerId` stored under `key` (for example, a file attached to an agent + * shared with the viewer). Runs in the viewer's tenant context. Without it, only + * owners can. + */ + canViewFile?: ( + viewer: StoredFileViewer, + file: { source: string; key: string; ownerId: string }, + ) => Promise; +} + +/** Raster images a browser displays; every other file is sent as a download. */ +const INLINE_TYPES: Record = { + png: 'image/png', + jpg: 'image/jpeg', + jpeg: 'image/jpeg', + jpe: 'image/jpeg', + gif: 'image/gif', + webp: 'image/webp', + bmp: 'image/bmp', + avif: 'image/avif', +}; + +/** + * The files are user uploads served from the app's own origin, so a document + * opened directly (an SVG or HTML file, say) must not run script there. + */ +const FILE_CSP = "default-src 'none'; style-src 'unsafe-inline'; sandbox"; + +/** `//`, each segment URL-encoded. */ +function decodePath(path: string): { source: string; key: string } | null { + const [source, ...segments] = path.replace(/^\/+/, '').split('/'); + if (!source || segments.length === 0) { + return null; + } + try { + const key = segments.map(decodeURIComponent).join('/'); + return key ? { source: decodeURIComponent(source), key } : null; + } catch { + return null; + } +} + +function getDisposition(fileName: string, extension: string): string { + if (INLINE_TYPES[extension]) { + return 'inline'; + } + return `attachment; filename="${sanitizeContentDispositionFilename(fileName)}"`; +} + +/** + * Serves stored files through LibreChat, so browsers never need access to the + * storage service and it can refuse requests from outside the deployment's + * network (an S3 bucket policy on aws:SourceVpce, say). Links carry the file's + * source and object key; the session cookie names the viewer. Within the + * viewer's tenant, the route serves: + * - the owner named in the key's user segment; + * - avatars to any signed-in user, as CloudFront avatar cookies do; + * - anyone `canViewFile` admits under the stored file's own access rules. + * Everyone else gets a 404. Mounted at `STORED_FILES_ROUTE`. + */ +export function createStoredFileHandler({ authenticate, sources, canViewFile }: StoredFileDeps) { + const isAllowed = async ( + viewer: StoredFileViewer, + file: { source: string; key: string }, + owner: StoredFileKey, + ): Promise => { + if (owner.userId === viewer.userId) { + return true; + } + if ((owner.tenantId ?? '') !== (viewer.tenantId ?? '')) { + return false; + } + if (owner.avatar) { + return true; + } + if (!canViewFile) { + return false; + } + const check = () => canViewFile(viewer, { ...file, ownerId: owner.userId }); + return viewer.tenantId + ? tenantStorage.run({ tenantId: viewer.tenantId, userId: viewer.userId }, check) + : check(); + }; + + return async function serveStoredFile(req: Request, res: Response): Promise { + if (req.method !== 'GET' && req.method !== 'HEAD') { + res.set('Allow', 'GET, HEAD').status(405).end(); + return; + } + + try { + const auth = await authenticate(req); + if (auth.status !== 'authenticated') { + res.status(auth.status === 'missing' ? 401 : 403).send('Unauthorized'); + return; + } + + const file = decodePath(req.path); + const source = file ? sources[file.source] : undefined; + const owner = file && source ? source.parseKey(file.key) : null; + const viewer: StoredFileViewer = { + userId: auth.userId, + role: auth.role, + tenantId: auth.tenantId, + }; + if (!file || !source || !owner || !(await isAllowed(viewer, file, owner))) { + res.status(404).send('Not Found'); + return; + } + + const controller = new AbortController(); + res.on('close', () => controller.abort()); + const head = req.method === 'HEAD'; + let object: StoredObject; + try { + object = await source.read(file.key, { head, signal: controller.signal }); + } catch (error) { + if (source.isNotFound(error)) { + res.status(404).send('Not Found'); + return; + } + throw error; + } + + const extension = owner.fileName.split('.').pop()?.toLowerCase() ?? ''; + res.set({ + 'Content-Type': INLINE_TYPES[extension] ?? 'application/octet-stream', + 'Content-Disposition': getDisposition(owner.fileName, extension), + 'Content-Security-Policy': FILE_CSP, + 'X-Content-Type-Options': 'nosniff', + 'Cache-Control': 'private, max-age=3600', + }); + if (object.contentLength != null) { + res.set('Content-Length', String(object.contentLength)); + } + if (head || !object.body) { + res.status(200).end(); + return; + } + res.status(200); + await pipeline(object.body, res); + } catch (error) { + if (res.headersSent) { + if (!res.writableFinished) { + logger.warn('[storedFiles] Stream ended early', getSafeErrorMetadata(error)); + } + return; + } + logger.error('[storedFiles] Error serving file', getSafeErrorMetadata(error)); + res.status(500).send('Internal Server Error'); + } + }; +} diff --git a/packages/api/src/storage/proxy/index.ts b/packages/api/src/storage/proxy/index.ts new file mode 100644 index 00000000000..27b7c883979 --- /dev/null +++ b/packages/api/src/storage/proxy/index.ts @@ -0,0 +1,2 @@ +export * from './link'; +export * from './handler'; diff --git a/packages/api/src/storage/proxy/link.ts b/packages/api/src/storage/proxy/link.ts new file mode 100644 index 00000000000..decef661f39 --- /dev/null +++ b/packages/api/src/storage/proxy/link.ts @@ -0,0 +1,48 @@ +import { getBasePath } from '~/utils/path'; +import { isEnabled } from '~/utils/common'; + +/** + * Where LibreChat serves stored files itself when `STORAGE_PROXY_FILES` is on. + * A link is this route, the file's storage source, then its object key, so it + * never expires; the route decides who may read it (see `createStoredFileHandler`). + */ +export const STORED_FILES_ROUTE = '/api/stored-files'; + +/** Whether storage strategies link files to LibreChat instead of to the storage service. */ +export function isStoredFileProxyEnabled(): boolean { + return isEnabled(process.env.STORAGE_PROXY_FILES); +} + +/** LibreChat's link to an object in a storage source. */ +export function getStoredFileURL(source: string, key: string): string { + const path = key + .replace(/^\/+/, '') + .split('/') + .map((segment) => encodeURIComponent(segment)) + .join('/'); + return `${getBasePath()}${STORED_FILES_ROUTE}/${encodeURIComponent(source)}/${path}`; +} + +/** The source and object key behind one of LibreChat's links, or `null` for any other value. */ +export function parseStoredFileURL(value: string): { source: string; key: string } | null { + const prefix = `${getBasePath()}${STORED_FILES_ROUTE}/`; + if (!value.startsWith(prefix)) { + return null; + } + const [source, ...segments] = value.slice(prefix.length).split(/[?#]/, 1)[0].split('/'); + if (!source || segments.length === 0) { + return null; + } + try { + const key = segments.map(decodeURIComponent).join('/'); + return key ? { source: decodeURIComponent(source), key } : null; + } catch { + return null; + } +} + +/** The object key behind one of LibreChat's links to `source`, or `null`. */ +export function getKeyFromStoredFileURL(value: string, source: string): string | null { + const parsed = parseStoredFileURL(value); + return parsed?.source === source ? parsed.key : null; +} diff --git a/packages/api/src/storage/s3/__tests__/source.test.ts b/packages/api/src/storage/s3/__tests__/source.test.ts new file mode 100644 index 00000000000..0fa9532ad24 --- /dev/null +++ b/packages/api/src/storage/s3/__tests__/source.test.ts @@ -0,0 +1,139 @@ +import { Readable } from 'stream'; +import { mockClient } from 'aws-sdk-client-mock'; +import { sdkStreamMixin } from '@smithy/util-stream'; +import { FileSources } from 'librechat-data-provider'; +import type * as ClientS3 from '@aws-sdk/client-s3'; + +jest.mock('@aws-sdk/s3-request-presigner', () => ({ + getSignedUrl: jest.fn().mockResolvedValue('https://bucket.s3.amazonaws.com/key?signed=true'), +})); + +jest.mock('@librechat/data-schemas', () => ({ + logger: { debug: jest.fn(), info: jest.fn(), warn: jest.fn(), error: jest.fn() }, +})); + +const owner = 'a1b2c3d4e5f6a1b2c3d4e5f6'; +const key = `images/${owner}/file-1__photo one.png`; + +type Modules = { + client: typeof ClientS3; + crud: typeof import('../crud'); + source: typeof import('../source'); + link: typeof import('~/storage/proxy/link'); +}; + +const signedUrl = () => { + const date = new Date(Date.now() - 60 * 1000).toISOString().replace(/[-:]/g, '').split('.')[0]; + return `https://bucket.s3.amazonaws.com/${key}?X-Amz-Signature=abc&X-Amz-Date=${date}Z&X-Amz-Expires=604800`; +}; + +describe('S3 with stored file links', () => { + const originalEnv = { ...process.env }; + let modules: Modules; + let s3Mock: ReturnType; + + beforeAll(() => { + jest.isolateModules(() => { + Object.assign(process.env, { + AWS_REGION: 'eu-west-2', + AWS_BUCKET_NAME: 'test-bucket', + DOMAIN_CLIENT: 'https://chat.example.com', + }); + modules = { + /** The isolated registry loads its own client, which is the one to mock. */ + client: jest.requireActual('@aws-sdk/client-s3'), + crud: jest.requireActual('../crud'), + source: jest.requireActual('../source'), + link: jest.requireActual('~/storage/proxy/link'), + }; + }); + s3Mock = mockClient(modules.client.S3Client); + }); + + beforeEach(() => { + s3Mock?.reset(); + process.env.STORAGE_PROXY_FILES = 'true'; + }); + + afterAll(() => { + process.env = originalEnv; + s3Mock.restore(); + }); + + const storedLink = () => modules.link.getStoredFileURL(FileSources.s3, key); + + describe('strategy', () => { + it('stores LibreChat links instead of presigned URLs and reads them back to keys', async () => { + const { getS3URL, extractKeyFromS3Url, needsRefresh } = modules.crud; + const url = await getS3URL({ userId: owner, fileName: 'file-1__photo one.png' }); + expect(url).toBe(storedLink()); + expect(extractKeyFromS3Url(url)).toBe(key); + expect(needsRefresh(url, 3600)).toBe(false); + }); + + it('replaces presigned links when the proxy is on, and stored links when it is off', async () => { + const { needsRefresh, refreshS3Url } = modules.crud; + expect(needsRefresh(signedUrl(), 0)).toBe(true); + await expect(refreshS3Url({ source: FileSources.s3, filepath: signedUrl() })).resolves.toBe( + storedLink(), + ); + + process.env.STORAGE_PROXY_FILES = 'false'; + expect(needsRefresh(storedLink(), 3600)).toBe(true); + await expect(refreshS3Url({ source: FileSources.s3, filepath: storedLink() })).resolves.toBe( + 'https://bucket.s3.amazonaws.com/key?signed=true', + ); + }); + + it('gives downloads no direct link, so callers stream through their own route', async () => { + await expect( + modules.crud.getS3DownloadURL({ + file: { filepath: storedLink() }, + customFilename: 'photo.png', + } as Parameters[0]), + ).resolves.toBe(''); + }); + }); + + describe('s3FileSource', () => { + const signal = new AbortController().signal; + + it('parses S3 keys, including tenant-scoped ones', () => { + const { s3FileSource } = modules.source; + expect(s3FileSource.parseKey(key)).toEqual( + expect.objectContaining({ basePath: 'images', userId: owner, avatar: false }), + ); + expect(s3FileSource.parseKey(`t/tenant-a/avatars/${owner}/avatar-1.png`)).toEqual( + expect.objectContaining({ userId: owner, tenantId: 'tenant-a', avatar: true }), + ); + expect(s3FileSource.parseKey(owner)).toBeNull(); + }); + + it('streams GetObject and answers HEAD with HeadObject', async () => { + const body = sdkStreamMixin(Readable.from([Buffer.from('image')])); + s3Mock.on(modules.client.GetObjectCommand).resolves({ Body: body, ContentLength: 5 }); + s3Mock.on(modules.client.HeadObjectCommand).resolves({ ContentLength: 5 }); + const { s3FileSource } = modules.source; + + const object = await s3FileSource.read(key, { head: false, signal }); + expect(object.contentLength).toBe(5); + expect(s3Mock.commandCalls(modules.client.GetObjectCommand)[0].args[0].input).toEqual({ + Bucket: 'test-bucket', + Key: key, + }); + + await expect(s3FileSource.read(key, { head: true, signal })).resolves.toEqual({ + contentLength: 5, + }); + expect(s3Mock.commandCalls(modules.client.HeadObjectCommand)).toHaveLength(1); + }); + + it('recognises missing objects', () => { + const { isNotFound } = modules.source.s3FileSource; + expect(isNotFound({ name: 'NoSuchKey' })).toBe(true); + expect(isNotFound({ name: 'NotFound' })).toBe(true); + expect(isNotFound({ $metadata: { httpStatusCode: 404 } })).toBe(true); + expect(isNotFound(new Error('AccessDenied'))).toBe(false); + }); + }); +}); diff --git a/packages/api/src/storage/s3/crud.ts b/packages/api/src/storage/s3/crud.ts index 5ddcf9cfa71..631320d81ad 100644 --- a/packages/api/src/storage/s3/crud.ts +++ b/packages/api/src/storage/s3/crud.ts @@ -51,6 +51,11 @@ import { assertPathSegment, sanitizeContentDispositionFilename, } from '~/storage/validation'; +import { + getStoredFileURL, + getKeyFromStoredFileURL, + isStoredFileProxyEnabled, +} from '~/storage/proxy/link'; import { getSafeErrorMetadata } from '~/utils/errors'; import { initializeS3 } from '~/cdn/s3'; import { deleteRagFile } from '~/files'; @@ -295,6 +300,10 @@ async function getS3URLForKey({ customFilename?: string | null; contentType?: string | null; }): Promise { + if (isStoredFileProxyEnabled()) { + return getStoredFileURL(FileSources.s3, key); + } + const params: GetObjectCommandInput = { Bucket: bucketName, Key: key }; if (customFilename) { @@ -664,6 +673,11 @@ export function extractKeyFromS3Url(fileUrlOrKey: string): string { throw new Error('Invalid input: URL or key is empty'); } + const proxiedKey = getKeyFromStoredFileURL(fileUrlOrKey, FileSources.s3); + if (proxiedKey != null) { + return proxiedKey; + } + if (!fileUrlOrKey.startsWith('http://') && !fileUrlOrKey.startsWith('https://')) { return fileUrlOrKey.startsWith('/') ? fileUrlOrKey.substring(1) : fileUrlOrKey; } @@ -879,6 +893,11 @@ export async function getS3DownloadURL({ customFilename = null, contentType = null, }: DownloadURLParams): Promise { + /** No direct link: the app serves the bytes either way, and the caller's own + * streaming route applies its access rules (a shared link's, for one). */ + if (isStoredFileProxyEnabled()) { + return ''; + } const key = resolveStoredS3Key(file); if (!key) { throw new Error('[getS3DownloadURL] Unable to extract S3 key from file path'); @@ -887,12 +906,20 @@ export async function getS3DownloadURL({ } export function needsRefresh(signedUrl: string, bufferSeconds: number): boolean { + /** LibreChat's own links never expire; with the proxy off they go back to presigned ones. */ + if (getKeyFromStoredFileURL(signedUrl, FileSources.s3) != null) { + return !isStoredFileProxyEnabled(); + } try { const url = new URL(signedUrl); if (!url.searchParams.has('X-Amz-Signature')) { return false; } + /** A presigned link stored before the proxy was turned on is replaced with LibreChat's. */ + if (isStoredFileProxyEnabled()) { + return true; + } const expiresParam = url.searchParams.get('X-Amz-Expires'); const dateParam = url.searchParams.get('X-Amz-Date'); diff --git a/packages/api/src/storage/s3/index.ts b/packages/api/src/storage/s3/index.ts index f9b6bca1080..70cff895f2f 100644 --- a/packages/api/src/storage/s3/index.ts +++ b/packages/api/src/storage/s3/index.ts @@ -1 +1,2 @@ export * from './crud'; +export * from './source'; diff --git a/packages/api/src/storage/s3/source.ts b/packages/api/src/storage/s3/source.ts new file mode 100644 index 00000000000..70102a9fa61 --- /dev/null +++ b/packages/api/src/storage/s3/source.ts @@ -0,0 +1,35 @@ +import { GetObjectCommand, HeadObjectCommand } from '@aws-sdk/client-s3'; +import type { Readable } from 'stream'; +import type { StoredFileSource } from '~/storage/proxy'; +import { AVATAR_BASE_PATH } from '~/storage/constants'; +import { initializeS3 } from '~/cdn/s3'; +import { s3Config } from './s3Config'; +import { parseS3Key } from './crud'; + +/** Serves S3 objects through LibreChat's stored-file route. */ +export const s3FileSource: StoredFileSource = { + parseKey(key) { + const parsed = parseS3Key(key); + return parsed && { ...parsed, avatar: parsed.basePath === AVATAR_BASE_PATH }; + }, + async read(key, { head, signal }) { + const s3 = initializeS3(); + if (!s3) { + throw new Error('[s3FileSource] S3 not initialized'); + } + const params = { Bucket: s3Config.AWS_BUCKET_NAME, Key: key }; + if (head) { + const data = await s3.send(new HeadObjectCommand(params), { abortSignal: signal }); + return { contentLength: data.ContentLength }; + } + const data = await s3.send(new GetObjectCommand(params), { abortSignal: signal }); + return { body: data.Body as Readable | undefined, contentLength: data.ContentLength }; + }, + isNotFound(error) { + const { name, $metadata } = (error ?? {}) as { + name?: string; + $metadata?: { httpStatusCode?: number }; + }; + return name === 'NoSuchKey' || name === 'NotFound' || $metadata?.httpStatusCode === 404; + }, +};