diff --git a/api/package.json b/api/package.json index 6e9c002a699..6fe29a0d997 100644 --- a/api/package.json +++ b/api/package.json @@ -44,6 +44,8 @@ "@azure/identity": "^4.13.1", "@azure/search-documents": "^12.0.0", "@azure/storage-blob": "^12.30.0", + "@firecrawl/anydoc": "0.1.3", + "@firecrawl/pdf-inspector": "1.12.0", "@google/genai": "^2.8.0", "@keyv/redis": "5.1.6", "@librechat/agents": "^4.0.1", diff --git a/api/server/routes/__tests__/share.spec.js b/api/server/routes/__tests__/share.spec.js index 060c463577a..83505f21ab0 100644 --- a/api/server/routes/__tests__/share.spec.js +++ b/api/server/routes/__tests__/share.spec.js @@ -1897,6 +1897,26 @@ describe('share-scoped file routes', () => { expect(getFiles).toHaveBeenCalledWith({ file_id: 'file-1' }, null, {}); }); + it('serves a parsed snapshot as text on both the preview and the file route', async () => { + getSharedLinkFile.mockResolvedValue({ + file: { file_id: 'file-1', source: 'text', filename: 'report.docx' }, + hasSnapshots: true, + }); + getFiles.mockResolvedValue([{ status: 'ready', text: '# Parsed report', textFormat: 'text' }]); + + const app = buildApp(); + const preview = await request(app).get('/api/share/share-123/files/file-1/preview'); + expect(preview.status).toBe(200); + expect(preview.body.text).toBe('# Parsed report'); + + const binary = await request(app).get('/api/share/share-123/files/file-1'); + expect(binary.status).toBe(200); + expect(binary.headers['content-type']).toContain('text/plain'); + expect(binary.text).toBe('# Parsed report'); + /** No durable object exists for a parsed record, so no storage strategy is consulted. */ + expect(mockGetStrategyFunctions).not.toHaveBeenCalled(); + }); + it('404s for a file not in the snapshot without rebuilding it', async () => { getSharedLinkFile.mockResolvedValue({ file: null, hasSnapshots: true }); diff --git a/api/server/routes/files/files.agents.test.js b/api/server/routes/files/files.agents.test.js index d1afe540567..2a0a13014b5 100644 --- a/api/server/routes/files/files.agents.test.js +++ b/api/server/routes/files/files.agents.test.js @@ -1298,6 +1298,37 @@ describe('File Routes - Agent Files Endpoint', () => { errorLogSpy.mockRestore(); }); + it('returns a stable 413 for an oversized document parser input', async () => { + processAgentFileUpload.mockRejectedValueOnce( + Object.assign(new Error('PRIVATE-UPLOAD.docx exceeds the 15MB parser limit'), { + code: 'PARSER_INPUT_LIMIT', + userErrorStatusCode: 413, + }), + ); + const errorLogSpy = jest.spyOn(logger, 'error').mockImplementation(() => {}); + const testApp = createAppWithUser(otherUserId, SystemRoles.USER, { + filters: { + files: { + pii: { + fields: ['name'], + }, + }, + }, + }); + + const response = await request(testApp).post('/files').send({ + endpoint: 'agents', + file_id: uuidv4(), + }); + + expect(response.status).toBe(413); + expect(response.body).toEqual({ + message: 'Document exceeds the supported file size limit', + }); + expect(JSON.stringify(response.body)).not.toContain('PRIVATE-UPLOAD'); + errorLogSpy.mockRestore(); + }); + it('preserves legacy upload error details when file protection is inactive', async () => { const legacyMessage = 'Invalid file format: .legacy'; processAgentFileUpload.mockRejectedValueOnce( diff --git a/api/server/routes/files/files.js b/api/server/routes/files/files.js index 8e90bd15fc3..d2ac2a0da06 100644 --- a/api/server/routes/files/files.js +++ b/api/server/routes/files/files.js @@ -154,7 +154,19 @@ router.get('/agent/:agent_id', async (req, res) => { router.get('/config', async (req, res) => { try { const appConfig = req.config; - res.status(200).json(appConfig.fileConfig); + res.status(200).json({ + ...appConfig.fileConfig, + ocr: { + ...appConfig.fileConfig?.ocr, + enabled: appConfig.ocr != null, + }, + /* Whether a RAG text service exists to route documents to. The client mirrors the + * upload path's routing, and that decision reads the environment, not the config. */ + text: { + ...appConfig.fileConfig?.text, + enabled: !!process.env.RAG_API_URL, + }, + }); } catch (error) { logger.error('[/files] Error getting fileConfig', error); res.status(400).json({ message: 'Error in request', error: error.message }); diff --git a/api/server/routes/files/files.test.js b/api/server/routes/files/files.test.js index 9df2acb0623..8a183787863 100644 --- a/api/server/routes/files/files.test.js +++ b/api/server/routes/files/files.test.js @@ -1420,6 +1420,58 @@ describe('File Routes - Delete with Agent Access', () => { }); }); + /** + * The client mirrors the upload path's routing to decide what it may offer, and both + * of these read the environment rather than the YAML, so the config endpoint is where + * that knowledge crosses over. + */ + describe('GET /files/config', () => { + const originalRagUrl = process.env.RAG_API_URL; + + afterEach(() => { + requestConfig = {}; + if (originalRagUrl === undefined) { + delete process.env.RAG_API_URL; + } else { + process.env.RAG_API_URL = originalRagUrl; + } + }); + + it('reports whether an OCR strategy and a RAG text service are configured', async () => { + process.env.RAG_API_URL = 'http://rag-api.test'; + requestConfig = { ocr: { strategy: 'mistral_ocr' }, fileConfig: {} }; + + const response = await request(app).get('/files/config'); + + expect(response.status).toBe(200); + expect(response.body.ocr.enabled).toBe(true); + expect(response.body.text.enabled).toBe(true); + }); + + it('reports both as absent when neither is configured', async () => { + delete process.env.RAG_API_URL; + requestConfig = { fileConfig: {} }; + + const response = await request(app).get('/files/config'); + + expect(response.status).toBe(200); + expect(response.body.ocr.enabled).toBe(false); + expect(response.body.text.enabled).toBe(false); + }); + + it('keeps the configured text allowlist alongside the derived flag', async () => { + process.env.RAG_API_URL = 'http://rag-api.test'; + requestConfig = { fileConfig: { text: { supportedMimeTypes: ['^text/plain$'] } } }; + + const response = await request(app).get('/files/config'); + + expect(response.body.text).toEqual({ + supportedMimeTypes: ['^text/plain$'], + enabled: true, + }); + }); + }); + describe('POST /files/usage', () => { const createQueuedFile = async (expiresAt) => { const ownFileId = uuidv4(); diff --git a/api/server/routes/files/index.js b/api/server/routes/files/index.js index e192ac02caf..86249f87327 100644 --- a/api/server/routes/files/index.js +++ b/api/server/routes/files/index.js @@ -24,7 +24,11 @@ const initialize = async () => { router.use(checkBan); router.use(uaParser); - const upload = await createMulterInstance(); + /* Per-request staging: two uploads of the same filename by the same user otherwise + * share one temporary path, and every stage that reopens it (the archive guard, the + * parser child, storage, the cleanup that deletes it) can read or remove bytes that + * belong to the other request. */ + const upload = await createMulterInstance({ uniqueTempPath: true }); router.post('/speech/stt', upload.single('audio'), restoreTenantContextFromReq); /* Important: speech route must be added before the upload limiters */ diff --git a/api/server/routes/files/index.limiters.test.js b/api/server/routes/files/index.limiters.test.js index 0e7d1cf0ec7..f0a9b9fad45 100644 --- a/api/server/routes/files/index.limiters.test.js +++ b/api/server/routes/files/index.limiters.test.js @@ -65,12 +65,17 @@ jest.mock('~/server/routes/assistants/v1', () => ({ describe('file route limiter wiring', () => { let app; + /* Captured during initialization: the suite clears mocks before each test, so the + * call that built the upload middleware is gone by the time one runs. */ + let multerOptions; beforeAll(async () => { const { initialize } = require('./index'); + const { createMulterInstance } = require('./multer'); app = express(); app.use(express.json()); app.use('/api/files', await initialize()); + multerOptions = createMulterInstance.mock.calls.at(-1)?.[0]; }); beforeEach(() => { @@ -79,6 +84,13 @@ describe('file route limiter wiring', () => { hits.usage = 0; }); + /* Shared staging is a correctness problem, not a tuning choice: two uploads of the + * same filename by one user would otherwise read and delete each other's bytes + * through every stage that reopens the temporary path. */ + it('stages uploads at a request-owned temporary path', () => { + expect(multerOptions).toEqual(expect.objectContaining({ uniqueTempPath: true })); + }); + it('meters POST /usage with the usage limiter, never leaving it unlimited', async () => { const response = await request(app) .post('/api/files/usage') diff --git a/api/server/routes/files/multer.js b/api/server/routes/files/multer.js index 5f375f39cd6..d5a24a677e3 100644 --- a/api/server/routes/files/multer.js +++ b/api/server/routes/files/multer.js @@ -2,14 +2,13 @@ const fs = require('fs'); const path = require('path'); const crypto = require('crypto'); const multer = require('multer'); -const { sanitizeFilename, createCustomError } = require('@librechat/api'); +const { sanitizeFilename, createCustomError, isAdmissibleUploadType } = require('@librechat/api'); const { logger } = require('@librechat/data-schemas'); const { mergeFileConfig, - inferMimeType, isAgentsEndpoint, + resolveEffectiveMimeType, getEndpointFileConfig, - fileConfig: defaultFileConfig, } = require('librechat-data-provider'); const { getAppConfig } = require('~/server/services/Config'); @@ -71,8 +70,13 @@ const collectSupportedMimeTypes = (customFileConfig, endpointFileConfig) => { return merged; }; +/** + * Admission reads the upload exactly as routing does. A client that types by magic bytes + * calls a `.docx` an archive, and resolving that here keeps a narrowed endpoint allowlist + * from refusing a document the parser downstream would have accepted. + */ const normalizeUploadMimeType = (file) => { - const mimeType = inferMimeType(file.originalname || '', file.mimetype || ''); + const mimeType = resolveEffectiveMimeType(file.originalname || '', file.mimetype || ''); if (mimeType && file.mimetype !== mimeType) { file.mimetype = mimeType; } @@ -103,8 +107,15 @@ const createFileFilter = (customFileConfig, resolveEndpoint) => { const resolved = resolveEndpoint?.(req); const endpoint = resolved?.endpoint ?? req.body.endpoint; const endpointType = resolved?.endpointType ?? req.body.endpointType; + /* The principal-merged config, which `configMiddleware` puts on the request before + * this route runs. The instance-level one was resolved once at startup, so a tenant, + * role or user override would be invisible here and the upload refused before the + * post-upload gate could apply the configuration that actually governs it. */ + const effectiveFileConfig = req.config?.fileConfig + ? mergeFileConfig(req.config.fileConfig) + : customFileConfig; const endpointFileConfig = getEndpointFileConfig({ - fileConfig: customFileConfig, + fileConfig: effectiveFileConfig, endpoint, endpointType, }); @@ -115,10 +126,24 @@ const createFileFilter = (customFileConfig, resolveEndpoint) => { * question is only whether any configured endpoint accepts the type. Narrowing to * `agents` would make the later provider check able to reject but never to permit. */ const supportedMimeTypes = isAgentsEndpoint(endpoint) - ? collectSupportedMimeTypes(customFileConfig, endpointFileConfig) + ? collectSupportedMimeTypes(effectiveFileConfig, endpointFileConfig) : endpointFileConfig.supportedMimeTypes; - if (!defaultFileConfig.checkType(mimeType, supportedMimeTypes)) { + /* An admin who names a type in `documentParser.supportedMimeTypes` has said the + * server parses it, so this filter admits those too. Deliberately not scoped to + * context uploads here: this runs while the file part is still streaming, before the + * fields that follow it have been parsed, and a multipart client may legally send + * `tool_resource` after the file. `filterFile` applies that scope on a complete body + * a moment later, before any provider is handed the upload, so the only thing this + * admits is a temporary file that the next gate deletes. */ + const admitted = isAdmissibleUploadType({ + mimeType, + fileConfig: effectiveFileConfig, + endpointMimeTypes: supportedMimeTypes, + admitParserTypes: true, + }); + + if (!admitted) { return cb( createCustomError(415, 'Unsupported file type: ' + (file.mimetype || mimeType)), false, diff --git a/api/server/routes/files/multer.spec.js b/api/server/routes/files/multer.spec.js index 351ed34a028..a74672dd9e9 100644 --- a/api/server/routes/files/multer.spec.js +++ b/api/server/routes/files/multer.spec.js @@ -426,6 +426,167 @@ describe('Multer Configuration', () => { expect(error.body).toEqual({ message: 'Unsupported file type: application/x-msdownload' }); }); + it.each(['application/octet-stream', 'binary/octet-stream'])( + 'should canonicalize generic MIME type %s before endpoint admission', + (genericType) => { + const { mergeFileConfig } = require('librechat-data-provider'); + const fileFilter = createFileFilter(mergeFileConfig()); + const documentFile = { + ...mockFile, + originalname: 'report.docx', + mimetype: genericType, + }; + const cb = jest.fn(); + + fileFilter(mockReq, documentFile, cb); + + expect(cb).toHaveBeenCalledWith(null, true); + expect(documentFile.mimetype).toBe( + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + ); + }, + ); + + /** + * A client that types uploads by magic bytes calls a `.docx` an archive. Admission + * and routing have to read it the same way: resolving only downstream leaves a + * narrowed endpoint allowlist refusing a document the parser would have accepted. + */ + it.each(['application/zip', 'application/x-zip-compressed'])( + 'should canonicalize a document sent as %s before endpoint admission', + (archiveType) => { + const { mergeFileConfig } = require('librechat-data-provider'); + const fileFilter = createFileFilter(mergeFileConfig()); + const documentFile = { + ...mockFile, + originalname: 'report.docx', + mimetype: archiveType, + }; + const cb = jest.fn(); + + fileFilter(mockReq, documentFile, cb); + + expect(cb).toHaveBeenCalledWith(null, true); + expect(documentFile.mimetype).toBe( + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + ); + }, + ); + + it('should leave an ordinary archive as an archive', () => { + const { mergeFileConfig } = require('librechat-data-provider'); + const fileFilter = createFileFilter(mergeFileConfig()); + const archive = { ...mockFile, originalname: 'bundle.zip', mimetype: 'application/zip' }; + const cb = jest.fn(); + + fileFilter(mockReq, archive, cb); + + expect(cb).toHaveBeenCalledWith(null, true); + expect(archive.mimetype).toBe('application/zip'); + }); + + /** + * An admin who names a type in `documentParser.supportedMimeTypes` has said the + * server parses it, and both the client's upload options and the routing act on + * that. Admission was the only place that did not, so a vendor alias was offered to + * the user and then refused at the door. + */ + /** + * The instance-level config is resolved once at startup, so a tenant, role or user + * override lives only on the request. Reading the startup copy refused the upload + * before the post-upload gate could apply the configuration that governs it. + */ + it('should admit a vendor MIME a per-request config override names', () => { + const { mergeFileConfig } = require('librechat-data-provider'); + const fileFilter = createFileFilter(mergeFileConfig()); + mockReq.config = { + ...mockReq.config, + fileConfig: { + documentParser: { supportedMimeTypes: ['^application/vnd\\.tenant\\.word$'] }, + }, + }; + const documentFile = { + ...mockFile, + originalname: 'report.docx', + mimetype: 'application/vnd.tenant.word', + }; + const cb = jest.fn(); + + fileFilter(mockReq, documentFile, cb); + + expect(cb).toHaveBeenCalledWith(null, true); + }); + + it('should admit a vendor MIME the document-parser allowlist names', () => { + const { mergeFileConfig } = require('librechat-data-provider'); + const fileFilter = createFileFilter( + mergeFileConfig({ + documentParser: { supportedMimeTypes: ['^application/vnd\\.vendor\\.word$'] }, + }), + ); + const documentFile = { + ...mockFile, + originalname: 'report.docx', + mimetype: 'application/vnd.vendor.word', + }; + const cb = jest.fn(); + + fileFilter(mockReq, documentFile, cb); + + expect(cb).toHaveBeenCalledWith(null, true); + }); + + /** + * This filter runs while the file part is still streaming, so a field a multipart + * client sends after the file is not there yet. Reading the tool resource here would + * refuse a valid upload from any client that orders its fields differently; + * `filterFile` applies that scope a moment later on a complete body. + */ + it.each([ + ['a different tool resource', 'file_search'], + ['no tool resource at all', undefined], + ])( + 'should admit a parser-named MIME regardless of field order, given %s', + (_l, toolResource) => { + const { mergeFileConfig } = require('librechat-data-provider'); + const fileFilter = createFileFilter( + mergeFileConfig({ + documentParser: { supportedMimeTypes: ['^application/vnd\\.vendor\\.word$'] }, + }), + ); + mockReq.body.tool_resource = toolResource; + const documentFile = { + ...mockFile, + originalname: 'report.docx', + mimetype: 'application/vnd.vendor.word', + }; + const cb = jest.fn(); + + fileFilter(mockReq, documentFile, cb); + + expect(cb).toHaveBeenCalledWith(null, true); + }, + ); + + it('should still reject a type no allowlist names', () => { + const { mergeFileConfig } = require('librechat-data-provider'); + const fileFilter = createFileFilter( + mergeFileConfig({ + documentParser: { supportedMimeTypes: ['^application/vnd\\.vendor\\.word$'] }, + }), + ); + const unknownFile = { + ...mockFile, + originalname: 'data.xyz', + mimetype: 'application/x-unknown', + }; + const cb = jest.fn(); + + fileFilter(mockReq, unknownFile, cb); + + expect(cb).toHaveBeenCalledWith(expect.any(Error), false); + }); + it('should use real mergeFileConfig function', async () => { const { mergeFileConfig, mbToBytes } = require('librechat-data-provider'); @@ -468,6 +629,34 @@ describe('Multer Configuration', () => { expect(multerInstance).toBeDefined(); }); + it('stages concurrent same-name uploads at separate paths when asked to', async () => { + const upload = await createMulterInstance({ fileConfig: {}, uniqueTempPath: true }); + const app = express(); + app.use((req, _res, next) => { + req.user = { id: 'test-user-123' }; + req.config = { paths: { uploads: tempDir } }; + next(); + }); + app.post('/upload', upload.single('file'), (req, res) => { + res.json({ path: req.file.path, bytes: fs.readFileSync(req.file.path, 'utf8') }); + }); + + const [first, second] = await Promise.all( + ['first body', 'second body'].map((body) => + request(app).post('/upload').attach('file', Buffer.from(body), { + filename: 'notes.txt', + contentType: 'text/plain', + }), + ), + ); + + expect(first.status).toBe(200); + expect(second.status).toBe(200); + expect(first.body.path).not.toBe(second.body.path); + expect(first.body.bytes).toBe('first body'); + expect(second.body.bytes).toBe('second body'); + }); + it('should create multer instance with expected interface', async () => { const multerInstance = await createMulterInstance(); diff --git a/api/server/routes/files/preview.spec.js b/api/server/routes/files/preview.spec.js index 3363c1b08ae..2ea927cf20e 100644 --- a/api/server/routes/files/preview.spec.js +++ b/api/server/routes/files/preview.spec.js @@ -78,12 +78,15 @@ const filesRouter = require('./files'); * Mount the router with a per-request user injector so we can simulate * a logged-in user without spinning up the full auth stack. */ -function buildApp({ user = { id: 'user-123', role: 'user' } } = {}) { +function buildApp({ + user = { id: 'user-123', role: 'user' }, + config = { fileStrategy: 'local' }, +} = {}) { const app = express(); app.use(express.json()); app.use((req, _res, next) => { req.user = user; - req.config = { fileStrategy: 'local' }; + req.config = config; next(); }); app.use('/files', filesRouter); @@ -92,6 +95,28 @@ function buildApp({ user = { id: 'user-123', role: 'user' } } = {}) { const OWNER_USER_ID = 'user-123'; +describe('GET /files/config', () => { + it('marks OCR enabled only when an OCR strategy exists', async () => { + const enabled = await request( + buildApp({ + config: { + ocr: { strategy: 'mistral_ocr' }, + fileConfig: { ocr: { supportedMimeTypes: ['^application/pdf$'] } }, + }, + }), + ).get('/files/config'); + expect(enabled.body.ocr).toEqual({ + enabled: true, + supportedMimeTypes: ['^application/pdf$'], + }); + + const disabled = await request( + buildApp({ config: { fileConfig: { ocr: { supportedMimeTypes: ['^application/pdf$'] } } } }), + ).get('/files/config'); + expect(disabled.body.ocr.enabled).toBe(false); + }); +}); + describe('GET /files/:file_id/preview', () => { beforeEach(() => { mockFindFileById.mockReset(); diff --git a/api/server/services/Files/OpenAI/crud.js b/api/server/services/Files/OpenAI/crud.js index 674c493a34e..534477dc7bf 100644 --- a/api/server/services/Files/OpenAI/crud.js +++ b/api/server/services/Files/OpenAI/crud.js @@ -1,5 +1,5 @@ -const fs = require('fs'); const { sleep } = require('@librechat/agents'); +const { openNamedUpload } = require('@librechat/api'); const { logger } = require('@librechat/data-schemas'); const { FilePurpose } = require('librechat-data-provider'); @@ -16,7 +16,7 @@ async function uploadOpenAIFile({ req, file, openai }) { const { height, width } = req.body; const isImage = height && width; const uploadedFile = await openai.files.create({ - file: fs.createReadStream(file.path), + file: await openNamedUpload(file.path, file.originalname), purpose: isImage ? FilePurpose.Vision : FilePurpose.Assistants, }); diff --git a/api/server/services/Files/OpenAI/crud.spec.js b/api/server/services/Files/OpenAI/crud.spec.js new file mode 100644 index 00000000000..c981f74ac85 --- /dev/null +++ b/api/server/services/Files/OpenAI/crud.spec.js @@ -0,0 +1,39 @@ +const fs = require('fs'); +const os = require('os'); +const path = require('path'); + +jest.mock('@librechat/agents', () => ({ + ...jest.requireActual('@librechat/agents'), + sleep: jest.fn(), +})); + +const { uploadOpenAIFile } = require('./crud'); + +describe('uploadOpenAIFile', () => { + let dir; + + beforeAll(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'openai-upload-')); + }); + + afterAll(() => { + fs.rmSync(dir, { recursive: true, force: true }); + }); + + it('sends the upload under its own name rather than the staged path', async () => { + const stagedPath = path.join(dir, 'req-123__report.pdf'); + fs.writeFileSync(stagedPath, 'pdf bytes'); + const create = jest.fn(async () => ({ id: 'file-1', status: 'processed' })); + + await uploadOpenAIFile({ + req: { body: {}, user: { id: 'user-1' } }, + file: { path: stagedPath, originalname: 'report.pdf' }, + openai: { files: { create } }, + }); + + const sent = create.mock.calls[0][0].file; + expect(sent).toBeInstanceOf(File); + expect(sent.name).toBe('report.pdf'); + expect(await sent.text()).toBe('pdf bytes'); + }); +}); diff --git a/api/server/services/Files/VectorDB/crud.js b/api/server/services/Files/VectorDB/crud.js index 9f43f0e4ae4..07374feee76 100644 --- a/api/server/services/Files/VectorDB/crud.js +++ b/api/server/services/Files/VectorDB/crud.js @@ -73,7 +73,10 @@ async function uploadVectors({ req, file, file_id, entity_id, storageMetadata, s const jwtToken = generateShortLivedToken(req.user.id); const formData = new FormData(); formData.append('file_id', file_id); - formData.append('file', fs.createReadStream(file.path)); + formData.append('file', fs.createReadStream(file.path), { + filename: file.originalname, + contentType: file.mimetype, + }); if (entity_id != null && entity_id) { formData.append('entity_id', entity_id); } diff --git a/api/server/services/Files/process.js b/api/server/services/Files/process.js index 35117ab248e..95fa92bfd82 100644 --- a/api/server/services/Files/process.js +++ b/api/server/services/Files/process.js @@ -26,25 +26,34 @@ const { isResponsesApiUpload, isSpeechProviderConfigured, getCustomEndpointProvider, + resolveEffectiveMimeType, } = require('librechat-data-provider'); const { logger, runAsSystem } = require('@librechat/data-schemas'); const { sanitizeFilename, parseText, + parseTextNative, processAudioFile, extractInspectableFileText, assertExtractedTextInspectable, getFileExtractionLogDetails, - getUploadExtractedTextPlan, resolveUploadFallbackText, - UPLOAD_EXTRACTED_TEXT_PLANS, MAX_STORED_EXTRACTED_TEXT_BYTES, + planDocumentExtraction, + isAdmissibleUploadType, + resolveDocumentExtraction, + isDocumentParserRefusal, + isNoDocumentTextError, + isPartialDocumentText, + isDelimitedTextType, inspectContent, extractFileContent, hasActiveFileFieldPolicy, sendUploadSuccess, getStorageMetadata, contentFilterBlockResponse, + annotateMissingPages, + summarizeMissingPages, sweepExpiredFiles: sweepExpiredFilesWithDeps, startExpiredFileSweep: startExpiredFileSweepWithDeps, resolveToolRoleGrants, @@ -96,6 +105,10 @@ const createSanitizedUploadWrapper = (uploadFunction) => { const hasCodeEnvRef = (file) => file?.metadata?.codeEnvRef != null || file?.metadata?.codeEnvRefs != null; +/** Same resolution the client validates and offers upload options with. */ +const resolveUploadMimeType = (file) => + resolveEffectiveMimeType(file?.originalname ?? '', file?.mimetype ?? ''); + const isMissingStorageError = (err) => { const code = err?.code ?? err?.status ?? err?.statusCode ?? err?.response?.status; if ([404, '404', 'ENOENT', 'NoSuchKey', 'NotFound', 'ResourceNotFound'].includes(code)) { @@ -1010,68 +1023,213 @@ const processAgentFileUpload = async ({ req, res, metadata, sseStream }) => { }; const fileConfig = mergeFileConfig(appConfig.fileConfig); - const extractedTextPlan = getUploadExtractedTextPlan({ + const effectiveMimeType = resolveUploadMimeType(file); + const { + parserMimeType: parserResolvedMimeType, + parserEligible: isDocumentParserEligible, + isBuiltInDocumentType: isKnownDocumentType, + useConfiguredText: shouldUseConfiguredText, + useConfiguredOCR: shouldUseConfiguredOCR, + useDocumentParser: shouldUseDocumentParser, + } = planDocumentExtraction({ endpoint: metadata.endpoint, toolResource: effectiveToolResource, - mimeType: file.mimetype, + mimeType: effectiveMimeType, + fileName: file.originalname, fileConfig, ocrConfigured: appConfig?.ocr != null, ragConfigured: !!process.env.RAG_API_URL, }); - const shouldUseConfiguredOCR = extractedTextPlan === UPLOAD_EXTRACTED_TEXT_PLANS.configuredOCR; - const shouldUseConfiguredText = extractedTextPlan === UPLOAD_EXTRACTED_TEXT_PLANS.configuredRAG; - const shouldUseDocumentParser = - extractedTextPlan === UPLOAD_EXTRACTED_TEXT_PLANS.documentParser; - - const shouldUseOCR = shouldUseConfiguredOCR || shouldUseDocumentParser; const resolveDocumentText = async () => { - if (shouldUseConfiguredOCR) { - try { - const ocrStrategy = appConfig?.ocr?.strategy ?? FileSources.document_parser; - const { handleFileUpload } = getStrategyFunctions(ocrStrategy); - return await handleFileUpload({ req, file, loadAuthValues }); - } catch (err) { - const { errorMetadata } = getExtractionLogDetails(err); - logger.error( - `[processAgentFileUpload] Configured OCR failed for ${extractionFileLabel}, falling back to document_parser:`, - errorMetadata, - ); - } + if (!isDocumentParserEligible) { + return; } + /* A user who removes the file from the composer closes this request, and every + * stage of the parse takes a signal: without one the archive walk and the native + * child hold an admission slot for their whole deadline and go on to produce text + * for a document nobody is waiting for. */ + const cancellation = new AbortController(); + const abortOnDisconnect = () => cancellation.abort(); + res.once('close', abortOnDisconnect); try { const { handleFileUpload } = getStrategyFunctions(FileSources.document_parser); - return await handleFileUpload({ req, file, loadAuthValues }); + return await handleFileUpload({ + req, + file, + loadAuthValues, + signal: cancellation.signal, + maxFileSize: fileConfig.documentParser?.fileSizeLimit, + timeoutMs: fileConfig.documentParser?.timeoutMs, + maxPageCount: fileConfig.documentParser?.maxPageCount, + archiveEntrySizeLimit: fileConfig.documentParser?.archiveEntrySizeLimit, + archiveTotalSizeLimit: fileConfig.documentParser?.archiveTotalSizeLimit, + maxRecoveredPageCount: fileConfig.documentParser?.maxRecoveredPageCount, + archiveEntryCountLimit: fileConfig.documentParser?.archiveEntryCountLimit, + maxConcurrentParsers: fileConfig.documentParser?.maxConcurrentParsers, + maxQueuedParsers: fileConfig.documentParser?.maxQueuedParsers, + classifierTimeoutMs: fileConfig.documentParser?.classifierTimeoutMs, + /* The engines log their own successful-path diagnostics, and only this route + * knows whether the filename may appear in them. */ + fileLabel: extractionFileLabel, + /* Recovery hides the failure from the code below, and the engine has no way + * to know whether this deployment redacts filenames and parser errors. */ + onEngineFallback: (err) => { + const { errorMetadata } = getExtractionLogDetails(err); + logger.warn( + `[processAgentFileUpload] Native PDF extraction failed for ${extractionFileLabel}, recovered with pdfjs:`, + errorMetadata, + ); + }, + }); } catch (err) { + if (isDocumentParserRefusal(err) || cancellation.signal.aborted) { + throw err; + } + /* The parser read the document and found no text in it. That is the case a + * configured OCR service exists for, and without one it is the image-based + * refusal below, so it must not surface as a parser failure. */ + if (isNoDocumentTextError(err)) { + return; + } const { errorMetadata } = getExtractionLogDetails(err); logger.error( `[processAgentFileUpload] Document parser failed for ${extractionFileLabel}:`, errorMetadata, ); throw err; + } finally { + res.off('close', abortOnDisconnect); + } + }; + + const resolveConfiguredOCR = async ({ throwOnMissingCapability = true } = {}) => { + if (!shouldUseConfiguredOCR) { + return; + } + if (!(await checkCapability(req, AgentCapabilities.ocr))) { + if (throwOnMissingCapability) { + throw new Error('OCR capability is not enabled for Agents'); + } + return; + } + try { + const ocrStrategy = appConfig?.ocr?.strategy ?? FileSources.mistral_ocr; + const { handleFileUpload } = getStrategyFunctions(ocrStrategy); + return await handleFileUpload({ req, file, loadAuthValues }); + } catch (err) { + const { errorMetadata } = getExtractionLogDetails(err); + logger.error( + `[processAgentFileUpload] Configured OCR failed for ${extractionFileLabel}:`, + errorMetadata, + ); + } + }; + + /** + * Single entry point for every extraction result, so the parser branch, the OCR + * branch and the RAG fallback below store the same shape: omitted pages are always + * logged and annotated into the text. + * + * Storage fields (`bytes`, `filepath`, `source`, `type`) belong to `createTextFile`, + * which uploads the original document and records what it stored; an extraction + * result only supplies the text and what the extractor could not read. + * + * Model routing is unaffected: parsed records are short-circuited on + * `source === text` both in `filterFilesByEndpointConfig` (packages/api/src/files/filter.ts, + * which runs first, during agent initialization) and in `BaseClient#categorizeAttachments`, + * before any type-based categorization. + * + * @param {MistralOCRUploadResult} result + * @return {Promise} + */ + const createDocumentTextFile = async ({ text, pagesNeedingOcr, mayOmitContent }) => { + if (pagesNeedingOcr?.length) { + const pageSummary = summarizeMissingPages(pagesNeedingOcr); + logger.warn( + `[processAgentFileUpload] ${extractionFileLabel} has no extractable text on page(s) ${pageSummary}; those pages were omitted.`, + ); + } + /* Logged rather than written into the text: unlike an omitted page, embedded + * artwork is not evidence that anything was lost. Most documents that carry it + * carry a logo, so a notice in the model's copy would assert an omission that + * usually did not happen, on the majority of office uploads. */ + if (mayOmitContent === true) { + logger.warn( + `[processAgentFileUpload] ${extractionFileLabel} embeds images the local parser reads no text from; configure an OCR service to recover any text they hold.`, + ); + } + const annotated = annotateMissingPages(text, pagesNeedingOcr); + return await createTextFile({ text: annotated }); + }; + + /** + * Resolves a delimited file as the bytes it already was when conversion cannot ship. + * + * Reads the file directly rather than through `parseText`: with a RAG service + * configured that would spend a health check and an extraction request before + * reaching the bytes, and could come back with transformed content, or with another + * extraction too large for the same limit that sent us here. The point of this path + * is the bytes themselves. + */ + const resolveDelimitedTextAsIs = async () => { + const { text, bytes } = await parseTextNative(file); + if (!text?.trim()) { + return; } + return { text, bytes, rawDelimitedText: true }; }; - if (shouldUseConfiguredOCR && !(await checkCapability(req, AgentCapabilities.ocr))) { - throw new Error('OCR capability is not enabled for Agents'); + if (shouldUseDocumentParser) { + const documentResult = await extractInspectableFileText({ + filters: appConfig?.filters, + extract: () => + resolveDocumentExtraction({ + delimitedText: isDelimitedTextType(parserResolvedMimeType), + parse: resolveDocumentText, + runConfiguredOCR: shouldUseConfiguredOCR ? resolveConfiguredOCR : undefined, + readRawText: resolveDelimitedTextAsIs, + assertPartialTextAllowed: () => + assertExtractedTextInspectable({ filters: appConfig?.filters, text: undefined }), + }), + }); + + if (documentResult?.text?.trim()) { + if (documentResult.rawDelimitedText === true) { + return await createTextFile({ text: documentResult.text }); + } + return await createDocumentTextFile(documentResult); + } + + assertExtractedTextInspectable({ + filters: appConfig?.filters, + text: documentResult?.text, + }); + + throw new Error( + `Unable to extract text from "${file.originalname}". The document may be image-based and requires an OCR service to process.`, + ); } - if (shouldUseOCR) { + if (shouldUseConfiguredOCR) { const ocrResult = await extractInspectableFileText({ filters: appConfig?.filters, - extract: resolveDocumentText, + extract: resolveConfiguredOCR, }); - if (ocrResult) { - const { text } = ocrResult; - return await createTextFile({ text }); + if (ocrResult?.text?.trim()) { + return await createDocumentTextFile(ocrResult); } throw new Error( - `Unable to extract text from "${file.originalname}". The document may be image-based and requires an OCR service to process.`, + `Unable to extract text from "${file.originalname}" with the configured OCR service.`, ); } + if (isKnownDocumentType && !shouldUseConfiguredText) { + throw new Error(`File type ${effectiveMimeType} is not enabled for document parsing.`); + } + const shouldUseSTT = fileConfig.checkType( - file.mimetype, + effectiveMimeType, fileConfig.stt?.supportedMimeTypes || [], ); @@ -1082,12 +1240,12 @@ const processAgentFileUpload = async ({ req, res, metadata, sseStream }) => { } const shouldUseText = fileConfig.checkType( - file.mimetype, + effectiveMimeType, fileConfig.text?.supportedMimeTypes || [], ); if (!shouldUseText) { - throw new Error(`File type ${file.mimetype} is not supported for text parsing.`); + throw new Error(`File type ${effectiveMimeType} is not supported for text parsing.`); } /** @@ -1116,8 +1274,13 @@ const processAgentFileUpload = async ({ req, res, metadata, sseStream }) => { `Unable to extract text from "${file.originalname}". RAG text extraction was unavailable and the built-in parser produced no result.`, ); } - const { text } = documentText; - return await createTextFile({ text }); + /* The same fail-closed rule the primary parser path applies at its own partial + * result: no OCR service is configured on this route, so there is nothing left + * to complete the text with. */ + if (isPartialDocumentText(documentText)) { + assertExtractedTextInspectable({ filters: appConfig?.filters, text: undefined }); + } + return await createDocumentTextFile(documentText); } return await createTextFile({ text: configuredText.text }); } @@ -1149,6 +1312,8 @@ const processAgentFileUpload = async ({ req, res, metadata, sseStream }) => { isMessageAttachment: messageAttachment, endpointConfig, filters: appConfig?.filters, + documentParser: fileConfig.documentParser, + connection: res, }); // Dual storage pattern for RAG files: Storage + Vector DB @@ -1711,10 +1876,17 @@ function filterFile({ req, image, isAvatar, endpoint: endpointOverride }) { ); } - const isSupportedMimeType = fileConfig.checkType( - file.mimetype, - endpointFileConfig.supportedMimeTypes, - ); + /* Admission and routing both honor `documentParser.supportedMimeTypes`, so this gate + * has to as well: an admin who names a vendor MIME there has said the server parses + * it, and a second check that disagrees only moves the refusal one step later. The + * request body is complete here, so this is the authoritative place to scope it to + * the context path, the only one that reaches the parser. */ + const isSupportedMimeType = isAdmissibleUploadType({ + mimeType: file.mimetype, + fileConfig, + endpointMimeTypes: endpointFileConfig.supportedMimeTypes, + toolResource: req.body?.tool_resource, + }); if (!isSupportedMimeType) { throw new Error('Unsupported file type'); diff --git a/api/server/services/Files/process.spec.js b/api/server/services/Files/process.spec.js index 33ec48b7861..a39b8df21b1 100644 --- a/api/server/services/Files/process.spec.js +++ b/api/server/services/Files/process.spec.js @@ -98,26 +98,128 @@ jest.mock('@librechat/api', () => { errorMetadata: contentProtected ? getSafeErrorMetadata(error) : error, }; }); - const getUploadExtractedTextPlan = jest.fn( - ({ mimeType, fileConfig, ocrConfigured, ragConfigured }) => { + /** + * Stands in for the real planner in `packages/api`, whose own suite owns its + * precedence rules. What these specs exercise is what `process.js` does with the + * answer, so the mock reproduces only the decisions the route branches on. + */ + const planDocumentExtraction = jest.fn( + ({ mimeType, fileName, fileConfig, ocrConfigured, ragConfigured }) => { const checkType = fileConfig.checkType; - if (ocrConfigured && checkType(mimeType, fileConfig.ocr?.supportedMimeTypes ?? [])) { - return UPLOAD_EXTRACTED_TEXT_PLANS.configuredOCR; - } - const isDocumentParserEligible = actualDataProvider.documentParserMimeTypes.some((pattern) => + const parserMimeTypes = + fileConfig.documentParser?.supportedMimeTypes ?? actualDataProvider.documentParserMimeTypes; + const isKnownDocumentType = actualDataProvider.documentParserMimeTypes.some((pattern) => pattern.test(mimeType), ); - if (!isDocumentParserEligible) { - return null; + const parserEligible = checkType(mimeType, parserMimeTypes); + const plan = (() => { + if (ocrConfigured && checkType(mimeType, fileConfig.ocr?.supportedMimeTypes ?? [])) { + return UPLOAD_EXTRACTED_TEXT_PLANS.configuredOCR; + } + if (!isKnownDocumentType && !parserEligible) { + return null; + } + if ( + ragConfigured && + !actualDataProvider.isPermissiveMimeConfig(fileConfig.text?.supportedMimeTypes) && + checkType(mimeType, fileConfig.text?.supportedMimeTypes ?? []) + ) { + return UPLOAD_EXTRACTED_TEXT_PLANS.configuredRAG; + } + return parserEligible ? UPLOAD_EXTRACTED_TEXT_PLANS.documentParser : null; + })(); + const useConfiguredText = plan === UPLOAD_EXTRACTED_TEXT_PLANS.configuredRAG; + const isAlias = !isKnownDocumentType && parserEligible; + const parserMimeType = isAlias + ? actualDataProvider.resolveEffectiveMimeType(fileName ?? '', '') + : mimeType; + const aliasSupportsOCR = + isAlias && + !useConfiguredText && + checkType(parserMimeType, fileConfig.ocr?.supportedMimeTypes ?? []); + return { + plan, + parserMimeType, + isBuiltInDocumentType: isKnownDocumentType, + parserEligible, + useConfiguredText, + useConfiguredOCR: + ocrConfigured && (plan === UPLOAD_EXTRACTED_TEXT_PLANS.configuredOCR || aliasSupportsOCR), + useDocumentParser: !useConfiguredText && parserEligible, + }; + }, + ); + /** + * The same arrangement for the extraction chain: `packages/api` owns the order the + * engines run in and its own suite proves it, and this reproduces the outcomes so + * these specs stay about what the route hands in and does with the answer. + */ + const isPartialDocumentText = (result) => + !!result?.pagesNeedingOcr?.length || result?.mayOmitContent === true; + const isDelimitedTextType = (mimeType) => /^(?:text|application)\/csv$/i.test(mimeType ?? ''); + const errorMatches = (error, code, name) => error?.code === code || error?.name === name; + const isNoDocumentTextError = (error) => + errorMatches(error, 'NO_DOCUMENT_TEXT', 'NoDocumentTextError'); + const isDocumentParserRefusal = (error) => + errorMatches(error, 'ZIP_BOMB', 'ZipBombError') || + errorMatches(error, 'ARCHIVE_INVALID', 'ArchiveValidationError') || + errorMatches(error, 'PDF_PAGE_LIMIT', 'PdfPageLimitError') || + errorMatches(error, 'CONCURRENCY_LIMIT', 'ConcurrencyLimitError') || + errorMatches(error, 'PARSER_INPUT_LIMIT', 'ParserInputLimitError') || + errorMatches(error, 'PARSER_OUTPUT_LIMIT', 'ParserOutputLimitError') || + errorMatches(error, 'UNSUPPORTED_DOCUMENT_TYPE', 'UnsupportedDocumentTypeError'); + const resolveDocumentExtraction = jest.fn( + async ({ delimitedText, parse, runConfiguredOCR, readRawText, assertPartialTextAllowed }) => { + let parsed; + try { + parsed = await parse(); + } catch (error) { + if (errorMatches(error, 'PARSER_OUTPUT_LIMIT', 'ParserOutputLimitError') && delimitedText) { + const raw = await readRawText?.(); + if (raw) { + return raw; + } + } + throw error; } - if ( - ragConfigured && - !actualDataProvider.isPermissiveMimeConfig(fileConfig.text?.supportedMimeTypes) && - checkType(mimeType, fileConfig.text?.supportedMimeTypes ?? []) - ) { - return UPLOAD_EXTRACTED_TEXT_PLANS.configuredRAG; + const hasText = !!parsed?.text?.trim(); + const needsOCR = !hasText || isPartialDocumentText(parsed); + if (hasText && !needsOCR) { + return parsed; + } + if (needsOCR && runConfiguredOCR) { + const ocr = await runConfiguredOCR({ throwOnMissingCapability: !hasText }); + if (ocr?.text?.trim()) { + return ocr; + } + } + if (hasText) { + assertPartialTextAllowed(); + return parsed; + } + return delimitedText ? await readRawText?.() : undefined; + }, + ); + /** + * The shared admission predicate both upload gates call. `packages/api` owns its rules + * and proves them; this reproduces the answer so these specs stay about the scope + * `filterFile` hands in. + */ + const isAdmissibleUploadType = jest.fn( + ({ mimeType, fileConfig, endpointMimeTypes, admitParserTypes, toolResource }) => { + const checkType = fileConfig.checkType; + if (checkType == null) { + return false; + } + if (checkType(mimeType, endpointMimeTypes ?? [])) { + return true; } - return UPLOAD_EXTRACTED_TEXT_PLANS.documentParser; + const parserMimeTypes = fileConfig.documentParser?.supportedMimeTypes; + const reachesParser = + admitParserTypes === true || toolResource === 'context' || toolResource === 'ocr'; + return reachesParser && parserMimeTypes != null + ? checkType(mimeType, parserMimeTypes) + : false; }, ); return { @@ -128,10 +230,17 @@ jest.mock('@librechat/api', () => { /** Stores no fallback text unless a test opts in; its own rules are covered in packages/api. */ resolveUploadFallbackText: jest.fn(async () => undefined), MAX_STORED_EXTRACTED_TEXT_BYTES: 15 * 1024 * 1024, + parseTextNative: jest.fn().mockResolvedValue({ text: '', bytes: 0 }), processAudioFile: jest.fn(), extractInspectableFileText: jest.fn(async ({ extract }) => extract()), assertExtractedTextInspectable: jest.fn(), - getUploadExtractedTextPlan, + planDocumentExtraction, + isAdmissibleUploadType, + resolveDocumentExtraction, + isDocumentParserRefusal, + isNoDocumentTextError, + isPartialDocumentText, + isDelimitedTextType, UPLOAD_EXTRACTED_TEXT_PLANS, getFileExtractionLogDetails, getSafeErrorMetadata, @@ -153,6 +262,17 @@ jest.mock('@librechat/api', () => { res.status(200).json({ message, ...result }); }), getStorageMetadata: jest.fn(() => ({})), + /* Stub, deliberately not the production wording: the notice text is covered by + * packages/api's own tests. What matters here is that process.js forwards the + * pages it was given, so assertions target the call, not the string. */ + annotateMissingPages: jest.fn((text, pages) => + pages?.length ? `${text}\n[omitted:${pages.join(',')}]` : text, + ), + summarizeMissingPages: jest.fn((pages) => { + const listed = pages.slice(0, 20); + const remaining = pages.length - listed.length; + return remaining ? `${listed.join(', ')} and ${remaining} more` : listed.join(', '); + }), getRetentionExpiry, createCodeApiRateLimitBudget, getCodeApiUploadOptions, @@ -250,25 +370,29 @@ jest.mock('./VectorDB/crud', () => ({ const { getRetentionExpiry, getAgentFileRetentionExpiry, + annotateMissingPages, + summarizeMissingPages, sweepExpiredFiles: sweepExpiredFilesWithDeps, startExpiredFileSweep: startExpiredFileSweepWithDeps, } = require('@librechat/api'); const { EModelEndpoint, EToolResources, + DocumentParser, FileSources, FileContext, RetentionMode, AgentCapabilities, } = require('librechat-data-provider'); +const { logger } = require('@librechat/data-schemas'); const { mergeFileConfig } = require('librechat-data-provider'); const { checkCapability } = require('~/server/services/Config'); const { loadAuthValues } = require('~/server/services/Tools/credentials'); const { getStrategyFunctions } = require('~/server/services/Files/strategies'); const { uploadVectors } = require('./VectorDB/crud'); -const { logger } = require('@librechat/data-schemas'); const db = require('~/models'); const { + filterFile, processAgentFileUpload, processImageFile, processDeleteRequest, @@ -277,7 +401,6 @@ const { retrieveAndProcessFile, sweepExpiredFiles, startExpiredFileSweep, - filterFile, } = require('./process'); const { inspectContent, @@ -314,6 +437,7 @@ const makeUninspectableExtractedTextError = () => const makeReq = ({ mimetype = PDF_MIME, + originalname = 'upload.bin', ocrConfig = null, interfaceConfig, filters, @@ -323,7 +447,7 @@ const makeReq = ({ user: { id: 'user-123', tenantId: 'tenant-a' }, file: { path: '/tmp/upload.bin', - originalname: 'upload.bin', + originalname, filename: 'upload-uuid.bin', mimetype, }, @@ -348,18 +472,26 @@ const makeMetadata = () => ({ const mockRes = { status: jest.fn().mockReturnThis(), json: jest.fn().mockReturnValue({}), + /* The upload route cancels a parse when the connection closes, so the response has to + * behave like one: an emitter it can subscribe to and unsubscribe from. */ + once: jest.fn().mockReturnThis(), + off: jest.fn().mockReturnThis(), }; const makeFileConfig = ({ ocrSupportedMimeTypes = [], sttSupportedMimeTypes = [], textSupportedMimeTypes = [], + documentParserSupportedMimeTypes, } = {}) => ({ checkType: (mime, types) => (types ?? []).some((t) => (typeof t === 'string' ? t === mime : t.test(mime))), ocr: { supportedMimeTypes: ocrSupportedMimeTypes }, stt: { supportedMimeTypes: sttSupportedMimeTypes }, text: { supportedMimeTypes: textSupportedMimeTypes }, + ...(documentParserSupportedMimeTypes + ? { documentParser: { supportedMimeTypes: documentParserSupportedMimeTypes } } + : {}), }); const setupStoredFileUpload = (result = {}) => { @@ -373,6 +505,25 @@ const setupStoredFileUpload = (result = {}) => { return handleFileUpload; }; +/** + * `createTextFile` uploads the original document through the storage strategy, so the + * extraction mock and the storage mock have to be separate: `getFileStrategy` is stubbed + * to `'local'`, while parsing and configured OCR answer on their own sources. + */ +const setupDocumentStrategies = ({ documentParser, ocr } = {}) => { + const storageUpload = jest.fn().mockResolvedValue({ + bytes: 4096, + filename: 'upload.bin', + filepath: '/uploads/user-1/upload.bin', + }); + const extractionUpload = (source) => + source === FileSources.document_parser ? documentParser : ocr; + getStrategyFunctions.mockImplementation((source) => ({ + handleFileUpload: source === 'local' ? storageUpload : extractionUpload(source), + })); + return storageUpload; +}; + describe('upload retention scheduling', () => { beforeEach(() => { jest.clearAllMocks(); @@ -465,10 +616,12 @@ describe('processAgentFileUpload', () => { checkCapability.mockResolvedValue(true); loadAuthValues.mockResolvedValue({ CODE_API_KEY: 'code-key' }); uploadVectors.mockResolvedValue({ embedded: true, filename: 'embedded-upload.bin' }); - getStrategyFunctions.mockReturnValue({ - handleFileUpload: jest - .fn() - .mockResolvedValue({ text: 'extracted text', bytes: 42, filepath: 'doc://result' }), + setupDocumentStrategies({ + documentParser: jest.fn().mockResolvedValue({ + text: 'extracted text', + bytes: 42, + filepath: FileSources.document_parser, + }), }); mergeFileConfig.mockReturnValue(makeFileConfig()); inspectContent.mockReturnValue(null); @@ -628,7 +781,140 @@ describe('processAgentFileUpload', () => { }); }); - describe('OCR strategy selection', () => { + describe('local document extraction and OCR selection', () => { + /** + * AnyDoc has no page numbers to report, so a DOCX whose content is a scanned image + * comes back looking complete. Embedded artwork is the signal that its Markdown may + * be missing something, and a configured OCR service is exactly what recovers it. + */ + test('escalates to configured OCR when the local parser reports embedded media', async () => { + mergeFileConfig.mockReturnValue(makeFileConfig({ ocrSupportedMimeTypes: [DOCX_MIME] })); + const localUpload = jest.fn().mockResolvedValue({ + text: 'text layer only', + bytes: 15, + filepath: DocumentParser.anydoc, + mayOmitContent: true, + }); + const remoteOCR = jest.fn().mockResolvedValue({ + text: 'text layer plus the scanned page', + bytes: 31, + filepath: FileSources.mistral_ocr, + }); + setupDocumentStrategies({ documentParser: localUpload, ocr: remoteOCR }); + const req = makeReq({ + mimetype: DOCX_MIME, + ocrConfig: { strategy: FileSources.mistral_ocr }, + }); + + await processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() }); + + expect(localUpload).toHaveBeenCalledTimes(1); + expect(remoteOCR).toHaveBeenCalledTimes(1); + expect(db.createFile.mock.calls[0][0].text).toBe('text layer plus the scanned page'); + }); + + test('uses the canonical parser format to check OCR support for a configured MIME alias', async () => { + const vendorMime = 'application/vnd.vendor.word'; + mergeFileConfig.mockReturnValue( + makeFileConfig({ + documentParserSupportedMimeTypes: [vendorMime], + ocrSupportedMimeTypes: [DOCX_MIME], + }), + ); + const localUpload = jest.fn().mockResolvedValue({ + text: 'text layer only', + bytes: 15, + filepath: DocumentParser.anydoc, + mayOmitContent: true, + }); + const remoteOCR = jest.fn().mockResolvedValue({ + text: 'text layer plus the scanned page', + bytes: 31, + filepath: FileSources.mistral_ocr, + }); + setupDocumentStrategies({ documentParser: localUpload, ocr: remoteOCR }); + const req = makeReq({ + mimetype: vendorMime, + originalname: 'report.docx', + ocrConfig: { strategy: FileSources.mistral_ocr }, + }); + + await processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() }); + + expect(localUpload).toHaveBeenCalledTimes(1); + expect(remoteOCR).toHaveBeenCalledTimes(1); + expect(db.createFile.mock.calls[0][0].text).toBe('text layer plus the scanned page'); + }); + + test('keeps the local text when embedded media is reported and no OCR is configured', async () => { + const localUpload = jest.fn().mockResolvedValue({ + text: 'text layer only', + bytes: 15, + filepath: DocumentParser.anydoc, + mayOmitContent: true, + }); + setupDocumentStrategies({ documentParser: localUpload }); + const req = makeReq({ mimetype: DOCX_MIME, ocrConfig: null }); + + await processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() }); + + expect(checkCapability).not.toHaveBeenCalledWith(expect.anything(), AgentCapabilities.ocr); + expect(db.createFile.mock.calls[0][0]).toEqual( + expect.objectContaining({ text: 'text layer only' }), + ); + }); + + test('does not escalate a document that embeds no media', async () => { + mergeFileConfig.mockReturnValue(makeFileConfig({ ocrSupportedMimeTypes: [DOCX_MIME] })); + const localUpload = jest.fn().mockResolvedValue({ + text: 'the whole document', + bytes: 18, + filepath: DocumentParser.anydoc, + }); + const remoteOCR = jest.fn(); + setupDocumentStrategies({ documentParser: localUpload, ocr: remoteOCR }); + const req = makeReq({ + mimetype: DOCX_MIME, + ocrConfig: { strategy: FileSources.mistral_ocr }, + }); + + await processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() }); + + expect(remoteOCR).not.toHaveBeenCalled(); + expect(db.createFile.mock.calls[0][0].text).toBe('the whole document'); + }); + + /* A user who removes the file from the composer closes the request while the parser + * is still working: the archive walk and the native child both take this signal, and + * without it they keep an admission slot until their deadline. */ + test('cancels the parse when the upload connection closes', async () => { + let parsedSignal; + const localUpload = jest.fn(async ({ signal }) => { + parsedSignal = signal; + return { text: 'the whole document', bytes: 18, filepath: DocumentParser.anydoc }; + }); + setupDocumentStrategies({ documentParser: localUpload }); + const closeHandlers = []; + mockRes.once.mockImplementation((event, handler) => { + if (event === 'close') { + closeHandlers.push(handler); + } + return mockRes; + }); + const req = makeReq({ mimetype: DOCX_MIME, ocrConfig: null }); + + await processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() }); + + expect(parsedSignal).toBeInstanceOf(AbortSignal); + expect(parsedSignal.aborted).toBe(false); + expect(closeHandlers).toHaveLength(1); + closeHandlers[0](); + expect(parsedSignal.aborted).toBe(true); + /* Removed once the parse is over, so a long-lived response does not accumulate one + * listener per upload. */ + expect(mockRes.off).toHaveBeenCalledWith('close', closeHandlers[0]); + }); + test.each([ ['PDF', PDF_MIME], ['DOCX', DOCX_MIME], @@ -646,6 +932,178 @@ describe('processAgentFileUpload', () => { expect(getStrategyFunctions).toHaveBeenCalledWith(FileSources.document_parser); }); + test('routes a generic MIME through the parser when the filename identifies an allowed document', async () => { + const req = makeReq({ + mimetype: 'application/octet-stream', + originalname: 'report.docx', + ocrConfig: null, + }); + const { parseText } = require('@librechat/api'); + + await processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() }); + + expect(getStrategyFunctions).toHaveBeenCalledWith(FileSources.document_parser); + expect(parseText).not.toHaveBeenCalled(); + expect(db.createFile.mock.calls[0][0].type).toBe('application/octet-stream'); + }); + + /** + * OOXML/ODF/EPUB documents are zip containers, so a client that types uploads by + * content announces a `.docx` as an archive. That is not a generic type, so nothing + * re-infers it, and the file used to miss every parser check and land in text + * parsing, which stores the archive's bytes as the model's copy of the document. + */ + test('routes a zip-typed office document through the parser on its extension', async () => { + const req = makeReq({ + mimetype: 'application/zip', + originalname: 'report.docx', + ocrConfig: null, + }); + const { parseText } = require('@librechat/api'); + + await processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() }); + + expect(getStrategyFunctions).toHaveBeenCalledWith(FileSources.document_parser); + expect(parseText).not.toHaveBeenCalled(); + expect(db.createFile.mock.calls[0][0].type).toBe('application/zip'); + }); + + test('leaves an ordinary archive alone', async () => { + mergeFileConfig.mockReturnValue( + makeFileConfig({ textSupportedMimeTypes: [/^[\w.-]+\/[\w.-]+$/] }), + ); + const req = makeReq({ + mimetype: 'application/zip', + originalname: 'bundle.zip', + ocrConfig: null, + }); + const { parseText } = require('@librechat/api'); + parseText.mockResolvedValueOnce({ text: 'archive listing', bytes: 15 }); + + await processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() }); + + expect(getStrategyFunctions).not.toHaveBeenCalledWith(FileSources.document_parser); + expect(parseText).toHaveBeenCalled(); + }); + + /** + * The parser only reformats a delimited file as a Markdown table, and the pipes and + * padding it adds to every cell can push a source file inside the storage limit past + * it. Failing the upload would lose a document whose bytes were readable all along. + */ + test('stores the raw bytes when the parser cannot convert a delimited file', async () => { + mergeFileConfig.mockReturnValue( + makeFileConfig({ textSupportedMimeTypes: [/^[\w.-]+\/[\w.-]+$/] }), + ); + const parserError = Object.assign( + new Error('anydoc failed: extracted 22MB of text, over the 15MB limit'), + { name: 'ParserOutputLimitError', code: 'PARSER_OUTPUT_LIMIT' }, + ); + const parserUpload = jest.fn().mockRejectedValue(parserError); + setupDocumentStrategies({ documentParser: parserUpload }); + const { parseText, parseTextNative } = require('@librechat/api'); + parseTextNative.mockResolvedValueOnce({ text: 'a,b\n1,2\n', bytes: 8 }); + const req = makeReq({ mimetype: 'text/csv', originalname: 'rows.csv', ocrConfig: null }); + + await processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() }); + + expect(db.createFile.mock.calls[0][0]).toEqual( + expect.objectContaining({ text: 'a,b\n1,2\n', type: 'text/csv' }), + ); + /* Read straight from disk: routing through the RAG path would spend a health check + * and an extraction request to arrive back at the bytes, or return something else. */ + expect(parseText).not.toHaveBeenCalled(); + }); + + /** + * The same reformatting problem, reported as the refusal it is rather than swallowed. + * The delimited file still falls back to its bytes; anything else has to surface, + * since nothing downstream should rebuild the string the parser declined to hand over. + */ + test('falls back to the bytes when a delimited conversion is refused on size', async () => { + mergeFileConfig.mockReturnValue( + makeFileConfig({ textSupportedMimeTypes: [/^[\w.-]+\/[\w.-]+$/] }), + ); + const refusal = Object.assign(new Error('anydoc extracted 22MB of text'), { + name: 'ParserOutputLimitError', + code: 'PARSER_OUTPUT_LIMIT', + }); + const parserUpload = jest.fn().mockRejectedValue(refusal); + setupDocumentStrategies({ documentParser: parserUpload }); + const { parseText, parseTextNative } = require('@librechat/api'); + parseTextNative.mockResolvedValueOnce({ text: 'a,b\n1,2\n', bytes: 8 }); + const req = makeReq({ mimetype: 'text/csv', originalname: 'rows.csv', ocrConfig: null }); + + await processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() }); + + expect(db.createFile.mock.calls[0][0].text).toBe('a,b\n1,2\n'); + expect(parseText).not.toHaveBeenCalled(); + }); + + /** + * A configured alias admits the upload and anydoc routes it by extension, so the + * same source file has to reach the same fallback. The alias itself never matches + * the delimited predicate; only the canonical type the parser resolved does. + */ + test('falls back to the bytes for a configured CSV alias refused on size', async () => { + const vendorMime = 'application/vnd.example.csv'; + mergeFileConfig.mockReturnValue( + makeFileConfig({ + documentParserSupportedMimeTypes: [vendorMime], + textSupportedMimeTypes: [/^[\w.-]+\/[\w.-]+$/], + }), + ); + const refusal = Object.assign(new Error('anydoc extracted 22MB of text'), { + name: 'ParserOutputLimitError', + code: 'PARSER_OUTPUT_LIMIT', + }); + const parserUpload = jest.fn().mockRejectedValue(refusal); + setupDocumentStrategies({ documentParser: parserUpload }); + const { parseText, parseTextNative } = require('@librechat/api'); + parseTextNative.mockResolvedValueOnce({ text: 'a,b\n1,2\n', bytes: 8 }); + const req = makeReq({ mimetype: vendorMime, originalname: 'rows.csv', ocrConfig: null }); + + await processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() }); + + expect(db.createFile.mock.calls[0][0].text).toBe('a,b\n1,2\n'); + expect(parseText).not.toHaveBeenCalled(); + }); + + test('surfaces a size refusal for a document with no readable raw form', async () => { + const refusal = Object.assign(new Error('pdf-inspector extracted 22MB of text'), { + name: 'ParserOutputLimitError', + code: 'PARSER_OUTPUT_LIMIT', + }); + getStrategyFunctions.mockReturnValue({ + handleFileUpload: jest.fn().mockRejectedValue(refusal), + }); + const { parseText } = require('@librechat/api'); + const req = makeReq({ mimetype: PDF_MIME, originalname: 'huge.pdf', ocrConfig: null }); + + await expect( + processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() }), + ).rejects.toThrow('pdf-inspector extracted 22MB of text'); + expect(parseText).not.toHaveBeenCalled(); + }); + + test('still refuses a binary document the parser cannot read', async () => { + /* The counterpart: a DOCX has no readable raw form, so storing its bytes would + * hand the model an archive. The refusal is the honest outcome there. */ + mergeFileConfig.mockReturnValue( + makeFileConfig({ textSupportedMimeTypes: [/^[\w.-]+\/[\w.-]+$/] }), + ); + const parserUpload = jest.fn().mockRejectedValue(new Error('anydoc failed')); + setupDocumentStrategies({ documentParser: parserUpload }); + const { parseText } = require('@librechat/api'); + const req = makeReq({ mimetype: DOCX_MIME, originalname: 'report.docx', ocrConfig: null }); + + await expect( + processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() }), + ).rejects.toThrow('anydoc failed'); + expect(db.createFile).not.toHaveBeenCalled(); + expect(parseText).not.toHaveBeenCalled(); + }); + test('does not check OCR capability when using automatic document_parser fallback', async () => { const req = makeReq({ mimetype: PDF_MIME, ocrConfig: null }); @@ -655,8 +1113,150 @@ describe('processAgentFileUpload', () => { expect(getStrategyFunctions).toHaveBeenCalledWith(FileSources.document_parser); }); - test('uses the configured OCR strategy when OCR is set up for the file type', async () => { + test('annotates the stored text when the parser reports pages needing OCR', async () => { + const parserUpload = jest.fn().mockResolvedValue({ + text: 'page one text', + bytes: 13, + filepath: FileSources.document_parser, + pagesNeedingOcr: [2, 3], + }); + setupDocumentStrategies({ documentParser: parserUpload }); + const req = makeReq({ mimetype: PDF_MIME, ocrConfig: null }); + + await processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() }); + + expect(annotateMissingPages).toHaveBeenCalledWith('page one text', [2, 3]); + /* A part-scanned document must not be stored as if it were complete: the + * persisted text carries the notice and bytes are recounted to match. */ + const created = db.createFile.mock.calls[0][0]; + expect(created.text).toBe('page one text\n[omitted:2,3]'); + }); + + test('caps the missing-page list written to logs', async () => { + const pagesNeedingOcr = Array.from({ length: 5000 }, (_, index) => index + 1); + const parserUpload = jest.fn().mockResolvedValue({ + text: 'partial text', + bytes: 12, + filepath: FileSources.document_parser, + pagesNeedingOcr, + }); + setupDocumentStrategies({ documentParser: parserUpload }); + const req = makeReq({ mimetype: PDF_MIME, ocrConfig: null }); + + await processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() }); + + expect(summarizeMissingPages).toHaveBeenCalledWith(pagesNeedingOcr); + const warning = logger.warn.mock.calls.find(([message]) => + message.includes('has no extractable text'), + )?.[0]; + expect(warning).toContain('20 and 4980 more'); + expect(warning).not.toContain('21, 22'); + expect(warning.length).toBeLessThan(300); + }); + + test("stores the parsed record with the document's real MIME type", async () => { + /* Stored as text/plain, the record forgets what kind of document it was, + * and the client's extracted-text affordances key on that type. Model + * routing is unaffected: BaseClient short-circuits on source === text. */ + const req = makeReq({ mimetype: PDF_MIME, ocrConfig: null }); + + await processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() }); + + expect(db.createFile.mock.calls[0][0].type).toBe(PDF_MIME); + }); + + test.each([DocumentParser.pdf_inspector, DocumentParser.anydoc])( + 'keeps the real MIME type for documents parsed by %s', + async (marker) => { + /* The request MIME is retained regardless of which local parser produced the + * extracted text; production multer has already normalized it upstream. */ + const parserUpload = jest + .fn() + .mockResolvedValue({ text: 'parsed', bytes: 6, filepath: marker }); + setupDocumentStrategies({ documentParser: parserUpload }); + const req = makeReq({ mimetype: PDF_MIME, ocrConfig: null }); + + await processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() }); + + expect(db.createFile.mock.calls[0][0].type).toBe(PDF_MIME); + }, + ); + + test('an admin document-parser allowlist overrides the built-in defaults', async () => { + /* The one extraction list that had no admin surface. Narrowing it must be able + * to take a type out of the parser's reach, the same way `fileConfig.ocr` and + * `fileConfig.text` already can. */ + mergeFileConfig.mockReturnValue( + makeFileConfig({ documentParserSupportedMimeTypes: ['application/vnd.ms-excel'] }), + ); + const req = makeReq({ mimetype: PDF_MIME, ocrConfig: null }); + + await expect( + processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() }), + ).rejects.toThrow(); + }); + + test('keeps the admin parser allowlist authoritative for a generic document MIME', async () => { + mergeFileConfig.mockReturnValue( + makeFileConfig({ documentParserSupportedMimeTypes: [PDF_MIME] }), + ); + const req = makeReq({ + mimetype: 'application/octet-stream', + originalname: 'report.docx', + ocrConfig: null, + }); + const { parseText } = require('@librechat/api'); + + await expect( + processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() }), + ).rejects.toThrow(`File type ${DOCX_MIME} is not enabled for document parsing.`); + + expect(getStrategyFunctions).not.toHaveBeenCalled(); + expect(parseText).not.toHaveBeenCalled(); + }); + + test('falls back to the built-in defaults when no admin allowlist is set', async () => { + mergeFileConfig.mockReturnValue(makeFileConfig()); + const req = makeReq({ mimetype: PDF_MIME, ocrConfig: null }); + + await processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() }); + + expect(db.createFile.mock.calls[0][0].type).toBe(PDF_MIME); + }); + + test('keeps the request MIME type for a configured OCR result on an image', async () => { + const mime = 'image/png'; + mergeFileConfig.mockReturnValue(makeFileConfig({ ocrSupportedMimeTypes: [mime] })); + const ocrUpload = jest.fn().mockResolvedValue({ + text: 'ocr text', + bytes: 8, + filepath: FileSources.mistral_ocr, + }); + setupDocumentStrategies({ ocr: ocrUpload }); + const req = makeReq({ mimetype: mime, ocrConfig: { strategy: FileSources.mistral_ocr } }); + + await processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() }); + + /* Production multer normalizes the request MIME upstream; this unit test bypasses + * multer, so the record keeps the MIME carried by the request. */ + const created = db.createFile.mock.calls[0][0]; + expect(created.type).toBe(mime); + }); + + test('keeps an OCR fallback result on the request MIME type for a partially readable PDF', async () => { mergeFileConfig.mockReturnValue(makeFileConfig({ ocrSupportedMimeTypes: [PDF_MIME] })); + const localUpload = jest.fn().mockResolvedValue({ + text: 'local page one', + bytes: 14, + filepath: DocumentParser.pdf_inspector, + pagesNeedingOcr: [2], + }); + const remoteUpload = jest.fn().mockResolvedValue({ + text: 'OCR recovered both pages', + bytes: 24, + filepath: FileSources.mistral_ocr, + }); + setupDocumentStrategies({ documentParser: localUpload, ocr: remoteUpload }); const req = makeReq({ mimetype: PDF_MIME, ocrConfig: { strategy: FileSources.mistral_ocr }, @@ -664,12 +1264,106 @@ describe('processAgentFileUpload', () => { await processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() }); - expect(checkCapability).toHaveBeenCalledWith(expect.anything(), AgentCapabilities.ocr); - expect(getStrategyFunctions).toHaveBeenCalledWith(FileSources.mistral_ocr); + const created = db.createFile.mock.calls[0][0]; + expect(created.type).toBe(PDF_MIME); + expect(created.text).toBe('OCR recovered both pages'); + }); + + test('leaves extracted text untouched when every page was extracted', async () => { + const localUpload = jest.fn().mockResolvedValue({ + text: 'complete text', + bytes: 42, + filepath: FileSources.document_parser, + }); + setupDocumentStrategies({ documentParser: localUpload }); + const req = makeReq({ mimetype: PDF_MIME, ocrConfig: null }); + + await processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() }); + + const created = db.createFile.mock.calls[0][0]; + expect(created.text).toBe('complete text'); }); - test('uses document_parser as default when OCR is configured but no strategy is specified', async () => { + test('uses local document extraction before configured OCR for a text-based PDF', async () => { mergeFileConfig.mockReturnValue(makeFileConfig({ ocrSupportedMimeTypes: [PDF_MIME] })); + const localUpload = jest.fn().mockResolvedValue({ + text: 'local PDF text', + bytes: 14, + filepath: DocumentParser.pdf_inspector, + }); + const remoteOCR = jest.fn().mockResolvedValue({ + text: 'remote OCR text', + bytes: 15, + filepath: FileSources.mistral_ocr, + }); + setupDocumentStrategies({ documentParser: localUpload, ocr: remoteOCR }); + const req = makeReq({ + mimetype: PDF_MIME, + ocrConfig: { strategy: FileSources.mistral_ocr }, + }); + + await processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() }); + + expect(localUpload).toHaveBeenCalledTimes(1); + expect(remoteOCR).not.toHaveBeenCalled(); + expect(checkCapability).not.toHaveBeenCalledWith(expect.anything(), AgentCapabilities.ocr); + expect(db.createFile.mock.calls[0][0]).toEqual( + expect.objectContaining({ + text: 'local PDF text', + type: PDF_MIME, + }), + ); + }); + + test.each([ + FileSources.mistral_ocr, + FileSources.azure_mistral_ocr, + FileSources.vertexai_mistral_ocr, + ])( + 'uses configured OCR service %s after local extraction reports a missing page', + async (strategy) => { + mergeFileConfig.mockReturnValue(makeFileConfig({ ocrSupportedMimeTypes: [PDF_MIME] })); + const localUpload = jest.fn().mockResolvedValue({ + text: 'local page one', + bytes: 14, + filepath: DocumentParser.pdf_inspector, + pagesNeedingOcr: [2], + }); + const remoteUpload = jest.fn().mockResolvedValue({ + text: 'OCR recovered both pages', + bytes: 24, + filepath: strategy, + }); + setupDocumentStrategies({ documentParser: localUpload, ocr: remoteUpload }); + const req = makeReq({ + mimetype: PDF_MIME, + ocrConfig: { strategy }, + }); + + await processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() }); + + expect(getStrategyFunctions).toHaveBeenNthCalledWith(1, FileSources.document_parser); + expect(getStrategyFunctions).toHaveBeenNthCalledWith(2, strategy); + expect(localUpload).toHaveBeenCalledTimes(1); + expect(remoteUpload).toHaveBeenCalledTimes(1); + expect(checkCapability).toHaveBeenCalledWith(expect.anything(), AgentCapabilities.ocr); + }, + ); + + test('defaults missing OCR strategy to Mistral after local extraction needs OCR', async () => { + mergeFileConfig.mockReturnValue(makeFileConfig({ ocrSupportedMimeTypes: [PDF_MIME] })); + const localUpload = jest.fn().mockResolvedValue({ + text: '', + bytes: 0, + filepath: FileSources.document_parser, + pagesNeedingOcr: [1], + }); + const remoteUpload = jest.fn().mockResolvedValue({ + text: 'OCR text', + bytes: 8, + filepath: FileSources.mistral_ocr, + }); + setupDocumentStrategies({ documentParser: localUpload, ocr: remoteUpload }); const req = makeReq({ mimetype: PDF_MIME, ocrConfig: { supportedMimeTypes: [PDF_MIME] }, @@ -678,16 +1372,25 @@ describe('processAgentFileUpload', () => { await processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() }); expect(checkCapability).toHaveBeenCalledWith(expect.anything(), AgentCapabilities.ocr); - expect(getStrategyFunctions).toHaveBeenCalledWith(FileSources.document_parser); + expect(getStrategyFunctions).toHaveBeenNthCalledWith(1, FileSources.document_parser); + expect(getStrategyFunctions).toHaveBeenNthCalledWith(2, FileSources.mistral_ocr); }); - test('throws when configured OCR capability is not enabled for the agent', async () => { + test('throws when OCR is needed but the agent capability is not enabled', async () => { mergeFileConfig.mockReturnValue(makeFileConfig({ ocrSupportedMimeTypes: [PDF_MIME] })); /* Only OCR is under test here; disabling every capability would trip the * separate context guard first. */ checkCapability.mockImplementation( async (_req, capability) => capability !== AgentCapabilities.ocr, ); + const localUpload = jest.fn().mockResolvedValue({ + text: '', + bytes: 0, + filepath: FileSources.document_parser, + pagesNeedingOcr: [1], + }); + const remoteUpload = jest.fn(); + setupDocumentStrategies({ documentParser: localUpload, ocr: remoteUpload }); const req = makeReq({ mimetype: PDF_MIME, ocrConfig: { strategy: FileSources.mistral_ocr }, @@ -698,6 +1401,90 @@ describe('processAgentFileUpload', () => { ).rejects.toThrow('OCR capability is not enabled for Agents'); }); + test('preserves partial local text when configured OCR capability is not enabled', async () => { + mergeFileConfig.mockReturnValue(makeFileConfig({ ocrSupportedMimeTypes: [PDF_MIME] })); + checkCapability.mockImplementation( + async (_req, capability) => capability !== AgentCapabilities.ocr, + ); + const localUpload = jest.fn().mockResolvedValue({ + text: 'local page one', + bytes: 14, + filepath: DocumentParser.pdf_inspector, + pagesNeedingOcr: [2], + }); + const remoteUpload = jest.fn().mockResolvedValue({ + text: 'remote OCR text', + bytes: 15, + filepath: FileSources.mistral_ocr, + }); + setupDocumentStrategies({ documentParser: localUpload, ocr: remoteUpload }); + const req = makeReq({ + mimetype: PDF_MIME, + ocrConfig: { strategy: FileSources.mistral_ocr }, + }); + + await processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() }); + + expect(checkCapability).toHaveBeenCalledWith(expect.anything(), AgentCapabilities.ocr); + expect(remoteUpload).not.toHaveBeenCalled(); + expect(db.createFile.mock.calls[0][0]).toEqual( + expect.objectContaining({ + text: 'local page one\n[omitted:2]', + type: PDF_MIME, + }), + ); + }); + + test.each([ + ['missing PDF pages', PDF_MIME, DocumentParser.pdf_inspector, { pagesNeedingOcr: [2] }], + ['embedded document media', DOCX_MIME, DocumentParser.anydoc, { mayOmitContent: true }], + ])( + 'fails closed when strict extracted-text policy cannot inspect %s', + async (_label, mimetype, filepath, incompleteResult) => { + mergeFileConfig.mockReturnValue(makeFileConfig({ ocrSupportedMimeTypes: [mimetype] })); + checkCapability.mockImplementation( + async (_req, capability) => capability !== AgentCapabilities.ocr, + ); + const localUpload = jest.fn().mockResolvedValue({ + text: 'local partial text', + bytes: 18, + filepath, + ...incompleteResult, + }); + const remoteOCR = jest.fn(); + setupDocumentStrategies({ documentParser: localUpload, ocr: remoteOCR }); + assertExtractedTextInspectable.mockImplementationOnce(({ text }) => { + expect(text).toBeUndefined(); + throw makeUninspectableExtractedTextError(); + }); + const req = makeReq({ + mimetype, + originalname: mimetype === PDF_MIME ? 'mixed.pdf' : 'scanned.docx', + ocrConfig: { strategy: FileSources.mistral_ocr }, + filters: { + files: { + pii: { + fields: ['extracted_text'], + uninspectable: 'block', + }, + }, + }, + }); + + await expect( + processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() }), + ).rejects.toMatchObject({ + code: 'content_filter_uninspectable', + body: { field: 'extracted_text' }, + }); + + expect(localUpload).toHaveBeenCalledTimes(1); + expect(remoteOCR).not.toHaveBeenCalled(); + expect(db.addAgentResourceFile).not.toHaveBeenCalled(); + expect(db.createFile).not.toHaveBeenCalled(); + }, + ); + test('uses document_parser (no capability check) when OCR capability returns false but no OCR config', async () => { /* Only OCR is under test here; disabling every capability would trip the * separate context guard first. */ @@ -739,9 +1526,13 @@ describe('processAgentFileUpload', () => { test.each([ ['ODT', ODT_MIME], ['ODP', ODP_MIME], - ['ODG', ODG_MIME], - ])('routes %s through configured OCR when OCR supports the type', async (_, mime) => { - mergeFileConfig.mockReturnValue(makeFileConfig({ ocrSupportedMimeTypes: [mime] })); + ])('extracts %s locally before considering configured OCR', async (_, mime) => { + mergeFileConfig.mockReturnValue( + makeFileConfig({ + ocrSupportedMimeTypes: [mime], + documentParserSupportedMimeTypes: [mime], + }), + ); const req = makeReq({ mimetype: mime, ocrConfig: { strategy: FileSources.mistral_ocr }, @@ -749,8 +1540,30 @@ describe('processAgentFileUpload', () => { await processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() }); + expect(getStrategyFunctions).toHaveBeenCalledWith(FileSources.document_parser); + expect(getStrategyFunctions).not.toHaveBeenCalledWith(FileSources.mistral_ocr); + expect(checkCapability).not.toHaveBeenCalledWith(expect.anything(), AgentCapabilities.ocr); + }); + + test('routes ODG directly through configured OCR because it is not a local document type', async () => { + mergeFileConfig.mockReturnValue(makeFileConfig({ ocrSupportedMimeTypes: [ODG_MIME] })); + const ocrUpload = jest.fn().mockResolvedValue({ + text: 'OCR text', + bytes: 8, + filepath: FileSources.mistral_ocr, + }); + setupDocumentStrategies({ ocr: ocrUpload }); + const req = makeReq({ + mimetype: ODG_MIME, + ocrConfig: { strategy: FileSources.mistral_ocr }, + }); + + await processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() }); + expect(checkCapability).toHaveBeenCalledWith(expect.anything(), AgentCapabilities.ocr); - expect(getStrategyFunctions).toHaveBeenCalledWith(FileSources.mistral_ocr); + expect(getStrategyFunctions).toHaveBeenCalledTimes(2); + expect(getStrategyFunctions).toHaveBeenNthCalledWith(1, FileSources.mistral_ocr); + expect(getStrategyFunctions).toHaveBeenNthCalledWith(2, 'local'); }); test('throws instead of falling back to parseText when document_parser fails for a document MIME type', async () => { @@ -879,15 +1692,20 @@ describe('processAgentFileUpload', () => { expect(db.createFile).not.toHaveBeenCalled(); }); - test('falls back to document_parser when configured OCR fails for a document MIME type', async () => { + test('uses configured OCR when local extraction returns no text for a document MIME type', async () => { mergeFileConfig.mockReturnValue(makeFileConfig({ ocrSupportedMimeTypes: [PDF_MIME] })); - const failingUpload = jest.fn().mockRejectedValue(new Error('OCR API returned 500')); - const fallbackUpload = jest - .fn() - .mockResolvedValue({ text: 'parsed text', bytes: 11, filepath: 'doc://result' }); - getStrategyFunctions - .mockReturnValueOnce({ handleFileUpload: failingUpload }) - .mockReturnValueOnce({ handleFileUpload: fallbackUpload }); + const localUpload = jest.fn().mockResolvedValue({ + text: '', + bytes: 0, + filepath: DocumentParser.pdf_inspector, + pagesNeedingOcr: [1], + }); + const remoteUpload = jest.fn().mockResolvedValue({ + text: 'OCR text', + bytes: 8, + filepath: FileSources.mistral_ocr, + }); + setupDocumentStrategies({ documentParser: localUpload, ocr: remoteUpload }); const req = makeReq({ mimetype: PDF_MIME, ocrConfig: { strategy: FileSources.mistral_ocr }, @@ -897,11 +1715,128 @@ describe('processAgentFileUpload', () => { processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() }), ).resolves.not.toThrow(); + expect(getStrategyFunctions).toHaveBeenNthCalledWith(1, FileSources.document_parser); + expect(getStrategyFunctions).toHaveBeenNthCalledWith(2, FileSources.mistral_ocr); + expect(db.createFile.mock.calls[0][0]).toEqual(expect.objectContaining({ text: 'OCR text' })); + }); + + test('propagates a ZIP-bomb refusal without sending the archive to configured OCR', async () => { + mergeFileConfig.mockReturnValue(makeFileConfig({ ocrSupportedMimeTypes: [DOCX_MIME] })); + const zipBombError = Object.assign(new Error('zip bomb suspected'), { + name: 'ZipBombError', + code: 'ZIP_BOMB', + }); + const localUpload = jest.fn().mockRejectedValue(zipBombError); + const remoteUpload = jest.fn().mockResolvedValue({ + text: 'remote OCR text', + bytes: 15, + filepath: FileSources.mistral_ocr, + }); + getStrategyFunctions.mockImplementation((source) => ({ + handleFileUpload: source === FileSources.document_parser ? localUpload : remoteUpload, + })); + const req = makeReq({ + mimetype: DOCX_MIME, + originalname: 'hostile.docx', + ocrConfig: { strategy: FileSources.mistral_ocr }, + }); + + await expect( + processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() }), + ).rejects.toBe(zipBombError); + + expect(localUpload).toHaveBeenCalledTimes(1); + expect(remoteUpload).not.toHaveBeenCalled(); + expect(checkCapability).not.toHaveBeenCalledWith(expect.anything(), AgentCapabilities.ocr); + }); + + test('propagates a parser input-limit refusal without sending the file to configured OCR', async () => { + mergeFileConfig.mockReturnValue(makeFileConfig({ ocrSupportedMimeTypes: [DOCX_MIME] })); + const inputLimitError = Object.assign(new Error('report.docx exceeds the 15MB limit'), { + name: 'ParserInputLimitError', + code: 'PARSER_INPUT_LIMIT', + userErrorStatusCode: 413, + }); + const localUpload = jest.fn().mockRejectedValue(inputLimitError); + const remoteUpload = jest.fn().mockResolvedValue({ + text: 'remote OCR text', + bytes: 15, + filepath: FileSources.mistral_ocr, + }); + getStrategyFunctions.mockImplementation((source) => ({ + handleFileUpload: source === FileSources.document_parser ? localUpload : remoteUpload, + })); + const req = makeReq({ + mimetype: DOCX_MIME, + originalname: 'report.docx', + ocrConfig: { strategy: FileSources.mistral_ocr }, + }); + + await expect( + processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() }), + ).rejects.toBe(inputLimitError); + + expect(localUpload).toHaveBeenCalledTimes(1); + expect(remoteUpload).not.toHaveBeenCalled(); + expect(checkCapability).not.toHaveBeenCalledWith(expect.anything(), AgentCapabilities.ocr); + }); + + test('propagates a PDF page-limit refusal without sending the PDF to configured OCR', async () => { + mergeFileConfig.mockReturnValue(makeFileConfig({ ocrSupportedMimeTypes: [PDF_MIME] })); + const pageLimitError = Object.assign( + new Error('PDF contains 4000 pages, exceeding the 250-page fallback limit'), + { + name: 'PdfPageLimitError', + code: 'PDF_PAGE_LIMIT', + }, + ); + const localUpload = jest.fn().mockRejectedValue(pageLimitError); + const remoteUpload = jest.fn().mockResolvedValue({ + text: 'remote OCR text', + bytes: 15, + filepath: FileSources.mistral_ocr, + }); + getStrategyFunctions.mockImplementation((source) => ({ + handleFileUpload: source === FileSources.document_parser ? localUpload : remoteUpload, + })); + const req = makeReq({ + mimetype: PDF_MIME, + originalname: 'page-flood.pdf', + ocrConfig: { strategy: FileSources.mistral_ocr }, + }); + + await expect( + processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() }), + ).rejects.toBe(pageLimitError); + + expect(localUpload).toHaveBeenCalledTimes(1); + expect(remoteUpload).not.toHaveBeenCalled(); + expect(checkCapability).not.toHaveBeenCalledWith(expect.anything(), AgentCapabilities.ocr); + }); + + test('does not bypass the document-parser allowlist when configured OCR fails', async () => { + mergeFileConfig.mockReturnValue( + makeFileConfig({ + ocrSupportedMimeTypes: [PDF_MIME], + documentParserSupportedMimeTypes: [DOCX_MIME], + }), + ); + const failingUpload = jest.fn().mockRejectedValue(new Error('OCR API returned 500')); + getStrategyFunctions.mockReturnValue({ handleFileUpload: failingUpload }); + const req = makeReq({ + mimetype: PDF_MIME, + ocrConfig: { strategy: FileSources.mistral_ocr }, + }); + + await expect( + processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() }), + ).rejects.toThrow(/configured OCR service/); + expect(getStrategyFunctions).toHaveBeenCalledWith(FileSources.mistral_ocr); - expect(getStrategyFunctions).toHaveBeenCalledWith(FileSources.document_parser); + expect(getStrategyFunctions).not.toHaveBeenCalledWith(FileSources.document_parser); }); - test('throws when both configured OCR and document_parser fallback fail', async () => { + test('throws when both local extraction and configured OCR fail', async () => { mergeFileConfig.mockReturnValue(makeFileConfig({ ocrSupportedMimeTypes: [PDF_MIME] })); getStrategyFunctions.mockReturnValue({ handleFileUpload: jest.fn().mockRejectedValue(new Error('failure')), @@ -953,6 +1888,31 @@ describe('processAgentFileUpload', () => { expect(getStrategyFunctions).not.toHaveBeenCalledWith(FileSources.document_parser); }); + /** + * Naming a type in the text allowlist while removing it from the parser allowlist is + * how an admin says "RAG handles this one". Reading local-parser eligibility to decide + * the RAG route turned that pair of settings into a refused upload. + */ + test('routes to RAG /text for a type the admin removed from the local parser', async () => { + process.env.RAG_API_URL = 'http://rag-api.test'; + mergeFileConfig.mockReturnValue( + makeFileConfig({ + textSupportedMimeTypes: DOCX_TEXT_REGEX, + documentParserSupportedMimeTypes: [/^application\/pdf$/], + }), + ); + const { parseText } = require('@librechat/api'); + parseText.mockResolvedValueOnce({ text: 'rag extracted', bytes: 13 }); + const req = makeReq({ mimetype: DOCX_MIME, ocrConfig: null }); + + await processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() }); + + expect(parseText).toHaveBeenCalledWith( + expect.objectContaining({ allowNativeFallback: false }), + ); + expect(db.createFile.mock.calls[0][0].text).toBe('rag extracted'); + }); + test('keeps the built-in document parser when text config is the permissive default', async () => { process.env.RAG_API_URL = 'http://rag-api.test'; mergeFileConfig.mockReturnValue( @@ -1034,6 +1994,78 @@ describe('processAgentFileUpload', () => { expect(db.createFile).not.toHaveBeenCalled(); }); + test('annotates omitted pages and keeps the real MIME type on the RAG fallback', async () => { + process.env.RAG_API_URL = 'http://rag-api.test'; + mergeFileConfig.mockReturnValue(makeFileConfig({ textSupportedMimeTypes: DOCX_TEXT_REGEX })); + const { parseText } = require('@librechat/api'); + parseText.mockRejectedValueOnce(new Error('native fallback is disabled')); + const parserUpload = jest.fn().mockResolvedValue({ + text: 'page one text', + bytes: 13, + filepath: FileSources.document_parser, + pagesNeedingOcr: [2, 3], + }); + setupDocumentStrategies({ documentParser: parserUpload }); + const req = makeReq({ mimetype: DOCX_MIME, ocrConfig: null }); + + await processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() }); + + /* The extraction result supplies the annotation and MIME remains the request's + * effective type; storage fields come from the separate local upload. */ + expect(annotateMissingPages).toHaveBeenCalledWith('page one text', [2, 3]); + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('page(s) 2, 3')); + const created = db.createFile.mock.calls[0][0]; + expect(created.text).toBe('page one text\n[omitted:2,3]'); + expect(created.type).toBe(DOCX_MIME); + }); + + /** + * The fallback's own partial result: RAG is gone, this route configures no OCR + * service to complete the text with, and the parser read only part of the document. + * A nonempty string is not evidence the document was inspected, so the strict + * policy has to refuse it here exactly as it does on the primary parser path. + */ + test('fails closed when the RAG fallback can only extract part of the document', async () => { + process.env.RAG_API_URL = 'http://rag-api.test'; + mergeFileConfig.mockReturnValue(makeFileConfig({ textSupportedMimeTypes: DOCX_TEXT_REGEX })); + const { parseText } = require('@librechat/api'); + parseText.mockRejectedValueOnce(new Error('native fallback is disabled')); + getStrategyFunctions.mockReturnValue({ + handleFileUpload: jest.fn().mockResolvedValue({ + text: 'page one text', + bytes: 13, + filepath: FileSources.document_parser, + pagesNeedingOcr: [2, 3], + }), + }); + assertExtractedTextInspectable.mockImplementationOnce(({ text }) => { + expect(text).toBeUndefined(); + throw makeUninspectableExtractedTextError(); + }); + const req = makeReq({ + mimetype: DOCX_MIME, + ocrConfig: null, + filters: { + files: { + pii: { + fields: ['extracted_text'], + uninspectable: 'block', + }, + }, + }, + }); + + await expect( + processAgentFileUpload({ req, res: mockRes, metadata: makeMetadata() }), + ).rejects.toMatchObject({ + code: 'content_filter_uninspectable', + body: { field: 'extracted_text' }, + }); + + expect(db.addAgentResourceFile).not.toHaveBeenCalled(); + expect(db.createFile).not.toHaveBeenCalled(); + }); + test('surfaces a persistence failure without retrying via the document parser', async () => { process.env.RAG_API_URL = 'http://rag-api.test'; mergeFileConfig.mockReturnValue(makeFileConfig({ textSupportedMimeTypes: DOCX_TEXT_REGEX })); @@ -1059,7 +2091,7 @@ describe('processAgentFileUpload', () => { handleFileUpload: jest.fn().mockResolvedValue({ text: oversizedText, bytes: Buffer.byteLength(oversizedText, 'utf8'), - filepath: 'doc://result', + filepath: FileSources.document_parser, }), }); const req = makeReq({ mimetype: PDF_MIME, ocrConfig: null }); @@ -1078,7 +2110,7 @@ describe('processAgentFileUpload', () => { handleFileUpload: jest.fn().mockResolvedValue({ text: okText, bytes: Buffer.byteLength(okText, 'utf8'), - filepath: 'doc://result', + filepath: FileSources.document_parser, }), }); const req = makeReq({ mimetype: PDF_MIME, ocrConfig: null }); @@ -1408,8 +2440,10 @@ describe('processAgentFileUpload', () => { it('defers an inferred file-search destination until tool execution', async () => { const { uploadVectors } = require('~/server/services/Files/VectorDB/crud'); setupStoredFileUpload(); + /* A type file search can read and no extractor can: a presentation itself is + * parsed to text now, which would route it to context instead of deferring. */ const req = makeReq({ - mimetype: 'application/vnd.openxmlformats-officedocument.presentationml.presentation', + mimetype: 'application/vnd.openxmlformats-officedocument.presentationml.template', ocrConfig: null, }); req.body.endpoint = EModelEndpoint.agents; @@ -1898,7 +2932,7 @@ describe('processAgentFileUpload', () => { }); test('plans extraction with the promoted context resource for auto-routed text uploads', async () => { - const { getUploadExtractedTextPlan } = require('@librechat/api'); + const { planDocumentExtraction } = require('@librechat/api'); mergeFileConfig.mockReturnValue(makeFileConfig()); const req = makeReq({ mimetype: DOCX_MIME, ocrConfig: null }); @@ -1910,7 +2944,7 @@ describe('processAgentFileUpload', () => { metadata: { agent_id: 'agent-abc', file_id: 'file-uuid-123' }, }).catch(() => {}); - expect(getUploadExtractedTextPlan).toHaveBeenCalledWith( + expect(planDocumentExtraction).toHaveBeenCalledWith( expect.objectContaining({ toolResource: EToolResources.context }), ); }); @@ -3377,3 +4411,44 @@ describe('fallback text for uploads left to tools', () => { ); }); }); + +/** + * The authoritative MIME gate. Multer's filter runs while the file part is still + * streaming, so it cannot read a `tool_resource` sent after the file; this one has the + * complete body and runs before any provider is handed the upload. + */ +describe('filterFile', () => { + const VENDOR_MIME = 'application/vnd.vendor.word'; + + const makeFilterReq = (toolResource) => ({ + file: { size: 1024, mimetype: VENDOR_MIME, originalname: 'report.docx' }, + body: { + endpoint: 'agents', + file_id: '11111111-1111-4111-8111-111111111111', + ...(toolResource ? { tool_resource: toolResource } : {}), + }, + config: { fileConfig: {} }, + }); + + beforeEach(() => { + mergeFileConfig.mockReturnValue({ + ...makeFileConfig(), + documentParser: { supportedMimeTypes: [new RegExp(`^${VENDOR_MIME}$`)] }, + endpoints: { default: { supportedMimeTypes: [/^application\/pdf$/] } }, + }); + }); + + it.each([EToolResources.context, EToolResources.ocr])( + 'admits a parser-named MIME for a %s upload', + (toolResource) => { + expect(() => filterFile({ req: makeFilterReq(toolResource) })).not.toThrow(); + }, + ); + + it.each([ + ['a different tool resource', 'file_search'], + ['no tool resource at all', undefined], + ])('refuses a parser-named MIME for %s', (_label, toolResource) => { + expect(() => filterFile({ req: makeFilterReq(toolResource) })).toThrow('Unsupported file type'); + }); +}); diff --git a/api/server/utils/import/fork.js b/api/server/utils/import/fork.js index dc141a63199..3e22dee47b7 100644 --- a/api/server/utils/import/fork.js +++ b/api/server/utils/import/fork.js @@ -9,7 +9,7 @@ const { transferNativeCopyProvenance, } = require('@librechat/api'); const { logger, tenantStorage } = require('@librechat/data-schemas'); -const { EModelEndpoint, Constants, ForkOptions } = require('librechat-data-provider'); +const { EModelEndpoint, Constants, ContentTypes, ForkOptions } = require('librechat-data-provider'); const { getConvo, getMessages, getSharedMessages } = require('~/models'); const { createImportBatchBuilder } = require('./importBatchBuilder'); const { getAppConfig } = require('~/server/services/Config'); @@ -324,18 +324,38 @@ function splitAtTargetLevel(messages, targetMessageId) { * sharer's files into the viewer's run. Dropping the ids keeps a fork's file * access no broader than viewing the read-only share, while leaving render-only * metadata (e.g. `filepath`, `toolCallId`) intact. + * * @param {TMessage} message - The shared message to sanitize. * @returns {TMessage} The message with file identifiers removed. */ +const stripSharedFileRef = ({ file_id: _fileId, ...file }) => file; + +/** Steer parts carry the same user file refs inline in `content`, so they get the same treatment. */ +function stripSharedSteerFileIds(content) { + let result = null; + for (let i = 0; i < content.length; i++) { + const part = content[i]; + if (part?.type !== ContentTypes.STEER || !Array.isArray(part.files)) { + continue; + } + if (result == null) { + result = [...content]; + } + result[i] = { ...part, files: part.files.map(stripSharedFileRef) }; + } + return result ?? content; +} + function stripSharedFileIds(message) { const sanitized = { ...message }; if (Array.isArray(sanitized.files)) { - sanitized.files = sanitized.files.map(({ file_id: _fileId, ...file }) => file); + sanitized.files = sanitized.files.map(stripSharedFileRef); } if (Array.isArray(sanitized.attachments)) { - sanitized.attachments = sanitized.attachments.map( - ({ file_id: _fileId, ...attachment }) => attachment, - ); + sanitized.attachments = sanitized.attachments.map(stripSharedFileRef); + } + if (Array.isArray(sanitized.content)) { + sanitized.content = stripSharedSteerFileIds(sanitized.content); } return sanitized; } diff --git a/api/server/utils/import/fork.spec.js b/api/server/utils/import/fork.spec.js index 8bf578fcfef..8379d17187f 100644 --- a/api/server/utils/import/fork.spec.js +++ b/api/server/utils/import/fork.spec.js @@ -1,4 +1,4 @@ -const { Constants, ForkOptions, RetentionMode } = require('librechat-data-provider'); +const { Constants, ContentTypes, ForkOptions, RetentionMode } = require('librechat-data-provider'); const mockLogger = { debug: jest.fn(), @@ -969,6 +969,46 @@ describe('forkSharedConversation', () => { expect(message.attachments[0].toolCallId).toBe('tool_1'); }); + /** + * Steer parts carry user file refs inline in `content`, so a fork that only cleaned + * `files`/`attachments` would still persist the sharer's ids and let the next turn's + * file-resend path rehydrate them into the viewer's run. + */ + test('should strip file ids carried by steer parts inside content', async () => { + getSharedMessages.mockResolvedValue({ + ...mockShare, + messages: [ + { + messageId: 'msg_a', + parentMessageId: Constants.NO_PARENT, + text: 'Message with a parsed document', + isCreatedByUser: true, + createdAt: '2021-01-01', + files: [{ file_id: 'owner-doc-1', filename: 'report.docx' }], + content: [ + { + type: ContentTypes.STEER, + files: [{ file_id: 'owner-doc-2', filename: 'notes.pdf' }], + }, + ], + }, + ], + }); + + await forkSharedConversation({ + shareId: 'share123', + requestUserId: 'user1', + }); + + const [message] = bulkSaveMessages.mock.calls[0][0]; + expect(message.files[0]).not.toHaveProperty('file_id'); + expect(message.files[0].filename).toBe('report.docx'); + + const [steerPart] = message.content; + expect(steerPart.files[0]).not.toHaveProperty('file_id'); + expect(steerPart.files[0].filename).toBe('notes.pdf'); + }); + test('should resolve interfaceConfig from the app config and pass it to the builder', async () => { const interfaceConfig = { retentionMode: 'all', retention: { days: 30 } }; const loadAppConfig = jest.fn().mockResolvedValue({ interfaceConfig }); diff --git a/client/src/common/types.ts b/client/src/common/types.ts index a3e0a9b3191..006ab93c2b0 100644 --- a/client/src/common/types.ts +++ b/client/src/common/types.ts @@ -600,9 +600,9 @@ export interface ExtendedFile { preview?: string; progress: number; source?: FileSources; + llmDeliveryPath?: t.TFile['llmDeliveryPath']; attached?: boolean; embedded?: boolean; - llmDeliveryPath?: t.TFile['llmDeliveryPath']; tool_resource?: string; metadata?: t.TFile['metadata']; } diff --git a/client/src/components/Chat/Input/Files/ExtractedTextPanel.tsx b/client/src/components/Chat/Input/Files/ExtractedTextPanel.tsx new file mode 100644 index 00000000000..5f5378aa181 --- /dev/null +++ b/client/src/components/Chat/Input/Files/ExtractedTextPanel.tsx @@ -0,0 +1,94 @@ +import React, { useState } from 'react'; +import copy from 'copy-to-clipboard'; +import { Spinner } from '@librechat/client'; +import CopyButton from '~/components/Messages/Content/CopyButton'; +import { useFilePreview } from '~/data-provider'; +import { useLocalize } from '~/hooks'; + +interface ExtractedTextPanelProps { + fileId?: string; + /** Gates the request so the text is only fetched while it is on screen. */ + enabled: boolean; + /** Routes the fetch through the share endpoint when the conversation is a shared snapshot. */ + shareId?: string; +} + +/** + * Renders the text a document parser extracted from an upload. + * + * Shown verbatim in a monospace block rather than rendered as Markdown: the point + * is to see exactly what the model receives, structure included. + */ +export default function ExtractedTextPanel({ fileId, enabled, shareId }: ExtractedTextPanelProps) { + const localize = useLocalize(); + const [isCopied, setIsCopied] = useState(false); + const { data, isInitialLoading, isError } = useFilePreview( + fileId, + { + enabled: enabled && !!fileId, + /* Extracted text is immutable once the parse finishes and runs to several MB, + * so a full refetch on every dialog open is pure waste. Scoped to this panel: + * the hook's default stays refetch-on-mount for the polling callers. */ + staleTime: Infinity, + }, + shareId, + ); + + const text = data?.text ?? ''; + + const handleCopy = () => { + copy(text, { format: 'text/plain' }); + setIsCopied(true); + setTimeout(() => setIsCopied(false), 2000); + }; + + /* `pending` means the parse is still running server-side; the query polls it + * every 2.5s, so this is a loading state rather than an empty one. */ + if (isInitialLoading || data?.status === 'pending') { + return ( +
+ + {localize('com_ui_loading')} +
+ ); + } + + if (isError) { + return ( +

+ {localize('com_ui_extracted_text_error')} +

+ ); + } + + if (!text) { + return ( +

+ {localize('com_ui_extracted_text_none')} +

+ ); + } + + return ( +
+
+ +
+ {/* Focusable so the overflow region can be scrolled from the keyboard: + * Chrome and Safari do not focus scroll containers on their own. */} + 
+        {text}
+
+
+ ); +} diff --git a/client/src/components/Chat/Input/Files/FileNameCell.tsx b/client/src/components/Chat/Input/Files/FileNameCell.tsx new file mode 100644 index 00000000000..2b6e1a2b0b8 --- /dev/null +++ b/client/src/components/Chat/Input/Files/FileNameCell.tsx @@ -0,0 +1,64 @@ +import React, { useState } from 'react'; +import { FileSources, isParsedDocument } from 'librechat-data-provider'; +import type { TFile } from 'librechat-data-provider'; +import FileTextDialog from '~/components/Chat/Input/Files/FileTextDialog'; +import FilePreview from '~/components/Chat/Input/Files/FilePreview'; +import { getFileType } from '~/utils'; +import { useLocalize } from '~/hooks'; + +/** + * Whether the record actually holds extracted text. + * + * A record whose text replaces its bytes for the model is stamped + * `llmDeliveryPath: 'text'` by the upload path; records written before storage kept + * the original document say the same thing with `FileSources.text`. The same PDF + * uploaded to file_search or execute_code is stored as a binary with no text at all, + * so the MIME check alone would offer a control that always dead-ends in an empty state. + */ +const hasExtractedText = (file: TFile): boolean => + (file.llmDeliveryPath === 'text' || file.source === FileSources.text) && + isParsedDocument(file.type, file.filename); + +/** + * Filename cell for the file manager. + * + * For parsed documents the name becomes a button that opens the extracted text, + * which is otherwise invisible: it is sent to the model but stripped from every + * file listing. Other file types render as plain text, unchanged. + */ +export default function FileNameCell({ file }: { file: TFile }) { + const localize = useLocalize(); + const [open, setOpen] = useState(false); + const fileType = getFileType(file.type); + + if (!hasExtractedText(file)) { + return ( +
+ {fileType && } + {file.filename} +
+ ); + } + + return ( + <> +
+ {fileType && } + +
+ + + ); +} diff --git a/client/src/components/Chat/Input/Files/FileRow.tsx b/client/src/components/Chat/Input/Files/FileRow.tsx index 95d43b2ac93..a47877bcac1 100644 --- a/client/src/components/Chat/Input/Files/FileRow.tsx +++ b/client/src/components/Chat/Input/Files/FileRow.tsx @@ -1,14 +1,33 @@ -import { useEffect } from 'react'; +import { useState, useEffect } from 'react'; import { useToastContext } from '@librechat/client'; -import { EToolResources } from 'librechat-data-provider'; +import { EToolResources, FileSources, isParsedDocument } from 'librechat-data-provider'; import type { ExtendedFile } from '~/common'; import { useDeleteFilesMutation } from '~/data-provider'; import { logger, getCachedPreview } from '~/utils'; import { useFileDeletion } from '~/hooks/Files'; +import FileTextDialog from './FileTextDialog'; import FileContainer from './FileContainer'; import { useLocalize } from '~/hooks'; import Image from './Image'; +/** + * Whether a staged upload already has extracted text to show. + * + * Parsing happens server-side on upload, so the text exists as soon as the + * progress bar completes, before the message is ever sent. The routing fields are + * copied onto the staged record by the same update that sets `progress` to 1. + * + * `llmDeliveryPath: 'text'` is what the upload path stamps on a record whose text + * replaces its bytes for the model; `FileSources.text` is the same statement from + * records written before storage kept the original document. The same PDF sent to + * file_search or execute_code carries neither and is stored as a binary with no text. + */ +const hasExtractedText = (file: ExtendedFile): boolean => + !!file.file_id && + file.progress >= 1 && + (file.llmDeliveryPath === 'text' || file.source === FileSources.text) && + isParsedDocument(file.type, file.filename); + /** * Shared wrapper with a stable module-scope identity. Passing an inline arrow as * `Wrapper` makes it a new component type on every render, so React remounts the @@ -59,6 +78,7 @@ export default function FileRow({ }) { const localize = useLocalize(); const { showToast } = useToastContext(); + const [textFile, setTextFile] = useState(null); const files = Array.from(_files?.values() ?? []).filter((file) => fileFilter ? fileFilter(file) : true, ); @@ -166,6 +186,20 @@ export default function FileRow({ file.progress >= 1 && isPastedTextFile(file) && !isPasteActionPending?.(file); + const showsText = hasExtractedText(file); + /** One chip, one primary action: returning a paste to the composer takes + * precedence over opening extracted text, and an inert chip gets neither. */ + let onChipClick: (() => void) | undefined; + let chipAriaLabel: string | undefined; + if (isEditablePaste) { + onChipClick = () => onEditPastedText(file); + chipAriaLabel = localize('com_ui_pasted_text_edit_chip', { 0: file.filename ?? '' }); + } else if (showsText) { + onChipClick = () => setTextFile(file); + chipAriaLabel = localize('com_ui_view_extracted_text_var', { + 0: file.filename ?? '', + }); + } return ( /* Sized by its content, not parked in a fixed 70px slot: a 56px @@ -191,12 +225,8 @@ export default function FileRow({ file={file} onDelete={handleDelete} buttonClassName="h-[58px]" - onClick={isEditablePaste ? () => onEditPastedText(file) : undefined} - ariaLabel={ - isEditablePaste - ? localize('com_ui_pasted_text_edit_chip', { 0: file.filename ?? '' }) - : undefined - } + onClick={onChipClick} + ariaLabel={chipAriaLabel} subtitleAction={ isEditablePaste && onMovePastedTextInline != null ? { @@ -214,9 +244,28 @@ export default function FileRow({ ); }; + const textDialog = ( + !open && setTextFile(null)} + fileId={textFile?.file_id ?? ''} + filename={textFile?.filename ?? ''} + /> + ); + if (Wrapper) { - return {renderFiles()}; + return ( + <> + {renderFiles()} + {textDialog} + + ); } - return renderFiles(); + return ( + <> + {renderFiles()} + {textDialog} + + ); } diff --git a/client/src/components/Chat/Input/Files/FileTextDialog.tsx b/client/src/components/Chat/Input/Files/FileTextDialog.tsx new file mode 100644 index 00000000000..3310816cc2b --- /dev/null +++ b/client/src/components/Chat/Input/Files/FileTextDialog.tsx @@ -0,0 +1,46 @@ +import React from 'react'; +import { + OGDialog, + OGDialogTitle, + OGDialogHeader, + OGDialogContent, + OGDialogDescription, +} from '@librechat/client'; +import ExtractedTextPanel from '~/components/Chat/Input/Files/ExtractedTextPanel'; +import { useLocalize } from '~/hooks'; + +interface FileTextDialogProps { + open: boolean; + onOpenChange: (open: boolean) => void; + fileId: string; + filename: string; +} + +/** + * Shows the text a document parser extracted from an upload. + * + * The file list strips `text` to keep payloads small, so the content is fetched + * on demand and only while the dialog is open. + */ +export default function FileTextDialog({ + open, + onOpenChange, + fileId, + filename, +}: FileTextDialogProps) { + const localize = useLocalize(); + + return ( + + + + + {filename} + + {localize('com_ui_extracted_text_description')} + + + + + ); +} diff --git a/client/src/components/Chat/Input/Files/Table/Columns.tsx b/client/src/components/Chat/Input/Files/Table/Columns.tsx index 2c2c6985a73..13b23b8775f 100644 --- a/client/src/components/Chat/Input/Files/Table/Columns.tsx +++ b/client/src/components/Chat/Input/Files/Table/Columns.tsx @@ -12,10 +12,10 @@ import { import type { ColumnDef } from '@tanstack/react-table'; import type { TFile } from 'librechat-data-provider'; import ImagePreview from '~/components/Chat/Input/Files/ImagePreview'; -import FilePreview from '~/components/Chat/Input/Files/FilePreview'; +import FileNameCell from '~/components/Chat/Input/Files/FileNameCell'; import { TranslationKeys, useLocalize } from '~/hooks'; import { SortFilterHeader } from './SortFilterHeader'; -import { formatDate, getFileType } from '~/utils'; +import { formatDate } from '~/utils'; const contextMap: Record = { [FileContext.avatar]: 'com_ui_avatar', @@ -119,13 +119,7 @@ export const columns: ColumnDef[] = [ ); } - const fileType = getFileType(file.type); - return ( -
- {fileType && } - {file.filename} -
- ); + return ; }, }, { diff --git a/client/src/components/Chat/Input/Files/__tests__/ExtractedTextPanel.spec.tsx b/client/src/components/Chat/Input/Files/__tests__/ExtractedTextPanel.spec.tsx new file mode 100644 index 00000000000..79261411fab --- /dev/null +++ b/client/src/components/Chat/Input/Files/__tests__/ExtractedTextPanel.spec.tsx @@ -0,0 +1,121 @@ +import React from 'react'; +import '@testing-library/jest-dom'; +import userEvent from '@testing-library/user-event'; +import { render, screen } from '@testing-library/react'; +import type { TFilePreview } from 'librechat-data-provider'; +import ExtractedTextPanel from '../ExtractedTextPanel'; + +const mockUseFilePreview = jest.fn(); + +jest.mock('~/data-provider', () => ({ + useFilePreview: ( + fileId: string | undefined, + config?: Record, + shareId?: string, + ) => mockUseFilePreview(fileId, config, shareId), +})); + +jest.mock('copy-to-clipboard', () => ({ + __esModule: true, + default: jest.fn(() => true), +})); + +const mockCopy = jest.requireMock('copy-to-clipboard').default as jest.Mock; + +interface PreviewResult { + data?: TFilePreview; + isInitialLoading?: boolean; + isError?: boolean; +} + +const setPreview = (result: PreviewResult) => + mockUseFilePreview.mockReturnValue({ + isInitialLoading: false, + isError: false, + ...result, + }); + +const ready = (text?: string): TFilePreview => ({ file_id: 'f1', status: 'ready', text }); + +describe('ExtractedTextPanel', () => { + beforeEach(() => { + jest.clearAllMocks(); + setPreview({ data: ready('Line one\nLine two') }); + }); + + test('waits on a parse that is still running', () => { + /* `pending` is not a terminal state: the query polls it every 2.5s, so + * reporting "no extracted text" here would be wrong the moment it resolves. */ + setPreview({ data: { file_id: 'f1', status: 'pending' } }); + + render(); + + expect(screen.getByRole('status')).toBeInTheDocument(); + expect(screen.queryByText(/No extracted text/i)).not.toBeInTheDocument(); + }); + + test('waits while the first fetch is in flight', () => { + setPreview({ data: undefined, isInitialLoading: true }); + + render(); + + expect(screen.getByRole('status')).toBeInTheDocument(); + }); + + test('tells a failed fetch apart from a document with no text', () => { + setPreview({ data: undefined, isError: true }); + + render(); + + expect(screen.getByRole('alert')).toHaveTextContent(/could not be loaded/i); + expect(screen.queryByText(/No extracted text/i)).not.toBeInTheDocument(); + }); + + test('reports an empty document as having no text', () => { + setPreview({ data: ready('') }); + + render(); + + expect(screen.getByText(/No extracted text/i)).toBeInTheDocument(); + }); + + test('the text is a labelled region a keyboard can scroll', () => { + /* Chrome and Safari do not focus overflow containers on their own, so without + * a tab stop everything past the panel's max height is unreachable. */ + render(); + + const region = screen.getByRole('region', { name: /extracted text/i }); + expect(region).toHaveAttribute('tabindex', '0'); + expect(region).toHaveTextContent('Line one'); + }); + + test('copies through the clipboard fallback rather than the secure-context API', async () => { + /* `navigator.clipboard` is undefined on plain-HTTP self-hosts, where calling + * it throws straight out of the click handler. */ + render(); + + await userEvent.click(screen.getByRole('button')); + + expect(mockCopy).toHaveBeenCalledWith('Line one\nLine two', { format: 'text/plain' }); + }); + + test('routes the fetch through the share endpoint and never restales the text', () => { + render(); + + expect(mockUseFilePreview).toHaveBeenCalledWith( + 'f1', + expect.objectContaining({ enabled: true, staleTime: Infinity }), + 'share-1', + ); + }); + + test('stays idle without a file', () => { + render(); + + expect(mockUseFilePreview).toHaveBeenCalledWith( + undefined, + expect.objectContaining({ enabled: false }), + undefined, + ); + }); +}); diff --git a/client/src/components/Chat/Input/Files/__tests__/FileNameCell.spec.tsx b/client/src/components/Chat/Input/Files/__tests__/FileNameCell.spec.tsx new file mode 100644 index 00000000000..461ebb41d36 --- /dev/null +++ b/client/src/components/Chat/Input/Files/__tests__/FileNameCell.spec.tsx @@ -0,0 +1,80 @@ +import React from 'react'; +import { FileSources } from 'librechat-data-provider'; +import { render, screen } from '@testing-library/react'; +import type { TFile } from 'librechat-data-provider'; +import FileNameCell from '../FileNameCell'; + +jest.mock('~/data-provider', () => ({ + useFilePreview: () => ({ data: undefined, isInitialLoading: false, isError: false }), +})); + +jest.mock('~/components/Chat/Input/Files/FilePreview', () => ({ + __esModule: true, + default: () =>
, +})); + +const makeFile = (type: string, source: FileSources = FileSources.text): TFile => + ({ + file_id: 'file-1', + filename: 'report.bin', + type, + filepath: '/uploads/report.bin', + source, + }) as TFile; + +describe('FileNameCell', () => { + test.each([ + ['application/pdf', 'pdf'], + ['application/vnd.openxmlformats-officedocument.wordprocessingml.document', 'docx'], + ['application/vnd.openxmlformats-officedocument.presentationml.presentation', 'pptx'], + ['application/vnd.oasis.opendocument.text', 'odt'], + ])('offers the extracted text for %s', (mimetype) => { + render(); + + expect(screen.getByRole('button', { name: /view text extracted from/i })).toBeInTheDocument(); + }); + + test.each([ + ['image/png', 'image'], + ['text/plain', 'plain text'], + ['application/zip', 'archive'], + ])('renders %s as plain text with no affordance', (mimetype) => { + render(); + + /* Only parsed documents have hidden extracted text worth surfacing; adding a + * control for every row would promise content that is not there. */ + expect(screen.queryByRole('button')).not.toBeInTheDocument(); + expect(screen.getByText('report.bin')).toBeInTheDocument(); + }); + + test.each([ + [FileSources.local, 'file_search'], + [FileSources.execute_code, 'code interpreter'], + [FileSources.s3, 'object storage'], + ])('offers nothing for a PDF stored as a binary via %s', (source) => { + /* Only the parser writes `text`, and every record it writes carries + * `FileSources.text`. The same PDF uploaded to file_search or execute_code is + * stored whole, so the control would always land on an empty state. */ + render(); + + expect(screen.queryByRole('button')).not.toBeInTheDocument(); + expect(screen.getByText('report.bin')).toBeInTheDocument(); + }); + + test('offers nothing when the record carries no source at all', () => { + const file = makeFile('application/pdf'); + delete (file as Partial).source; + + render(); + + expect(screen.queryByRole('button')).not.toBeInTheDocument(); + }); + + test('the trigger names the file for screen readers', () => { + render(); + + expect( + screen.getByRole('button', { name: 'View text extracted from report.bin' }), + ).toBeInTheDocument(); + }); +}); diff --git a/client/src/components/Chat/Input/Files/__tests__/FileRow.spec.tsx b/client/src/components/Chat/Input/Files/__tests__/FileRow.spec.tsx index c4cb734091b..21ca14094b8 100644 --- a/client/src/components/Chat/Input/Files/__tests__/FileRow.spec.tsx +++ b/client/src/components/Chat/Input/Files/__tests__/FileRow.spec.tsx @@ -1,11 +1,30 @@ import React from 'react'; -import userEvent from '@testing-library/user-event'; -import { render, screen } from '@testing-library/react'; + import '@testing-library/jest-dom'; +import userEvent from '@testing-library/user-event'; import { FileSources } from 'librechat-data-provider'; +import { render, screen } from '@testing-library/react'; import type { ExtendedFile } from '~/common'; import FileRow from '../FileRow'; +interface MockImageProps { + url?: string; + progress: number; + source?: FileSources; +} + +interface MockFileContainerProps { + file: Partial; + ariaLabel?: string; + onClick?: React.MouseEventHandler; + subtitleAction?: { label: string; onClick: () => void }; +} + +interface MockFileTextDialogProps { + open: boolean; + filename: string; +} + jest.mock('~/hooks', () => ({ useLocalize: jest.fn(), })); @@ -26,7 +45,7 @@ jest.mock('~/utils', () => ({ })); jest.mock('../Image', () => { - return function MockImage({ url, progress, source }: any) { + return function MockImage({ url, progress, source }: MockImageProps) { return (
{url} @@ -37,10 +56,17 @@ jest.mock('../Image', () => { }; }); +/* Renders a real button so the click path into FileRow's dialog is exercised + * rather than only the presence of an `onClick` prop. */ jest.mock('../FileContainer', () => { - return function MockFileContainer({ file, onClick, ariaLabel, subtitleAction }: any) { + return function MockFileContainer({ + file, + onClick, + ariaLabel, + subtitleAction, + }: MockFileContainerProps) { return ( -
+
{file.filename} {onClick != null && (