From d734b4ed56ac4ec348900e655b5d5d913aa747fe Mon Sep 17 00:00:00 2001 From: lishuceo Date: Sun, 17 May 2026 09:49:33 +0800 Subject: [PATCH 1/4] =?UTF-8?q?feat:=20=E5=A4=A7=E6=96=87=E6=9C=AC?= =?UTF-8?q?=E6=96=87=E4=BB=B6=20lazy=20loading=20+=20=E5=85=B1=E4=BA=AB=20?= =?UTF-8?q?file-cache?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 抽取 src/feishu/file-cache.ts: saveMessageFileToCache + cleanupOldDownloads,复用 tmpdir/feishu-downloads - feishu_download_message_file MCP 工具改为复用该 helper - 当前消息文本附件: ≤64KB 内嵌 prompt;>64KB 落盘并注入路径元数据,让 agent 用 Read 工具按需 offset/limit 分段读 - 历史消息文本附件: 同样小文件内嵌、大文件落盘 - 文本文件上限从 1MB 提升到 30MB(与 PDF 一致) - index.ts 周期 cleanup 接入 24h 过期清理 Co-Authored-By: Claude Opus 4.7 --- src/feishu/event-handler.ts | 40 ++++++++++++++++------- src/feishu/file-cache.ts | 63 +++++++++++++++++++++++++++++++++++++ src/feishu/tools/message.ts | 13 ++------ src/index.ts | 2 ++ 4 files changed, 96 insertions(+), 22 deletions(-) create mode 100644 src/feishu/file-cache.ts diff --git a/src/feishu/event-handler.ts b/src/feishu/event-handler.ts index af33e04..709f6b3 100644 --- a/src/feishu/event-handler.ts +++ b/src/feishu/event-handler.ts @@ -10,6 +10,7 @@ import { buildStatusCard, buildCancelledCard, buildPipelineCard, buildPipelineCo import type { AskUserQuestionItem } from './message-builder.js'; import { TOTAL_PHASES } from '../pipeline/types.js'; import { feishuClient, feishuClientContext, runWithAccountId } from './client.js'; +import { saveMessageFileToCache } from './file-cache.js'; import { config, isMultiBotMode } from '../config.js'; import { checkAndRequestApproval, handleApprovalTextCommand, handleApprovalCardAction, setOnApproved } from './approval.js'; import { resolveThreadContext } from './thread-context.js'; @@ -1518,7 +1519,8 @@ async function downloadHistoryFiles( const toProcess = uniqueRefs.slice(-MAX_HISTORY_FILES); const MAX_PDF_SIZE = 30 * 1024 * 1024; - const MAX_TEXT_SIZE = 1 * 1024 * 1024; + const MAX_TEXT_SIZE = 30 * 1024 * 1024; + const INLINE_HISTORY_TEXT_THRESHOLD = 64 * 1024; const documents: DocumentAttachment[] = []; const fileTexts: string[] = []; @@ -1542,10 +1544,19 @@ async function downloadHistoryFiles( } } else if (isTextFile(ref.fileName)) { const buf = await feishuClient.downloadMessageFile(ref.messageId, ref.fileKey); - if (buf.length <= MAX_TEXT_SIZE) { + if (buf.length > MAX_TEXT_SIZE) { + logger.warn({ messageId: ref.messageId, fileName: ref.fileName, sizeBytes: buf.length }, 'History text file too large, skipping'); + return; + } + if (buf.length <= INLINE_HISTORY_TEXT_THRESHOLD) { const content = buf.toString('utf-8'); fileTexts.push(`[历史消息中的文件: ${ref.fileName}]\n\n\n${content}\n`); - logger.info({ messageId: ref.messageId, fileName: ref.fileName, sizeBytes: buf.length }, 'History text file downloaded'); + logger.info({ messageId: ref.messageId, fileName: ref.fileName, sizeBytes: buf.length }, 'History text file embedded inline'); + } else { + const filePath = await saveMessageFileToCache(ref.messageId, ref.fileKey, buf, ref.fileName); + const sizeKB = (buf.length / 1024).toFixed(1); + fileTexts.push(`[历史消息中的文件: ${ref.fileName}（${sizeKB} KB），已保存到本地: ${filePath}\n请使用 Read 工具按需读取该文件，支持 offset/limit 分段；文件保留 24 小时。]`); + logger.info({ messageId: ref.messageId, fileName: ref.fileName, sizeBytes: buf.length, filePath }, 'History text file saved to cache for lazy read'); } } } catch (err) { @@ -3404,9 +3415,9 @@ async function parseMessage(data: MessageEventData): Promise MAX_TEXT_FILE_SIZE_BYTES) { + if (buf.length > MAX_FILE_SIZE_BYTES) { logger.warn({ messageId: message.message_id, sizeBytes: buf.length, fileName }, 'Text file too large, skipping'); - await feishuClient.replyText(message.message_id, `⚠️ 文本文件太大（${(buf.length / 1024 / 1024).toFixed(1)}MB），请压缩到 1MB 以内后重试`); + await feishuClient.replyText(message.message_id, `⚠️ 文本文件太大（${(buf.length / 1024 / 1024).toFixed(1)}MB），上限 ${MAX_FILE_SIZE_BYTES / 1024 / 1024}MB`); return null; } - const fileContent = buf.toString('utf-8'); - text = `[用户发送了文件: ${fileName}]\n\n\n${fileContent}\n`; - logger.info({ messageId: message.message_id, fileName, sizeBytes: buf.length }, 'Text file downloaded and embedded in prompt'); + if (buf.length <= INLINE_TEXT_FILE_THRESHOLD) { + const fileContent = buf.toString('utf-8'); + text = `[用户发送了文件: ${fileName}]\n\n\n${fileContent}\n`; + logger.info({ messageId: message.message_id, fileName, sizeBytes: buf.length }, 'Text file embedded inline'); + } else { + const filePath = await saveMessageFileToCache(message.message_id, fileKey, buf, fileName); + const sizeKB = (buf.length / 1024).toFixed(1); + text = `[用户发送了文件: ${fileName}（${sizeKB} KB），已保存到本地: ${filePath}\n请使用 Read 工具按需读取该文件，支持 offset/limit 分段；文件保留 24 小时。]`; + logger.info({ messageId: message.message_id, fileName, sizeBytes: buf.length, filePath }, 'Text file saved to cache for lazy read'); + } } else { text = `[用户发送了文件: ${fileName}，该文件类型暂不支持。支持的类型：PDF、常见文本/代码文件（.md, .txt, .json, .log, .py, .ts 等）]`; } diff --git a/src/feishu/file-cache.ts b/src/feishu/file-cache.ts new file mode 100644 index 0000000..1e5c614 --- /dev/null +++ b/src/feishu/file-cache.ts @@ -0,0 +1,63 @@ +import { writeFile, mkdir, readdir, stat, unlink } from 'fs/promises'; +import { join, extname } from 'path'; +import { tmpdir } from 'os'; +import { logger } from '../utils/logger.js'; + +export const DOWNLOAD_DIR = join(tmpdir(), 'feishu-downloads'); + +const SAFE_NAME_RE = /[^a-zA-Z0-9_-]/g; + +/** + * 把飞书消息附件落盘到共享缓存目录，返回绝对路径。 + * + * 用于 lazy-loading 文本类附件：上传后只把路径注入 prompt，让 agent 用 Read 工具按需 offset/limit 读取。 + */ +export async function saveMessageFileToCache( + messageId: string, + fileKey: string, + buf: Buffer, + originalFileName?: string, +): Promise { + await mkdir(DOWNLOAD_DIR, { recursive: true }); + const base = `${messageId}-${fileKey}`.replace(SAFE_NAME_RE, '_'); + const ext = originalFileName ? extname(originalFileName).toLowerCase() : ''; + const safeExt = /^\.[a-z0-9]{1,8}$/.test(ext) ? ext : ''; + const filePath = join(DOWNLOAD_DIR, base + safeExt); + await writeFile(filePath, buf); + return filePath; +} + +/** + * 清理 DOWNLOAD_DIR 中早于 maxAgeMs 的文件。默认 24 小时。 + * + * 由 index.ts 的周期性 cleanup interval 调用，硬盘充裕，给 agent 留足回头读的窗口。 + */ +export async function cleanupOldDownloads(maxAgeMs: number = 24 * 60 * 60 * 1000): Promise { + let cleaned = 0; + let entries: string[]; + try { + entries = await readdir(DOWNLOAD_DIR); + } catch (err) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') return 0; + logger.warn({ err }, 'cleanupOldDownloads: readdir failed'); + return 0; + } + + const cutoff = Date.now() - maxAgeMs; + await Promise.all(entries.map(async (name) => { + const filePath = join(DOWNLOAD_DIR, name); + try { + const st = await stat(filePath); + if (!st.isFile()) return; + if (st.mtimeMs < cutoff) { + await unlink(filePath); + cleaned += 1; + } + } catch (err) { + logger.debug({ err, filePath }, 'cleanupOldDownloads: stat/unlink failed'); + } + })); + + if (cleaned > 0) logger.info({ cleaned, maxAgeMs }, 'Old feishu-downloads cleaned'); + return cleaned; +} diff --git a/src/feishu/tools/message.ts b/src/feishu/tools/message.ts index 932d1cd..3aa796e 100644 --- a/src/feishu/tools/message.ts +++ b/src/feishu/tools/message.ts @@ -1,12 +1,9 @@ import { tool } from '@anthropic-ai/claude-agent-sdk'; import { z } from 'zod'; -import { writeFile, mkdir } from 'fs/promises'; -import { join } from 'path'; -import { tmpdir } from 'os'; import { feishuClient } from '../client.js'; +import { saveMessageFileToCache } from '../file-cache.js'; import { logger } from '../../utils/logger.js'; -const DOWNLOAD_DIR = join(tmpdir(), 'feishu-downloads'); const MAX_FILE_SIZE = 30 * 1024 * 1024; // 30MB /** @@ -43,13 +40,7 @@ export function feishuMessageFileTool() { }; } - // 确保下载目录存在 - await mkdir(DOWNLOAD_DIR, { recursive: true }); - - // 文件名：messageId-fileKey 避免冲突 - const safeFileName = `${args.message_id}-${args.file_key}`.replace(/[^a-zA-Z0-9_-]/g, '_'); - const filePath = join(DOWNLOAD_DIR, safeFileName); - await writeFile(filePath, buf); + const filePath = await saveMessageFileToCache(args.message_id, args.file_key, buf); logger.info( { messageId: args.message_id, fileKey: args.file_key, sizeBytes: buf.length, filePath }, diff --git a/src/index.ts b/src/index.ts index 3ec67b8..7d48bd0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,6 +6,7 @@ import { startServer, closeServer } from './server.js'; import { sessionManager } from './session/manager.js'; import { claudeExecutor } from './claude/executor.js'; import { cleanupTmpDirs, cleanupExpiredCaches } from './workspace/cache.js'; +import { cleanupOldDownloads } from './feishu/file-cache.js'; import { pipelineStore } from './pipeline/store.js'; import { recoverInterruptedPipelines } from './pipeline/runner.js'; import { killOrphanedClaudeProcesses } from './utils/process-cleanup.js'; @@ -274,6 +275,7 @@ async function main(): Promise { pipelineStore.cleanExpired(30); cleanupExpiredApprovals(); chatBotRegistry.cleanup(); + void cleanupOldDownloads().catch((err) => logger.warn({ err }, 'cleanupOldDownloads failed')); if (config.memory.enabled) { runMemoryMaintenance(); } From 349a1d718333be187a3c825b62a3f0bda717a915 Mon Sep 17 00:00:00 2001 From: lishuceo Date: Sun, 17 May 2026 09:49:40 +0800 Subject: [PATCH 2/4] =?UTF-8?q?test:=20file-cache=20helper=20+=20lazy=20te?= =?UTF-8?q?xt=20=E6=96=87=E4=BB=B6=E5=8A=A0=E8=BD=BD=E9=98=88=E5=80=BC?= =?UTF-8?q?=E8=A6=86=E7=9B=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - saveMessageFileToCache 路径/扩展名 sanitize - cleanupOldDownloads 24h 过期清理 + 缺失目录 no-op - 历史文本附件 ≤64KB 内嵌、>64KB 落盘注入 Read 提示 - 30MB 硬上限丢弃 Co-Authored-By: Claude Opus 4.7 --- src/__tests__/lazy-text-files.test.ts | 141 ++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 src/__tests__/lazy-text-files.test.ts diff --git a/src/__tests__/lazy-text-files.test.ts b/src/__tests__/lazy-text-files.test.ts new file mode 100644 index 0000000..c289bbd --- /dev/null +++ b/src/__tests__/lazy-text-files.test.ts @@ -0,0 +1,141 @@ +/** + * Tests for lazy loading of large text files. + * + * Small text files (≤64KB) are embedded inline in the prompt. + * Large text files (>64KB) are saved to a cache dir and the prompt only + * carries a path pointer + instructions to use the Read tool's offset/limit. + * + * Covers both: shared file-cache helpers and the history-file processor. + */ +// @ts-nocheck — test file +import { describe, it, expect, vi, beforeEach, afterAll } from 'vitest'; +import { mkdtempSync, rmSync, writeFileSync, statSync, existsSync, utimesSync } from 'fs'; +import { tmpdir } from 'os'; +import { join } from 'path'; + +const mockDownloadMessageFile = vi.fn(); + +vi.mock('../feishu/client.js', () => ({ + feishuClient: { + downloadMessageFile: (...args: unknown[]) => mockDownloadMessageFile(...args), + }, +})); + +vi.mock('../utils/logger.js', () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, +})); + +import { saveMessageFileToCache, cleanupOldDownloads, DOWNLOAD_DIR } from '../feishu/file-cache.js'; +import { _testDownloadHistoryFiles as downloadHistoryFiles } from '../feishu/event-handler.js'; + +describe('file-cache', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + afterAll(() => { + // Best-effort cleanup of any leftovers from this test + try { + rmSync(DOWNLOAD_DIR, { recursive: true, force: true }); + } catch { + /* ignore */ + } + }); + + it('saveMessageFileToCache writes buffer and returns absolute path', async () => { + const buf = Buffer.from('hello-content'); + const filePath = await saveMessageFileToCache('om_test_save', 'file_test_save', buf, 'notes.txt'); + + expect(filePath.startsWith(DOWNLOAD_DIR)).toBe(true); + expect(filePath.endsWith('.txt')).toBe(true); + expect(existsSync(filePath)).toBe(true); + expect(statSync(filePath).size).toBe(buf.length); + }); + + it('saveMessageFileToCache sanitizes filename, drops weird extensions', async () => { + const buf = Buffer.from('x'); + const filePath = await saveMessageFileToCache('om_a/b', 'file_c:d', buf, 'weird.ext-with-bad-stuff!!'); + // No slashes/colons leak through; weird ext is dropped + const base = filePath.slice(DOWNLOAD_DIR.length + 1); + expect(base).not.toMatch(/[\/:!]/); + expect(base.endsWith('.ext-with-bad-stuff!!')).toBe(false); + }); + + it('cleanupOldDownloads removes files older than cutoff, keeps fresh ones', async () => { + const fresh = await saveMessageFileToCache('om_fresh', 'file_fresh', Buffer.from('new'), 'fresh.txt'); + const stale = await saveMessageFileToCache('om_stale', 'file_stale', Buffer.from('old'), 'stale.txt'); + // Backdate stale file by 25 hours + const oldTime = (Date.now() - 25 * 60 * 60 * 1000) / 1000; + utimesSync(stale, oldTime, oldTime); + + const cleaned = await cleanupOldDownloads(24 * 60 * 60 * 1000); + expect(cleaned).toBeGreaterThanOrEqual(1); + expect(existsSync(stale)).toBe(false); + expect(existsSync(fresh)).toBe(true); + }); + + it('cleanupOldDownloads is a no-op when dir does not exist', async () => { + const tmp = mkdtempSync(join(tmpdir(), 'fc-empty-')); + rmSync(tmp, { recursive: true, force: true }); + // Different dir — cleanup on DOWNLOAD_DIR shouldn't throw even if missing + const cleaned = await cleanupOldDownloads(1); + expect(typeof cleaned).toBe('number'); + }); +}); + +describe('downloadHistoryFiles lazy text threshold', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + function makeMsg(id: string, fileRefs?: Array<{ fileKey: string; fileName: string }>) { + return { messageId: id, ...(fileRefs ? { fileRefs } : {}) }; + } + + it('embeds small text files (≤64KB) inline', async () => { + const small = Buffer.from('console.log("tiny")'); + mockDownloadMessageFile.mockResolvedValue(small); + + const result = await downloadHistoryFiles([ + makeMsg('t1', [{ fileKey: 'fk_small', fileName: 'tiny.ts' }]), + ], 0); + + expect(result.fileTexts).toHaveLength(1); + expect(result.fileTexts[0]).toContain('console.log("tiny")'); + expect(result.fileTexts[0]).not.toContain('Read 工具'); + }); + + it('writes large text files (>64KB) to cache and injects path metadata only', async () => { + const big = Buffer.alloc(80 * 1024, 'a'); // 80KB + mockDownloadMessageFile.mockResolvedValue(big); + + const result = await downloadHistoryFiles([ + makeMsg('t_big', [{ fileKey: 'fk_big', fileName: 'big.log' }]), + ], 0); + + expect(result.fileTexts).toHaveLength(1); + const meta = result.fileTexts[0]; + expect(meta).toContain('big.log'); + expect(meta).toContain('Read 工具'); + expect(meta).toContain(DOWNLOAD_DIR); + // raw content is NOT embedded + expect(meta).not.toContain('aaaa'); + + // The path mentioned in metadata should actually exist on disk + const match = meta.match(/已保存到本地: (\S+)/); + expect(match).not.toBeNull(); + expect(existsSync(match![1])).toBe(true); + }); + + it('drops history text files above 30MB hard cap', async () => { + const huge = Buffer.alloc(31 * 1024 * 1024, 'a'); + mockDownloadMessageFile.mockResolvedValue(huge); + + const result = await downloadHistoryFiles([ + makeMsg('t_huge', [{ fileKey: 'fk_huge', fileName: 'huge.log' }]), + ], 0); + + expect(result.fileTexts).toHaveLength(0); + expect(result.documents).toHaveLength(0); + }); +}); From fbfe024628af53d79e35359524feef7b529b93e9 Mon Sep 17 00:00:00 2001 From: lishuceo Date: Sun, 17 May 2026 14:52:01 +0800 Subject: [PATCH 3/4] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=20CI=20lint=20?= =?UTF-8?q?=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 移除未使用的 writeFileSync import - 字符类中的 / 不需要转义 Co-Authored-By: Claude Opus 4.7 --- src/__tests__/lazy-text-files.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/__tests__/lazy-text-files.test.ts b/src/__tests__/lazy-text-files.test.ts index c289bbd..9816a40 100644 --- a/src/__tests__/lazy-text-files.test.ts +++ b/src/__tests__/lazy-text-files.test.ts @@ -9,7 +9,7 @@ */ // @ts-nocheck — test file import { describe, it, expect, vi, beforeEach, afterAll } from 'vitest'; -import { mkdtempSync, rmSync, writeFileSync, statSync, existsSync, utimesSync } from 'fs'; +import { mkdtempSync, rmSync, statSync, existsSync, utimesSync } from 'fs'; import { tmpdir } from 'os'; import { join } from 'path'; @@ -57,7 +57,7 @@ describe('file-cache', () => { const filePath = await saveMessageFileToCache('om_a/b', 'file_c:d', buf, 'weird.ext-with-bad-stuff!!'); // No slashes/colons leak through; weird ext is dropped const base = filePath.slice(DOWNLOAD_DIR.length + 1); - expect(base).not.toMatch(/[\/:!]/); + expect(base).not.toMatch(/[/:!]/); expect(base.endsWith('.ext-with-bad-stuff!!')).toBe(false); }); From 39db4249bde93fcf0fd2baba4100dfebb8b24a58 Mon Sep 17 00:00:00 2001 From: lishuceo Date: Sun, 17 May 2026 14:52:44 +0800 Subject: [PATCH 4/4] =?UTF-8?q?fix:=20=E5=BC=95=E7=94=A8=E5=9B=9E=E5=A4=8D?= =?UTF-8?q?=E8=B7=AF=E5=BE=84=E5=90=8C=E6=AD=A5=E4=BD=BF=E7=94=A8=20lazy?= =?UTF-8?q?=20=E6=96=87=E4=BB=B6=E5=8A=A0=E8=BD=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 第三处文本附件入口（引用回复父消息的文件）此前仍走 1MB silent-skip 的旧逻辑。现在与直接上传和历史消息路径行为一致：30MB 上限、>64KB 落盘 + 注入 Read 提示。 Co-Authored-By: Claude Opus 4.7 --- src/feishu/event-handler.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/feishu/event-handler.ts b/src/feishu/event-handler.ts index 709f6b3..e6f29ad 100644 --- a/src/feishu/event-handler.ts +++ b/src/feishu/event-handler.ts @@ -3505,14 +3505,20 @@ async function parseMessage(data: MessageEventData): Promise MAX_TEXT_SIZE) { + logger.warn({ messageId: message.message_id, sizeBytes: buf.length, fileName }, 'Quoted text file too large, skipping'); + } else if (buf.length <= INLINE_THRESHOLD) { const fileContent = buf.toString('utf-8'); text = `${text}\n\n[引用的文件: ${fileName}]\n\n\n${fileContent}\n`; - logger.info({ messageId: message.message_id, parentId: message.parent_id, fileName, sizeBytes: buf.length }, 'Text file downloaded from quoted parent message'); + logger.info({ messageId: message.message_id, parentId: message.parent_id, fileName, sizeBytes: buf.length }, 'Quoted text file embedded inline'); } else { - logger.warn({ messageId: message.message_id, sizeBytes: buf.length, fileName }, 'Quoted text file too large, skipping'); + const filePath = await saveMessageFileToCache(parent.message_id, fileKey, buf, fileName); + const sizeKB = (buf.length / 1024).toFixed(1); + text = `${text}\n\n[引用的文件: ${fileName}（${sizeKB} KB），已保存到本地: ${filePath}\n请使用 Read 工具按需读取该文件，支持 offset/limit 分段；文件保留 24 小时。]`; + logger.info({ messageId: message.message_id, parentId: message.parent_id, fileName, sizeBytes: buf.length, filePath }, 'Quoted text file saved to cache for lazy read'); } } }