feat: 大文本文件 lazy loading + 24h 文件缓存

src/__tests__/lazy-text-files.test.ts

@@ -0,0 +1,141 @@
+/**
+ * Tests for lazy loading of large text files.
+ *
+ * Small text files (≤64KB) are embedded inline in the prompt.
+ * Large text files (>64KB) are saved to a cache dir and the prompt only
+ * carries a path pointer + instructions to use the Read tool's offset/limit.
+ *
+ * Covers both: shared file-cache helpers and the history-file processor.
+ */
+// @ts-nocheck — test file
+import { describe, it, expect, vi, beforeEach, afterAll } from 'vitest';
+import { mkdtempSync, rmSync, statSync, existsSync, utimesSync } from 'fs';
+import { tmpdir } from 'os';
+import { join } from 'path';
+
+const mockDownloadMessageFile = vi.fn();
+
+vi.mock('../feishu/client.js', () => ({
+  feishuClient: {
+    downloadMessageFile: (...args: unknown[]) => mockDownloadMessageFile(...args),
+  },
+}));
+
+vi.mock('../utils/logger.js', () => ({
+  logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() },
+}));
+
+import { saveMessageFileToCache, cleanupOldDownloads, DOWNLOAD_DIR } from '../feishu/file-cache.js';
+import { _testDownloadHistoryFiles as downloadHistoryFiles } from '../feishu/event-handler.js';
+
+describe('file-cache', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  afterAll(() => {
+    // Best-effort cleanup of any leftovers from this test
+    try {
+      rmSync(DOWNLOAD_DIR, { recursive: true, force: true });
+    } catch {
+      /* ignore */
+    }
+  });
+
+  it('saveMessageFileToCache writes buffer and returns absolute path', async () => {
+    const buf = Buffer.from('hello-content');
+    const filePath = await saveMessageFileToCache('om_test_save', 'file_test_save', buf, 'notes.txt');
+
+    expect(filePath.startsWith(DOWNLOAD_DIR)).toBe(true);
+    expect(filePath.endsWith('.txt')).toBe(true);
+    expect(existsSync(filePath)).toBe(true);
+    expect(statSync(filePath).size).toBe(buf.length);
+  });
+
+  it('saveMessageFileToCache sanitizes filename, drops weird extensions', async () => {
+    const buf = Buffer.from('x');
+    const filePath = await saveMessageFileToCache('om_a/b', 'file_c:d', buf, 'weird.ext-with-bad-stuff!!');
+    // No slashes/colons leak through; weird ext is dropped
+    const base = filePath.slice(DOWNLOAD_DIR.length + 1);
+    expect(base).not.toMatch(/[/:!]/);
+    expect(base.endsWith('.ext-with-bad-stuff!!')).toBe(false);
+  });
+
+  it('cleanupOldDownloads removes files older than cutoff, keeps fresh ones', async () => {
+    const fresh = await saveMessageFileToCache('om_fresh', 'file_fresh', Buffer.from('new'), 'fresh.txt');
+    const stale = await saveMessageFileToCache('om_stale', 'file_stale', Buffer.from('old'), 'stale.txt');
+    // Backdate stale file by 25 hours
+    const oldTime = (Date.now() - 25 * 60 * 60 * 1000) / 1000;
+    utimesSync(stale, oldTime, oldTime);
+
+    const cleaned = await cleanupOldDownloads(24 * 60 * 60 * 1000);
+    expect(cleaned).toBeGreaterThanOrEqual(1);
+    expect(existsSync(stale)).toBe(false);
+    expect(existsSync(fresh)).toBe(true);
+  });
+
+  it('cleanupOldDownloads is a no-op when dir does not exist', async () => {
+    const tmp = mkdtempSync(join(tmpdir(), 'fc-empty-'));
+    rmSync(tmp, { recursive: true, force: true });
+    // Different dir — cleanup on DOWNLOAD_DIR shouldn't throw even if missing
+    const cleaned = await cleanupOldDownloads(1);
+    expect(typeof cleaned).toBe('number');
+  });
+});
+
+describe('downloadHistoryFiles lazy text threshold', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  function makeMsg(id: string, fileRefs?: Array<{ fileKey: string; fileName: string }>) {
+    return { messageId: id, ...(fileRefs ? { fileRefs } : {}) };
+  }
+
+  it('embeds small text files (≤64KB) inline', async () => {
+    const small = Buffer.from('console.log("tiny")');
+    mockDownloadMessageFile.mockResolvedValue(small);
+
+    const result = await downloadHistoryFiles([
+      makeMsg('t1', [{ fileKey: 'fk_small', fileName: 'tiny.ts' }]),
+    ], 0);
+
+    expect(result.fileTexts).toHaveLength(1);
+    expect(result.fileTexts[0]).toContain('console.log("tiny")');
+    expect(result.fileTexts[0]).not.toContain('Read 工具');
+  });
+
+  it('writes large text files (>64KB) to cache and injects path metadata only', async () => {
+    const big = Buffer.alloc(80 * 1024, 'a'); // 80KB
+    mockDownloadMessageFile.mockResolvedValue(big);
+
+    const result = await downloadHistoryFiles([
+      makeMsg('t_big', [{ fileKey: 'fk_big', fileName: 'big.log' }]),
+    ], 0);
+
+    expect(result.fileTexts).toHaveLength(1);
+    const meta = result.fileTexts[0];
+    expect(meta).toContain('big.log');
+    expect(meta).toContain('Read 工具');
+    expect(meta).toContain(DOWNLOAD_DIR);
+    // raw content is NOT embedded
+    expect(meta).not.toContain('aaaa');
+
+    // The path mentioned in metadata should actually exist on disk
+    const match = meta.match(/已保存到本地: (\S+)/);
+    expect(match).not.toBeNull();
+    expect(existsSync(match![1])).toBe(true);
+  });
+
+  it('drops history text files above 30MB hard cap', async () => {
+    const huge = Buffer.alloc(31 * 1024 * 1024, 'a');
+    mockDownloadMessageFile.mockResolvedValue(huge);
+
+    const result = await downloadHistoryFiles([
+      makeMsg('t_huge', [{ fileKey: 'fk_huge', fileName: 'huge.log' }]),
+    ], 0);
+
+    expect(result.fileTexts).toHaveLength(0);
+    expect(result.documents).toHaveLength(0);
+  });
+});

src/feishu/event-handler.ts

@@ -10,6 +10,7 @@ import { buildStatusCard, buildCancelledCard, buildPipelineCard, buildPipelineCo
 import type { AskUserQuestionItem } from './message-builder.js';
 import { TOTAL_PHASES } from '../pipeline/types.js';
 import { feishuClient, feishuClientContext, runWithAccountId } from './client.js';
+import { saveMessageFileToCache } from './file-cache.js';
 import { config, isMultiBotMode } from '../config.js';
 import { checkAndRequestApproval, handleApprovalTextCommand, handleApprovalCardAction, setOnApproved } from './approval.js';
 import { resolveThreadContext } from './thread-context.js';
@@ -1518,7 +1519,8 @@ async function downloadHistoryFiles(
 
   const toProcess = uniqueRefs.slice(-MAX_HISTORY_FILES);
   const MAX_PDF_SIZE = 30 * 1024 * 1024;
-  const MAX_TEXT_SIZE = 1 * 1024 * 1024;
+  const MAX_TEXT_SIZE = 30 * 1024 * 1024;
+  const INLINE_HISTORY_TEXT_THRESHOLD = 64 * 1024;
 
   const documents: DocumentAttachment[] = [];
   const fileTexts: string[] = [];
@@ -1542,10 +1544,19 @@ async function downloadHistoryFiles(
         }
       } else if (isTextFile(ref.fileName)) {
         const buf = await feishuClient.downloadMessageFile(ref.messageId, ref.fileKey);
-        if (buf.length <= MAX_TEXT_SIZE) {
+        if (buf.length > MAX_TEXT_SIZE) {
+          logger.warn({ messageId: ref.messageId, fileName: ref.fileName, sizeBytes: buf.length }, 'History text file too large, skipping');
+          return;
+        }
+        if (buf.length <= INLINE_HISTORY_TEXT_THRESHOLD) {
           const content = buf.toString('utf-8');
           fileTexts.push(`[历史消息中的文件: ${ref.fileName}]\n\n<file name="${ref.fileName}">\n${content}\n</file>`);
-          logger.info({ messageId: ref.messageId, fileName: ref.fileName, sizeBytes: buf.length }, 'History text file downloaded');
+          logger.info({ messageId: ref.messageId, fileName: ref.fileName, sizeBytes: buf.length }, 'History text file embedded inline');
+        } else {
+          const filePath = await saveMessageFileToCache(ref.messageId, ref.fileKey, buf, ref.fileName);
+          const sizeKB = (buf.length / 1024).toFixed(1);
+          fileTexts.push(`[历史消息中的文件: ${ref.fileName}（${sizeKB} KB），已保存到本地: ${filePath}\n请使用 Read 工具按需读取该文件，支持 offset/limit 分段；文件保留 24 小时。]`);
+          logger.info({ messageId: ref.messageId, fileName: ref.fileName, sizeBytes: buf.length, filePath }, 'History text file saved to cache for lazy read');
         }
       }
     } catch (err) {
@@ -3404,9 +3415,9 @@ async function parseMessage(data: MessageEventData): Promise<ParsedMessage | nul
       return null;
     }
   } else if (message.message_type === 'file') {
-    // 文件消息：支持 PDF（多模态）和文本类文件（嵌入 prompt）
+    // 文件消息：支持 PDF（多模态）和文本类文件（小文件嵌入 prompt，大文件落盘 lazy load）
     const MAX_FILE_SIZE_BYTES = 30 * 1024 * 1024; // 30MB
-    const MAX_TEXT_FILE_SIZE_BYTES = 1 * 1024 * 1024; // 1MB for text files
+    const INLINE_TEXT_FILE_THRESHOLD = 64 * 1024; // <= 64KB 直接嵌入 prompt，省一轮工具调用
     try {
       const content = JSON.parse(message.content);
       const fileKey = content.file_key as string | undefined;
@@ -3430,18 +3441,25 @@ async function parseMessage(data: MessageEventData): Promise<ParsedMessage | nul
         documents = [{ data: buf.toString('base64'), mediaType: 'application/pdf', fileName }];
         logger.info({ messageId: message.message_id, fileName, sizeBytes: buf.length }, 'PDF file downloaded');
       } else if (isTextFile(fileName)) {
-        // 文本类文件：下载后作为文本嵌入 prompt
+        // 文本类文件：小文件直接嵌入 prompt；大文件落盘，让 agent 用 Read 按需 offset/limit 分段读
         const buf = await feishuClient.downloadMessageFile(message.message_id, fileKey);
 
-        if (buf.length > MAX_TEXT_FILE_SIZE_BYTES) {
+        if (buf.length > MAX_FILE_SIZE_BYTES) {
           logger.warn({ messageId: message.message_id, sizeBytes: buf.length, fileName }, 'Text file too large, skipping');
-          await feishuClient.replyText(message.message_id, `⚠️ 文本文件太大（${(buf.length / 1024 / 1024).toFixed(1)}MB），请压缩到 1MB 以内后重试`);
+          await feishuClient.replyText(message.message_id, `⚠️ 文本文件太大（${(buf.length / 1024 / 1024).toFixed(1)}MB），上限 ${MAX_FILE_SIZE_BYTES / 1024 / 1024}MB`);
           return null;
         }
 
-        const fileContent = buf.toString('utf-8');
-        text = `[用户发送了文件: ${fileName}]\n\n<file name="${fileName}">\n${fileContent}\n</file>`;
-        logger.info({ messageId: message.message_id, fileName, sizeBytes: buf.length }, 'Text file downloaded and embedded in prompt');
+        if (buf.length <= INLINE_TEXT_FILE_THRESHOLD) {
+          const fileContent = buf.toString('utf-8');
+          text = `[用户发送了文件: ${fileName}]\n\n<file name="${fileName}">\n${fileContent}\n</file>`;
+          logger.info({ messageId: message.message_id, fileName, sizeBytes: buf.length }, 'Text file embedded inline');
+        } else {
+          const filePath = await saveMessageFileToCache(message.message_id, fileKey, buf, fileName);
+          const sizeKB = (buf.length / 1024).toFixed(1);
+          text = `[用户发送了文件: ${fileName}（${sizeKB} KB），已保存到本地: ${filePath}\n请使用 Read 工具按需读取该文件，支持 offset/limit 分段；文件保留 24 小时。]`;
+          logger.info({ messageId: message.message_id, fileName, sizeBytes: buf.length, filePath }, 'Text file saved to cache for lazy read');
+        }
       } else {
         text = `[用户发送了文件: ${fileName}，该文件类型暂不支持。支持的类型：PDF、常见文本/代码文件（.md, .txt, .json, .log, .py, .ts 等）]`;
       }
@@ -3487,14 +3505,20 @@ async function parseMessage(data: MessageEventData): Promise<ParsedMessage | nul
                 logger.warn({ messageId: message.message_id, sizeBytes: buf.length, fileName }, 'Quoted file too large, skipping');
               }
             } else if (isTextFile(fileName)) {
-              const MAX_TEXT_SIZE = 1 * 1024 * 1024;
+              const MAX_TEXT_SIZE = 30 * 1024 * 1024;
+              const INLINE_THRESHOLD = 64 * 1024;
               const buf = await feishuClient.downloadMessageFile(parent.message_id, fileKey);
-              if (buf.length <= MAX_TEXT_SIZE) {
+              if (buf.length > MAX_TEXT_SIZE) {
+                logger.warn({ messageId: message.message_id, sizeBytes: buf.length, fileName }, 'Quoted text file too large, skipping');
+              } else if (buf.length <= INLINE_THRESHOLD) {
                 const fileContent = buf.toString('utf-8');
                 text = `${text}\n\n[引用的文件: ${fileName}]\n\n<file name="${fileName}">\n${fileContent}\n</file>`;
-                logger.info({ messageId: message.message_id, parentId: message.parent_id, fileName, sizeBytes: buf.length }, 'Text file downloaded from quoted parent message');
+                logger.info({ messageId: message.message_id, parentId: message.parent_id, fileName, sizeBytes: buf.length }, 'Quoted text file embedded inline');
               } else {
-                logger.warn({ messageId: message.message_id, sizeBytes: buf.length, fileName }, 'Quoted text file too large, skipping');
+                const filePath = await saveMessageFileToCache(parent.message_id, fileKey, buf, fileName);
+                const sizeKB = (buf.length / 1024).toFixed(1);
+                text = `${text}\n\n[引用的文件: ${fileName}（${sizeKB} KB），已保存到本地: ${filePath}\n请使用 Read 工具按需读取该文件，支持 offset/limit 分段；文件保留 24 小时。]`;
+                logger.info({ messageId: message.message_id, parentId: message.parent_id, fileName, sizeBytes: buf.length, filePath }, 'Quoted text file saved to cache for lazy read');
               }
             }
           }

src/feishu/file-cache.ts

@@ -0,0 +1,63 @@
+import { writeFile, mkdir, readdir, stat, unlink } from 'fs/promises';
+import { join, extname } from 'path';
+import { tmpdir } from 'os';
+import { logger } from '../utils/logger.js';
+
+export const DOWNLOAD_DIR = join(tmpdir(), 'feishu-downloads');
+
+const SAFE_NAME_RE = /[^a-zA-Z0-9_-]/g;
+
+/**
+ * 把飞书消息附件落盘到共享缓存目录，返回绝对路径。
+ *
+ * 用于 lazy-loading 文本类附件：上传后只把路径注入 prompt，让 agent 用 Read 工具按需 offset/limit 读取。
+ */
+export async function saveMessageFileToCache(
+  messageId: string,
+  fileKey: string,
+  buf: Buffer,
+  originalFileName?: string,
+): Promise<string> {
+  await mkdir(DOWNLOAD_DIR, { recursive: true });
+  const base = `${messageId}-${fileKey}`.replace(SAFE_NAME_RE, '_');
+  const ext = originalFileName ? extname(originalFileName).toLowerCase() : '';
+  const safeExt = /^\.[a-z0-9]{1,8}$/.test(ext) ? ext : '';
+  const filePath = join(DOWNLOAD_DIR, base + safeExt);
+  await writeFile(filePath, buf);
+  return filePath;
+}
+
+/**
+ * 清理 DOWNLOAD_DIR 中早于 maxAgeMs 的文件。默认 24 小时。
+ *
+ * 由 index.ts 的周期性 cleanup interval 调用，硬盘充裕，给 agent 留足回头读的窗口。
+ */
+export async function cleanupOldDownloads(maxAgeMs: number = 24 * 60 * 60 * 1000): Promise<number> {
+  let cleaned = 0;
+  let entries: string[];
+  try {
+    entries = await readdir(DOWNLOAD_DIR);
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException).code === 'ENOENT') return 0;
+    logger.warn({ err }, 'cleanupOldDownloads: readdir failed');
+    return 0;
+  }
+
+  const cutoff = Date.now() - maxAgeMs;
+  await Promise.all(entries.map(async (name) => {
+    const filePath = join(DOWNLOAD_DIR, name);
+    try {
+      const st = await stat(filePath);
+      if (!st.isFile()) return;
+      if (st.mtimeMs < cutoff) {
+        await unlink(filePath);
+        cleaned += 1;
+      }
+    } catch (err) {
+      logger.debug({ err, filePath }, 'cleanupOldDownloads: stat/unlink failed');
+    }
+  }));
+
+  if (cleaned > 0) logger.info({ cleaned, maxAgeMs }, 'Old feishu-downloads cleaned');
+  return cleaned;
+}

src/feishu/tools/message.ts

@@ -1,12 +1,9 @@
 import { tool } from '@anthropic-ai/claude-agent-sdk';
 import { z } from 'zod';
-import { writeFile, mkdir } from 'fs/promises';
-import { join } from 'path';
-import { tmpdir } from 'os';
 import { feishuClient } from '../client.js';
+import { saveMessageFileToCache } from '../file-cache.js';
 import { logger } from '../../utils/logger.js';
 
-const DOWNLOAD_DIR = join(tmpdir(), 'feishu-downloads');
 const MAX_FILE_SIZE = 30 * 1024 * 1024; // 30MB
 
 /**
@@ -43,13 +40,7 @@ export function feishuMessageFileTool() {
           };
         }
 
-        // 确保下载目录存在
-        await mkdir(DOWNLOAD_DIR, { recursive: true });
-
-        // 文件名：messageId-fileKey 避免冲突
-        const safeFileName = `${args.message_id}-${args.file_key}`.replace(/[^a-zA-Z0-9_-]/g, '_');
-        const filePath = join(DOWNLOAD_DIR, safeFileName);
-        await writeFile(filePath, buf);
+        const filePath = await saveMessageFileToCache(args.message_id, args.file_key, buf);
 
         logger.info(
           { messageId: args.message_id, fileKey: args.file_key, sizeBytes: buf.length, filePath },

src/index.ts

@@ -6,6 +6,7 @@ import { startServer, closeServer } from './server.js';
 import { sessionManager } from './session/manager.js';
 import { claudeExecutor } from './claude/executor.js';
 import { cleanupTmpDirs, cleanupExpiredCaches } from './workspace/cache.js';
+import { cleanupOldDownloads } from './feishu/file-cache.js';
 import { pipelineStore } from './pipeline/store.js';
 import { recoverInterruptedPipelines } from './pipeline/runner.js';
 import { killOrphanedClaudeProcesses } from './utils/process-cleanup.js';
@@ -274,6 +275,7 @@ async function main(): Promise<void> {
     pipelineStore.cleanExpired(30);
     cleanupExpiredApprovals();
     chatBotRegistry.cleanup();
+    void cleanupOldDownloads().catch((err) => logger.warn({ err }, 'cleanupOldDownloads failed'));
     if (config.memory.enabled) {
       runMemoryMaintenance();
     }