diff --git a/src/__tests__/fetch-topic-root-images.test.ts b/src/__tests__/fetch-topic-root-images.test.ts new file mode 100644 index 0000000..99ce09e --- /dev/null +++ b/src/__tests__/fetch-topic-root-images.test.ts @@ -0,0 +1,202 @@ +/** + * Tests for fetchTopicRootImages — separately fetched topic-root multimodal images. + * + * 覆盖: + * - msg_type='image' / msg_type='post' 解析 + * - LRU 缓存命中重排 + 上限淘汰 + * - 失败/空 / 删除等场景写哨兵, 避免每轮 resume 重复打 API + * - imageKeys 截长防御 (MAX_HISTORY_IMAGES) + */ +// @ts-nocheck — test file +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +const mockGetMessageById = vi.fn(); +const mockDownloadMessageImage = vi.fn(); +const mockSaveMessageFileToCache = vi.fn(); + +vi.mock('../feishu/client.js', () => ({ + feishuClient: { + getMessageById: (...args: unknown[]) => mockGetMessageById(...args), + downloadMessageImage: (...args: unknown[]) => mockDownloadMessageImage(...args), + }, +})); + +vi.mock('../feishu/file-cache.js', async (importOriginal) => { + const original = await importOriginal(); + return { + ...original, + saveMessageFileToCache: (...args: unknown[]) => mockSaveMessageFileToCache(...args), + }; +}); + +vi.mock('../utils/logger.js', () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, +})); + +vi.mock('../utils/image-compress.js', () => ({ + compressImage: vi.fn(), + compressImageForHistory: vi.fn(async (buf: Buffer, mediaType: string) => ({ + data: buf, + mediaType, + })), +})); + +import { + _testFetchTopicRootImages as fetchTopicRootImages, + _testClearTopicRootCache as clearCache, +} from '../feishu/event-handler.js'; + +const JPEG_PREFIX = Buffer.from([0xff, 0xd8, 0xff, 0xe0]); +const imageBuf = () => Buffer.concat([JPEG_PREFIX, Buffer.from('fake')]); + +describe('fetchTopicRootImages', () => { + beforeEach(() => { + vi.clearAllMocks(); + clearCache(); + mockDownloadMessageImage.mockResolvedValue(imageBuf()); + mockSaveMessageFileToCache.mockImplementation(async (msgId, key) => + `/tmp/cache/${msgId}-${key}.jpg`, + ); + }); + + it('extracts image_key from msg_type=image root', async () => { + mockGetMessageById.mockResolvedValue([ + { + message_id: 'root1', + msg_type: 'image', + body: { content: '{"image_key":"ik_root"}' }, + }, + ]); + + const res = await fetchTopicRootImages('root1'); + expect(res.rootMessageId).toBe('root1'); + expect(res.images).toHaveLength(1); + expect(res.images[0].label).toBe('话题首条消息的图片'); + expect(res.savedPaths).toEqual(['/tmp/cache/root1-ik_root.jpg']); + expect(mockDownloadMessageImage).toHaveBeenCalledWith('root1', 'ik_root'); + }); + + it('extracts img tags from msg_type=post root', async () => { + const postContent = JSON.stringify({ + zh_cn: { + title: 'hi', + content: [ + [{ tag: 'text', text: 'before' }, { tag: 'img', image_key: 'ik_a' }], + [{ tag: 'img', image_key: 'ik_b' }, { tag: 'text', text: 'after' }], + ], + }, + }); + mockGetMessageById.mockResolvedValue([ + { message_id: 'root2', msg_type: 'post', body: { content: postContent } }, + ]); + + const res = await fetchTopicRootImages('root2'); + expect(res.images).toHaveLength(2); + expect(mockDownloadMessageImage).toHaveBeenCalledWith('root2', 'ik_a'); + expect(mockDownloadMessageImage).toHaveBeenCalledWith('root2', 'ik_b'); + }); + + it('caps imageKeys at MAX_HISTORY_IMAGES (5)', async () => { + // post 含 7 张图,只取前 5 + const content = [ + [{ tag: 'img', image_key: 'k1' }, { tag: 'img', image_key: 'k2' }], + [{ tag: 'img', image_key: 'k3' }, { tag: 'img', image_key: 'k4' }], + [{ tag: 'img', image_key: 'k5' }, { tag: 'img', image_key: 'k6' }, { tag: 'img', image_key: 'k7' }], + ]; + mockGetMessageById.mockResolvedValue([ + { message_id: 'r', msg_type: 'post', body: { content: JSON.stringify({ zh_cn: { content } }) } }, + ]); + + const res = await fetchTopicRootImages('r'); + expect(res.images).toHaveLength(5); + expect(mockDownloadMessageImage).toHaveBeenCalledTimes(5); + }); + + it('skips non-string image_key values defensively', async () => { + const content = [[ + { tag: 'img', image_key: null }, + { tag: 'img', image_key: 123 }, + { tag: 'img' /* missing */ }, + { tag: 'img', image_key: 'good' }, + ]]; + mockGetMessageById.mockResolvedValue([ + { message_id: 'r', msg_type: 'post', body: { content: JSON.stringify({ content }) } }, + ]); + + const res = await fetchTopicRootImages('r'); + expect(res.images).toHaveLength(1); + expect(mockDownloadMessageImage).toHaveBeenCalledWith('r', 'good'); + }); + + it('caches result and returns it on second call without re-fetching', async () => { + mockGetMessageById.mockResolvedValue([ + { message_id: 'root1', msg_type: 'image', body: { content: '{"image_key":"ik"}' } }, + ]); + + const first = await fetchTopicRootImages('root1'); + const second = await fetchTopicRootImages('root1'); + + expect(mockGetMessageById).toHaveBeenCalledTimes(1); + expect(mockDownloadMessageImage).toHaveBeenCalledTimes(1); + expect(second.images).toEqual(first.images); + }); + + it('caches empty sentinel on getMessageById returning empty (negative cache)', async () => { + mockGetMessageById.mockResolvedValue([]); + + const first = await fetchTopicRootImages('t_empty'); + const second = await fetchTopicRootImages('t_empty'); + + expect(first.rootMessageId).toBeUndefined(); + expect(first.images).toHaveLength(0); + // 关键: 第二次不再 hit Feishu API + expect(mockGetMessageById).toHaveBeenCalledTimes(1); + expect(second.images).toHaveLength(0); + }); + + it('caches empty sentinel on getMessageById throwing (transient errors should not hammer API)', async () => { + mockGetMessageById.mockRejectedValue(new Error('network')); + + const first = await fetchTopicRootImages('t_err'); + const second = await fetchTopicRootImages('t_err'); + + expect(first.images).toHaveLength(0); + expect(mockGetMessageById).toHaveBeenCalledTimes(1); + expect(second.images).toHaveLength(0); + }); + + it('caches empty result when root message has no images', async () => { + mockGetMessageById.mockResolvedValue([ + { message_id: 'root_text', msg_type: 'text', body: { content: '{"text":"hi"}' } }, + ]); + + await fetchTopicRootImages('root_text'); + await fetchTopicRootImages('root_text'); + + expect(mockGetMessageById).toHaveBeenCalledTimes(1); + }); + + it('handles malformed body content gracefully', async () => { + mockGetMessageById.mockResolvedValue([ + { message_id: 'r', msg_type: 'image', body: { content: 'not-json' } }, + ]); + + const res = await fetchTopicRootImages('r'); + expect(res.images).toHaveLength(0); + }); + + it('returns partial results when some downloads fail', async () => { + const content = [[{ tag: 'img', image_key: 'ok' }, { tag: 'img', image_key: 'fail' }]]; + mockGetMessageById.mockResolvedValue([ + { message_id: 'r', msg_type: 'post', body: { content: JSON.stringify({ content }) } }, + ]); + mockDownloadMessageImage.mockImplementation(async (_msgId, key) => { + if (key === 'fail') throw new Error('boom'); + return imageBuf(); + }); + + const res = await fetchTopicRootImages('r'); + expect(res.images).toHaveLength(1); + expect(res.savedPaths).toHaveLength(1); + }); +}); diff --git a/src/__tests__/lazy-history-images.test.ts b/src/__tests__/lazy-history-images.test.ts index c934073..9cd8223 100644 --- a/src/__tests__/lazy-history-images.test.ts +++ b/src/__tests__/lazy-history-images.test.ts @@ -1,5 +1,9 @@ /** - * Tests for lazy loading of parent chat images in history context. + * Tests for downloadHistoryImages — split outputs: + * - Parent-chat images → lazyHints (metadata only) + * - Topic-root images → handled separately by fetchTopicRootImages (excluded here) + * - Pure history images → persisted to cache and returned as historyImagePaths (text-hint only, + * not embedded in multimodal to avoid context pollution) * * 与 lazy-history-files.test.ts 对齐:当 buildChatHistoryContext / buildDirectTaskHistory * 从父群补充消息时,父群中的图片附件不应被自动下载并嵌入 prompt。 @@ -13,6 +17,7 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; const mockDownloadMessageImage = vi.fn(); +const mockSaveMessageFileToCache = vi.fn(); vi.mock('../feishu/client.js', () => ({ feishuClient: { @@ -20,6 +25,14 @@ vi.mock('../feishu/client.js', () => ({ }, })); +vi.mock('../feishu/file-cache.js', async (importOriginal) => { + const original = await importOriginal(); + return { + ...original, + saveMessageFileToCache: (...args: unknown[]) => mockSaveMessageFileToCache(...args), + }; +}); + vi.mock('../utils/logger.js', () => ({ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, })); @@ -44,13 +57,16 @@ function makeImageBuf(payload = 'fake'): Buffer { return Buffer.concat([JPEG_PREFIX, Buffer.from(payload)]); } -describe('downloadHistoryImages with parentMsgCount (lazy loading)', () => { +describe('downloadHistoryImages — text-hint output for pure history', () => { beforeEach(() => { vi.clearAllMocks(); mockDownloadMessageImage.mockResolvedValue(makeImageBuf()); + mockSaveMessageFileToCache.mockImplementation(async (msgId, imageKey) => + `/tmp/cache/${msgId}-${imageKey}.jpg`, + ); }); - it('downloads all images when parentMsgCount is 0 (default)', async () => { + it('persists all images to cache (text-hint) when parentMsgCount is 0 (default)', async () => { const messages = [ makeMsg('m1', [{ imageKey: 'ik1' }]), makeMsg('m2', [{ imageKey: 'ik2' }]), @@ -59,7 +75,11 @@ describe('downloadHistoryImages with parentMsgCount (lazy loading)', () => { const result = await downloadHistoryImages(messages); expect(mockDownloadMessageImage).toHaveBeenCalledTimes(2); - expect(result.images).toHaveLength(2); + expect(result.historyImagePaths).toHaveLength(2); + expect(result.historyImagePaths).toEqual(expect.arrayContaining([ + '/tmp/cache/m1-ik1.jpg', + '/tmp/cache/m2-ik2.jpg', + ])); expect(result.lazyHints).toHaveLength(0); }); @@ -77,7 +97,9 @@ describe('downloadHistoryImages with parentMsgCount (lazy loading)', () => { expect(mockDownloadMessageImage).toHaveBeenCalledTimes(1); expect(mockDownloadMessageImage).toHaveBeenCalledWith('thread_msg_target_resume', 'ik_target_resume'); - expect(result.images).toHaveLength(1); + // 话题图片走 historyImagePaths(文本路径,不进多模态) + expect(result.historyImagePaths).toHaveLength(1); + expect(result.historyImagePaths[0]).toContain('thread_msg_target_resume'); // 父群图片变成元数据 expect(result.lazyHints).toHaveLength(1); @@ -95,9 +117,8 @@ describe('downloadHistoryImages with parentMsgCount (lazy loading)', () => { const result = await downloadHistoryImages(messages, 2); - // 全部父群图片都不下载 expect(mockDownloadMessageImage).not.toHaveBeenCalled(); - expect(result.images).toHaveLength(0); + expect(result.historyImagePaths).toHaveLength(0); expect(result.lazyHints).toHaveLength(2); }); @@ -109,7 +130,7 @@ describe('downloadHistoryImages with parentMsgCount (lazy loading)', () => { const result = await downloadHistoryImages(messages, 0); expect(mockDownloadMessageImage).toHaveBeenCalledTimes(1); - expect(result.images).toHaveLength(1); + expect(result.historyImagePaths).toHaveLength(1); expect(result.lazyHints).toHaveLength(0); }); @@ -119,7 +140,7 @@ describe('downloadHistoryImages with parentMsgCount (lazy loading)', () => { const result = await downloadHistoryImages(messages, 1); expect(mockDownloadMessageImage).not.toHaveBeenCalled(); - expect(result.images).toHaveLength(0); + expect(result.historyImagePaths).toHaveLength(0); expect(result.lazyHints).toHaveLength(0); }); @@ -148,8 +169,8 @@ describe('downloadHistoryImages with parentMsgCount (lazy loading)', () => { // 父群图片仍输出元数据 expect(result.lazyHints).toHaveLength(1); expect(result.lazyHints[0]).toContain('p1'); - // 话题图片下载失败,images 为空 - expect(result.images).toHaveLength(0); + // 话题图片下载失败,historyImagePaths 为空 + expect(result.historyImagePaths).toHaveLength(0); }); it('multiple imageRefs in one message are tracked separately by source', async () => { @@ -165,8 +186,38 @@ describe('downloadHistoryImages with parentMsgCount (lazy loading)', () => { expect(result.lazyHints[0]).toContain('ik_p1'); expect(result.lazyHints[1]).toContain('ik_p2'); - // 话题图片正常下载 + // 话题图片正常下载 + 落盘 → historyImagePaths expect(mockDownloadMessageImage).toHaveBeenCalledTimes(1); expect(mockDownloadMessageImage).toHaveBeenCalledWith('thread_msg', 'ik_t1'); + expect(result.historyImagePaths).toHaveLength(1); + }); + + it('excludes topic-root images (those are fetched separately by fetchTopicRootImages)', async () => { + const messages = [ + // 话题首条 — 应该被 fetchTopicRootImages 单独处理,此处不重复下载 + makeMsg('topic_root_msg', [{ imageKey: 'ik_root' }]), + // 话题中间的其他图片 — 走 historyImagePaths + makeMsg('thread_msg_mid', [{ imageKey: 'ik_mid' }]), + ]; + + const result = await downloadHistoryImages(messages, 0, 'topic_root_msg'); + + // 话题首条图片不再下载/落盘 (由 fetchTopicRootImages 接管) + expect(mockDownloadMessageImage).toHaveBeenCalledTimes(1); + expect(mockDownloadMessageImage).toHaveBeenCalledWith('thread_msg_mid', 'ik_mid'); + expect(result.historyImagePaths).toHaveLength(1); + expect(result.historyImagePaths[0]).toContain('thread_msg_mid'); + }); + + it('falls back gracefully when saveMessageFileToCache throws', async () => { + mockSaveMessageFileToCache.mockRejectedValue(new Error('disk full')); + + const messages = [makeMsg('m1', [{ imageKey: 'ik1' }])]; + + const result = await downloadHistoryImages(messages); + + // 下载仍然进行,但落盘失败 → 不计入 historyImagePaths + expect(mockDownloadMessageImage).toHaveBeenCalledTimes(1); + expect(result.historyImagePaths).toHaveLength(0); }); }); diff --git a/src/__tests__/multimodal-image-labels.test.ts b/src/__tests__/multimodal-image-labels.test.ts new file mode 100644 index 0000000..4c241a5 --- /dev/null +++ b/src/__tests__/multimodal-image-labels.test.ts @@ -0,0 +1,122 @@ +/** + * Tests for buildMultimodalPrompt label injection. + * + * 验证标签机制:有 label 的图片在 content block 序列中前置一个文本块说明来源, + * 防止 agent 混淆"用户当前消息的图片" / "话题首条" / "引用消息" 等不同图片来源。 + */ +// @ts-nocheck — test file + +import { describe, it, expect, vi } from 'vitest'; + +// 屏蔽重型模块:executor.ts 顶层会 import MCP server / memory / cron 等, +// 测试 buildMultimodalPrompt 这个纯函数时无需真实初始化。 +vi.mock('@anthropic-ai/claude-agent-sdk', () => ({ + query: vi.fn(), +})); +vi.mock('../workspace/tool.js', () => ({ createWorkspaceMcpServer: vi.fn() })); +vi.mock('../feishu/tools/index.js', () => ({ createFeishuToolsMcpServer: vi.fn() })); +vi.mock('../memory/tools/memory-search.js', () => ({ createMemorySearchMcpServer: vi.fn() })); +vi.mock('../memory/init.js', () => ({ + getMemoryStore: vi.fn(), + getHybridSearch: vi.fn(), + isMemoryEnabled: () => false, +})); +vi.mock('../cron/tool.js', () => ({ createCronMcpServer: vi.fn() })); +vi.mock('../cron/init.js', () => ({ getCronScheduler: vi.fn() })); +vi.mock('../feishu/client.js', () => ({ + feishuClientContext: { getStore: () => undefined }, +})); +vi.mock('../workspace/isolation.js', () => ({ + isAutoWorkspacePath: vi.fn(), + isServiceOwnRepo: vi.fn(), + isInsideSourceRepo: vi.fn(), +})); +vi.mock('../utils/runtime.js', () => ({ detectRuntime: vi.fn() })); +vi.mock('../utils/logger.js', () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, +})); +vi.mock('../config.js', () => ({ + config: { claude: { defaultModel: 'claude-opus-4-6' } }, +})); + +import { _testBuildMultimodalPrompt } from '../claude/executor.js'; + +async function collectFirstMessage(gen: AsyncIterable) { + for await (const msg of gen) return msg as { message: { content: Array<{ type: string; text?: string; source?: { media_type: string } }> } }; + throw new Error('no message yielded'); +} + +const IMG_DATA = 'fake-base64'; + +describe('buildMultimodalPrompt — image labels', () => { + it('inserts a text block before each labeled image', async () => { + const images = [ + { data: IMG_DATA, mediaType: 'image/png', label: '用户当前消息的图片' }, + { data: IMG_DATA, mediaType: 'image/jpeg', label: '话题首条消息的图片' }, + ]; + + const msg = await collectFirstMessage(_testBuildMultimodalPrompt('hello', images)); + const blocks = msg.message.content; + + // 期望顺序:[label-text, image, label-text, image, main-text] + expect(blocks).toHaveLength(5); + expect(blocks[0]).toMatchObject({ type: 'text', text: '[图片说明: 用户当前消息的图片]' }); + expect(blocks[1]).toMatchObject({ type: 'image' }); + expect((blocks[1] as any).source.media_type).toBe('image/png'); + expect(blocks[2]).toMatchObject({ type: 'text', text: '[图片说明: 话题首条消息的图片]' }); + expect(blocks[3]).toMatchObject({ type: 'image' }); + expect((blocks[3] as any).source.media_type).toBe('image/jpeg'); + expect(blocks[4]).toMatchObject({ type: 'text', text: 'hello' }); + }); + + it('omits the label text block when image has no label', async () => { + const images = [{ data: IMG_DATA, mediaType: 'image/png' }]; + const msg = await collectFirstMessage(_testBuildMultimodalPrompt('main text', images)); + const blocks = msg.message.content; + + // 期望:[image, main-text],无前置 label 文本块 + expect(blocks).toHaveLength(2); + expect(blocks[0]).toMatchObject({ type: 'image' }); + expect(blocks[1]).toMatchObject({ type: 'text', text: 'main text' }); + }); + + it('mixes labeled and unlabeled images correctly', async () => { + const images = [ + { data: IMG_DATA, mediaType: 'image/png', label: '用户当前消息的图片' }, + { data: IMG_DATA, mediaType: 'image/jpeg' }, + { data: IMG_DATA, mediaType: 'image/webp', label: '用户引用的消息中的图片' }, + ]; + + const msg = await collectFirstMessage(_testBuildMultimodalPrompt('q', images)); + const blocks = msg.message.content; + + expect(blocks).toHaveLength(6); + expect(blocks[0]).toMatchObject({ type: 'text', text: '[图片说明: 用户当前消息的图片]' }); + expect(blocks[1]).toMatchObject({ type: 'image' }); + expect(blocks[2]).toMatchObject({ type: 'image' }); // 无 label, 不前置 + expect(blocks[3]).toMatchObject({ type: 'text', text: '[图片说明: 用户引用的消息中的图片]' }); + expect(blocks[4]).toMatchObject({ type: 'image' }); + expect(blocks[5]).toMatchObject({ type: 'text', text: 'q' }); + }); + + it('keeps documents → images → text ordering with labels', async () => { + const documents = [{ data: 'pdf-base64', mediaType: 'application/pdf' as const, fileName: 'a.pdf' }]; + const images = [{ data: IMG_DATA, mediaType: 'image/png', label: '用户当前消息的图片' }]; + + const msg = await collectFirstMessage(_testBuildMultimodalPrompt('main', images, documents)); + const blocks = msg.message.content; + + expect(blocks).toHaveLength(4); + expect(blocks[0]).toMatchObject({ type: 'document' }); + expect(blocks[1]).toMatchObject({ type: 'text', text: '[图片说明: 用户当前消息的图片]' }); + expect(blocks[2]).toMatchObject({ type: 'image' }); + expect(blocks[3]).toMatchObject({ type: 'text', text: 'main' }); + }); + + it('returns just the main text block when no images and no documents', async () => { + const msg = await collectFirstMessage(_testBuildMultimodalPrompt('only text', [])); + const blocks = msg.message.content; + expect(blocks).toHaveLength(1); + expect(blocks[0]).toMatchObject({ type: 'text', text: 'only text' }); + }); +}); diff --git a/src/claude/executor.ts b/src/claude/executor.ts index b544fcf..8a1fe15 100644 --- a/src/claude/executor.ts +++ b/src/claude/executor.ts @@ -525,6 +525,9 @@ async function* buildMultimodalPrompt( } for (const img of images) { + if (img.label) { + contentBlocks.push({ type: 'text', text: `[图片说明: ${img.label}]` }); + } contentBlocks.push({ type: 'image', source: { @@ -550,6 +553,9 @@ async function* buildMultimodalPrompt( } as import('@anthropic-ai/claude-agent-sdk').SDKUserMessage; } +/** 仅测试用:导出 buildMultimodalPrompt */ +export const _testBuildMultimodalPrompt = buildMultimodalPrompt; + export class ClaudeExecutor { /** 运行中的 query 实例 (用于 abort) */ private runningQueries = new Map(); diff --git a/src/claude/types.ts b/src/claude/types.ts index cfd8669..69c4e67 100644 --- a/src/claude/types.ts +++ b/src/claude/types.ts @@ -102,6 +102,8 @@ export interface ImageAttachment { data: string; /** MIME 类型 */ mediaType: 'image/jpeg' | 'image/png' | 'image/gif' | 'image/webp'; + /** 可选说明,会作为前置文本块插入到该图片前,帮助 agent 区分图片来源 */ + label?: string; } /** 文档附件 (从飞书消息下载的 PDF 等) */ diff --git a/src/feishu/__tests__/inject-quoted-message.test.ts b/src/feishu/__tests__/inject-quoted-message.test.ts index ab0c2a5..c09b731 100644 --- a/src/feishu/__tests__/inject-quoted-message.test.ts +++ b/src/feishu/__tests__/inject-quoted-message.test.ts @@ -182,6 +182,8 @@ describe('injectQuotedMessage', () => { expect(result.prompt).toContain('引用了一张图片'); expect(result.images).toHaveLength(1); expect(result.images![0].mediaType).toBe('image/png'); + // 引用图片必须带标签,buildMultimodalPrompt 才能在图片前插入说明文本块 + expect(result.images![0].label).toBe('用户引用的消息中的图片'); expect(mockDownloadMessageImage).toHaveBeenCalledWith('img1', 'img_key_123'); }); diff --git a/src/feishu/event-handler.ts b/src/feishu/event-handler.ts index c215a61..0da3116 100644 --- a/src/feishu/event-handler.ts +++ b/src/feishu/event-handler.ts @@ -106,6 +106,20 @@ export function formatRestartImageHints(paths: string[]): string { ].join('\n'); } +/** + * 把历史消息中的图片落盘路径拼成文本提示。与 formatRestartImageHints 同形式但措辞不同: + * 历史图片不再走多模态(避免污染上下文),需要时让 agent 用 Read 工具按需读取。 + */ +export function formatHistoryImageHints(paths: string[]): string { + if (!paths.length) return ''; + const unique = Array.from(new Set(paths)); + const lines = unique.map(p => `- ${p}`); + return [ + '[历史聊天图片] 历史消息中的图片未自动展开,已落盘到本地,如需查看请使用 Read 工具读取:', + ...lines, + ].join('\n'); +} + // ============================================================ // AskUserQuestion 待回答存储 // 当 Claude 调用 AskUserQuestion 时,canUseTool 拦截并发送飞书卡片, @@ -1393,7 +1407,8 @@ const MAX_HISTORY_IMAGES = 5; async function downloadHistoryImages( messages: Array<{ messageId: string; imageRefs?: Array<{ imageKey: string }> }>, parentMsgCount = 0, -): Promise<{ images: ImageAttachment[]; lazyHints: string[]; savedImagePaths: string[] }> { + topicRootMessageId?: string, +): Promise<{ historyImagePaths: string[]; lazyHints: string[] }> { // 收集所有图片引用,标记是否来自父群 const refs: Array<{ messageId: string; imageKey: string; fromParent: boolean }> = []; for (let i = 0; i < messages.length; i++) { @@ -1405,7 +1420,7 @@ async function downloadHistoryImages( } } } - if (refs.length === 0) return { images: [], lazyHints: [], savedImagePaths: [] }; + if (refs.length === 0) return { historyImagePaths: [], lazyHints: [] }; // 父群图片:lazy loading(注入元数据,不下载) const lazyHints: string[] = []; @@ -1416,6 +1431,10 @@ async function downloadHistoryImages( ); return false; } + // 话题首条图片由 fetchTopicRootImages 单独处理(走多模态),这里跳过避免重复下载 + if (topicRootMessageId && ref.messageId === topicRootMessageId) { + return false; + } return true; }); @@ -1423,10 +1442,11 @@ async function downloadHistoryImages( logger.info({ parentImageCount: lazyHints.length }, 'Parent chat images skipped (lazy loading), metadata injected'); } - if (downloadable.length === 0) return { images: [], lazyHints, savedImagePaths: [] }; + if (downloadable.length === 0) return { historyImagePaths: [], lazyHints }; const toDownload = downloadable.slice(-MAX_HISTORY_IMAGES); + // 历史图片仅落盘 → 文本路径提示;不再嵌入多模态,避免污染上下文 const results = await Promise.all(toDownload.map(async (ref) => { try { const buf = await feishuClient.downloadMessageImage(ref.messageId, ref.imageKey); @@ -1435,41 +1455,181 @@ async function downloadHistoryImages( return null; } const mediaType = detectImageMediaType(buf); - const compressed = await compressImageForHistory(buf, mediaType); - // 同时落盘原图,方便 workspace 切换后 agent 通过 Read 工具按需重新加载 - // (restart query 不重传多模态 images 参数,避免重复消耗 token;落盘路径作为文本兜底) - let savedPath: string | undefined; try { - savedPath = await saveMessageFileToCache( + const savedPath = await saveMessageFileToCache( ref.messageId, ref.imageKey, buf, `image${mediaTypeToExt(mediaType)}`, ); + return savedPath; } catch (saveErr) { logger.debug({ err: saveErr, messageId: ref.messageId, imageKey: ref.imageKey }, 'Failed to persist history image to cache (non-fatal)'); + return null; } - return { - image: { - data: compressed.data.toString('base64'), - mediaType: compressed.mediaType, - } as ImageAttachment, - savedPath, - }; } catch (err) { logger.warn({ err, messageId: ref.messageId, imageKey: ref.imageKey }, 'Failed to download history image, skipping'); return null; } })); - const successful = results.filter((r): r is { image: ImageAttachment; savedPath: string | undefined } => r !== null); - const images = successful.map(r => r.image); - const savedImagePaths = successful.map(r => r.savedPath).filter((p): p is string => !!p); + const historyImagePaths = results.filter((p): p is string => !!p); + + if (historyImagePaths.length > 0) { + logger.info({ count: historyImagePaths.length, totalRefs: refs.length, parentSkipped: lazyHints.length }, 'Persisted history images (text-hint only)'); + } + + return { historyImagePaths, lazyHints }; +} + +/** + * 话题首条消息的图片单独 fetch + 下载,作为多模态图片(带标签)注入。 + * 只在话题模式下调用(threadId 有值)。返回的图片自带 label='话题首条消息的图片'。 + * + * 用 threadId 单独取根消息比依赖历史窗口更稳定 —— 历史窗口在 resume 时可能已经把根消息滑出去了, + * 而根消息往往承载用户问题的核心图片(如"看这张图有什么问题")。 + * + * 进程内 LRU 缓存按 threadId 缓存结果,避免每轮 resume 都重复 fetch。 + */ +const topicRootImagesCache = new Map(); +const TOPIC_ROOT_CACHE_MAX = 100; + +async function fetchTopicRootImages(threadId: string): Promise<{ + rootMessageId?: string; + images: ImageAttachment[]; + savedPaths: string[]; +}> { + const cached = topicRootImagesCache.get(threadId); + if (cached) { + // LRU: 命中后移到最近 + topicRootImagesCache.delete(threadId); + topicRootImagesCache.set(threadId, cached); + // '' 是哨兵 (表示"已确认无可用首条图片"),对外暴露为 undefined + return { + rootMessageId: cached.rootMessageId || undefined, + images: cached.images, + savedPaths: cached.savedPaths, + }; + } + + // 负面缓存哨兵:任何无法定位/无图片/失败的情况都用空 entry 占位, + // 避免每轮 resume 都重复打 Feishu API。rootMessageId 为空串表示"已确认无图" + const cacheEmpty = (): { rootMessageId?: string; images: ImageAttachment[]; savedPaths: string[] } => { + const empty = { rootMessageId: '', images: [] as ImageAttachment[], savedPaths: [] as string[] }; + _putTopicRootCache(threadId, empty); + return { rootMessageId: undefined, images: [], savedPaths: [] }; + }; + + try { + const items = await feishuClient.getMessageById(threadId); + if (!items || items.length === 0) return cacheEmpty(); + const rootMsg = items.find(m => m.message_id === threadId) ?? items[0]; + if (!rootMsg) return cacheEmpty(); + + const rootMessageId = rootMsg.message_id; + if (!rootMessageId) return cacheEmpty(); + const msgType = rootMsg.msg_type || 'text'; + const imageKeys: string[] = []; + + if (msgType === 'image') { + try { + const body = JSON.parse(rootMsg.body?.content ?? '{}') as Record; + const key = body.image_key; + if (typeof key === 'string' && key.length > 0) imageKeys.push(key); + } catch { /* ignore */ } + } else if (msgType === 'post') { + try { + const body = JSON.parse(rootMsg.body?.content ?? '{}') as Record; + // 飞书 post 可能直接含 content 数组,也可能按语言 key (zh_cn/en_us/ja_jp) 嵌套 + const localized = (body.zh_cn || body.en_us || body.ja_jp) as Record | undefined; + const postBody: Record | undefined = Array.isArray(body.content) + ? body + : (localized && typeof localized === 'object' ? localized : undefined); + const paragraphs = postBody?.content; + if (Array.isArray(paragraphs)) { + for (const para of paragraphs) { + if (!Array.isArray(para)) continue; + for (const el of para) { + if (el && typeof el === 'object' && (el as Record).tag === 'img') { + const key = (el as Record).image_key; + if (typeof key === 'string' && key.length > 0) imageKeys.push(key); + } + } + } + } + } catch { /* ignore */ } + } + + if (imageKeys.length === 0) { + const empty = { rootMessageId, images: [] as ImageAttachment[], savedPaths: [] as string[] }; + _putTopicRootCache(threadId, empty); + return { rootMessageId, images: [], savedPaths: [] }; + } + + // 防御:截断异常的超长 imageKeys 列表,避免 post 携带巨量 img 元素时 + // 把多模态 payload 撑爆 + 拖慢首条加载 + const cappedKeys = imageKeys.slice(0, MAX_HISTORY_IMAGES); + if (cappedKeys.length < imageKeys.length) { + logger.warn({ threadId, total: imageKeys.length, kept: cappedKeys.length }, 'Topic-root image count capped'); + } + const results = await Promise.all(cappedKeys.map(async (imageKey) => { + try { + const buf = await feishuClient.downloadMessageImage(rootMessageId, imageKey); + if (buf.length > MAX_IMAGE_SIZE_BYTES) { + logger.warn({ rootMessageId, imageKey, sizeBytes: buf.length }, 'Topic-root image too large, skipping'); + return null; + } + const mediaType = detectImageMediaType(buf); + const compressed = await compressImageForHistory(buf, mediaType); + let savedPath: string | undefined; + try { + savedPath = await saveMessageFileToCache( + rootMessageId, + imageKey, + buf, + `image${mediaTypeToExt(mediaType)}`, + ); + } catch (saveErr) { + logger.debug({ err: saveErr, rootMessageId, imageKey }, 'Failed to persist topic-root image to cache (non-fatal)'); + } + const image: ImageAttachment = { + data: compressed.data.toString('base64'), + mediaType: compressed.mediaType, + label: '话题首条消息的图片', + }; + return { image, savedPath }; + } catch (err) { + logger.warn({ err, rootMessageId, imageKey }, 'Failed to download topic-root image'); + return null; + } + })); + + const ok = results.filter((r): r is { image: ImageAttachment; savedPath: string | undefined } => r !== null); + const images = ok.map(r => r.image); + const savedPaths = ok.map(r => r.savedPath).filter((p): p is string => !!p); + + if (images.length > 0) { + logger.info({ threadId, rootMessageId, count: images.length }, 'Fetched topic-root images'); + } - if (images.length > 0) { - logger.info({ count: images.length, totalRefs: refs.length, parentSkipped: lazyHints.length, savedCount: savedImagePaths.length }, 'Downloaded history images'); + const result = { rootMessageId, images, savedPaths }; + _putTopicRootCache(threadId, result); + return result; + } catch (err) { + logger.warn({ err, threadId }, 'Failed to fetch topic-root message'); + // 失败也缓存哨兵 - 避免 transient 错误每轮 resume 都重复 hit Feishu API + return cacheEmpty(); } +} - return { images, lazyHints, savedImagePaths }; +function _putTopicRootCache( + threadId: string, + value: { rootMessageId: string; images: ImageAttachment[]; savedPaths: string[] }, +): void { + if (topicRootImagesCache.size >= TOPIC_ROOT_CACHE_MAX) { + const oldest = topicRootImagesCache.keys().next().value; + if (oldest) topicRootImagesCache.delete(oldest); + } + topicRootImagesCache.set(threadId, value); } /** mediaType → 文件扩展名(与 detectImageMediaType 配对) */ @@ -1724,19 +1884,26 @@ async function buildHistoryContext( ); } + // 话题模式下先单独 fetch 话题首条消息的图片(走多模态,带标签), + // 并把首条 messageId 传给 downloadHistoryImages 用于排重(避免重复下载/落盘)。 + const topicRoot = threadId + ? await fetchTopicRootImages(threadId) + : { rootMessageId: undefined as string | undefined, images: [] as ImageAttachment[], savedPaths: [] as string[] }; + const [text, imagesResult, historyFiles] = await Promise.all([ formatHistoryMessages(messages, chatId, selfBotOpenIds, parentMsgCount > 0 ? { parentMsgCount } : undefined), - downloadHistoryImages(messages, parentMsgCount), + downloadHistoryImages(messages, parentMsgCount, topicRoot.rootMessageId), downloadHistoryFiles(messages, parentMsgCount), ]); const fileTexts = [...historyFiles.fileTexts, ...imagesResult.lazyHints]; + const historyImagePaths = [...topicRoot.savedPaths, ...imagesResult.historyImagePaths]; return { text: text ?? undefined, newestMsgId, - ...(imagesResult.images.length > 0 ? { images: imagesResult.images } : {}), + ...(topicRoot.images.length > 0 ? { topicRootImages: topicRoot.images } : {}), ...(historyFiles.documents.length > 0 ? { documents: historyFiles.documents } : {}), ...(fileTexts.length > 0 ? { fileTexts } : {}), - ...(imagesResult.savedImagePaths.length > 0 ? { savedImagePaths: imagesResult.savedImagePaths } : {}), + ...(historyImagePaths.length > 0 ? { historyImagePaths } : {}), }; } catch (err) { logger.error({ err, chatId, threadId }, errorLabel); @@ -1860,6 +2027,9 @@ export const _testFormatHistoryMessages = formatHistoryMessages; /** 仅测试用:导出 downloadHistoryFiles */ export const _testDownloadHistoryFiles = downloadHistoryFiles; export const _testDownloadHistoryImages = downloadHistoryImages; +export const _testFetchTopicRootImages = fetchTopicRootImages; +/** 仅测试用:清空话题首条图片缓存,防止用例之间相互干扰 */ +export const _testClearTopicRootCache = () => topicRootImagesCache.clear(); /** * 格式化历史消息为上下文文本(共享逻辑)。 @@ -2014,7 +2184,7 @@ async function injectQuotedMessage( if (buf.length <= MAX_IMAGE_SIZE_BYTES) { const mediaType = detectImageMediaType(buf); const compressed = await compressImage(buf, mediaType); - quotedImage = { data: compressed.data.toString('base64'), mediaType: compressed.mediaType }; + quotedImage = { data: compressed.data.toString('base64'), mediaType: compressed.mediaType, label: '用户引用的消息中的图片' }; rootContent = '[用户引用了一张图片]'; logger.info({ rootId, imageSize: buf.length, compressedSize: compressed.data.length }, 'Downloaded quoted image'); // 同时落盘原图,工作区切换 restart 时通过 Read 工具兜底加载 @@ -2281,8 +2451,8 @@ export async function executeClaudeTask( const afterMsgId = activeConversationId ? _historyDedup.get(sessionKey) : undefined; const history = await buildChatHistoryContext(chatId, threadId, messageId, afterMsgId, selfBotOpenIds); - if (history.savedImagePaths?.length) { - restartImagePaths.push(...history.savedImagePaths); + if (history.historyImagePaths?.length) { + restartImagePaths.push(...history.historyImagePaths); } if (history.text) { effectivePrompt = history.text + '\n\n---\n\n' + promptWithTime; @@ -2293,9 +2463,9 @@ export async function executeClaudeTask( // Resume 时跳过历史文件附件:SDK 会重放所有前序 turn,文件已在对话中, // 重复附加会导致 payload 累积膨胀(N turns × PDF size → 超 30MB 限制) if (activeConversationId) { - if (history.images?.length || history.documents?.length) { + if (history.topicRootImages?.length || history.documents?.length) { logger.info( - { historyImages: history.images?.length ?? 0, historyDocs: history.documents?.length ?? 0 }, + { topicRootImages: history.topicRootImages?.length ?? 0, historyDocs: history.documents?.length ?? 0 }, 'Skipping history file attachments on resume — already in conversation', ); } @@ -2309,9 +2479,20 @@ export async function executeClaudeTask( } } else { // 非 resume:正常合并历史文件 - // 合并历史消息中的图片 - if (history.images && history.images.length > 0) { - images = [...(history.images), ...(images ?? [])]; + // 当前消息图片打标签(用户主动发的图片) + if (images?.length) { + images = images.map(img => ({ ...img, label: img.label ?? '用户当前消息的图片' })); + } + // 话题首条图片(自带 label)合并进多模态 + if (history.topicRootImages && history.topicRootImages.length > 0) { + images = [...(images ?? []), ...history.topicRootImages]; + } + // 纯历史图片转成文本提示,前置到 effectivePrompt(避免污染多模态) + if (history.historyImagePaths && history.historyImagePaths.length > 0) { + const hint = formatHistoryImageHints(history.historyImagePaths); + if (hint) { + effectivePrompt = hint + '\n\n---\n\n' + effectivePrompt; + } } // 合并历史消息中的文档(PDF),按 fileName 去重 + 大小截断 if (history.documents && history.documents.length > 0) { @@ -2879,9 +3060,9 @@ export async function executeDirectTask( // Resume 时跳过历史文件附件:SDK 会重放所有前序 turn,文件已在对话中, // 重复附加会导致 payload 累积膨胀(N turns × PDF size → 超 30MB 限制) if (canResume) { - if (history.images?.length || history.documents?.length) { + if (history.topicRootImages?.length || history.documents?.length) { logger.info( - { historyImages: history.images?.length ?? 0, historyDocs: history.documents?.length ?? 0 }, + { topicRootImages: history.topicRootImages?.length ?? 0, historyDocs: history.documents?.length ?? 0 }, 'Skipping history file attachments on resume — already in conversation', ); } @@ -2895,9 +3076,18 @@ export async function executeDirectTask( } } else { // 非 resume:正常合并历史文件 - // 合并历史消息中的图片 - if (history.images && history.images.length > 0) { - images = [...(history.images), ...(images ?? [])]; + // 当前消息图片打默认标签 + if (images?.length) { + images = images.map(img => ({ ...img, label: img.label ?? '用户当前消息的图片' })); + } + // 话题首条图片走多模态(已带标签) + if (history.topicRootImages && history.topicRootImages.length > 0) { + images = [...(images ?? []), ...history.topicRootImages]; + } + // 纯历史图片走文本路径提示,前置到 prompt + if (history.historyImagePaths && history.historyImagePaths.length > 0) { + const hint = formatHistoryImageHints(history.historyImagePaths); + if (hint) effectivePrompt = hint + '\n\n---\n\n' + effectivePrompt; } // 合并历史消息中的文档(PDF),按 fileName 去重 + 大小截断 if (history.documents && history.documents.length > 0) { @@ -3032,14 +3222,14 @@ interface HistoryResult { text?: string; /** 本次注入的最新 messageId(用于下次去重) */ newestMsgId?: string; - /** 历史消息中提取的图片(已压缩) */ - images?: ImageAttachment[]; + /** 话题首条消息的图片(带 label,走多模态)。仅话题模式有值。 */ + topicRootImages?: ImageAttachment[]; /** 历史消息中提取的文档附件(PDF) */ documents?: DocumentAttachment[]; /** 历史消息中提取的文本文件内容(已格式化,可拼入 prompt) */ fileTexts?: string[]; - /** 历史图片原图落盘路径,用于 workspace 切换后通过 Read 工具按需重新加载 */ - savedImagePaths?: string[]; + /** 历史消息中纯历史图片(含话题首条)的落盘路径,走文本提示 + restartImagePaths 兜底 */ + historyImagePaths?: string[]; } /**