From f507f6196e653cd9c637c964db5634c8616349af Mon Sep 17 00:00:00 2001 From: 7Sageer <7sageer@djwcb.cn> Date: Tue, 4 Aug 2026 15:31:22 +0800 Subject: [PATCH 01/40] feat: engine-native image references via kimi-file:// media resolver --- .changeset/sdk-upload-file.md | 5 + .../src/tui/controllers/editor-keyboard.ts | 41 ++- apps/kimi-code/src/tui/kimi-tui.ts | 3 +- .../src/tui/utils/image-attachment-store.ts | 9 + .../src/tui/utils/image-placeholder.ts | 43 ++- .../editor-keyboard-image-paste.test.ts | 95 ++++- .../tui/input/image-attachment-store.test.ts | 12 + .../test/tui/input/image-placeholder.test.ts | 72 ++++ .../agent/llmRequester/llmRequesterService.ts | 13 +- .../src/agent/media/file-type.ts | 39 +-- .../src/agent/media/kimiFileUrl.ts | 57 +-- .../src/agent/media/mediaResolver.ts | 32 ++ .../src/agent/media/mediaResolverService.ts | 325 ++++++++++++++++++ .../src/agent/media/videoResolver.ts | 29 +- .../src/agent/media/videoResolverService.ts | 250 +------------- packages/agent-core-v2/src/index.ts | 4 + .../src/kosong/contract/mediaRef.ts | 216 ++++++++++++ .../llmRequester/llmRequesterService.test.ts | 4 +- .../test/agent/media/image-compress.test.ts | 13 + ...Resolver.test.ts => mediaResolver.test.ts} | 218 ++++++++---- .../test/kosong/contract/mediaRef.test.ts | 148 ++++++++ packages/kap-server/src/routes/prompts.ts | 72 +++- .../services/messages/messageProjection.ts | 27 +- packages/kap-server/test/prompts.test.ts | 111 +++++- .../messages/messageProjection.test.ts | 14 + packages/klient/AGENTS.md | 6 +- packages/klient/src/contract/global/files.ts | 46 +++ packages/klient/src/contract/index.ts | 2 + packages/klient/src/core/facade/global.ts | 43 +++ packages/klient/src/index.ts | 3 + .../src/transports/memory/dispatcher.ts | 37 ++ .../src/transports/memory/serviceRegistry.ts | 2 + packages/klient/test/contract-parity.ts | 13 + packages/klient/test/facade.test.ts | 65 ++++ packages/klient/test/helpers/conformance.ts | 23 ++ packages/node-sdk/src/index.ts | 15 + packages/node-sdk/src/kimi-harness.ts | 11 + packages/node-sdk/src/rpc.ts | 15 + packages/node-sdk/src/sdk-rpc-client-v2.ts | 23 +- packages/node-sdk/src/types.ts | 9 + .../node-sdk/test/sdk-rpc-client-v2.test.ts | 27 ++ packages/transcript/test/layers.test.ts | 77 ++++- 42 files changed, 1791 insertions(+), 478 deletions(-) create mode 100644 .changeset/sdk-upload-file.md create mode 100644 packages/agent-core-v2/src/agent/media/mediaResolver.ts create mode 100644 packages/agent-core-v2/src/agent/media/mediaResolverService.ts create mode 100644 packages/agent-core-v2/src/kosong/contract/mediaRef.ts rename packages/agent-core-v2/test/agent/media/{videoResolver.test.ts => mediaResolver.test.ts} (56%) create mode 100644 packages/agent-core-v2/test/kosong/contract/mediaRef.test.ts create mode 100644 packages/klient/src/contract/global/files.ts diff --git a/.changeset/sdk-upload-file.md b/.changeset/sdk-upload-file.md new file mode 100644 index 0000000000..dddb41bac9 --- /dev/null +++ b/.changeset/sdk-upload-file.md @@ -0,0 +1,5 @@ +--- +"@moonshot-ai/kimi-code-sdk": minor +--- + +Add `uploadFile` to the harness plus daemon file-reference helpers, so SDK consumers can upload media to the engine's file store and reference it by path in prompts. Requires the v2 harness (`createKimiHarnessV2`). diff --git a/apps/kimi-code/src/tui/controllers/editor-keyboard.ts b/apps/kimi-code/src/tui/controllers/editor-keyboard.ts index 76d0363f80..b64bd77231 100644 --- a/apps/kimi-code/src/tui/controllers/editor-keyboard.ts +++ b/apps/kimi-code/src/tui/controllers/editor-keyboard.ts @@ -15,7 +15,7 @@ import { } from '../constant/kimi-tui'; import { formatErrorMessage } from '../utils/event-payload'; import type { ImageAttachmentStore } from '../utils/image-attachment-store'; -import { extractMediaAttachments } from '../utils/image-placeholder'; +import { extractMediaAttachments, imageExtensionForMime } from '../utils/image-placeholder'; import type { PendingExit, QueuedMessage, SteerInputItem } from '../types'; import type { TUIState } from '../tui-state'; import type { BtwPanelController } from './btw-panel'; @@ -30,6 +30,12 @@ export interface EditorKeyboardHost { * env/built-in default. */ harness?: KimiHarness | undefined; + /** + * True when the TUI runs on the agent-core-v2 engine (startup-selected). + * Gates the paste-time upload to the daemon file store; the v1 engine has + * no file store and keeps the submit-time inline base64 form. + */ + readonly engineV2?: boolean | undefined; handleUserInput(text: string): void; readonly btwPanelController: BtwPanelController; @@ -468,6 +474,12 @@ export class EditorKeyboardController { }, }); const sessionDir = this.host.session?.summary?.sessionDir; + // v2 only: upload the final bytes to the daemon file store so submit-time + // expansion emits a `kimi-file://` reference instead of inline base64. + const fileId = await this.uploadImageToDaemonFileStore( + compressed.changed ? compressed.data : media.bytes, + compressed.changed ? compressed.mimeType : meta.mime, + ); // Dimensions come from the compression result, not parseImageMeta: the // compressor reports display space (EXIF orientation applied) — the space // the sent image, the caption, and ReadMediaFile region readback share — @@ -489,12 +501,15 @@ export class EditorKeyboardController { byteLength: media.bytes.length, mime: meta.mime, }, + fileId, ) : this.imageStore.addImage( media.bytes, meta.mime, compressed.width || meta.width, compressed.height || meta.height, + undefined, + fileId, ); this.host.state.editor.insertTextAtCursor?.(`${attachment.placeholder} `); this.host.state.ui.requestRender(); @@ -502,6 +517,30 @@ export class EditorKeyboardController { return true; } + /** + * Paste-time upload of the final image bytes to the engine's daemon file + * store (agent-core-v2 only). Best effort: any failure returns undefined, + * so the attachment keeps no `fileId` and submit-time expansion falls back + * to the inline base64 form — the paste never blocks on the upload. + */ + private async uploadImageToDaemonFileStore( + bytes: Uint8Array, + mime: string, + ): Promise { + if (this.host.engineV2 !== true) return undefined; + const harness = this.host.harness; + if (harness === undefined) return undefined; + try { + const meta = await harness.uploadFile(bytes, { + name: `pasted-image.${imageExtensionForMime(mime)}`, + mimeType: mime, + }); + return meta.id; + } catch { + return undefined; + } + } + private async openExternalEditor(): Promise { const { state } = this.host; if (state.externalEditorRunning) return; diff --git a/apps/kimi-code/src/tui/kimi-tui.ts b/apps/kimi-code/src/tui/kimi-tui.ts index c64c65dff4..50bc23075f 100644 --- a/apps/kimi-code/src/tui/kimi-tui.ts +++ b/apps/kimi-code/src/tui/kimi-tui.ts @@ -328,7 +328,8 @@ export class KimiTUI { private backgroundRefreshPromise: Promise | undefined; private readonly migrationPlan: MigrationPlan | null; private readonly migrateOnly: boolean; - private readonly engineV2: boolean; + /** agent-core-v2 engine flag (startup-selected); read by controllers via host interfaces. */ + readonly engineV2: boolean; private startupNotice: string | undefined; private lastActivityMode: string | undefined; private currentLoadingTip: { kind: LoadingTipKind; tip: string | undefined } | undefined = diff --git a/apps/kimi-code/src/tui/utils/image-attachment-store.ts b/apps/kimi-code/src/tui/utils/image-attachment-store.ts index 8d653159a2..61d8d18c06 100644 --- a/apps/kimi-code/src/tui/utils/image-attachment-store.ts +++ b/apps/kimi-code/src/tui/utils/image-attachment-store.ts @@ -42,6 +42,13 @@ export interface ImageAttachment { * knows it received a downsampled copy. Absent for untouched pastes. */ readonly original?: ImageAttachmentOriginal | undefined; + /** + * Daemon file-store id, set when the bytes were uploaded at paste time + * (v2 engine only). Submit-time expansion then emits a `kimi-file://` + * reference plus an `` tag instead of inline base64; absent + * means the inline form is used. + */ + readonly fileId?: string; /** Rendered placeholder string, e.g. `[image #1 (640×480)]`. */ readonly placeholder: string; } @@ -69,6 +76,7 @@ export class ImageAttachmentStore { width: number, height: number, original?: ImageAttachmentOriginal, + fileId?: string, ): ImageAttachment { const id = this.nextId; this.nextId += 1; @@ -80,6 +88,7 @@ export class ImageAttachmentStore { width, height, original, + fileId, placeholder: formatPlaceholder(id, width, height), }; this.byId.set(id, attachment); diff --git a/apps/kimi-code/src/tui/utils/image-placeholder.ts b/apps/kimi-code/src/tui/utils/image-placeholder.ts index 87d53eabc2..daafb187b8 100644 --- a/apps/kimi-code/src/tui/utils/image-placeholder.ts +++ b/apps/kimi-code/src/tui/utils/image-placeholder.ts @@ -5,7 +5,14 @@ * `extractMediaAttachments` (sync) is the single expansion path for prompts: * - image placeholders expand to inline image content parts (preceded by a * compression caption when paste-time compression shrank the bytes — see - * `ImageAttachment.original`); + * `ImageAttachment.original`). When the paste was uploaded to the daemon + * file store (`ImageAttachment.fileId`, v2 engine only), the placeholder + * instead expands — mirroring the kap-server REST edge — to the + * `` tag text (pointing at a cache copy of the bytes, so the + * model always has a path it can re-open) plus a + * `kimi-file://?path=…` image part the engine resolves at request + * time; without a `fileId` the inline base64 form is emitted unchanged + * (the only form the v1 engine accepts); * - video placeholders are copied into the shared cache (`getCacheDir()`) * and expand to a `video_url` part pointing at the cache copy with a * `file://` url. The v1 engine resolves that local reference inside the @@ -34,7 +41,11 @@ import { join } from 'node:path'; import { pathToFileURL } from 'node:url'; import type { PromptPart } from '@moonshot-ai/kimi-code-sdk'; -import { buildImageCompressionCaption } from '@moonshot-ai/kimi-code-sdk'; +import { + buildDaemonFileUrl, + buildImageCompressionCaption, + buildMediaPathTag, +} from '@moonshot-ai/kimi-code-sdk'; import { getCacheDir } from '#/utils/paths'; @@ -94,7 +105,21 @@ export function extractMediaAttachments( if (attachment.original !== undefined) { pushText(parts, captionForCompressedImage(attachment)); } - parts.push(imagePartForAttachment(attachment)); + if (attachment.fileId !== undefined) { + // The bytes were uploaded to the daemon file store at paste time + // (v2): reference them by a `kimi-file://` url the engine resolves + // at request time, preceded by the `` tag text so the + // model always has a path it can re-open. The tag's path is a cache + // copy of the same (already-compressed) bytes. + const cachePath = materializeImageToCache(attachment); + parts.push({ type: 'text', text: buildMediaPathTag('image', cachePath) }); + parts.push({ + type: 'image_url', + imageUrl: { url: buildDaemonFileUrl(attachment.fileId, cachePath) }, + }); + } else { + parts.push(imagePartForAttachment(attachment)); + } imageAttachmentIds.push(id); } hasMedia = true; @@ -242,13 +267,21 @@ const IMAGE_MIME_EXTENSION: Readonly> = { 'image/tiff': 'tif', }; +/** + * File-extension hint for an image MIME (`image/png` → `png`). The real + * format is always sniffed from the bytes, so this only names files (cache + * copies, daemon upload labels). + */ +export function imageExtensionForMime(mime: string): string { + return IMAGE_MIME_EXTENSION[mime.trim().toLowerCase()] ?? 'img'; +} + function materializeImageToCache(att: ImageAttachment): string { const cacheDir = getCacheDir(); mkdirSync(cacheDir, { recursive: true }); // ReadMediaFile sniffs the real format from the bytes, so the extension // only needs to be a reasonable hint. - const ext = IMAGE_MIME_EXTENSION[att.mime.trim().toLowerCase()] ?? 'img'; - const target = join(cacheDir, `${randomUUID()}.${ext}`); + const target = join(cacheDir, `${randomUUID()}.${imageExtensionForMime(att.mime)}`); writeFileSync(target, att.bytes); return target; } diff --git a/apps/kimi-code/test/tui/controllers/editor-keyboard-image-paste.test.ts b/apps/kimi-code/test/tui/controllers/editor-keyboard-image-paste.test.ts index b87b2d4d59..d84ab6a11d 100644 --- a/apps/kimi-code/test/tui/controllers/editor-keyboard-image-paste.test.ts +++ b/apps/kimi-code/test/tui/controllers/editor-keyboard-image-paste.test.ts @@ -10,6 +10,9 @@ * point the model at the full-fidelity bytes * - a within-budget paste is stored byte-for-byte (fast path), with no * original recorded + * - on the v2 engine the final bytes are uploaded to the daemon file store + * and the attachment carries the returned `fileId`; an upload failure + * leaves the paste on the inline base64 fallback */ import { mkdtemp, readFile, rm, unlink } from 'node:fs/promises'; @@ -42,7 +45,17 @@ interface PasteHarness { pasteImage(): Promise; } -function createPasteHarness(options: { sessionDir?: string; imageLimits?: ImageLimits } = {}): PasteHarness { +function createPasteHarness( + options: { + sessionDir?: string; + imageLimits?: ImageLimits; + engineV2?: boolean; + uploadFile?: ( + data: Uint8Array, + opts: { name: string; mimeType?: string }, + ) => Promise<{ id: string }>; + } = {}, +): PasteHarness { const editor: Record unknown) | undefined> = { setHistoryFilter: vi.fn() as unknown as (...args: never[]) => unknown, }; @@ -61,14 +74,16 @@ function createPasteHarness(options: { sessionDir?: string; imageLimits?: ImageL ? undefined : { summary: { sessionDir: options.sessionDir } }, btwPanelController: { closeOrCancel: vi.fn(() => false) }, + engineV2: options.engineV2, track, showError: vi.fn(), openUndoSelector: vi.fn(), cancelRunningShellCommand: vi.fn(), } as unknown as EditorKeyboardHost; - if (options.imageLimits !== undefined) { + if (options.imageLimits !== undefined || options.uploadFile !== undefined) { (host as unknown as { harness: KimiHarness }).harness = { imageLimits: options.imageLimits, + uploadFile: options.uploadFile, } as unknown as KimiHarness; } @@ -98,6 +113,11 @@ async function solidJpeg(width: number, height: number): Promise { ); } +/** Typed `uploadFile` stub so `mock.calls` keeps the (data, options) tuple. */ +function uploadFileMock(id: string) { + return vi.fn(async (_data: Uint8Array, _opts: { name: string; mimeType?: string }) => ({ id })); +} + /** * Insert a minimal EXIF APP1 segment carrying only an Orientation tag right * after the JPEG SOI marker (jimp itself never writes EXIF). Mirrors the @@ -296,4 +316,75 @@ describe('clipboard image paste compression', () => { expect(props['source']).toBe('tui_paste'); expect(props['outcome']).toBe('compressed'); }); + + it('uploads the final bytes to the daemon file store on the v2 engine', async () => { + const small = await solidPng(80, 80); + readClipboardMedia.mockResolvedValue({ kind: 'image', bytes: small, mimeType: 'image/png' }); + const uploadFile = uploadFileMock('file-1'); + + const { store, pasteImage } = createPasteHarness({ engineV2: true, uploadFile }); + await pasteImage(); + + const att = store.get(1); + if (att?.kind !== 'image') throw new Error('expected image attachment'); + expect(att.fileId).toBe('file-1'); + expect(uploadFile).toHaveBeenCalledTimes(1); + const [data, opts] = uploadFile.mock.calls[0]!; + expect(new Uint8Array(data)).toEqual(small); + expect(opts).toEqual({ name: 'pasted-image.png', mimeType: 'image/png' }); + // The bytes stay on the attachment for the inline fallback / cache copy. + expect(att.bytes).toBe(small); + }); + + it('uploads the compressed bytes when paste-time compression changed them (v2)', async () => { + const big = await solidPng(3600, 1800); + readClipboardMedia.mockResolvedValue({ kind: 'image', bytes: big, mimeType: 'image/png' }); + const uploadFile = uploadFileMock('file-9'); + + const { store, pasteImage } = createPasteHarness({ engineV2: true, uploadFile }); + await pasteImage(); + + const att = store.get(1); + if (att?.kind !== 'image') throw new Error('expected image attachment'); + expect(att.fileId).toBe('file-9'); + // The upload carries exactly what the attachment stores — the compressed + // bytes, not the clipboard original. + const [data] = uploadFile.mock.calls[0]!; + expect(data).toBe(att.bytes); + expect(att.bytes).not.toBe(big); + await unlink(att.original!.path!).catch(() => undefined); + }); + + it('keeps the paste on the inline fallback when the daemon upload fails (v2)', async () => { + const small = await solidPng(80, 80); + readClipboardMedia.mockResolvedValue({ kind: 'image', bytes: small, mimeType: 'image/png' }); + const uploadFile = vi.fn( + async (_data: Uint8Array, _opts: { name: string; mimeType?: string }): Promise<{ id: string }> => { + throw new Error('daemon down'); + }, + ); + + const { store, pasteImage } = createPasteHarness({ engineV2: true, uploadFile }); + await pasteImage(); // must not throw + + const att = store.get(1); + if (att?.kind !== 'image') throw new Error('expected image attachment'); + expect(att.fileId).toBeUndefined(); + expect(att.bytes).toBe(small); + }); + + it('never uploads on the v1 engine', async () => { + const small = await solidPng(80, 80); + readClipboardMedia.mockResolvedValue({ kind: 'image', bytes: small, mimeType: 'image/png' }); + const uploadFile = uploadFileMock('file-1'); + + // engineV2 unset — the v1 host shape. + const { store, pasteImage } = createPasteHarness({ uploadFile }); + await pasteImage(); + + expect(uploadFile).not.toHaveBeenCalled(); + const att = store.get(1); + if (att?.kind !== 'image') throw new Error('expected image attachment'); + expect(att.fileId).toBeUndefined(); + }); }); diff --git a/apps/kimi-code/test/tui/input/image-attachment-store.test.ts b/apps/kimi-code/test/tui/input/image-attachment-store.test.ts index 6add1e428e..ce60a986b6 100644 --- a/apps/kimi-code/test/tui/input/image-attachment-store.test.ts +++ b/apps/kimi-code/test/tui/input/image-attachment-store.test.ts @@ -49,6 +49,18 @@ describe('ImageAttachmentStore', () => { expect(att.mime).toBe('image/jpeg'); }); + it('records the daemon file-store id when the paste was uploaded (v2)', () => { + const s = new ImageAttachmentStore(); + const att = s.addImage(new Uint8Array([1]), 'image/png', 10, 20, undefined, 'file-abc'); + expect(att.fileId).toBe('file-abc'); + }); + + it('leaves fileId undefined for attachments that were not uploaded', () => { + const s = new ImageAttachmentStore(); + const att = s.addImage(new Uint8Array([1]), 'image/png', 10, 20); + expect(att.fileId).toBeUndefined(); + }); + it('clear() resets ids and empties storage', () => { const s = new ImageAttachmentStore(); s.addImage(new Uint8Array(), 'image/png', 10, 10); diff --git a/apps/kimi-code/test/tui/input/image-placeholder.test.ts b/apps/kimi-code/test/tui/input/image-placeholder.test.ts index 85755641c6..3990677031 100644 --- a/apps/kimi-code/test/tui/input/image-placeholder.test.ts +++ b/apps/kimi-code/test/tui/input/image-placeholder.test.ts @@ -5,6 +5,8 @@ import { fileURLToPath } from 'node:url'; import { describe, it, expect } from 'vitest'; +import { parseDaemonFileUrl } from '@moonshot-ai/kimi-code-sdk'; + import { KIMI_CODE_HOME_ENV } from '#/constant/app'; import { ImageAttachmentStore } from '#/tui/utils/image-attachment-store'; import { @@ -229,6 +231,76 @@ describe('extractMediaAttachments', () => { expect(r.parts).toHaveLength(1); expect(r.parts[0]?.type).toBe('image_url'); }); + + it('expands an uploaded (fileId) image into an tag plus a kimi-file reference', () => { + const { cleanup } = setupTempCache(); + try { + const bytes = new Uint8Array([0x89, 0x50, 0x4e, 0x47]); + const store = new ImageAttachmentStore(); + const att = store.addImage(bytes, 'image/png', 640, 480, undefined, 'file-1'); + const r = extractMediaAttachments(`describe ${att.placeholder} please`, store); + expect(r.hasMedia).toBe(true); + expect(r.imageAttachmentIds).toEqual([1]); + expect(r.parts).toHaveLength(4); + expect(r.parts[0]).toEqual({ type: 'text', text: 'describe ' }); + const tag = r.parts[1]; + if (tag?.type !== 'text') throw new Error('expected text part'); + const m = /^<\/image>$/.exec(tag.text); + if (!m) throw new Error(`no image tag found in: ${tag.text}`); + const cachePath = m[1]!; + expect(cachePath.startsWith(getCacheDir())).toBe(true); + expect(cachePath.endsWith('.png')).toBe(true); + // The cache copy carries the attachment bytes the reference resolves to. + expect(new Uint8Array(readFileSync(cachePath))).toEqual(bytes); + const image = r.parts[2]; + if (image?.type !== 'image_url') throw new Error('expected image_url part'); + // The `?path=` query is the URL-encoded cache path. + expect(image.imageUrl.url).toBe( + `kimi-file://file-1?path=${encodeURIComponent(cachePath)}`, + ); + expect(parseDaemonFileUrl(image.imageUrl.url)).toEqual({ + fileId: 'file-1', + path: cachePath, + }); + expect(r.parts[3]).toEqual({ type: 'text', text: ' please' }); + } finally { + cleanup(); + } + }); + + it('emits the compression caption before the tag and kimi-file reference', () => { + const { cleanup } = setupTempCache(); + try { + const store = new ImageAttachmentStore(); + const att = store.addImage( + new Uint8Array([1, 2, 3]), + 'image/png', + 2000, + 2000, + { + path: '/tmp/kimi-code-original-images/abc.png', + width: 2600, + height: 2600, + byteLength: 123456, + mime: 'image/png', + }, + 'file-2', + ); + const r = extractMediaAttachments(att.placeholder, store); + expect(r.parts).toHaveLength(3); + const caption = r.parts[0]; + if (caption?.type !== 'text') throw new Error('expected leading text part'); + expect(caption.text).toContain('Image compressed'); + const tag = r.parts[1]; + if (tag?.type !== 'text') throw new Error('expected text part'); + expect(tag.text).toMatch(/^<\/image>$/); + const image = r.parts[2]; + if (image?.type !== 'image_url') throw new Error('expected image_url part'); + expect(image.imageUrl.url.startsWith('kimi-file://file-2?path=')).toBe(true); + } finally { + cleanup(); + } + }); }); describe('rewriteMediaPlaceholders', () => { diff --git a/packages/agent-core-v2/src/agent/llmRequester/llmRequesterService.ts b/packages/agent-core-v2/src/agent/llmRequester/llmRequesterService.ts index 2bd32c3520..6409c4a62d 100644 --- a/packages/agent-core-v2/src/agent/llmRequester/llmRequesterService.ts +++ b/packages/agent-core-v2/src/agent/llmRequester/llmRequesterService.ts @@ -9,9 +9,10 @@ * resolved from `IModelCatalog`: one primary `requester.request(input, signal, * params)` attempt plus projection rebuilds for request structure or media * compatibility. Before each request the projected messages pass through `media`'s - * video resolver, which rewrites every `kimi-file://` prompt-video reference - * to a provider-acceptable part (uploaded `ms://`, inline base64, or a - * `