feat(server): remove the 50 MiB upload size cap and stream uploads to disk

.changeset/upload-no-size-cap.md

@@ -0,0 +1,5 @@
+---
+"@moonshot-ai/kimi-code": minor
+---
+
+Remove the 50 MB size limit on file uploads to the built-in server, so large attachments (for example in the web UI) no longer fail with an upload-too-large error. Uploads now stream to disk instead of being buffered in memory.

packages/agent-core-v2/src/_base/utils/fs.ts

@@ -112,3 +112,48 @@ export async function atomicWrite(
     }
   }
 }
+
+/**
+ * Streamed variant of `atomicWrite`: same tmp + fsync + rename discipline, but
+ * the content arrives as an `AsyncIterable` so arbitrarily large values never
+ * sit in memory at once.
+ */
+export async function atomicWriteStream(
+  filePath: string,
+  source: AsyncIterable<Uint8Array>,
+  mode?: number,
+): Promise<void> {
+  const hex = randomBytes(4).toString('hex');
+  const tmpPath = `${filePath}.tmp.${process.pid}.${hex}`;
+  let renamed = false;
+  try {
+    const fh = await open(tmpPath, 'w', mode);
+    try {
+      for await (const chunk of source) {
+        if (chunk.byteLength > 0) {
+          await fh.writeFile(chunk);
+        }
+      }
+      await fh.sync();
+    } finally {
+      await fh.close();
+    }
+    if (process.platform === 'win32') {
+      try {
+        await unlink(filePath);
+      } catch (error) {
+        const code = (error as NodeJS.ErrnoException).code;
+        if (code !== 'ENOENT') throw error;
+      }
+    }
+    await rename(tmpPath, filePath);
+    renamed = true;
+  } finally {
+    if (!renamed) {
+      try {
+        await unlink(tmpPath);
+      } catch {
+      }
+    }
+  }
+}

packages/agent-core-v2/src/app/file/fileService.ts

@@ -25,8 +25,6 @@ export const fileMetaSchema = z.object({
 });
 export type FileMeta = z.infer<typeof fileMetaSchema>;
 
-export const DEFAULT_MAX_UPLOAD_BYTES = 50 * 1024 * 1024;
-
 export interface SaveOptions {
   readonly name?: string;
   readonly mimeType?: string;
@@ -57,7 +55,6 @@ export const IFileService: ServiceIdentifier<IFileService> = createDecorator<IFi
 export const FileErrors = {
   codes: {
     FILE_NOT_FOUND: 'file.not_found',
-    FILE_TOO_LARGE: 'file.too_large',
   },
   info: {
     'file.not_found': {
@@ -66,12 +63,6 @@ export const FileErrors = {
       public: true,
       action: 'Check the file_id or upload the file again.',
     },
-    'file.too_large': {
-      title: 'Upload too large',
-      retryable: false,
-      public: true,
-      action: 'Upload a smaller file (limit is 50 MiB).',
-    },
   },
 } as const satisfies ErrorDomain;
 
@@ -92,14 +83,6 @@ export function fileNotFoundError(fileId: string): FileError {
   return new FileError(FileErrors.codes.FILE_NOT_FOUND, `file not found: ${fileId}`, { fileId });
 }
 
-export function fileTooLargeError(seen: number, limit: number): FileError {
-  return new FileError(
-    FileErrors.codes.FILE_TOO_LARGE,
-    `upload size ${seen} bytes exceeds limit ${limit} bytes`,
-    { seen, limit },
-  );
-}
-
 export function isFileError(error: unknown, code: (typeof FileErrors.codes)[keyof typeof FileErrors.codes]): boolean {
   return error instanceof Error2 && error.code === code;
 }

packages/agent-core-v2/src/app/file/fileServiceImpl.ts

@@ -2,10 +2,12 @@
  * `file` domain (L2) — `IFileService` implementation.
  *
  * Streams uploads into the `IBlobStore` under the `files` scope and keeps a
- * JSON `FileMeta` index in the same store under the `file` scope.
- * Enforces the 50 MiB upload cap while collecting the stream, prunes the
- * index when a referenced blob is missing, and hands downloads back as a lazy
- * `Readable` over `getStream`. Bound at App scope.
+ * JSON `FileMeta` index in the same store under the `file` scope. Uploads are
+ * written incrementally (`putStream`), so their size is bounded by disk, not
+ * memory; the service counts bytes as they flow through to record
+ * `FileMeta.size`. Prunes the index when a referenced blob is missing, and
+ * hands downloads back as a lazy `Readable` over `getStream`. Bound at App
+ * scope.
  */
 
 import { randomUUID } from 'node:crypto';
@@ -16,10 +18,8 @@ import type { FileMeta } from './fileService';
 import { LifecycleScope, ScopeActivation, registerScopedService } from '#/_base/di/scope';
 import { IBlobStore } from '#/persistence/interface/blobStore';
 import {
-  DEFAULT_MAX_UPLOAD_BYTES,
   IFileService,
   fileNotFoundError,
-  fileTooLargeError,
   type FileReadRange,
   type GetResult,
   type SaveOptions,
@@ -70,26 +70,28 @@ export class FileServiceImpl implements IFileService {
     await this.ensureIndex();
 
     const id = `f_${randomUUID()}`;
-    const chunks: Buffer[] = [];
-    let bytes = 0;
-    for await (const chunk of source) {
-      const buf = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as string);
-      bytes += buf.length;
-      if (bytes > DEFAULT_MAX_UPLOAD_BYTES) {
-        throw fileTooLargeError(bytes, DEFAULT_MAX_UPLOAD_BYTES);
+    let size = 0;
+    const counting = async function* (): AsyncIterable<Uint8Array> {
+      for await (const chunk of source) {
+        const buf = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as string);
+        size += buf.length;
+        yield buf;
       }
-      chunks.push(buf);
+    };
+    try {
+      await this.blobs.putStream(BLOB_SCOPE, id, counting());
+    } catch (error) {
+      // best-effort cleanup of a partially written blob
+      await this.blobs.delete(BLOB_SCOPE, id).catch(() => undefined);
+      throw error;
     }
-    const data = Buffer.concat(chunks);
-
-    await this.blobs.put(BLOB_SCOPE, id, data);
 
     const now = Date.now();
     const meta: FileMeta = {
       id,
       name: options.name ?? filename,
       media_type: options.mimeType ?? 'application/octet-stream',
-      size: data.length,
+      size,
       created_at: new Date(now).toISOString(),
       ...(options.expiresInSec !== undefined
         ? { expires_at: new Date(now + options.expiresInSec * 1000).toISOString() }

packages/agent-core-v2/src/persistence/backends/memory/inMemoryStorageService.ts

@@ -69,6 +69,28 @@ export class InMemoryStorageService implements IFileSystemStorageService {
     this.notifyWatchers(scope, key);
   }
 
+  async writeStream(
+    scope: string,
+    key: string,
+    source: AsyncIterable<Uint8Array>,
+    _options: StorageWriteOptions = {},
+  ): Promise<void> {
+    const chunks: Uint8Array[] = [];
+    let total = 0;
+    for await (const chunk of source) {
+      chunks.push(chunk);
+      total += chunk.byteLength;
+    }
+    const merged = new Uint8Array(total);
+    let offset = 0;
+    for (const chunk of chunks) {
+      merged.set(chunk, offset);
+      offset += chunk.byteLength;
+    }
+    this.bucket(scope).set(key, merged);
+    this.notifyWatchers(scope, key);
+  }
+
   async append(
     scope: string,
     key: string,

packages/agent-core-v2/src/persistence/backends/node-fs/blobStoreService.ts

@@ -19,6 +19,10 @@ export class BlobStoreService implements IBlobStore {
     await this.storage.write(scope, key, data, { atomic: true });
   }
 
+  async putStream(scope: string, key: string, source: AsyncIterable<Uint8Array>): Promise<void> {
+    await this.storage.writeStream(scope, key, source, { atomic: true });
+  }
+
   async get(scope: string, key: string): Promise<Uint8Array | undefined> {
     return this.storage.read(scope, key);
   }

packages/agent-core-v2/src/persistence/backends/node-fs/fileStorageService.ts

@@ -8,6 +8,8 @@
  * Primitives:
  *   - `write`  → `atomicWrite` (tmp + fsync + rename) followed by a directory
  *                fsync, so the replacement is both atomic and durable.
+ *   - `writeStream` → the streamed form of `write` (`atomicWriteStream`), for
+ *                values too large to buffer in memory.
  *   - `append` → `open('a')` + write + `fh.sync()` (when `durable`), plus a
  *                one-time directory fsync per scope.
  *   - `watch`  → chokidar on the parent directory, filtered to the exact key and
@@ -29,7 +31,7 @@ import { dirname, join, normalize } from 'pathe';
 import { DisposableStore, combinedDisposable, toDisposable, type IDisposable } from '#/_base/di/lifecycle';
 import { Emitter, type Event } from '#/_base/event';
 import { onUnexpectedError } from '#/_base/errors/unexpectedError';
-import { atomicWrite, syncDir } from '#/_base/utils/fs';
+import { atomicWrite, atomicWriteStream, syncDir } from '#/_base/utils/fs';
 
 import type {
   IFileSystemStorageService,
@@ -102,6 +104,22 @@ export class FileStorageService implements IFileSystemStorageService {
     }
   }
 
+  async writeStream(
+    scope: string,
+    key: string,
+    source: AsyncIterable<Uint8Array>,
+    _options: StorageWriteOptions = {},
+  ): Promise<void> {
+    const filePath = this.path(scope, key);
+    try {
+      await mkdir(dirname(filePath), { recursive: true, mode: this.dirMode });
+      await atomicWriteStream(filePath, source, this.fileMode);
+      await this.syncDirOnce(dirname(filePath));
+    } catch (error) {
+      throw toStorageIoError(error, { path: filePath, op: 'write' });
+    }
+  }
+
   async append(
     scope: string,
     key: string,

packages/agent-core-v2/src/persistence/interface/blobStore.ts

@@ -15,6 +15,7 @@ export interface IBlobStore {
   readonly _serviceBrand: undefined;
 
   put(scope: string, key: string, data: Uint8Array): Promise<void>;
+  putStream(scope: string, key: string, source: AsyncIterable<Uint8Array>): Promise<void>;
   get(scope: string, key: string): Promise<Uint8Array | undefined>;
   getStream(scope: string, key: string, range?: BlobReadRange): AsyncIterable<Uint8Array>;
   has(scope: string, key: string): Promise<boolean>;

packages/agent-core-v2/src/persistence/interface/storage.ts

@@ -13,6 +13,10 @@
  * the last value" semantics. Keeping both as first-class primitives lets each
  * implementation implement them optimally (file: `open('a')` vs tmp+rename).
  *
+ * `writeStream` is the streamed form of `write` for values too large to hold
+ * in memory: same whole-value replacement semantics (tmp + rename on the file
+ * backend), but the bytes arrive as an `AsyncIterable`.
+ *
  * The service is byte-oriented and scope/key-addressed: `scope` maps to a
  * directory, `key` maps to a filename. It knows nothing about JSON, records,
  * configs, versions or framing. Those concerns live in the typed Store facades
@@ -124,6 +128,12 @@ export interface IFileSystemStorageService {
   read(scope: string, key: string): Promise<Uint8Array | undefined>;
   readStream(scope: string, key: string, range?: StorageReadRange): AsyncIterable<Uint8Array>;
   write(scope: string, key: string, data: Uint8Array, options?: StorageWriteOptions): Promise<void>;
+  writeStream(
+    scope: string,
+    key: string,
+    source: AsyncIterable<Uint8Array>,
+    options?: StorageWriteOptions,
+  ): Promise<void>;
   append(scope: string, key: string, data: Uint8Array, options?: StorageAppendOptions): Promise<void>;
   list(scope: string, prefix?: string): Promise<readonly string[]>;
   delete(scope: string, key: string): Promise<void>;

packages/agent-core-v2/test/agent/media/videoResolver.test.ts

@@ -57,6 +57,11 @@ function blobStore(): IBlobStore {
     put: async (scope, key, bytes) => {
       data.set(`${scope}/${key}`, bytes);
     },
+    putStream: async (scope, key, source) => {
+      const chunks: Uint8Array[] = [];
+      for await (const chunk of source) chunks.push(chunk);
+      data.set(`${scope}/${key}`, Buffer.concat(chunks));
+    },
     get: async (scope, key) => data.get(`${scope}/${key}`),
     getStream: async function* () {},
     has: async (scope, key) => data.has(`${scope}/${key}`),

packages/agent-core-v2/test/agent/task/taskService.test.ts

@@ -152,6 +152,7 @@ describe('AgentTaskService', () => {
       read: async () => undefined,
       readStream: async function* () {},
       write: async () => {},
+      writeStream: async () => {},
       append: async () => {},
       list: async () => [],
       delete: async () => {},
@@ -802,6 +803,7 @@ describe('AgentTaskService', () => {
       read: async () => undefined,
       readStream: async function* () {},
       write: async () => {},
+      writeStream: async () => {},
       append: async (_scope: string, _key: string, chunk: Uint8Array) => {
         persistedChars += chunk.byteLength;
       },

packages/agent-core-v2/test/app/file/fileService.test.ts

@@ -13,7 +13,7 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest';
 import { SyncDescriptor } from '#/_base/di/descriptors';
 import { DisposableStore } from '#/_base/di/lifecycle';
 import { createServices, type TestInstantiationService } from '#/_base/di/test';
-import { DEFAULT_MAX_UPLOAD_BYTES, FileErrors, IFileService } from '#/app/file/fileService';
+import { FileErrors, IFileService } from '#/app/file/fileService';
 import { FileServiceImpl } from '#/app/file/fileServiceImpl';
 import { IFileSystemStorageService } from '#/persistence/interface/storage';
 import { InMemoryStorageService } from '#/persistence/backends/memory/inMemoryStorageService';
@@ -115,11 +115,22 @@ describe('FileServiceImpl', () => {
     });
   });
 
-  it('rejects an upload that exceeds the cap', async () => {
-    const big = Buffer.alloc(DEFAULT_MAX_UPLOAD_BYTES + 1, 0);
-    await expect(store().save(readable(big), 'big.bin')).rejects.toMatchObject({
-      code: FileErrors.codes.FILE_TOO_LARGE,
-    });
+  it('streams a multi-chunk upload and records the total size', async () => {
+    const chunks = [Buffer.from('aaa'), Buffer.from('bbbb'), Buffer.from('cc')];
+    const meta = await store().save(Readable.from(chunks), 'chunked.bin');
+
+    expect(meta.size).toBe(9);
+    const { stream } = await store().get(meta.id);
+    expect((await readAll(stream())).toString()).toBe('aaabbbbcc');
+  });
+
+  it('cleans up the blob when the source stream fails mid-upload', async () => {
+    const failing = Readable.from((async function* () {
+      yield Buffer.from('partial');
+      throw new Error('source exploded');
+    })());
+
+    await expect(store().save(failing, 'broken.bin')).rejects.toThrow('source exploded');
     expect(await backend.list('files')).toHaveLength(0);
   });
 

packages/agent-core-v2/test/persistence/backends/node-fs/appendLogStore.test.ts

@@ -34,6 +34,7 @@ function chunkedStorage(chunks: Uint8Array[]): IFileSystemStorageService {
       for (const c of chunks) yield c;
     },
     write: async () => {},
+    writeStream: async () => {},
     append: async () => {},
     list: async () => [],
     delete: async () => {},

packages/agent-core-v2/test/persistence/backends/node-fs/fileStorageService.test.ts

@@ -87,3 +87,41 @@ describe('FileStorageService — error translation', () => {
     });
   });
 });
+
+describe('FileStorageService — writeStream', () => {
+  let dir: string;
+
+  beforeEach(async () => {
+    dir = await mkdtemp(join(tmpdir(), 'fss-stream-'));
+  });
+
+  afterEach(async () => {
+    await rm(dir, { recursive: true, force: true });
+  });
+
+  it('writes a chunked source and replaces the whole value', async () => {
+    const svc = new FileStorageService(dir);
+    await svc.write('scope', 'k.bin', encoder.encode('old'));
+    await svc.writeStream('scope', 'k.bin', (async function* () {
+      yield encoder.encode('aa');
+      yield encoder.encode('bbb');
+    })());
+
+    const chunks: Uint8Array[] = [];
+    for await (const chunk of svc.readStream('scope', 'k.bin')) chunks.push(chunk);
+    expect(Buffer.concat(chunks).toString()).toBe('aabbb');
+  });
+
+  it('leaves no target file behind when the source fails mid-stream', async () => {
+    const svc = new FileStorageService(dir);
+    await expect(
+      svc.writeStream('scope', 'k.bin', (async function* () {
+        yield encoder.encode('partial');
+        throw new Error('boom');
+      })()),
+    ).rejects.toThrow();
+
+    expect(await svc.read('scope', 'k.bin')).toBeUndefined();
+    expect(await svc.list('scope')).toEqual([]);
+  });
+});