diff --git a/.agents/skills/agent-core-dev/edge-exposure.md b/.agents/skills/agent-core-dev/edge-exposure.md index 334c1e4546..5039201ac9 100644 --- a/.agents/skills/agent-core-dev/edge-exposure.md +++ b/.agents/skills/agent-core-dev/edge-exposure.md @@ -55,9 +55,9 @@ Read = `GET`, write = `POST`. `sid` = `session_id`, `aid` = `agent_id`. | resource | action | Service.method | verb | |---|---|---|---| -| `sessions` | `list` | ISessionIndex.list | GET | +| `sessions` | `listRecent` | ISessionIndex.listRecent | GET | | `sessions` | `get` | ISessionIndex.get | GET | -| `sessions` | `countActive` | ISessionIndex.countActive | GET | +| `sessions` | `count` | ISessionIndex.count | GET | | `workspaces` | `list` | IWorkspaceService.list | GET | | `workspaces` | `get` | IWorkspaceService.get | GET | | `workspaces` | `createOrTouch` | IWorkspaceService.createOrTouch | POST | diff --git a/.oxlintrc.json b/.oxlintrc.json index 003359f31d..51969ea255 100644 --- a/.oxlintrc.json +++ b/.oxlintrc.json @@ -90,6 +90,26 @@ "eslint/no-console": "off" } }, + { + // The stage-6 worker closure: these modules (and everything + // packages/minidb/src/worker/ pulls in) are loaded by a bare + // node:worker_threads Worker under Node's native type stripping with + // `execArgv: ['--experimental-transform-types']`, which requires + // explicit `.ts` import specifiers (the strip loader does not remap + // `.js` -> `.ts`). Keep the exception scoped to exactly that closure. + "files": [ + "packages/minidb/src/worker/**/*.ts", + "packages/minidb/src/codec.ts", + "packages/minidb/src/crc32.ts", + "packages/minidb/src/trigram.ts", + "packages/minidb/src/text-postings.ts", + "packages/minidb/src/text-index/tokenize.ts", + "packages/minidb/src/gen-codec.ts" + ], + "rules": { + "import/extensions": "off" + } + }, { "files": ["packages/kosong/src/providers/**/*.ts"], "rules": { diff --git a/AGENTS.md b/AGENTS.md index f6314bd015..b360bd3f86 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -26,11 +26,11 @@ This is a TypeScript monorepo built for agent-assisted development. Keep the roo - `packages/oauth`: Kimi OAuth and managed auth utilities. - `packages/telemetry`: shared client-side telemetry infrastructure. - `packages/transcript`: the isomorphic transcript rendering data layer — agent-granular L1 store, idempotent L2 operations, `off/turn/block/delta` L3 subscription granularity, framework-free L4 view registry, and turn-cursor pagination. Pure TypeScript (browser-safe, no engine imports) and the sole owner of all transcript contract types (`src/contract/`); consumed by `packages/kap-server` (engine events → transcript, REST + WS surface; live stores backfill history from the persisted per-agent wire records — main on first attach, any agent on demand, cold sessions rebuild any agent — with 0-based turn ordinals matching the engine's). The cold rebuild is a two-level fold over `wire.jsonl` as the single source of truth: `history/groupTurns.ts` (context messages → turn tree) plus `history/foldFacts.ts` (non-context records → tasks, interactions, todos, goal/plan/swarm meta, and end-appended markers/taskrefs; interactions left pending at shutdown fold to `cancelled`). Plan content is a recorded fact too: each ExitPlanMode review submission offloads the document to `agents//plan//v.md` and persists a reference-only `plan.revision` record (`{id, version, path, sha256, bytes}`), which projects — live and cold — to a `plan.revision` marker and the `modes.plan` badge (`{reviewPath, version}`). It also owns the op-batch sequencing contract (`transcriptSeqSchema` in `contract/schema.ts`): a per-(session, agent) monotonic batch `seq` on `transcript.ops` / `transcript.reset` / the REST transcript response, the `transcript_since` subscription cursor, and the `GET .../transcript/ops` catch-up response shape — every field optional so pre-seq peers fall back to loss-signal-driven refreshes. Beyond the timeline, the model carries wire-equivalent detail: steps carry `usage` / `finishReason` / `timing` (LLM latencies) / `retry` / interrupt reason, turns carry `durationMs` / `error` / `usage`, tool frames carry the streamed `inputText` and the latest `progress`, tasks carry subagent `resultSummary` / `error` / `stateReason` / `usage`, `meta.agent` mirrors the agent status slices (model / usage / context / permission / phase), a global `prompts` entity (op `prompt.upsert`) tracks the prompt queue, and `hook.result` lands as a `'hook'` marker. These live-projected fields are NOT backfilled by the cold rebuild (known limitation). -- `packages/kap-server`: the Kimi Code server, backed by the DI × Scope agent engine (`@moonshot-ai/agent-core-v2` — four scopes, App/Workspace/Session/Agent; session create/resume/fork routes compose `ISessionIndex` → `IWorkspaceLifecycleService.handlerFor` → the handler's `ISessionLifecycleService`, and the fs routes resolve session → handler → the Workspace-scope fs services, with one exception: `fs:search` also accepts a workspace reference (registered id or absolute root) in the `{session_id}` slot, so a not-yet-created draft session's `@` file mention resolves the workspace handler directly; the first-class session-less form is `POST /api/v1/workspace/fs:search` (the workspace reference travels in the body)). Exposes sessions over REST + WebSocket (`/api/v1` + `/api/v1/ws`); bootstrapped from `src/start.ts` and consumed by `apps/kimi-code`. The RPC surface is `/api/v1/debug/*` — a reflection dispatcher over the ENTIRE scoped DI registry (every Service callable, no whitelist, Workspace scope addressable alongside App/Session/Agent; `src/transport/registerDebugRoutes.ts` + `serviceDispatcherRoutes.ts`), mounted only with `--debug-endpoints` on a loopback bind and gated by the global bearer auth; repo dev scripts pass the flag. Its transcript surface implements the op-batch sequencing contract: `TranscriptService.dispatchOps` assigns every dispatched batch a per-agent consecutive `seq` and retains it in a bounded in-memory journal (`TRANSCRIPT_OPS_JOURNAL_CAPACITY`, dies with the live store); WS `transcript.ops`/`transcript.reset` payloads carry the seq/watermark, a `transcript_since` subscription cursor (carried, with the per-agent grades, by the `subscribe_v2` control frame — the only transcript subscription channel; its agent-grained counterpart `unsubscribe_v2` detaches listed agents' streams, or the whole session's when `agent_ids` is absent, letting the detached agents' legacy events flow again) replays journaled batches instead of a baseline reset when the journal covers it, and `GET /sessions/{id}/transcript/ops?since_seq=` serves point-to-point catch-up (`complete: false` = journal can't cover or session cold → caller falls back to a full refresh). Beside the paged route, `GET /sessions/{id}/transcript/plan?agent_id=[&tool_call_id=]` projects an agent's ExitPlanMode plan info (content / path / options / review outcome; `tool_call_id` narrows to one call, omitted lists every recoverable plan) from the first available fact — the linked approval interaction's persisted request display, the live tool frame's display, or the tool result output text. The baseline `transcript.reset` itself is items-empty (`TRANSCRIPT_RESET_TAIL_TURNS = 0`): it carries only global state + the watermark + `has_more_older`, because history always pages in over REST. When a WS connection subscribes to the transcript protocol (grade ≠ `off` for an agent), the broadcaster suppresses the transcript-projected `session_event` types for that connection × agent (`TRANSCRIPT_PROJECTED_EVENT_TYPES` + `suppressedByTranscript` in `sessionEventBroadcaster.ts`; cursor replay via `getBufferedSince` applies the same filter). Suppression is only a per-connection send view — the journal still records everything, and connections without transcript grades are unaffected. The session's work aggregate behind `event.session.work_changed` (`busy` / `main_turn_active` / `pending_interaction` / `last_turn_reason`) is owned by the core's `ISessionActivityView` (`sessionActivity` domain, Session scope): the broadcaster only schedules the wire emission around turn frames (`busy:false` lands after `turn.ended`), and `resolveSessionFacts` (`src/routes/sessions.ts`) reads the same view — never fold per-agent activity at the edge. Delivery split on `/api/v1/ws`: global events (`session.meta.updated` and the `event.session.*` / `event.workspace.*` / `event.config.*` families, including every activated session's `event.session.work_changed`) fan out to EVERY established connection — `WsConnectionV1` registers itself via `broadcaster.addGlobalTarget` on construction and unregisters on close — while session/agent-grained events only reach connections subscribed to that session (subject to `agent_filter` and the transcript suppression above); transcript frames are a separate channel governed by the per-agent grades alone and bypass `agent_filter` entirely. The global search surface is `POST /api/v1/search` (`src/search/` + `src/routes/search.ts`): a cross-session full-text search over user messages, assistant text, and session titles, backed by a single minidb database at `/search-index` (`IGlobalSearchService`, App scope — the write-lock holder is the indexer, other processes open read-only and catch up via WAL). It serves two modes: `terms` (the default — minidb's inverted text index over ASCII words + CJK uni/bigrams, no positions, term-level AND) and `literal` (substring-exact search: a hashed 2/3-gram index supplies candidates, every candidate's text is then confirmed with `includes`, so hits carry zero false positives; literal ignores `sort` and returns newest-first, and a candidate set truncated at `LITERAL_CANDIDATE_CAP` is flagged `incomplete: 'candidate_cap'`). When `container.session_id` is provided and that session is live in this process (`TranscriptService.forSessionLive` returns a store, wired via `setLiveTranscriptSource` in `start.ts`), BOTH modes instead scan the in-memory transcript store (turn prompts + assistant text frames, history established via `whenReady`/`ensureAgentHistory`) — no index involved; terms-mode live hits are scored Σ log(1+tf) (comparable only within a route, per the `GlobalSearchSource` contract), live-route errors never fall back to the index, and the response's `source: 'live' | 'index'` field (also mixed into the page-token fingerprint, so a mid-pagination route flip invalidates the old token) tells the caller which route served the page. +- `packages/kap-server`: the Kimi Code server, backed by the DI × Scope agent engine (`@moonshot-ai/agent-core-v2` — four scopes, App/Workspace/Session/Agent; session create/resume/fork routes compose `ISessionIndex` → `IWorkspaceLifecycleService.handlerFor` → the handler's `ISessionLifecycleService`, and the fs routes resolve session → handler → the Workspace-scope fs services, with one exception: `fs:search` also accepts a workspace reference (registered id or absolute root) in the `{session_id}` slot, so a not-yet-created draft session's `@` file mention resolves the workspace handler directly; the first-class session-less form is `POST /api/v1/workspace/fs:search` (the workspace reference travels in the body)). Exposes sessions over REST + WebSocket (`/api/v1` + `/api/v1/ws`); bootstrapped from `src/start.ts` and consumed by `apps/kimi-code`. The RPC surface is `/api/v1/debug/*` — a reflection dispatcher over the ENTIRE scoped DI registry (every Service callable, no whitelist, Workspace scope addressable alongside App/Session/Agent; `src/transport/registerDebugRoutes.ts` + `serviceDispatcherRoutes.ts`), mounted only with `--debug-endpoints` on a loopback bind and gated by the global bearer auth; repo dev scripts pass the flag. Its transcript surface implements the op-batch sequencing contract: `TranscriptService.dispatchOps` assigns every dispatched batch a per-agent consecutive `seq` and retains it in a bounded in-memory journal (`TRANSCRIPT_OPS_JOURNAL_CAPACITY`, dies with the live store); WS `transcript.ops`/`transcript.reset` payloads carry the seq/watermark, a `transcript_since` subscription cursor (carried, with the per-agent grades, by the `subscribe_v2` control frame — the only transcript subscription channel; its agent-grained counterpart `unsubscribe_v2` detaches listed agents' streams, or the whole session's when `agent_ids` is absent, letting the detached agents' legacy events flow again) replays journaled batches instead of a baseline reset when the journal covers it, and `GET /sessions/{id}/transcript/ops?since_seq=` serves point-to-point catch-up (`complete: false` = journal can't cover or session cold → caller falls back to a full refresh). Beside the paged route, `GET /sessions/{id}/transcript/plan?agent_id=[&tool_call_id=]` projects an agent's ExitPlanMode plan info (content / path / options / review outcome; `tool_call_id` narrows to one call, omitted lists every recoverable plan) from the first available fact — the linked approval interaction's persisted request display, the live tool frame's display, or the tool result output text. The baseline `transcript.reset` itself is items-empty (`TRANSCRIPT_RESET_TAIL_TURNS = 0`): it carries only global state + the watermark + `has_more_older`, because history always pages in over REST. When a WS connection subscribes to the transcript protocol (grade ≠ `off` for an agent), the broadcaster suppresses the transcript-projected `session_event` types for that connection × agent (`TRANSCRIPT_PROJECTED_EVENT_TYPES` + `suppressedByTranscript` in `sessionEventBroadcaster.ts`; cursor replay via `getBufferedSince` applies the same filter). Suppression is only a per-connection send view — the journal still records everything, and connections without transcript grades are unaffected. The session's work aggregate behind `event.session.work_changed` (`busy` / `main_turn_active` / `pending_interaction` / `last_turn_reason`) is owned by the core's `ISessionActivityView` (`sessionActivity` domain, Session scope): the broadcaster only schedules the wire emission around turn frames (`busy:false` lands after `turn.ended`), and `resolveSessionFacts` (`src/routes/sessions.ts`) reads the same view — never fold per-agent activity at the edge. Delivery split on `/api/v1/ws`: global events (`session.meta.updated` and the `event.session.*` / `event.workspace.*` / `event.config.*` families, including every activated session's `event.session.work_changed`) fan out to EVERY established connection — `WsConnectionV1` registers itself via `broadcaster.addGlobalTarget` on construction and unregisters on close — while session/agent-grained events only reach connections subscribed to that session (subject to `agent_filter` and the transcript suppression above); transcript frames are a separate channel governed by the per-agent grades alone and bypass `agent_filter` entirely. The global search surface is `POST /api/v1/search` (`src/search/` + `src/routes/search.ts`): a cross-session full-text search over user messages, assistant text, and session titles, backed by a single minidb database at `/search-index` (`IGlobalSearchService`, App scope — the write-lock holder is the indexer, other processes open read-only and catch up via WAL). It serves two modes: `terms` (the default — minidb's inverted text index over ASCII words + CJK uni/bigrams, no positions, term-level AND) and `literal` (substring-exact search: a hashed 2/3-gram index supplies candidates, every candidate's text is then confirmed with `includes`, so hits carry zero false positives; literal ignores `sort` and returns newest-first). The index route is fully bounded (stage 4): a search request serves the currently published generation and never awaits a sync/reopen/reindex — it kicks the single-flight + debounced background coordinator instead, and reports `index_state.stale` / `index_state.degraded` when serving a behind view or after a failed refresh, and `index_state.state: 'building'` while the served handle's text base is still being (re)built by the deferred fallback build (searches get the empty building page, never a partial result); every query runs under explicit budgets (max terms, postings visits via `MiniDb.searchBoundedAsync`, candidate caps, confirmation text volume, a match deadline) with over-budget pages flagged `incomplete: 'candidate_cap' | 'postings_budget' | 'deadline'`; pagination is keyset over `(time, key)` / `(score, time, key)` with versioned v2 tokens pinning the index generation (a rebuild/reopen/rescan invalidates old tokens with `invalid_page_token`; legacy v1 offset tokens are still accepted and upgraded), and per-session sync scans only that session's file-meta keys (`\0meta\file\\`, migrated from the pre-v2 hash-only keys by a one-time background pass). When `container.session_id` is provided and that session is live in this process (`TranscriptService.forSessionLive` returns a store, wired via `setLiveTranscriptSource` in `start.ts`), BOTH modes instead scan the in-memory transcript store (turn prompts + assistant text frames, history established via `whenReady`/`ensureAgentHistory`) — no index involved; terms-mode live hits are scored Σ log(1+tf) (comparable only within a route, per the `GlobalSearchSource` contract), live-route errors never fall back to the index, and the response's `source: 'live' | 'index'` field (also mixed into the page-token fingerprint, so a mid-pagination route flip invalidates the old token) tells the caller which route served the page. - `packages/klient`: the client SDK — a contract-driven facade over agent-core-v2 with aggregated `global.*` / `session(id).*` / `agent(id).*` methods, zod validation on every call, and klient-level typed event forwarding. Transport is chosen once at creation via subpath entry (`@moonshot-ai/klient/ipc|memory`); both return the same `Klient`. The package also hosts the e2e suites: the legacy `/api/v1` live suites (`test/e2e/legacy/`) and the docker e2e runner (`pnpm --filter @moonshot-ai/klient docker:e2e`). See `packages/klient/AGENTS.md`. - `packages/server-e2e`: live e2e tests and scenarios against a running server (`KIMI_SERVER_URL`, default `http://127.0.0.1:58627`). See `packages/server-e2e/AGENTS.md`. - `packages/tree-sitter-bash`: a pure-TypeScript bash parser (no runtime deps, no wasm) that produces a syntax tree with tree-sitter-bash 0.25.0 named-node type names and UTF-16 code-unit offsets. `parse(source, { timeoutMs, maxNodes })` runs under a deterministic budget (default 50 ms / 50k nodes, plus per-chain recursion depth caps) and returns a discriminated `ParseResult` (`{ ok, rootNode, hasError }` or `{ ok: false, reason: 'aborted' }`) — callers must treat aborted/hasError trees as "cannot analyze" and degrade. Parser only, no safety judgments; consumers (e.g. Bash tool permission matching) live elsewhere. Known deviations from the reference are tracked in the package README's "Known differences" section, pinned by differential fixtures tested against the real `tree-sitter-bash` wasm (dev-only). -- `packages/minidb`: the embedded JSON document store (`MiniDb`) behind kap-server's search index — snapshot + WAL persistence with an exclusive write lock (losers open read-only and catch up from the WAL), plus a larger-than-RAM full-text layer: `src/text-index.ts` is the inverted index (in-RAM dictionary + delta, on-disk postings in `src/text-postings.ts`, rebuilt from the Store on open and on compaction) with an injectable `tokenizer`/`queryTokenizer`; the default tokenizer keeps ASCII words and CJK uni/bigrams, while `src/trigram.ts` provides the hashed 2/3-gram tokenizer (NFKC + lowercase, code-point windows) that backs substring-exact search. Text-index definitions (including the tokenizer name) persist in `db.textindexes.json`. +- `packages/minidb`: the embedded JSON document store (`MiniDb`) behind kap-server's search index — snapshot + WAL persistence with an exclusive write lock (losers open read-only and catch up from the WAL), plus a larger-than-RAM full-text layer: `src/text-index/` is the inverted index (in-RAM dictionary + delta, on-disk postings in `src/text-postings.ts`; the module is split into `tokenize.ts` / `types.ts` / `builder.ts` / `image.ts` around the `TextIndex` core in `index.ts`) with an injectable `tokenizer`/`queryTokenizer`; the default tokenizer keeps ASCII words and CJK uni/bigrams, while `src/trigram.ts` provides the hashed 2/3-gram tokenizer (NFKC + lowercase, code-point windows) that backs substring-exact search. Text-index definitions (including the tokenizer name) persist in `db.textindexes.json`. Derived state (store image, dt/secondary/compound indexes, text dictionary + postings + doc table) is checkpointed as persistent index **generations** (`generations/g-NNNNNN/` + `CURRENT`, format v1 — `src/generation.ts` layout/manifest, `src/gen-codec.ts` binary images): the writer builds them into a `g-N.tmp-*` dir and atomically publishes (rename + CURRENT swap, fsyncs strict), each compaction's rotation and the generation publish form one transaction (replacing the old synchronous `rebuildTextPostings()` tail), and open loads the published generation + WAL delta replay instead of re-decoding every value / re-tokenizing the corpus / rewriting postings (the full recovery remains the automatic **fallback** for missing/invalid/unknown-version generations; `OpenOptions.indexGenerations: false` forces that path). On the fallback path the corpus-scale text rebuild is no longer awaited inside `open()`: it runs as a `'text-build'` maintenance task on the same bounded engine pinned at the recovery checkpoint (rollback: `OpenOptions.deferOpenTextBuilds: false`), searches on a not-yet-committed index raise `TextIndexBuildingError` (state surfaced via `MiniDb.textIndexBuilding` — the guard is a dedicated `basePending` flag, so staged builds over a live old base keep serving), and a read-only opener builds into a private scratch dir next to the db dir (`.ro-scratch/-*`, dropped on close) and adopts the disk base there instead of aggregating a full in-RAM base. Async base reads are commit-safe via a base-swap epoch (`TextIndex.baseEpoch`): a read straddling a base commit re-reads from the fresh base and never caches a stale list. A healthy writer also keeps a valid generation around at runtime — the per-write WAL-growth trigger (`MiniDb.maybeAutoGenerationBuild`, 4 MiB staleness rule, throttled with failure backoff) covers the started-from-empty window the open-time kick cannot, and `close()` publishes a missing/stale generation best-effort (`buildGeneration('close')`). Load-time integrity is per-file crc32 + definition hashes — a corrupt or definition-mismatched image rebuilds only the affected index from the loaded store. Heavy maintenance runs through the unified **maintenance scheduler** (`src/maintenance.ts` — one heavy task per database at a time, queue backpressure, disk free-space preflight, deadline/cancellation, shutdown drain-or-cancel, `MiniDb.maintenanceStatus()` read model; nested submissions from inside a task run inline via AsyncLocalStorage to avoid self-deadlock). Full-text generation artifacts (tokenization → bounded-memory aggregation → segmented external merge → postings/dictionary/base-docs) are produced **off the main thread** by a worker build (`src/worker/text-build-core.ts`, hosted by `src/worker/text-build.ts` + `src/worker/text-build-worker.ts` — Node-native type-stripping with `execArgv: ['--experimental-transform-types']`, explicit `.ts` import specifiers in the whole worker closure; worker writes only inside the tmp generation dir, the main thread verifies (sanity + streaming crc) and swaps the live base via `TextIndex.commitRebase` after `beginRebase`; `OpenOptions.textBuildWorker: false` is the rollback switch; a missing worker file or slot pressure hosts the SAME bounded core inline on the main thread instead — the in-thread staged aggregation is kept only for small corpora (< 4096 docs), custom function tokenizers, and the explicit rollback). The same bounded engine (worker-or-inline + rebase, `MiniDb.boundedTextBuild`) also backs the two full-corpus initial-build entries — `createTextIndex` and the open-time loader rebuild of a corrupt/definition-mismatched image — so first-time indexing of a large existing store no longer aggregates the whole term->postings map in RAM. The async read surface is additive: `getAsync` / `searchAsync` / `searchBoundedAsync` / `queryAsync` (`ValueReader.readAsync`, `PostingsFile.readAsync`, byte-bounded decoded-postings cache via `TextIndexOptions.cacheBytes`); recovery scans use the async sequential scanner (`scanFrameRefsFdAsync` — windowed reads, sliced CRC, periodic yields, AbortSignal, bounded corruption-resync candidate budget shared with the sync scanner); compaction's disk-mode snapshot groups live refs by (file, offset) and reads them with bounded-concurrency async positioned reads (`src/snapshot.ts`). ## Environment Requirements diff --git a/apps/kimi-code/.gitignore b/apps/kimi-code/.gitignore index 901b7a6d26..762220ab4c 100644 --- a/apps/kimi-code/.gitignore +++ b/apps/kimi-code/.gitignore @@ -8,4 +8,4 @@ agents/ src/generated/vis-web-asset.ts # Copied from packages/pi-tui/native at build time by scripts/copy-native-assets.mjs -native/ +/native/ diff --git a/apps/kimi-code/package.json b/apps/kimi-code/package.json index ea52280f2a..c67b35fda1 100644 --- a/apps/kimi-code/package.json +++ b/apps/kimi-code/package.json @@ -91,6 +91,7 @@ "@moonshot-ai/kimi-telemetry": "workspace:^", "@moonshot-ai/kimi-web": "workspace:^", "@moonshot-ai/migration-legacy": "workspace:^", + "@moonshot-ai/minidb": "workspace:^", "@moonshot-ai/pi-tui": "workspace:^", "@moonshot-ai/vis-server": "workspace:^", "@moonshot-ai/vis-web": "workspace:*", diff --git a/apps/kimi-code/scripts/native/01-bundle.mjs b/apps/kimi-code/scripts/native/01-bundle.mjs index df46acc110..9f917e0196 100644 --- a/apps/kimi-code/scripts/native/01-bundle.mjs +++ b/apps/kimi-code/scripts/native/01-bundle.mjs @@ -15,6 +15,12 @@ export async function runBundleStep() { // miss it (npm builds get it via the `prebuild` script). await run(process.execPath, [buildVisAssetPath]); await run(process.execPath, [tsdownCliPath, '--config', 'tsdown.native.config.ts']); + // Bundle the minidb text-build worker into one self-contained ESM file so + // it can ride the SEA blob as an asset (02-sea-blob.mjs) and be spawned + // from disk at runtime — bundled binaries otherwise lack the worker entry + // and heavy text-index builds degrade to the inline main-thread core. + // Runs after the main bundle with clean:false so both verified files remain. + await run(process.execPath, [tsdownCliPath, '--config', 'tsdown.worker.config.ts']); await run(process.execPath, [checkBundlePath]); } diff --git a/apps/kimi-code/scripts/native/assets.mjs b/apps/kimi-code/scripts/native/assets.mjs index 859262449f..3c9f3b2047 100644 --- a/apps/kimi-code/scripts/native/assets.mjs +++ b/apps/kimi-code/scripts/native/assets.mjs @@ -5,7 +5,12 @@ import { createRequire } from 'node:module'; import { dirname, extname, isAbsolute, join, relative, resolve } from 'node:path'; import { pathToFileURL } from 'node:url'; -import { NATIVE_ASSET_MANIFEST_VERSION, buildManifestKey } from './manifest.mjs'; +import { + MINIDB_TEXT_BUILD_WORKER_ASSET, + NATIVE_ASSET_MANIFEST_VERSION, + buildManifestKey, + buildRuntimeAssetKey, +} from './manifest.mjs'; import { resolveTargetDeps, SUPPORTED_TARGETS } from './native-deps.mjs'; export { NATIVE_ASSET_MANIFEST_VERSION }; @@ -229,7 +234,10 @@ async function packageManifestEntries({ packageName, packageRoot, files, target export const nativeAssetManifestKey = buildManifestKey; export function nativeAssetSummary(manifest) { - return manifest.packages.map((pkg) => `${pkg.name}: ${pkg.files.length} files`); + return [ + ...manifest.packages.map((pkg) => `${pkg.name}: ${pkg.files.length} files`), + `runtime: ${manifest.runtimeFiles.length} files`, + ]; } export async function collectNativeAssets({ appRoot, target }) { @@ -264,10 +272,25 @@ export async function collectNativeAssets({ appRoot, target }) { Object.assign(assets, result.assets); } + const workerSource = resolve(appRoot, 'dist-native', 'intermediates', 'text-build-worker.mjs'); + const workerBytes = await readFile(workerSource); + const workerAssetKey = buildRuntimeAssetKey(target, MINIDB_TEXT_BUILD_WORKER_ASSET.key); + const runtimeFiles = [ + { + key: MINIDB_TEXT_BUILD_WORKER_ASSET.key, + assetKey: workerAssetKey, + relativePath: MINIDB_TEXT_BUILD_WORKER_ASSET.relativePath, + sha256: sha256(workerBytes), + mode: MINIDB_TEXT_BUILD_WORKER_ASSET.mode, + }, + ]; + assets[workerAssetKey] = workerSource; + const manifest = { version: NATIVE_ASSET_MANIFEST_VERSION, target, packages: manifestPackages, + runtimeFiles, }; return { diff --git a/apps/kimi-code/scripts/native/check-bundle.mjs b/apps/kimi-code/scripts/native/check-bundle.mjs index 3cd10c278d..bf63064068 100644 --- a/apps/kimi-code/scripts/native/check-bundle.mjs +++ b/apps/kimi-code/scripts/native/check-bundle.mjs @@ -1,10 +1,8 @@ +import { existsSync, readFileSync } from 'node:fs'; import { builtinModules } from 'node:module'; -import { readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; -import { nativeJsBundlePath } from './paths.mjs'; - -const bundlePath = nativeJsBundlePath(); -const text = readFileSync(bundlePath, 'utf-8'); +import { nativeIntermediatesDir, nativeJsBundlePath } from './paths.mjs'; const builtins = new Set([ ...builtinModules, @@ -23,18 +21,8 @@ const optionalRuntimeRequires = new Set([ 'utf-8-validate', ]); const optionalRelativeRuntimeRequires = new Set(['./crypto/build/Release/sshcrypto.node']); -const handledNativeRuntimeRequires = new Set(); - -function isAllowedSpecifier(specifier) { - if (builtins.has(specifier) || specifier.startsWith('node:')) return true; - if (optionalRuntimeRequires.has(specifier)) return true; - if (handledNativeRuntimeRequires.has(specifier)) return true; - return false; -} -const errors = []; - -function executableLines() { +function executableLines(text) { return text .split('\n') .map((line) => line.trim()) @@ -45,48 +33,51 @@ function executableLines() { }); } -for (const line of executableLines()) { - for (const match of line.matchAll(/(? { if (specifier.startsWith('.') || specifier.startsWith('/')) { - if (optionalRelativeRuntimeRequires.has(specifier)) continue; - errors.push(`relative require remains: ${specifier}`); - continue; + if (!allowedRelative.has(specifier)) errors.push(`relative ${kind} remains: ${specifier}`); + return; } - if (!isAllowedSpecifier(specifier)) { - errors.push(`external require remains: ${specifier}`); + if (!builtins.has(specifier) && !specifier.startsWith('node:') && !allowedExternal.has(specifier)) { + errors.push(`external ${kind} remains: ${specifier}`); } - } + }; - for (const match of line.matchAll(/(? 0) { - console.error(`Native JS bundle check failed for ${bundlePath}:`); - for (const error of errors) { - console.error(`- ${error}`); - } - process.exit(1); +const bundles = [ + { path: nativeJsBundlePath(), worker: false }, + { path: resolve(nativeIntermediatesDir(), 'text-build-worker.mjs'), worker: true }, +]; +let failed = false; +for (const bundle of bundles) { + const errors = checkBundle(bundle.path, { worker: bundle.worker }); + if (errors.length === 0) continue; + failed = true; + console.error(`Native JS bundle check failed for ${bundle.path}:`); + for (const error of errors) console.error(`- ${error}`); } +if (failed) process.exit(1); diff --git a/apps/kimi-code/scripts/native/manifest.mjs b/apps/kimi-code/scripts/native/manifest.mjs index 30d5e9da32..1344a24f46 100644 --- a/apps/kimi-code/scripts/native/manifest.mjs +++ b/apps/kimi-code/scripts/native/manifest.mjs @@ -1,10 +1,20 @@ -export const NATIVE_ASSET_MANIFEST_VERSION = 1; +export const NATIVE_ASSET_MANIFEST_VERSION = 2; export const WEB_ASSET_MANIFEST_VERSION = 1; +export const MINIDB_TEXT_BUILD_WORKER_ASSET = Object.freeze({ + key: 'minidb-text-build-worker', + relativePath: 'runtime/minidb/text-build-worker.mjs', + mode: 0o644, +}); + export function buildManifestKey(target) { return `native/${target}/manifest.json`; } +export function buildRuntimeAssetKey(target, key) { + return `native/${target}/runtime/${key}`; +} + export function isManifestVersionSupported(version) { return version === NATIVE_ASSET_MANIFEST_VERSION; } diff --git a/apps/kimi-code/scripts/native/smoke.mjs b/apps/kimi-code/scripts/native/smoke.mjs index ed3a8624f7..0d0f2604b4 100644 --- a/apps/kimi-code/scripts/native/smoke.mjs +++ b/apps/kimi-code/scripts/native/smoke.mjs @@ -1,5 +1,5 @@ import { execFile } from 'node:child_process'; -import { readFile, stat } from 'node:fs/promises'; +import { mkdir, readFile, rm, stat } from 'node:fs/promises'; import { resolve } from 'node:path'; import { promisify } from 'node:util'; @@ -73,10 +73,19 @@ assertIncludes(helpOutput, 'Usage: kimi', '--help'); const exportHelpOutput = await runKimi(['export', '--help']); assertIncludes(exportHelpOutput, 'Usage: kimi export', 'export --help'); -const nativeAssetOutput = await runKimiWithEnv(['--version'], { - KIMI_CODE_HOME: smokeHome, - KIMI_CODE_NATIVE_ASSET_SMOKE: '1', -}); -assertIncludes(nativeAssetOutput, `Native asset smoke passed: ${target}`, 'native asset smoke'); +const smokeCache = resolve(smokeHome, 'cache'); +await rm(smokeHome, { recursive: true, force: true }); +await mkdir(smokeCache, { recursive: true }); +try { + const nativeAssetOutput = await runKimiWithEnv(['--version'], { + KIMI_CODE_CACHE_DIR: smokeCache, + KIMI_CODE_HOME: smokeHome, + KIMI_CODE_NATIVE_ASSET_SMOKE: '1', + }); + assertIncludes(nativeAssetOutput, `Native asset smoke passed: ${target}`, 'native asset smoke'); + assertIncludes(nativeAssetOutput, 'MiniDb worker build passed', 'MiniDb worker smoke'); +} finally { + await rm(smokeHome, { recursive: true, force: true }); +} console.log(`Native smoke passed: ${executablePath}`); diff --git a/apps/kimi-code/src/cli/run-shell.ts b/apps/kimi-code/src/cli/run-shell.ts index 3e1a9b89c2..35b0ca9e01 100644 --- a/apps/kimi-code/src/cli/run-shell.ts +++ b/apps/kimi-code/src/cli/run-shell.ts @@ -25,6 +25,7 @@ import type { TuiConfig } from '#/tui/config'; import { loadTuiConfig, TuiConfigParseError } from '#/tui/config'; import { CHROME_GUTTER } from '#/tui/constant/rendering'; import { KimiTUI } from '#/tui/index'; +import { startupTrace } from '#/utils/startup-trace'; import { currentTheme, getColorPalette } from '#/tui/theme'; import { toTerminalHyperlink } from '#/utils/terminal-hyperlink'; import { restoreTerminalModes } from '#/utils/terminal-restore'; @@ -87,6 +88,7 @@ export async function runShell( const harness = engineV2 ? createKimiHarnessV2(harnessOptions) : createKimiHarness(harnessOptions); + startupTrace('harness:created'); log.info('kimi-code starting', { version, uiMode: CLI_UI_MODE, @@ -107,6 +109,7 @@ export async function runShell( return; } const config = await harness.getConfig(); + startupTrace('config:loaded'); // Config diagnostics (deprecated keys, invalid sections, ...) are surfaced // by the TUI itself at `finishStartup` via `showConfigWarningsIfAny` — // folded into the dim startup notice they were too easy to miss. @@ -243,7 +246,9 @@ export async function runShell( }; try { const initStartedAt = Date.now(); + startupTrace('tui.start:begin'); await tui.start(); + startupTrace('tui.start:end'); const initMs = Date.now() - initStartedAt; const startupSessionId = tui.getCurrentSessionId(); const mcpMs = await tui.getStartupMcpMs(); diff --git a/apps/kimi-code/src/cli/v2/run-v2-print.ts b/apps/kimi-code/src/cli/v2/run-v2-print.ts index 891032b4d8..aac6062fce 100644 --- a/apps/kimi-code/src/cli/v2/run-v2-print.ts +++ b/apps/kimi-code/src/cli/v2/run-v2-print.ts @@ -330,8 +330,7 @@ async function resolveNativeSession( }; if (opts.session !== undefined) { - const page = await index.list({}); - const target = page.items.find((summary) => summary.id === opts.session); + const target = await index.get(opts.session); if (target === undefined) { throw new Error(`Session "${opts.session}" not found.`); } @@ -358,7 +357,7 @@ async function resolveNativeSession( } if (opts.continue) { - const page = await index.list({}); + const page = await index.listRecent({}); const previous = page.items.find((summary) => summary.cwd === workDir); if (previous !== undefined) { const session = await resumeById(previous.id); diff --git a/apps/kimi-code/src/main.ts b/apps/kimi-code/src/main.ts index 28ed63a900..7c4e1040b0 100644 --- a/apps/kimi-code/src/main.ts +++ b/apps/kimi-code/src/main.ts @@ -24,6 +24,7 @@ import { import { createProgram } from './cli/commands'; import { finalizeHeadlessRun } from './cli/headless-exit'; +import { startupTrace } from './utils/startup-trace'; import type { CLIOptions } from './cli/options'; import { OptionConflictError, validateOptions } from './cli/options'; import { runPrompt } from './cli/run-prompt'; @@ -36,6 +37,7 @@ import { runUpdatePreflight } from './cli/update/preflight'; import { createKimiCodeHostIdentity, getVersion } from './cli/version'; import { CLI_SHUTDOWN_TIMEOUT_MS, CLI_UI_MODE, PROCESS_NAME } from './constant/app'; import { cleanupStaleNativeCacheForCurrent } from './native/native-assets'; +import { installMinidbTextBuildWorker } from './native/minidb-worker'; import { installNativeModuleHook } from './native/module-hook'; import { runNativeAssetSmokeIfRequested } from './native/smoke'; @@ -56,6 +58,7 @@ export async function handleMainCommand( version: string, ): Promise { let validated: ReturnType; + startupTrace('main:enter'); try { validated = validateOptions(opts); } catch (error) { @@ -66,10 +69,12 @@ export async function handleMainCommand( throw error; } + startupTrace('preflight:begin'); const preflightResult = await runUpdatePreflight( version, validated.uiMode === 'print' ? { track, isTTY: false } : { track }, ); + startupTrace('preflight:end'); if (preflightResult === 'exit') { process.exit(0); } @@ -79,6 +84,7 @@ export async function handleMainCommand( return { headlessCompleted: true }; } + startupTrace('runShell:begin'); await runShell(validated.options, version); return { headlessCompleted: false }; } @@ -142,6 +148,16 @@ export function main(): void { // invalid proxy URL is reported and ignored rather than aborting startup. installGlobalProxyDispatcher(); installNativeModuleHook(); + // Best-effort SEA worker installation. Diagnostics are trace-only and avoid + // exposing the user's cache path; failure keeps MiniDb's bounded inline mode. + const workerInstall = installMinidbTextBuildWorker(); + startupTrace( + workerInstall.status === 'installed' + ? `minidb-worker:installed basename=${workerInstall.basename} sha256=${workerInstall.assetSha256}` + : workerInstall.status === 'failed' + ? `minidb-worker:failed code=${workerInstall.errorCode} sha256=${workerInstall.assetSha256 ?? 'unknown'}` + : `minidb-worker:${workerInstall.status}`, + ); if (runNativeAssetSmokeIfRequested()) return; // Start the background cleanup of stale native cache. Fire-and-forget; must not block startup or throw. diff --git a/apps/kimi-code/src/native/minidb-worker.ts b/apps/kimi-code/src/native/minidb-worker.ts new file mode 100644 index 0000000000..babad491e5 --- /dev/null +++ b/apps/kimi-code/src/native/minidb-worker.ts @@ -0,0 +1,69 @@ +import { basename } from 'node:path'; + +import { + configureTextBuildWorkerRuntime, + getTextBuildWorkerRuntimeState, +} from '@moonshot-ai/minidb/worker-runtime'; + +import { MINIDB_TEXT_BUILD_WORKER_ASSET } from '../../scripts/native/manifest.mjs'; +import { + getEmbeddedNativeAssetManifest, + getMinidbTextBuildWorkerFile, + getSeaAssetSource, + type NativeAssetOptions, +} from './native-assets'; + +export type MinidbTextBuildWorkerInstallStatus = + | { readonly status: 'not-sea' } + | { readonly status: 'asset-missing' } + | { + readonly status: 'installed'; + readonly assetSha256: string; + readonly basename: string; + } + | { + readonly status: 'failed'; + readonly errorCode: string; + readonly assetSha256?: string; + }; + +function errorCode(error: unknown): string { + const code = (error as NodeJS.ErrnoException | undefined)?.code; + if (typeof code === 'string' && code.length > 0) return code; + return error instanceof Error ? error.name : 'UNKNOWN'; +} + +/** Install the SEA-bundled worker without making optional extraction fatal. */ +export function installMinidbTextBuildWorker( + options: NativeAssetOptions = {}, +): MinidbTextBuildWorkerInstallStatus { + const source = options.source ?? getSeaAssetSource(); + if (source === null) return { status: 'not-sea' }; + + let assetSha256: string | undefined; + try { + const manifest = options.manifest ?? getEmbeddedNativeAssetManifest(source); + const file = manifest?.runtimeFiles.find( + (entry) => entry.key === MINIDB_TEXT_BUILD_WORKER_ASSET.key, + ); + if (manifest === null || file === undefined) return { status: 'asset-missing' }; + assetSha256 = file.sha256; + + const workerPath = getMinidbTextBuildWorkerFile({ ...options, source, manifest }); + if (workerPath === null) return { status: 'asset-missing' }; + configureTextBuildWorkerRuntime(workerPath); + const runtime = getTextBuildWorkerRuntimeState(); + if (!runtime.configured) throw new Error('MiniDb worker runtime was not configured'); + return { + status: 'installed', + assetSha256, + basename: basename(workerPath), + }; + } catch (error) { + return { + status: 'failed', + errorCode: errorCode(error), + assetSha256, + }; + } +} diff --git a/apps/kimi-code/src/native/native-assets.ts b/apps/kimi-code/src/native/native-assets.ts index d66547695c..a692246614 100644 --- a/apps/kimi-code/src/native/native-assets.ts +++ b/apps/kimi-code/src/native/native-assets.ts @@ -11,11 +11,15 @@ import { } from 'node:fs'; import { createRequire } from 'node:module'; import { homedir } from 'node:os'; -import { dirname, join, win32 as pathWin32 } from 'node:path'; +import { dirname, isAbsolute, join, relative, resolve, win32 as pathWin32 } from 'node:path'; import { join as joinPosix } from 'pathe'; import { KIMI_BUILD_INFO } from '#/cli/build-info'; -import { NATIVE_ASSET_MANIFEST_VERSION as MANIFEST_VERSION, buildManifestKey } from '../../scripts/native/manifest.mjs'; +import { + MINIDB_TEXT_BUILD_WORKER_ASSET, + NATIVE_ASSET_MANIFEST_VERSION as MANIFEST_VERSION, + buildManifestKey, +} from '../../scripts/native/manifest.mjs'; export const NATIVE_ASSET_MANIFEST_VERSION = MANIFEST_VERSION; @@ -32,10 +36,15 @@ export interface NativeAssetPackage { readonly files: readonly NativeAssetFile[]; } +export interface NativeRuntimeAssetFile extends NativeAssetFile { + readonly key: string; +} + export interface NativeAssetManifest { readonly version: typeof NATIVE_ASSET_MANIFEST_VERSION; readonly target: string; readonly packages: readonly NativeAssetPackage[]; + readonly runtimeFiles: readonly NativeRuntimeAssetFile[]; } export interface NativeAssetSource { @@ -53,10 +62,6 @@ export interface NativeAssetOptions { readonly version?: string; } -type RawNativeAssetManifest = Omit & { - readonly version: number; -}; - interface NodeSeaModule { isSea(): boolean; getAssetKeys(): string[]; @@ -97,6 +102,149 @@ function sha256(bytes: Buffer | Uint8Array | string): string { return createHash('sha256').update(bytes).digest('hex'); } +function manifestObject(value: unknown, label: string): Record { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new Error(`Invalid native asset manifest: ${label} must be an object`); + } + return value as Record; +} + +function manifestString(value: unknown, label: string): string { + if (typeof value !== 'string' || value.length === 0) { + throw new Error(`Invalid native asset manifest: ${label} must be a non-empty string`); + } + return value; +} + +function validateRelativePath(value: unknown, label: string): string { + const path = manifestString(value, label); + const segments = path.split(/[\\/]/); + if ( + isAbsolute(path) || + /^[a-zA-Z]:/.test(path) || + path.startsWith('\\\\') || + segments.some((segment) => segment.length === 0 || segment === '.' || segment === '..') + ) { + throw new Error(`Invalid native asset manifest: ${label} must be a safe relative path`); + } + return path; +} + +function validateAssetFile( + value: unknown, + label: string, + assetKeys: Set, + relativePaths: Set, +): NativeAssetFile { + const file = manifestObject(value, label); + const assetKey = manifestString(file['assetKey'], `${label}.assetKey`); + if (assetKeys.has(assetKey)) { + throw new Error(`Invalid native asset manifest: duplicate assetKey ${assetKey}`); + } + assetKeys.add(assetKey); + const relativePath = validateRelativePath(file['relativePath'], `${label}.relativePath`); + const portableRelativePath = relativePath.replaceAll('\\', '/'); + if (relativePaths.has(portableRelativePath)) { + throw new Error(`Invalid native asset manifest: duplicate relativePath ${relativePath}`); + } + relativePaths.add(portableRelativePath); + const fileSha256 = file['sha256']; + if (typeof fileSha256 !== 'string' || !/^[a-f0-9]{64}$/.test(fileSha256)) { + throw new Error(`Invalid native asset manifest: ${label}.sha256 must be 64 lowercase hex characters`); + } + const mode = file['mode']; + if ( + mode !== undefined && + (!Number.isInteger(mode) || (mode as number) < 0 || (mode as number) > 0o777) + ) { + throw new Error(`Invalid native asset manifest: ${label}.mode must be an integer between 0 and 0777`); + } + return { + assetKey, + relativePath, + sha256: fileSha256, + mode: mode as number | undefined, + }; +} + +export function validateNativeAssetManifest( + value: unknown, + expectedTarget?: string, +): NativeAssetManifest { + const manifest = manifestObject(value, 'root'); + if (manifest['version'] !== NATIVE_ASSET_MANIFEST_VERSION) { + throw new Error(`Unsupported native asset manifest version: ${String(manifest['version'])}`); + } + const target = manifestString(manifest['target'], 'target'); + if (expectedTarget !== undefined && target !== expectedTarget) { + throw new Error(`Native asset manifest target mismatch: ${target} !== ${expectedTarget}`); + } + const manifestPackages = manifest['packages']; + if (!Array.isArray(manifestPackages)) { + throw new TypeError('Invalid native asset manifest: packages must be an array'); + } + const manifestRuntimeFiles = manifest['runtimeFiles']; + if (!Array.isArray(manifestRuntimeFiles)) { + throw new TypeError('Invalid native asset manifest: runtimeFiles must be an array'); + } + + const assetKeys = new Set(); + const relativePaths = new Set(); + const packageNames = new Set(); + const packages = manifestPackages.map((value, packageIndex): NativeAssetPackage => { + const label = `packages[${packageIndex}]`; + const pkg = manifestObject(value, label); + const name = manifestString(pkg['name'], `${label}.name`); + if (packageNames.has(name)) { + throw new Error(`Invalid native asset manifest: duplicate package name ${name}`); + } + packageNames.add(name); + const root = validateRelativePath(pkg['root'], `${label}.root`); + const packageFiles = pkg['files']; + if (!Array.isArray(packageFiles)) { + throw new TypeError(`Invalid native asset manifest: ${label}.files must be an array`); + } + return { + name, + root, + files: packageFiles.map((file, fileIndex) => + validateAssetFile(file, `${label}.files[${fileIndex}]`, assetKeys, relativePaths), + ), + }; + }); + + const runtimeKeys = new Set(); + const runtimeFiles = manifestRuntimeFiles.map((value, index): NativeRuntimeAssetFile => { + const label = `runtimeFiles[${index}]`; + const raw = manifestObject(value, label); + const key = manifestString(raw['key'], `${label}.key`); + if (runtimeKeys.has(key)) { + throw new Error(`Invalid native asset manifest: duplicate runtime key ${key}`); + } + runtimeKeys.add(key); + return { + ...validateAssetFile(raw, label, assetKeys, relativePaths), + key, + }; + }); + + return { + version: NATIVE_ASSET_MANIFEST_VERSION, + target, + packages, + runtimeFiles, + }; +} + +function resolveAssetPath(cacheRoot: string, relativePath: string): string { + const path = resolve(cacheRoot, ...relativePath.split(/[\\/]/)); + const fromRoot = relative(cacheRoot, path); + if (fromRoot === '..' || fromRoot.startsWith('../') || fromRoot.startsWith('..\\') || isAbsolute(fromRoot)) { + throw new Error(`Native asset path escapes cache root: ${relativePath}`); + } + return path; +} + function optionalEnvValue(env: NodeJS.ProcessEnv, key: string): string | null { const value = env[key]; return typeof value === 'string' && value.length > 0 ? value : null; @@ -124,14 +272,9 @@ export function getEmbeddedNativeAssetManifest( const key = nativeAssetManifestKey(target); if (!source.getAssetKeys().includes(key)) return null; const raw = source.getRawAsset(key); - const manifest = JSON.parse(toBuffer(raw).toString('utf-8')) as RawNativeAssetManifest; - if (manifest.version !== NATIVE_ASSET_MANIFEST_VERSION) { - throw new Error(`Unsupported native asset manifest version: ${manifest.version}`); - } - if (manifest.target !== target) { - throw new Error(`Native asset manifest target mismatch: ${manifest.target} !== ${target}`); - } - return manifest as NativeAssetManifest; + const parsed: unknown = JSON.parse(toBuffer(raw).toString('utf-8')); + validateNativeAssetManifest(parsed, target); + return parsed as NativeAssetManifest; } export function getNativeCacheBase(options: NativeAssetOptions = {}): string { @@ -159,13 +302,14 @@ export function getNativeAssetCacheRoot( manifest: NativeAssetManifest, options: NativeAssetOptions = {}, ): string { + const validated = validateNativeAssetManifest(manifest); const version = sanitizeSegment(options.version ?? KIMI_BUILD_INFO.version ?? 'dev'); const manifestHash = sha256(JSON.stringify(manifest)); return join( getNativeCacheBase(options), 'native', version, - sanitizeSegment(manifest.target), + sanitizeSegment(validated.target), manifestHash, ); } @@ -219,27 +363,59 @@ export function ensureNativeAssetTree(options: NativeAssetOptions = {}): string const source = options.source ?? getSeaAssetSource(); if (source === null) return null; - const manifest = + const rawManifest = options.manifest ?? getEmbeddedNativeAssetManifest(source, currentTarget()); - if (manifest === null) return null; - - const cacheRoot = getNativeAssetCacheRoot(manifest, options); - for (const pkg of manifest.packages) { - for (const file of pkg.files) { - const bytes = toBuffer(source.getRawAsset(file.assetKey)); - const actualSha256 = sha256(bytes); - if (actualSha256 !== file.sha256) { - throw new Error( - `Native asset checksum mismatch for ${file.assetKey}: ${actualSha256} !== ${file.sha256}`, - ); - } - ensureFile(join(cacheRoot, file.relativePath), bytes, file.sha256, file.mode); + if (rawManifest === null) return null; + const manifest = validateNativeAssetManifest(rawManifest); + + const cacheRoot = getNativeAssetCacheRoot(rawManifest, options); + const sourceKeys = new Set(source.getAssetKeys()); + const files = [ + ...manifest.packages.flatMap((pkg) => pkg.files), + ...manifest.runtimeFiles, + ]; + for (const file of files) { + if (!sourceKeys.has(file.assetKey)) { + throw new Error(`Native asset is missing: ${file.assetKey}`); } + const bytes = toBuffer(source.getRawAsset(file.assetKey)); + const actualSha256 = sha256(bytes); + if (actualSha256 !== file.sha256) { + throw new Error( + `Native asset checksum mismatch for ${file.assetKey}: ${actualSha256} !== ${file.sha256}`, + ); + } + ensureFile(resolveAssetPath(cacheRoot, file.relativePath), bytes, file.sha256, file.mode); } ensureEntryFile(cacheRoot); return cacheRoot; } +export function getNativeRuntimeFile( + key: string, + options: NativeAssetOptions = {}, +): string | null { + const source = options.source ?? getSeaAssetSource(); + if (source === null) return null; + + const rawManifest = + options.manifest ?? getEmbeddedNativeAssetManifest(source, currentTarget()); + if (rawManifest === null) return null; + const manifest = validateNativeAssetManifest(rawManifest); + + const file = manifest.runtimeFiles.find((entry) => entry.key === key); + if (file === undefined) return null; + + const cacheRoot = ensureNativeAssetTree({ ...options, source, manifest: rawManifest }); + return cacheRoot === null ? null : resolveAssetPath(cacheRoot, file.relativePath); +} + +export function getMinidbTextBuildWorkerFile( + options: NativeAssetOptions = {}, +): string | null { + return getNativeRuntimeFile(MINIDB_TEXT_BUILD_WORKER_ASSET.key, options); +} + export function getNativePackageRoot( packageName: string, options: NativeAssetOptions = {}, @@ -247,15 +423,16 @@ export function getNativePackageRoot( const source = options.source ?? getSeaAssetSource(); if (source === null) return null; - const manifest = + const rawManifest = options.manifest ?? getEmbeddedNativeAssetManifest(source, currentTarget()); - if (manifest === null) return null; + if (rawManifest === null) return null; + const manifest = validateNativeAssetManifest(rawManifest); const pkg = manifest.packages.find((entry) => entry.name === packageName); if (pkg === undefined) return null; - const cacheRoot = ensureNativeAssetTree({ ...options, source, manifest }); - return cacheRoot === null ? null : join(cacheRoot, pkg.root); + const cacheRoot = ensureNativeAssetTree({ ...options, source, manifest: rawManifest }); + return cacheRoot === null ? null : resolveAssetPath(cacheRoot, pkg.root); } export function hasNativePackage(packageName: string, manifest: NativeAssetManifest): boolean { diff --git a/apps/kimi-code/src/native/smoke.ts b/apps/kimi-code/src/native/smoke.ts index c77f1419d0..1d330bc877 100644 --- a/apps/kimi-code/src/native/smoke.ts +++ b/apps/kimi-code/src/native/smoke.ts @@ -1,14 +1,17 @@ +import { mkdtempSync, mkdirSync, rmSync } from 'node:fs'; import { createRequire } from 'node:module'; import { dirname, join } from 'node:path'; -import { getEmbeddedNativeAssetManifest, getNativePackageRoot } from './native-assets'; +import { MiniDb } from '@moonshot-ai/minidb'; + +import { + getEmbeddedNativeAssetManifest, + getNativeCacheBase, + getNativePackageRoot, +} from './native-assets'; const smokePackages = ['@mariozechner/clipboard', '@moonshot-ai/pi-tui']; -// Verify pi-tui's native helper can actually be loaded through the module hook. -// pi-tui computes native helper paths from process.execPath and require()s them; -// those paths do not exist next to the SEA binary, so this only succeeds when -// installNativeModuleHook() redirects the require into the native-asset cache. function smokePiTuiNativeLoad(): void { const platform = process.platform; const arch = process.arch; @@ -18,42 +21,81 @@ function smokePiTuiNativeLoad(): void { } else if (platform === 'win32' && (arch === 'x64' || arch === 'arm64')) { rel = join('native', 'win32', 'prebuilds', `win32-${arch}`, 'win32-console-mode.node'); } - if (rel === undefined) return; // Linux: no native helper, nothing to load. + if (rel === undefined) return; const req = createRequire(import.meta.url); - const bogusPath = join(dirname(process.execPath), rel); - const helper = req(bogusPath) as { + const helper = req(join(dirname(process.execPath), rel)) as { isModifierPressed?: unknown; enableVirtualTerminalInput?: unknown; }; - const ok = - typeof helper.isModifierPressed === 'function' || - typeof helper.enableVirtualTerminalInput === 'function'; - if (!ok) { - throw new Error(`pi-tui native helper loaded but exports are unexpected: ${rel}`); + if ( + typeof helper.isModifierPressed !== 'function' && + typeof helper.enableVirtualTerminalInput !== 'function' + ) { + throw new TypeError(`pi-tui native helper exports are unexpected: ${rel}`); } } -export function runNativeAssetSmokeIfRequested(): boolean { - if (process.env['KIMI_CODE_NATIVE_ASSET_SMOKE'] !== '1') return false; - +async function smokeMinidbWorker(): Promise { + const cacheBase = getNativeCacheBase(); + mkdirSync(cacheBase, { recursive: true }); + const dir = mkdtempSync(join(cacheBase, 'sea-minidb-smoke-')); + let db: MiniDb> | null = null; try { - const manifest = getEmbeddedNativeAssetManifest(); - if (manifest === null) { - throw new Error('Native asset manifest is not available.'); + db = await MiniDb.open>({ dir, valueCodec: 'json' }); + const total = 4_200; + for (let base = 0; base < total; base += 500) { + await db.batch( + Array.from({ length: Math.min(500, total - base) }, (_, offset) => { + const id = base + offset; + return { + op: 'set' as const, + key: `doc-${id}`, + value: { text: `sea worker searchable document ${id}` }, + }; + }), + ); + } + await db.createTextIndex('smoke', { fields: ['text'] }); + if (db.stats.textWorkerBuilds < 1) { + throw new Error(`MiniDb worker did not run: ${JSON.stringify(db.stats)}`); + } + if (db.stats.textWorkerFallbacks !== 0) { + throw new Error( + `MiniDb worker unexpectedly fell back: ${db.stats.lastTextWorkerFallback ?? 'unknown'}`, + ); + } + if (!db.search('smoke', 'searchable').some((hit) => hit.key === 'doc-0')) { + throw new Error('MiniDb worker-built text index returned an incorrect search result'); } - for (const packageName of smokePackages) { - const packageRoot = getNativePackageRoot(packageName, { manifest }); - if (packageRoot === null) { - throw new Error(`Native package is not available: ${packageName}`); - } + } finally { + await db?.close().catch(() => {}); + rmSync(dir, { recursive: true, force: true }); + } +} + +async function runSmoke(): Promise { + const manifest = getEmbeddedNativeAssetManifest(); + if (manifest === null) throw new Error('Native asset manifest is not available.'); + for (const packageName of smokePackages) { + if (getNativePackageRoot(packageName, { manifest }) === null) { + throw new Error(`Native package is not available: ${packageName}`); } - smokePiTuiNativeLoad(); - process.stdout.write(`Native asset smoke passed: ${manifest.target}\n`); - process.exit(0); - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - process.stderr.write(`Native asset smoke failed: ${message}\n`); - process.exit(1); } + smokePiTuiNativeLoad(); + await smokeMinidbWorker(); + process.stdout.write(`Native asset smoke passed: ${manifest.target}; MiniDb worker build passed\n`); +} + +export function runNativeAssetSmokeIfRequested(): boolean { + if (process.env['KIMI_CODE_NATIVE_ASSET_SMOKE'] !== '1') return false; + void runSmoke().then( + () => process.exit(0), + (error: unknown) => { + const message = error instanceof Error ? error.message : String(error); + process.stderr.write(`Native asset smoke failed: ${message}\n`); + process.exit(1); + }, + ); + return true; } diff --git a/apps/kimi-code/src/tui/kimi-tui.ts b/apps/kimi-code/src/tui/kimi-tui.ts index 714dd7e4b7..a979de7219 100644 --- a/apps/kimi-code/src/tui/kimi-tui.ts +++ b/apps/kimi-code/src/tui/kimi-tui.ts @@ -144,6 +144,8 @@ import { formatErrorMessage } from './utils/event-payload'; import { pickForegroundTasks } from './utils/foreground-task'; import { ImageAttachmentStore, type ImageAttachment } from './utils/image-attachment-store'; import { extractMediaAttachments, rewriteMediaPlaceholders } from './utils/image-placeholder'; +import { installInputLatencyProbe } from './utils/input-latency'; +import { startupTrace } from '#/utils/startup-trace'; import { REPLAY_TURN_LIMIT } from './utils/message-replay'; import { hasPatchChanges } from './utils/object-patch'; import { sessionRowsForPicker } from './utils/session-picker-rows'; @@ -574,6 +576,7 @@ export class KimiTUI { // ========================================================================= async start(): Promise { + startupTrace('tui:start'); // Signal handlers must be installed before raw mode to avoid EIO loops. this.registerSignalHandlers(); // Outer try rolls back signal listeners on startup failure. @@ -601,16 +604,25 @@ export class KimiTUI { return; } + startupTrace('trustPrompt:begin'); const trustPromptStartedLoop = await this.maybeRunWorkspaceTrustPrompt(); + startupTrace('trustPrompt:end'); + startupTrace('initMainTui:begin'); const shouldReplayHistory = await this.initMainTui(); + startupTrace('initMainTui:end'); + // Debug-only input→render latency overlay (KIMI_TUI_INPUT_LATENCY=1). + if (process.env['KIMI_TUI_INPUT_LATENCY']) installInputLatencyProbe(this.state.ui); // When the trust prompt already started the event loop, starting it // again would re-run pi-tui's terminal.start() — stacking a second // Kitty keyboard-protocol push (leaking CSI-u mode past exit) and // duplicate stdin listeners. if (!trustPromptStartedLoop) this.startEventLoop(); + startupTrace('eventLoop:started'); try { this.startBackgroundFdAutocomplete(); + startupTrace('finishStartup:begin'); await this.finishStartup(shouldReplayHistory); + startupTrace('finishStartup:end'); } catch (error) { this.disposeTerminalTracking(); this.state.ui.stop(); diff --git a/apps/kimi-code/src/tui/utils/input-latency.ts b/apps/kimi-code/src/tui/utils/input-latency.ts new file mode 100644 index 0000000000..8ad69f718b --- /dev/null +++ b/apps/kimi-code/src/tui/utils/input-latency.ts @@ -0,0 +1,105 @@ +// src/tui/utils/input-latency.ts +// +// Debug-only input→render latency probe, enabled with KIMI_TUI_INPUT_LATENCY=1. +// Registers a pi-tui input listener (event timestamps) and mounts a +// non-capturing overlay in the top-right corner whose render() drains the +// queue: each pending input event is stamped against the frame that first +// renders after it, and the overlay shows the live stats (last / p50 / p95 / +// p99 / max, plus >100ms / >300ms / >1s counters and the five worst samples). +// Optional JSONL sink: KIMI_TUI_INPUT_LATENCY_LOG= appends one record +// per event for post-hoc analysis. +// +// The measured latency is "input event → start of the first frame rendered +// after it" — it includes input handling and the 16ms render throttle, and +// underestimates by the frame's own diff/write tail (sub-ms to a few ms), +// which is the right granularity for diagnosing >100ms stalls. + +import { appendFileSync, mkdirSync } from 'node:fs'; +import path from 'node:path'; +import type { Component, TUI } from '@moonshot-ai/pi-tui'; + +/** Rolling sample cap for the percentile window. */ +const MAX_SAMPLES = 500; + +export interface LatencySample { + latency: number; + at: string; +} + +/** The pure stats core (exported for tests): feed it input→render latencies + * and it keeps the rolling window, counters, and the five worst samples. */ +export class LatencyStats { + last = 0; + events = 0; + over100 = 0; + over300 = 0; + over1000 = 0; + readonly worst: LatencySample[] = []; + private readonly samples: number[] = []; + + record(latency: number, at: string): void { + this.last = latency; + this.events++; + if (latency > 100) this.over100++; + if (latency > 300) this.over300++; + if (latency > 1000) this.over1000++; + this.samples.push(latency); + if (this.samples.length > MAX_SAMPLES) this.samples.shift(); + const smallestKept = this.worst[this.worst.length - 1]?.latency ?? -1; + if (this.worst.length < 5 || latency >= smallestKept) { + this.worst.push({ latency, at }); + this.worst.sort((a, b) => b.latency - a.latency); + if (this.worst.length > 5) this.worst.length = 5; + } + } + + percentile(p: number): number { + if (this.samples.length === 0) return 0; + const sorted = [...this.samples].sort((a, b) => a - b); + return sorted[Math.min(sorted.length - 1, Math.ceil((p / 100) * sorted.length) - 1)]!; + } + + max(): number { + return this.samples.length === 0 ? 0 : Math.max(...this.samples); + } + + formatLines(): string[] { + if (this.events === 0) return [' input→render: (type something) ']; + const head = + ` io ${this.last.toFixed(0)}ms | p50 ${this.percentile(50).toFixed(0)} p95 ${this.percentile(95).toFixed(0)}` + + ` p99 ${this.percentile(99).toFixed(0)} max ${this.max().toFixed(0)}ms | n=${this.events}` + + ` >100:${this.over100} >300:${this.over300} >1s:${this.over1000} `; + const worstLine = ` worst: ${this.worst.map((w) => `${w.latency.toFixed(0)}ms@${w.at}`).join(' ')} `; + return [head, worstLine]; + } +} + +/** Install the probe on a running TUI (call only when the env flag is set). */ +export function installInputLatencyProbe(tui: TUI): void { + const stats = new LatencyStats(); + const pending: number[] = []; + const logPath = process.env['KIMI_TUI_INPUT_LATENCY_LOG']; + if (logPath) mkdirSync(path.dirname(logPath), { recursive: true }); + + tui.addInputListener(() => { + pending.push(performance.now()); + return undefined; + }); + + const overlay: Component = { + invalidate: () => {}, + render: () => { + if (pending.length > 0) { + const now = performance.now(); + const at = new Date().toISOString().slice(11, 23); + for (const t of pending.splice(0)) { + const latency = now - t; + stats.record(latency, at); + if (logPath) appendFileSync(logPath, `${JSON.stringify({ t: new Date().toISOString(), latencyMs: Math.round(latency) })}\n`); + } + } + return stats.formatLines(); + }, + }; + tui.showOverlay(overlay, { nonCapturing: true, anchor: 'top-right', margin: 0 }); +} diff --git a/apps/kimi-code/src/utils/startup-trace.ts b/apps/kimi-code/src/utils/startup-trace.ts new file mode 100644 index 0000000000..65ac7eb441 --- /dev/null +++ b/apps/kimi-code/src/utils/startup-trace.ts @@ -0,0 +1,34 @@ +// src/utils/startup-trace.ts +// +// Debug-only startup phase tracer, enabled with KIMI_STARTUP_TRACE=1. +// Each call appends one `